[antlr-interest] Lexer inheritance problem: nextToken ignores super implementation

[Sebastian Baltes]

Hello Antlers,

as a newbie I have a problem with inheritance of lexers and would ask for
some help from the experts. :-)

I have a lexer InCoTempAbstractLexer that parses a litte if - else grammar.
This works well. Then I tries to make a second lexer InCoTempLineLexer that
should use the same gramar and just extends it with a new token, but this
lexer does not recognize anything. As far as I understand the problem lies
within the nextToken method. The generated java class InCoTempLineLexer
extends InCoTempAbstractLexer, but the generated nextToken method does not
implement or call any logic of the parents implementation.

Is this a bug in my grammar or in ANTLR? I use eclipse ANTLR studio.

Best Regards and thank for any help

Sebastian Baltes

---- file incotemp/incotempAbstractLexer.g

header {
  package incotemp;
}

class InCoTempAbstractLexer extends Lexer;

options {
  k=2;
  exportVocab = InCoTempBase;
}  

tokens {
  IF               = "if";
  ELSE             = "else";
  END              = "end";
}

IDENT options { testLiterals = true;}
  : ('a'..'z'|'A'..'Z'|'0'..'9'|'_')+
  ;

WHITESPACE : ( ' ' | '\t' ) {$setType(Token.SKIP);} ;

---- file incotemp/incotempLineLexer.g

header {
  package incotemp;
}

class InCoTempLineLexer extends Lexer("incotemp.InCoTempAbstractLexer");

options {
  importVocab = InCoTempBase;
}


protected
NEWLINE :  ('\n' | '\r' ('\n')?)  {$setType(Token.SKIP); newline();
GlobalLexers.getInstance().pop(); } ;


---- incotemp/InCoTempAbstractLexer.java


public class InCoTempAbstractLexer extends
antlr.antlrStudio.ASDebugCharScanner implements InCoTempBaseTokenTypes,
TokenStream
{

...

public InCoTempAbstractLexer(LexerSharedInputState state) {
  super(state);
  caseSensitiveLiterals = true;
  setCaseSensitive(true);
  literals = new Hashtable();
  literals.put(new ANTLRHashString("else", this), new Integer(13));
  literals.put(new ANTLRHashString("if", this), new Integer(12));
  literals.put(new ANTLRHashString("end", this), new Integer(14));
}

public Token nextToken() throws TokenStreamException {
  Token theRetToken=null;
tryAgain:
  for (;;) {
    Token _token = null;
    int _ttype = Token.INVALID_TYPE;
    resetText();
    try {   // for char stream error handling
      try {   // for lexical error handling
        switch ( LA(1)) {
        case '0':  case '1':  case '2':  case '3':
        case '4':  case '5':  case '6':  case '7':
        case '8':  case '9':  case 'A':  case 'B':
        
        ....
        
      }
      catch (RecognitionException e) {
        throw new TokenStreamRecognitionException(e);
      }
    }
    catch (CharStreamException cse) {
      if ( cse instanceof CharStreamIOException ) {
        throw new TokenStreamIOException(((CharStreamIOException)cse).io);
      }
      else {
        throw new TokenStreamException(cse.getMessage());
      }
    }
  }
}

---- incotemp/InCoTempLineLexer.java

public class InCoTempLineLexer extends incotemp.InCoTempAbstractLexer
implements InCoTempLineLexerTokenTypes, TokenStream
{

public InCoTempLineLexer(LexerSharedInputState state) {
  super(state);
  caseSensitiveLiterals = true;
  setCaseSensitive(true);
  literals = new Hashtable();
  literals = new Hashtable();
  literals.put(new ANTLRHashString("else", this), new Integer(13));
  literals.put(new ANTLRHashString("if", this), new Integer(12));
  literals.put(new ANTLRHashString("end", this), new Integer(14));
}

public Token nextToken() throws TokenStreamException {
  try {uponEOF();}
  catch(CharStreamIOException csioe) {
    throw new TokenStreamIOException(csioe.io);
  }
  catch(CharStreamException cse) {
    throw new TokenStreamException(cse.getMessage());
  }
  return new CommonToken(Token.EOF_TYPE, "");
}