public Token nextToken ( ) { if ( lexer . _input == null ) { throw new IllegalStateException ( "nextToken requires a non-null input stream." ) ; } else { int tokenStartMarker = lexer . _input . mark ( ) ; try { Token ttype1 ; label110 : while ( ! lexer . _hitEOF ) { lexer . _token = null ; lexer . _channel = Token . DEFAULT_CHANNEL ; lexer . _tokenStartCharIndex = lexer . _input . index ( ) ; lexer . _tokenStartCharPositionInLine = ( ( LexerATNSimulator ) lexer . getInterpreter ( ) ) . getCharPositionInLine ( ) ; lexer . _tokenStartLine = ( ( LexerATNSimulator ) lexer . getInterpreter ( ) ) . getLine ( ) ; lexer . _text = null ; do { lexer . _type = 0 ; int ttype ; try { ttype = ( ( LexerATNSimulator ) lexer . getInterpreter ( ) ) . match ( lexer . _input , lexer . _mode ) ; } catch ( LexerNoViableAltException var7 ) { lexer . notifyListeners ( var7 ) ; lexer . recover ( var7 ) ; ttype = - 3 ; } if ( lexer . _input . LA ( 1 ) == - 1 ) { lexer . _hitEOF = true ; } if ( lexer . _type == 0 ) { lexer . _type = ttype ; } if ( lexer . _type == - 3 ) { continue label110 ; } } while ( lexer . _type == - 2 ) ; if ( lexer . _token == null ) { lexer . emit ( ) ; } ttype1 = lexer . _token ; //log.trace("return token: >" + ttype1.getText()+"< " + ttype1.getType()); return ttype1 ; } // recover from unexpected EOF eof = true ; if ( ! ls . isBalanced ( ) ) { return generateEOFRecover ( ) ; } log . trace ( "lexer state is balanced - emitEOF" ) ; lexer . emitEOF ( ) ; ttype1 = lexer . _token ; return ttype1 ; } finally { lexer . _input . release ( tokenStartMarker ) ; } } }
Implements Lexer s next token with extra token passing from recovery function