public CommonTokenStream lex ( final String cleanedCobolSource ) throws RecognizerException { if ( _log . isDebugEnabled ( ) ) { _log . debug ( "2. Lexing COBOL source code: {}" , cleanedCobolSource ) ; } try { CobolStructureLexer lex = new CobolStructureLexerImpl ( new ANTLRReaderStream ( new StringReader ( cleanedCobolSource ) ) , getErrorHandler ( ) ) ; CommonTokenStream tokens = new CommonTokenStream ( lex ) ; if ( lex . getNumberOfSyntaxErrors ( ) != 0 || tokens == null ) { throw ( new RecognizerException ( "Lexing failed. " + lex . getNumberOfSyntaxErrors ( ) + " syntax errors." + " Last error was " + getErrorHistory ( ) . get ( getErrorHistory ( ) . size ( ) - 1 ) ) ) ; } return tokens ; } catch ( IOException e ) { throw ( new RecognizerException ( e ) ) ; } }
Apply the lexer to produce a token stream from source .