public List < Token > scan ( ) throws Error { ArrayList < Token > tokens = new ArrayList <> ( ) ; int pos = 0 ; while ( pos < input . length ( ) ) { int start = pos ; for ( int i = 0 ; i != rules . length ; ++ i ) { Rule rule = rules [ i ] ; int left = input . length ( ) - pos ; if ( left >= rule . lookahead ( ) ) { Token t = rule . match ( input , pos ) ; if ( t != null ) { tokens . add ( t ) ; pos = pos + t . text . length ( ) ; break ; // restart rule application loop } } } if ( pos == start ) { throw new Error ( "unrecognised token encountered (" + input . charAt ( pos ) + ")" , pos ) ; } } return tokens ; }
Scan the given input stream and produce a list of tokens or an error .