public List < Token > scan ( ) { ArrayList < Token > tokens = new ArrayList <> ( ) ; pos = 0 ; while ( pos < input . length ( ) ) { char c = input . charAt ( pos ) ; if ( isDigit ( c ) ) { tokens . add ( scanNumericLiteral ( ) ) ; } else if ( c == ' ' ) { tokens . add ( scanStringLiteral ( ) ) ; } else if ( c == ' ' ) { tokens . add ( scanCharacterLiteral ( ) ) ; } else if ( isOperatorStart ( c ) ) { tokens . add ( scanOperator ( ) ) ; } else if ( isLetter ( c ) || c == ' ' ) { tokens . add ( scanIdentifier ( ) ) ; } else if ( Character . isWhitespace ( c ) ) { scanWhiteSpace ( tokens ) ; } else { syntaxError ( "unknown token encountered" , pos ) ; } } return tokens ; }
Scan all characters from the input stream and generate a corresponding list of tokens whilst discarding all whitespace and comments .