public IXPathToken nextToken ( ) { // some tokens start in another state than the START state mState = mStartState ; // reset startState mStartState = State . START ; mOutput = new StringBuilder ( ) ; mFinnished = false ; mType = TokenType . INVALID ; mLastPos = mPos ; do { mInput = mQuery . charAt ( mPos ) ; switch ( mState ) { case START : // specify token type according to first char scanStart ( ) ; break ; case NUMBER : // number scanNumber ( ) ; break ; case TEXT : // some text, could be a name scanText ( ) ; break ; case SPECIAL2 : // special character that could have 2 digits scanTwoDigitSpecial ( ) ; break ; case COMMENT : scanComment ( ) ; break ; case E_NUM : scanENum ( ) ; break ; default : mPos ++ ; mFinnished = true ; } } while ( ! mFinnished || mPos >= mQuery . length ( ) ) ; if ( mCommentCount > 0 ) { throw new IllegalStateException ( "Error in Query. Comment does not end." ) ; } return new VariableXPathToken ( mOutput . toString ( ) , mType ) ; }
Reads the string char by char and returns one token by call . The scanning starts in the start state if not further specified before and specifies the next scanner state and the type of the future token according to its first char . As soon as the current char does not fit the conditions for the current token type the token is generated and returned .