public void process ( Iterator < String > text ) { String nextToken = null , curToken = null ; // Base case for the next token buffer to ensure we always have two // valid tokens present if ( text . hasNext ( ) ) nextToken = text . next ( ) ; while ( text . hasNext ( ) ) { curToken = nextToken ; nextToken = text . next ( ) ; // Only process bigrams where the two tokens weren't excluded by the // token filter if ( ! ( excludeToken ( curToken ) || excludeToken ( nextToken ) ) ) processBigram ( curToken , nextToken ) ; } }
Processes the tokens in the iterator to gather statistics for any bigrams contained therein