private void process ( Iterator < String > tokens ) { // NOTE: this method is intentionally private to ensure that the // IteratorFactory.tokenize() tokenization scheme is enforced on the // input data long numTokens = 0 ; while ( tokens . hasNext ( ) ) { String token = tokens . next ( ) ; if ( doLowerCasing ) token = token . toLowerCase ( ) ; if ( token . matches ( "[0-9]+" ) ) token = "<NUM>" ; if ( token . matches ( "[^\\w\\s;:\\(\\)\\[\\]'!/&?\",\\.<>]" ) ) continue ; Integer count = tokenToCount . get ( token ) ; tokenToCount . put ( token , ( count == null ) ? 1 : 1 + count ) ; numTokens ++ ; if ( numTokens % UPDATE_INTERVAL == 0 && LOGGER . isLoggable ( Level . FINE ) ) LOGGER . fine ( "Processed " + numTokens + " tokens. Currently " + tokenToCount . size ( ) + " unique tokens" ) ; } }
Counts all of the tokens in the iterator