public StringWrapper prepare ( String s ) { BagOfTokens bag = new BagOfTokens ( s , tokenizer . tokenize ( s ) ) ; // reweight by -log( freq/collectionSize ) double normalizer = 0.0 ; for ( Iterator i = bag . tokenIterator ( ) ; i . hasNext ( ) ; ) { Token tok = ( Token ) i . next ( ) ; if ( collectionSize > 0 ) { Integer dfInteger = ( Integer ) documentFrequency . get ( tok ) ; // set previously unknown words to df==1, which gives them a high value double df = dfInteger == null ? 1.0 : dfInteger . intValue ( ) ; double w = - Math . log ( df / collectionSize ) ; bag . setWeight ( tok , w ) ; } else { bag . setWeight ( tok , Math . log ( 10 ) ) ; } } return bag ; }
Preprocess a string by finding tokens and giving them appropriate weights