final public StringWrapper prepare ( String s ) { BagOfTokens bag = new BagOfTokens ( s , tokenizer . tokenize ( s ) ) ; double totalWeight = bag . getTotalWeight ( ) ; for ( Iterator i = bag . tokenIterator ( ) ; i . hasNext ( ) ; ) { Token tok = ( Token ) i . next ( ) ; double freq = bag . getWeight ( tok ) ; bag . setWeight ( tok , smoothedProbability ( tok , freq , totalWeight ) ) ; } return bag ; }
Preprocess a string by finding tokens and giving them weights W such that W is the smoothed probability of the token appearing in the document .