public void freeze ( ) { if ( frozen ) return ; // train the TFIDF distance on all strings seen trainDistances ( ) ; // now, compute the maxScore of each token, and create an inverted index if ( DEBUG ) System . out . println ( "computing maxScore of " + tokenizer . maxTokenIndex ( ) + " tokens" ) ; invertedIndex = new Set [ tokenizer . maxTokenIndex ( ) + 1 ] ; maxTFIDFScore = new double [ tokenizer . maxTokenIndex ( ) + 1 ] ; for ( Iterator i = valueMap . keySet ( ) . iterator ( ) ; i . hasNext ( ) ; ) { String s = ( String ) i . next ( ) ; tfidfDistance . prepare ( s ) ; Token [ ] tokens = tfidfDistance . getTokens ( ) ; for ( int j = 0 ; j < tokens . length ; j ++ ) { Token tok = tokens [ j ] ; double w = tfidfDistance . getWeight ( tok ) ; maxTFIDFScore [ tok . getIndex ( ) ] = Math . max ( maxTFIDFScore [ tok . getIndex ( ) ] , w ) ; Set ii = invertedIndex [ tok . getIndex ( ) ] ; if ( ii == null ) ii = invertedIndex [ tok . getIndex ( ) ] = new HashSet ( ) ; ii . add ( s ) ; //if (DEBUG) System.out.println("adjust maxscore, invertedIndex for "+tok); } } // find out which tokens are similar to which other tokens if ( DEBUG ) System . out . println ( "computing similar-tokens for " + tokenizer . maxTokenIndex ( ) + " tokens, window=" + windowSize ) ; allTokens = new Token [ tokenizer . maxTokenIndex ( ) ] ; numTokens = 0 ; for ( Iterator i = tokenizer . tokenIterator ( ) ; i . hasNext ( ) ; ) { Token toki = ( Token ) i . next ( ) ; allTokens [ numTokens ++ ] = toki ; } Arrays . sort ( allTokens , LEXICAL_ORDER_FOR_TOKENS ) ; similarTokens = new Token [ tokenizer . maxTokenIndex ( ) + 1 ] [ ] ; for ( int i = 0 ; i < numTokens ; i ++ ) { Token toki = allTokens [ i ] ; Set likeTokI = findSimilarTokens ( toki . getValue ( ) , i ) ; similarTokens [ toki . getIndex ( ) ] = new Token [ likeTokI . size ( ) ] ; int k = 0 ; for ( Iterator j = likeTokI . iterator ( ) ; j . hasNext ( ) ; ) { Token tokj = ( Token ) j . next ( ) ; similarTokens [ toki . getIndex ( ) ] [ k ++ ] = tokj ; } } frozen = true ; }
Make it impossible to add new values but possible to perform lookups .