void sortVocab ( ) { Arrays . sort ( vocab , 0 , vocabSize ) ; // re-build vocabIndexMap final int size = vocabSize ; trainWords = 0 ; table = new int [ size ] ; for ( int i = 0 ; i < size ; i ++ ) { VocabWord word = vocab [ i ] ; // Words occuring less than min_count times will be discarded from the vocab if ( word . cn < config . getMinCount ( ) ) { table [ vocabIndexMap . get ( word . word ) ] = - 4 ; vocabSize -- ; } else { // Hash will be re-computed, as after the sorting it is not actual table [ vocabIndexMap . get ( word . word ) ] = i ; setVocabIndexMap ( word , i ) ; } } // lose weight vocabIndexMap = null ; VocabWord [ ] nvocab = new VocabWord [ vocabSize ] ; System . arraycopy ( vocab , 0 , nvocab , 0 , vocabSize ) ; }
Sorts the vocabulary by frequency using word counts