private void processBigram ( String left , String right ) { TokenStats leftStats = getStatsFor ( left ) ; TokenStats rightStats = getStatsFor ( right ) ; // mark that both appeared leftStats . count ++ ; rightStats . count ++ ; // Mark the respective positions of each leftStats . leftCount ++ ; rightStats . rightCount ++ ; // Increase the number of bigrams seen numBigramsInCorpus ++ ; // Update the bigram statistics // Map the two token's indices into a single long long bigram = ( ( ( long ) leftStats . index ) << 32 ) | rightStats . index ; Number curBigramCount = bigramCounts . get ( bigram ) ; int i = ( curBigramCount == null ) ? 1 : 1 + curBigramCount . intValue ( ) ; // Compact the count into the smallest numeric type that can represent // it. This hopefully results in some space savings. Number val = null ; if ( i < Byte . MAX_VALUE ) val = Byte . valueOf ( ( byte ) i ) ; else if ( i < Short . MAX_VALUE ) val = Short . valueOf ( ( short ) i ) ; else val = Integer . valueOf ( i ) ; bigramCounts . put ( bigram , val ) ; }
Updates the statistics for the bigram formed from the provided left and right token .