private int processIntDocument ( int termIndex , int [ ] document , Matrix contextMatrix , int rowStart , BitSet featuresForTerm ) { int contexts = 0 ; for ( int i = 0 ; i < document . length ; ++ i ) { int curToken = document [ i ] ; // Skip processing tokens that are not the current focus if ( curToken != termIndex ) continue ; // Buffer the count of how many times each feature appeared in the // context. SparseArray < Integer > contextCounts = new SparseHashArray < Integer > ( ) ; // Process all the tokes to the left (prior) to the current token; for ( int left = Math . max ( i - contextWindowSize , 0 ) ; left < i ; ++ left ) { // NOTE: this token value could be -1 if the token's original // text was filtered out from the corpus, i.e. was EMPTY_TOKEN int token = document [ left ] ; // Only count co-occurrences that are valid features for the // current token if ( token >= 0 && featuresForTerm . get ( token ) ) { Integer count = contextCounts . get ( token ) ; contextCounts . set ( token , ( count == null ) ? 1 : count + 1 ) ; } } // Process all the tokes to the right (after) to the current token; int end = Math . min ( i + contextWindowSize , document . length ) ; for ( int right = i + 1 ; right < end ; ++ right ) { int token = document [ right ] ; // Only count co-occurrences that are valid features for the // current token if ( token >= 0 && featuresForTerm . get ( token ) ) { Integer count = contextCounts . get ( token ) ; contextCounts . set ( token , ( count == null ) ? 1 : count + 1 ) ; } } // Each word in the document represents a new context, so the // specific context instance can be determined from the current word // and the number of previously process words int curContext = rowStart + contexts ; for ( int feat : contextCounts . getElementIndices ( ) ) { //System.out.println("setting row: " + curContext + ", col: " + feat); contextMatrix . set ( curContext , feat , contextCounts . get ( feat ) ) ; } // If the current token wasn't skipped, indicate that another // context was seen contexts ++ ; } return contexts ; }
Processes the compressed version of a document where each integer indicates that token s index and identifies all the contexts for the target word adding them as new rows to the context matrix .