private void senseInduce ( String term , Matrix contexts ) throws IOException { LOGGER . fine ( "Clustering " + contexts . rows ( ) + " contexts for " + term ) ; // For terms with fewer than seven contexts, set the number of potential // clusters lower int numClusters = Math . min ( 7 , contexts . rows ( ) ) ; // Cluster each of the rows into seven groups. Note that if the term is // not purely alphabetic (i.e. contains number of other symbols), don't // bother clustering it. This is done to reduce the computation time, // and to avoid clustering non-meaningful terms such as '.' or ''' if ( ! ( term . matches ( "[a-zA-z]+" ) && numClusters > 6 ) ) { // special case SparseDoubleVector meanSenseVector = new CompactSparseVector ( termToIndex . size ( ) ) ; int rows = contexts . rows ( ) ; for ( int row = 0 ; row < rows ; ++ row ) VectorMath . ( meanSenseVector , contexts . getRowVector ( row ) ) ; termToVector . put ( term , meanSenseVector ) ; return ; } Assignments clusterAssignment = new ClutoClustering ( ) . cluster ( contexts , numClusters , ClutoClustering . Method . AGGLOMERATIVE , ClutoClustering . Criterion . UPGMA ) ; LOGGER . fine ( "Generative sense vectors for " + term ) ; // For each of the clusters, compute the mean sense vector int [ ] clusterSize = new int [ numClusters ] ; // Use CompactSparseVector to conserve memory given the potentially // large number of sense vectors SparseDoubleVector [ ] meanSenseVectors = new CompactSparseVector [ numClusters ] ; for ( int i = 0 ; i < meanSenseVectors . length ; ++ i ) meanSenseVectors [ i ] = new CompactSparseVector ( termToIndex . size ( ) ) ; // For each of the contexts, determine which cluster it was in and sum // it value with the other contexts for ( int row = 0 ; row < clusterAssignment . size ( ) ; ++ row ) { // Check whether this row was assigned a cluster if ( clusterAssignment . get ( row ) . assignments ( ) . length == 0 ) continue ; int assignment = clusterAssignment . get ( row ) . assignments ( ) [ 0 ] ; clusterSize [ assignment ] ++ ; DoubleVector contextVector = contexts . getRowVector ( row ) ; VectorMath . add ( meanSenseVectors [ assignment ] , contextVector ) ; } // For each of the clusters with more than 2% of the contexts, generage // an average sense vectors. For those clusters with less than that // amount, discard them. int senseCounter = 0 ; for ( int i = 0 ; i < numClusters ; ++ i ) { int size = clusterSize [ i ] ; if ( size / ( double ) ( contexts . rows ( ) ) > 0.02 ) { String termWithSense = ( senseCounter == 0 ) ? term : term + "-" + senseCounter ; senseCounter ++ ; termToVector . put ( termWithSense , meanSenseVectors [ i ] ) ; } } LOGGER . fine ( "Discovered " + senseCounter + " senses for " + term ) ; }
Given a matrix for the term where each row is a different context clusters the rows to identify how many senses the word has .