private void processSpace ( ) throws IOException { LOGGER . info ( "generating reflective vectors" ) ; compressedDocumentsWriter . close ( ) ; int numDocuments = documentCounter . get ( ) ; termToIndexVector . clear ( ) ; indexToTerm = new String [ termToIndex . size ( ) ] ; for ( Map . Entry < String , Integer > e : termToIndex . entrySet ( ) ) indexToTerm [ . getValue ( ) ] = . getKey ( ) ; // Read in the compressed version of the corpus, re-processing each // document to build up the document vectors DataInputStream corpusReader = new DataInputStream ( new BufferedInputStream ( new FileInputStream ( compressedDocuments ) ) ) ; // Set up the concurrent data structures so we can reprocess the // documents concurrently using a work queue final BlockingQueue < Runnable > workQueue = new LinkedBlockingQueue < Runnable > ( ) ; for ( int i = 0 ; i < Runtime . getRuntime ( ) . availableProcessors ( ) ; ++ i ) { Thread t = new WorkerThread ( workQueue ) ; t . start ( ) ; } final Semaphore documentsRerocessed = new Semaphore ( 0 ) ; for ( int d = 0 ; d < numDocuments ; ++ d ) { final int docId = d ; // This value already has any filtered tokens taken into account, // i.e. in only counts those tokens that remain after filtering int tokensInDoc = corpusReader . readInt ( ) ; // Read in the document final int [ ] doc = new int [ tokensInDoc ] ; for ( int i = 0 ; i < tokensInDoc ; ++ i ) doc [ i ] = corpusReader . readInt ( ) ; workQueue . offer ( new Runnable ( ) { public void run ( ) { // This method creates the document vector and then adds // that document vector with the reflective semantic // vector for each word occurring in the document LOGGER . fine ( "reprocessing doc #" + docId ) ; processIntDocument ( docToVector . get ( docId ) , doc ) ; documentsRerocessed . release ( ) ; } } ) ; } corpusReader . close ( ) ; // Wait until all the documents have been processed try { documentsRerocessed . acquire ( numDocuments ) ; } catch ( InterruptedException ie ) { throw new Error ( "interrupted while waiting for documents to " + "finish reprocessing" , ie ) ; } LOGGER . fine ( "finished reprocessing all documents" ) ; }
Computes the reflective semantic vectors for word meanings