public Pair < int [ ] [ ] [ ] , int [ ] > documentToDataAndLabels ( List < IN > document ) { int docSize = document . size ( ) ; // first index is position in the document also the index of the // clique/factor table // second index is the number of elements in the clique/window these // features are for (starting with last element) // third index is position of the feature in the array that holds them // element in data[j][k][m] is the index of the mth feature occurring in // position k of the jth clique int [ ] [ ] [ ] data = new int [ docSize ] [ windowSize ] [ ] ; // index is the position in the document // element in labels[j] is the index of the correct label (if it exists) at // position j of document int [ ] labels = new int [ docSize ] ; if ( flags . useReverse ) { Collections . reverse ( document ) ; } // System.err.println("docSize:"+docSize); for ( int j = 0 ; j < docSize ; j ++ ) { CRFDatum < List < String > , CRFLabel > d = makeDatum ( document , j , featureFactory ) ; List < List < String > > features = d . asFeatures ( ) ; for ( int k = 0 , fSize = features . size ( ) ; k < fSize ; k ++ ) { Collection < String > cliqueFeatures = features . get ( k ) ; data [ j ] [ k ] = new int [ cliqueFeatures . size ( ) ] ; int m = 0 ; for ( String feature : cliqueFeatures ) { int index = featureIndex . indexOf ( feature ) ; if ( index >= 0 ) { data [ j ] [ k ] [ m ] = index ; m ++ ; } else { // this is where we end up when we do feature threshold cutoffs } } // Reduce memory use when some features were cut out by threshold if ( m < data [ j ] [ k ] . length ) { int [ ] f = new int [ m ] ; System . arraycopy ( data [ j ] [ k ] , 0 , f , 0 , m ) ; data [ j ] [ k ] = f ; } } IN wi = document . get ( j ) ; labels [ j ] = classIndex . indexOf ( wi . get ( AnswerAnnotation . class ) ) ; } if ( flags . useReverse ) { Collections . reverse ( document ) ; } // System.err.println("numClasses: "+classIndex.size()+" "+classIndex); // System.err.println("numDocuments: 1"); // System.err.println("numDatums: "+data.length); // System.err.println("numFeatures: "+featureIndex.size()); return new Pair < int [ ] [ ] [ ] , int [ ] > ( data , labels ) ; }
Convert a document List into arrays storing the data features and labels .