protected void addProcessedData ( List < List < CRFDatum < Collection < String > , String > > > processedData , int [ ] [ ] [ ] [ ] data , int [ ] [ ] labels , int offset ) { for ( int i = 0 , pdSize = processedData . size ( ) ; i < pdSize ; i ++ ) { int dataIndex = i + offset ; List < CRFDatum < Collection < String > , String > > document = processedData . get ( i ) ; int dsize = document . size ( ) ; labels [ dataIndex ] = new int [ dsize ] ; data [ dataIndex ] = new int [ dsize ] [ ] [ ] ; for ( int j = 0 ; j < dsize ; j ++ ) { CRFDatum < Collection < String > , String > crfDatum = document . get ( j ) ; // add label, they are offset by extra context labels [ dataIndex ] [ j ] = classIndex . indexOf ( crfDatum . label ( ) ) ; // add features List < Collection < String > > cliques = crfDatum . asFeatures ( ) ; int csize = cliques . size ( ) ; data [ dataIndex ] [ j ] = new int [ csize ] [ ] ; for ( int k = 0 ; k < csize ; k ++ ) { Collection < String > features = cliques . get ( k ) ; // Debug only: Remove // if (j < windowSize) { // System.err.println("addProcessedData: Features Size: " + // features.size()); // } data [ dataIndex ] [ j ] [ k ] = new int [ features . size ( ) ] ; int m = 0 ; try { for ( String feature : features ) { // System.err.println("feature " + feature); // if (featureIndex.indexOf(feature)) ; if ( featureIndex == null ) { System . out . println ( "Feature is NULL!" ) ; } data [ dataIndex ] [ j ] [ k ] [ m ] = featureIndex . indexOf ( feature ) ; m ++ ; } } catch ( Exception e ) { e . printStackTrace ( ) ; System . err . printf ( "[index=%d, j=%d, k=%d, m=%d]\n" , dataIndex , j , k , m ) ; System . err . println ( "data.length " + data . length ) ; System . err . println ( "data[dataIndex].length " + data [ dataIndex ] . length ) ; System . err . println ( "data[dataIndex][j].length " + data [ dataIndex ] [ j ] . length ) ; System . err . println ( "data[dataIndex][j][k].length " + data [ dataIndex ] [ j ] . length ) ; System . err . println ( "data[dataIndex][j][k][m] " + data [ dataIndex ] [ j ] [ k ] [ m ] ) ; return ; } } } } }
Adds the List of Lists of CRFDatums to the data and labels arrays treating each datum as if it were its own document . Adds context labels in addition to the target label for each datum meaning that for a particular document the number of labels will be windowSize - 1 greater than the number of datums .