public static void splitData ( final String originalTrainingDataFile ) { List < Tuple > trainingData = NaiveBayesClassifier . readTrainingData ( originalTrainingDataFile , "\\s" ) ; List < Tuple > wrongData = new ArrayList <> ( ) ; int lastTrainingDataSize ; int iterCount = 0 ; do { System . out . println ( "Iteration:\t" + ( ++ iterCount ) ) ; lastTrainingDataSize = trainingData . size ( ) ; NaiveBayesClassifier nbc = new NaiveBayesClassifier ( ) ; nbc . train ( trainingData ) ; Iterator < Tuple > trainingDataIter = trainingData . iterator ( ) ; while ( trainingDataIter . hasNext ( ) ) { Tuple t = trainingDataIter . next ( ) ; String actual = nbc . predictLabel ( t ) ; if ( ! t . label . equals ( actual ) && ! t . label . equals ( "1" ) ) { // preserve 1 since too few. wrongData . add ( t ) ; trainingDataIter . remove ( ) ; } } Iterator < Tuple > wrongDataIter = wrongData . iterator ( ) ; while ( wrongDataIter . hasNext ( ) ) { Tuple t = wrongDataIter . next ( ) ; String actual = nbc . predictLabel ( t ) ; if ( t . label . equals ( actual ) ) { trainingData . add ( t ) ; wrongDataIter . remove ( ) ; } } } while ( trainingData . size ( ) != lastTrainingDataSize ) ; writeToFile ( trainingData , originalTrainingDataFile + ".aligned" ) ; writeToFile ( wrongData , originalTrainingDataFile + ".wrong" ) ; }
Split the data between trainable and wrong .