<commit_before><commit_msg>train using moa v1<commit_after>package testBench.ids.kdd99;


import moa.classifiers.*;
import moa.classifiers.trees.HoeffdingTree;

import weka.core.Instance;
import weka.core.Instances;


import moa.streams.generators.*;
import moa.streams.*;

int numInstances=5 * 1000 * 1000;



String datasetName = "combined_createFullKdd99TrainingDatasetForBinary";
String datasetNameFullFileName = Finals.ARFF_SAVE_FOLDER + datasetName + ".arff";

println(datasetNameFullFileName);



Classifier learner = new HoeffdingTree ( ) ;

int classIndex = 42;
ArffFileStream stream = new ArffFileStream(datasetNameFullFileName, classIndex)

stream.prepareForUse();

 learner.setModelContext(stream.getHeader()) ;
 learner.prepareForUse() ;

 int numberSamplesCorrect=0;
 int numberSamples=0;
 boolean isTesting = true ;
 while( stream.hasMoreInstances () && numberSamples < numInstances){
     Instance trainInst = stream.nextInstance() ;
     if ( isTesting ){
         if ( learner.correctlyClassifies(trainInst )){
             numberSamplesCorrect++;
         }
     }
     numberSamples++;
     learner.trainOnInstance(trainInst) ;
 }



 double accuracy = 100.0*(double) numberSamplesCorrect / (double) numberSamples ;
 
 println (numberSamples+ " instances processed with " + accuracy+ "% accuracy" )


{'fraction_non_alphanumeric': 0.10378787878787879, 'fraction_numerical': 0.016666666666666666, 'mean_word_length': 4.669527896995708, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6206158', 'n_tokens_mistral': 431, 'n_tokens_neox': 387, 'n_words': 79}