<commit_before>package de.ugoe.cs.cpdp.training;

import java.util.Arrays;
import java.util.logging.Level;

import de.ugoe.cs.util.console.Console;
import weka.core.OptionHandler;
import weka.classifiers.Classifier;
import weka.classifiers.meta.CVParameterSelection;

public abstract class WekaBaseTraining2 implements WekaCompatibleTrainer {
	
	protected Classifier classifier = null;
	protected String classifierClassName;
	protected String classifierName;
	protected String[] classifierParams;
	
	@Override
	public void setParameter(String parameters) {
		String[] params = parameters.split(" ");

		// first is classifierName
		classifierName = params[0];
		
		// all following parameters can be copied from weka!
		
		// second param is classifierClassName
		classifierClassName = params[1];
	
		// rest are params to the specified classifier
		classifierParams = Arrays.copyOfRange(params, 2, params.length);
		
		classifier = setupClassifier();
	}

	@Override
	public Classifier getClassifier() {
		return classifier;
	}

	public Classifier setupClassifier() {
		Classifier cl = null;
		try{
			@SuppressWarnings("rawtypes")
			Class c = Class.forName(classifierClassName);
			Classifier obj = (Classifier) c.newInstance();
			
			// Filter -CVPARAM
			String[] param = Arrays.copyOf(classifierParams, classifierParams.length);
			String[] cvparam = {};
			boolean cv = false;
			for ( int i=0; i < classifierParams.length; i++ ) {
				if(classifierParams[i].equals("-CVPARAM")) {
					// rest of array are cvparam
					cvparam = Arrays.copyOfRange(classifierParams, i+1, classifierParams.length);
					
					// before this we have normal params
					param = Arrays.copyOfRange(classifierParams, 0, i);
					
					cv = true;
					break;
				}
			}
			
			// set classifier params
			((OptionHandler)obj).setOptions(param);
			cl = obj;
			
			// we have cross val params
			// cant check on cvparam.length may not be initialized			
			if(cv) {
				final CVParameterSelection ps = new CVParameterSelection();
				ps.setClassifier(obj);
				ps.setNumFolds(5);
				//ps.addCVParameter("I 5 25 5");
				ps.addCVParameter(Arrays.asList(cvparam).toString().replaceAll(", ", " ").replaceAll("^\\[|\\]$", ""));
				
				cl = ps;
			}

		}catch(ClassNotFoundException e) {
			Console.traceln(Level.WARNING, String.format("class not found: %s", e.toString()));
			e.printStackTrace();
		} catch (InstantiationException e) {
			Console.traceln(Level.WARNING, String.format("Instantiation Exception: %s", e.toString()));
			e.printStackTrace();<commit_msg>* changed WekaBaseTraining2 to allow multiple CV parameters<commit_after>package de.ugoe.cs.cpdp.training;

import java.util.Arrays;
import java.util.logging.Level;

import de.ugoe.cs.util.console.Console;
import weka.core.OptionHandler;
import weka.classifiers.Classifier;
import weka.classifiers.meta.CVParameterSelection;

public abstract class WekaBaseTraining2 implements WekaCompatibleTrainer {
	
	protected Classifier classifier = null;
	protected String classifierClassName;
	protected String classifierName;
	protected String[] classifierParams;
	
	@Override
	public void setParameter(String parameters) {
		String[] params = parameters.split(" ");

		// first is classifierName
		classifierName = params[0];
		
		// all following parameters can be copied from weka!
		
		// second param is classifierClassName
		classifierClassName = params[1];
	
		// rest are params to the specified classifier
		classifierParams = Arrays.copyOfRange(params, 2, params.length);
		
		classifier = setupClassifier();
	}

	@Override
	public Classifier getClassifier() {
		return classifier;
	}

	public Classifier setupClassifier() {
		Classifier cl = null;
		try{
			@SuppressWarnings("rawtypes")
			Class c = Class.forName(classifierClassName);
			Classifier obj = (Classifier) c.newInstance();
			
			// Filter -CVPARAM
			String[] param = Arrays.copyOf(classifierParams, classifierParams.length);
			String[] cvparam = {};
			boolean cv = false;
			for ( int i=0; i < classifierParams.length; i++ ) {
				if(classifierParams[i].equals("-CVPARAM")) {
					// rest of array are cvparam
					cvparam = Arrays.copyOfRange(classifierParams, i+1, classifierParams.length);
					
					// before this we have normal params
					param = Arrays.copyOfRange(classifierParams, 0, i);
					
					cv = true;
					break;
				}
			}
			
			// set classifier params
			((OptionHandler)obj).setOptions(param);
			cl = obj;
			
			// we have cross val params
			// cant check on cvparam.length may not be initialized			
			if(cv) {
				final CVParameterSelection ps = new CVParameterSelection();
				ps.setClassifier(obj);
				ps.setNumFolds(5);
				//ps.addCVParameter("I 5 25 5");
				for( int i=1 ; i<cvparam.length/4 ; i++ ) {
					ps.addCVParameter(Arrays.asList(Arrays.copyOfRange(cvparam, 0, 4*i)).toString().replaceAll(", ", " ").replaceAll("^\\[|\\]$", ""));
				}
				
				cl = ps;
			}

		}catch(ClassNotFoundException e) {
			Console.traceln(Level.WARNING, String.format("class not found: %s", e.toString()));
			e.printStackTrace();
		} catch (InstantiationException e) {
			Console.traceln(Level.WARNING, String.format("Instantiation Exception: %s", e.toString()));
			e.printStackTrace();
{'fraction_non_alphanumeric': 0.12062703116038999, 'fraction_numerical': 0.005543873064423628, 'mean_word_length': 3.7650273224043715, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3748768', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1543, 'n_words': 387}