<commit_before>package pl.edu.icm.cermine.tools.classification.svm;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import libsvm.*;
import org.apache.commons.collections.iterators.ArrayIterator;
import pl.edu.icm.cermine.structure.model.BxPage;
import pl.edu.icm.cermine.structure.model.BxZone;
import pl.edu.icm.cermine.structure.model.BxZoneLabel;
import pl.edu.icm.cermine.tools.classification.features.FeatureVector;
import pl.edu.icm.cermine.tools.classification.features.FeatureVectorBuilder;
import pl.edu.icm.cermine.tools.classification.general.*;

/**
 * @author <NAME> (p.<EMAIL>ek@
 *
 * @param <S> classified object's class
 * @param <T> context class
 * @param <E> target enumeration for labels
 */
public abstract class SVMClassifier<S, T, E extends Enum<E>> {
	final static protected svm_parameter defaultParameter = new svm_parameter();		
	static {
		// default values
		defaultParameter.svm_type = svm_parameter.C_SVC;
		defaultParameter.C = 8;
		defaultParameter.kernel_type = svm_parameter.POLY;
		defaultParameter.degree = 3;
		defaultParameter.gamma = 1.0/8.0; // 1/k
		defaultParameter.coef0 = 0.5;
		defaultParameter.nu = 0.5;
		defaultParameter.cache_size = 100;
		defaultParameter.eps = 1e-3;
		defaultParameter.p = 0.1;
		defaultParameter.shrinking = 1;
		defaultParameter.probability = 0;
		defaultParameter.nr_weight = 0;
		defaultParameter.weight_label = new int[0];
		defaultParameter.weight = new double[0];
	}
	protected FeatureVectorBuilder<S, T> featureVectorBuilder;
	protected FeatureVectorScaler scaler;
	protected String[] featuresNames;
		
	protected svm_parameter param;
	protected svm_problem problem;
	protected svm_model model;
	
	protected Class<E> enumClassObj;
	
	public SVMClassifier(FeatureVectorBuilder<S, T> featureVectorBuilder, Class<E> enumClassObj) 
	{
		this.featureVectorBuilder = featureVectorBuilder;
		this.enumClassObj = enumClassObj;
		Integer dimensions = featureVectorBuilder.size();
		
		Double scaledLowerBound = 0.0;
		Double scaledUpperBound = 1.0;
        FeatureVectorScalerImpl scaler = new FeatureVectorScalerImpl(dimensions, scaledLowerBound, scaledUpperBound);
		scaler.setStrategy(new LinearScaling());
		this.scaler = scaler;

		featuresNames = (String[])featureVectorBuilder.getFeatureNames().toArray(new String[0]);
		
		param = getDefaultParam();
	}
	
	protected static svm_parameter clone(svm_parameter param) {
		svm_parameter ret = new svm_parameter();
		// default values
		ret.svm_type = param.svm_type;
		ret.C = param.C;
		ret.kernel_type = param.kernel_type;
		ret.degree = param.degree;
		ret.gamma = param.gamma; // 1/k
		ret.coef0 = param.coef0;
		ret.nu = param.nu;
		ret.cache_size = param.cache_size;
		ret.eps = param.eps;
		ret.p = param.p;
		ret.shrinking = param.shrinking;
		ret.probability = param.probability;
		ret.nr_weight = param.nr_weight;
		ret.weight_label = param.weight_label;
		ret.weight = param.weight;
		return ret;
	}
	
	public static svm_parameter getDefaultParam() {
		return clone(defaultParameter);
	}
	
	public void buildClassifier(List<TrainingSample<E>> trainingElements) 
	{
		assert trainingElements.size() > 0;
		scaler.calculateFeatureLimits(trainingElements);
		problem = buildDatasetForTraining(trainingElements);
		model = libsvm.svm.svm_train(problem, param);
	}
	
	public BxZoneLabel predictLabel(S object, T context) {
		svm_node[] instance = buildDatasetForClassification(object, context);
		double predictedVal = svm.svm_predict(model, instance);
		return BxZoneLabel.values()[(int)predictedVal];
	}

	public E classify(S object, T context) {
		svm_node[] instance = buildDatasetForClassification(object, context);
		Integer predictedVal = ((Double)svm.svm_predict(model, instance)).intValue();
		return enumClassObj.getEnumConstants()[predictedVal];
	}

    public Map<E, Double> predictProbabilities(S object, T context) {
        svm_node[] instance = buildDatasetForClassification(object, context);
        double[] probEstimates = new double[enumClassObj.getEnumConstants().length];
        svm.svm_predict_probability(model, instance, probEstimates);

        Map<E, Double> result = new HashMap<E, Double>();
        for (int i = 0; i < probEstimates.length; ++i) {
            result.put(enumClassObj.getEnumConstants()[i], probEstimates[i]);
        }
        return result;
    }<commit_msg>fixing probabilities prediction<commit_after>package pl.edu.icm.cermine.tools.classification.svm;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import libsvm.*;
import org.apache.commons.collections.iterators.ArrayIterator;
import pl.edu.icm.cermine.structure.model.BxPage;
import pl.edu.icm.cermine.structure.model.BxZone;
import pl.edu.icm.cermine.structure.model.BxZoneLabel;
import pl.edu.icm.cermine.tools.classification.features.FeatureVector;
import pl.edu.icm.cermine.tools.classification.features.FeatureVectorBuilder;
import pl.edu.icm.cermine.tools.classification.general.*;

/**
 * @author <NAME> (p.szostek@
 *
 * @param <S> classified object's class
 * @param <T> context class
 * @param <E> target enumeration for labels
 */
public abstract class SVMClassifier<S, T, E extends Enum<E>> {
	final static protected svm_parameter defaultParameter = new svm_parameter();		
	static {
		// default values
		defaultParameter.svm_type = svm_parameter.C_SVC;
		defaultParameter.C = 8;
		defaultParameter.kernel_type = svm_parameter.POLY;
		defaultParameter.degree = 3;
		defaultParameter.gamma = 1.0/8.0; // 1/k
		defaultParameter.coef0 = 0.5;
		defaultParameter.nu = 0.5;
		defaultParameter.cache_size = 100;
		defaultParameter.eps = 1e-3;
		defaultParameter.p = 0.1;
		defaultParameter.shrinking = 1;
		defaultParameter.probability = 0;
		defaultParameter.nr_weight = 0;
		defaultParameter.weight_label = new int[0];
		defaultParameter.weight = new double[0];
	}
	protected FeatureVectorBuilder<S, T> featureVectorBuilder;
	protected FeatureVectorScaler scaler;
	protected String[] featuresNames;
		
	protected svm_parameter param;
	protected svm_problem problem;
	protected svm_model model;
	
	protected Class<E> enumClassObj;
	
	public SVMClassifier(FeatureVectorBuilder<S, T> featureVectorBuilder, Class<E> enumClassObj) 
	{
		this.featureVectorBuilder = featureVectorBuilder;
		this.enumClassObj = enumClassObj;
		Integer dimensions = featureVectorBuilder.size();
		
		Double scaledLowerBound = 0.0;
		Double scaledUpperBound = 1.0;
        FeatureVectorScalerImpl scaler = new FeatureVectorScalerImpl(dimensions, scaledLowerBound, scaledUpperBound);
		scaler.setStrategy(new LinearScaling());
		this.scaler = scaler;

		featuresNames = (String[])featureVectorBuilder.getFeatureNames().toArray(new String[0]);
		
		param = getDefaultParam();
	}
	
	protected static svm_parameter clone(svm_parameter param) {
		svm_parameter ret = new svm_parameter();
		// default values
		ret.svm_type = param.svm_type;
		ret.C = param.C;
		ret.kernel_type = param.kernel_type;
		ret.degree = param.degree;
		ret.gamma = param.gamma; // 1/k
		ret.coef0 = param.coef0;
		ret.nu = param.nu;
		ret.cache_size = param.cache_size;
		ret.eps = param.eps;
		ret.p = param.p;
		ret.shrinking = param.shrinking;
		ret.probability = param.probability;
		ret.nr_weight = param.nr_weight;
		ret.weight_label = param.weight_label;
		ret.weight = param.weight;
		return ret;
	}
	
	public static svm_parameter getDefaultParam() {
		return clone(defaultParameter);
	}
	
	public void buildClassifier(List<TrainingSample<E>> trainingElements) 
	{
		assert trainingElements.size() > 0;
		scaler.calculateFeatureLimits(trainingElements);
		problem = buildDatasetForTraining(trainingElements);
		model = libsvm.svm.svm_train(problem, param);
	}
	
	public BxZoneLabel predictLabel(S object, T context) {
		svm_node[] instance = buildDatasetForClassification(object, context);
		double predictedVal = svm.svm_predict(model, instance);
		return BxZoneLabel.values()[(int)predictedVal];
	}

	public E classify(S object, T context) {
		svm_node[] instance = buildDatasetForClassification(object, context);
		Integer predictedVal = ((Double)svm.svm_predict(model, instance)).intValue();
		return enumClassObj.getEnumConstants()[predictedVal];
	}

    public Map<E, Double> predictProbabilities(S object, T context) {
        svm_node[] instance = buildDatasetForClassification(object, context);
        double[] probEstimates = new double[enumClassObj.getEnumConstants().length];
        svm.svm_predict_probability(model, instance, probEstimates);

        Map<E, Double> result = new HashMap<E, Double>();
        for (int i = 0; i < probEstimates.length; ++i) {
            result.put(enumClassObj.getEnumConstants()[model.label[i]], probEstimates[i]);
        }
        return result;
    }
{'fraction_non_alphanumeric': 0.11019864609921208, 'fraction_numerical': 0.0075463322605704135, 'mean_word_length': 4.776923076923077, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1093902', 'n_tokens_mistral': 3068, 'n_tokens_neox': 2819, 'n_words': 576}