<commit_before>package com.wikia;

import com.beust.jcommander.internal.Lists;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.wikia.classifier.classifiers.model.PageWithType;
import com.wikia.classifier.classifiers.serialization.GZippedClassifierFileFormat;
import com.wikia.classifier.classifiers.training.ClassifierTrainer;
import com.wikia.classifier.classifiers.training.ClassifierTrainerFactory;
import com.wikia.classifier.classifiers.training.ClassifierTrainingResult;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Component;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import org.apache.maven.project.MavenProjectHelper;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.List;

/**
 * Goal which touches a timestamp file.
 */
@SuppressWarnings("unused")
@Mojo(name="train", defaultPhase = LifecyclePhase.GENERATE_RESOURCES)
public class TrainClassifierMojo extends AbstractMojo {

    @Parameter(defaultValue = "", required = true)
    private String trainingSet = null;
    @Parameter(defaultValue = "${project.build.directory}/generated-sources/")
    private File outputPath = null;
    @Component
    private MavenProjectHelper projectHelper = null;
    @Component
    private MavenProject mavenProject = null;

    public void execute() throws MojoExecutionException {
        if ( trainingSet == null || trainingSet.isEmpty() ) {
            getLog().info("No training file specified.");
            return;
        }

        List<PageWithType> inputSet;
        try {
            inputSet = readFile(trainingSet);
        } catch (IOException e) {
            throw new MojoExecutionException(String.format("Cannot read training set file. (file:%s)", trainingSet), e);
        }

        ClassifierTrainer trainer = new ClassifierTrainerFactory().create();
        ClassifierTrainingResult trainingResult = trainer.train(inputSet);

        getLog().info(String.format("SuccessRate: %.2f", trainingResult.getSuccessRate() * 100.0));

        if (outputPath.exists() || outputPath.mkdirs()) {
            try {
                new GZippedClassifierFileFormat().write(trainingResult.getClassifier(), outputPath + "/classifier.out");
            } catch (IOException e) {
                throw new MojoExecutionException("Cannot write output file.", e);
            }

            projectHelper.addResource(mavenProject, outputPath.getAbsolutePath(), Lists.newArrayList("**/*"), null);
        } else {
            throw new MojoExecutionException(String.format("Cannot create %s", outputPath));
        }
    }

    public List<PageWithType> readFile( String inputFile ) throws IOException {
        Gson gson = new Gson();
        try (Reader reader = new FileReader(inputFile)) {
            return gson.fromJson(reader, new TypeToken<List<PageWithType>>() {
            }.getType());
        }
    }
}<commit_msg>Cache trained classifier in mojo<commit_after>package com.wikia;

import com.beust.jcommander.internal.Lists;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.wikia.classifier.classifiers.model.PageWithType;
import com.wikia.classifier.classifiers.serialization.GZippedClassifierFileFormat;
import com.wikia.classifier.classifiers.training.ClassifierTrainer;
import com.wikia.classifier.classifiers.training.ClassifierTrainerFactory;
import com.wikia.classifier.classifiers.training.ClassifierTrainingResult;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Component;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import org.apache.maven.project.MavenProjectHelper;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.List;
import java.util.Map;

/**
 * Goal which touches a timestamp file.
 */
@SuppressWarnings("unused")
@Mojo(name="train", defaultPhase = LifecyclePhase.GENERATE_RESOURCES)
public class TrainClassifierMojo extends AbstractMojo {

    @Parameter(defaultValue = "", required = true)
    private String trainingSet = null;
    @Parameter(defaultValue = "${project.build.directory}/generated-sources/")
    private File outputPath = null;
    @Component
    private MavenProjectHelper projectHelper = null;
    @Component
    private MavenProject mavenProject = null;

    public void execute() throws MojoExecutionException {
        if ( trainingSet == null || trainingSet.isEmpty() ) {
            new MojoExecutionException("No training file specified.");
        }

        ClassifierTrainingResult trainingResult = train(trainingSet);

        getLog().info(String.format("SuccessRate: %.2f", trainingResult.getSuccessRate() * 100.0));

        if (outputPath.exists() || outputPath.mkdirs()) {
            try {
                new GZippedClassifierFileFormat().write(trainingResult.getClassifier(), outputPath + "/classifier.out");
            } catch (IOException e) {
                throw new MojoExecutionException("Cannot write output file.", e);
            }

            projectHelper.addResource(mavenProject, outputPath.getAbsolutePath(), Lists.newArrayList("**/*"), null);
        } else {
            throw new MojoExecutionException(String.format("Cannot create %s", outputPath));
        }
    }

    private static Map<String, ClassifierTrainingResult> trainCache = Maps.newHashMap();

    public static ClassifierTrainingResult train(String trainingSetFileName) throws MojoExecutionException {
        if (trainCache.containsKey(trainingSetFileName)) {
            return trainCache.get(trainingSetFileName);
        } else {
            List<PageWithType> inputSet;
            try {
                inputSet = readFile(trainingSetFileName);
            } catch (IOException e) {
                throw new MojoExecutionException(String.format("Cannot read training set file. (file:%s)", trainingSetFileName), e);
            }

            ClassifierTrainer trainer = new ClassifierTrainerFactory().create();
            ClassifierTrainingResult trainingResult = trainer.train(inputSet);
            trainCache.put(trainingSetFileName, trainingResult);
            return trainingResult;
        }
    }

    public static List<PageWithType> readFile( String inputFile ) throws IOException {
        Gson gson = new Gson();
        try (Reader reader = new FileReader(inputFile)) {
            return gson.fromJson(reader, new TypeToken<List<PageWithType>>() {
            }.getType());
        }
    }
}
{'fraction_non_alphanumeric': 0.10553977272727273, 'fraction_numerical': 0.0014204545454545455, 'mean_word_length': 3.62007874015748, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1373607', 'n_tokens_mistral': 1922, 'n_tokens_neox': 1860, 'n_words': 384}