<commit_before>package edu.jhu.thrax.hadoop.tools;

import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;

import edu.jhu.thrax.ThraxConfig;
import edu.jhu.thrax.hadoop.datatypes.RuleWritable;

import edu.jhu.thrax.hadoop.features.Feature;
import edu.jhu.thrax.hadoop.features.MapReduceFeature;
import edu.jhu.thrax.hadoop.features.FeatureFactory;

import edu.jhu.thrax.hadoop.output.OutputReducer;

public class OutputTool extends Configured implements Tool
{
    public int run(String [] argv) throws Exception
    {
        if (argv.length < 2) {
            System.err.println("usage: OutputTool <work dir> <f1,f2,...>");
            return 1;
        }
        Configuration conf = getConf();
        Job job = new Job(conf, "thrax-collect");

        String thraxConf = conf.getRaw("thrax_work");
        if (thraxConf.endsWith(Path.SEPARATOR))
            thraxConf += "thrax.config";
        else
            thraxConf += Path.SEPARATOR + "thrax.config";
        ThraxConfig.configure(thraxConf);

        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(OutputReducer.class);

        job.setInputFormatClass(SequenceFileInputFormat.class);

        job.setMapOutputKeyClass(RuleWritable.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(RuleWritable.class);
        job.setOutputValueClass(NullWritable.class);

        if (!argv[0].endsWith(Path.SEPARATOR))
            argv[0] += Path.SEPARATOR;
        for (String x : argv[1].split(",")) {
            FileInputFormat.addInputPath(job, new Path(argv[0] + x));<commit_msg>easier command line for OutputTool<commit_after>package edu.jhu.thrax.hadoop.tools;

import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;

import edu.jhu.thrax.ThraxConfig;
import edu.jhu.thrax.hadoop.datatypes.RuleWritable;

import edu.jhu.thrax.hadoop.features.Feature;
import edu.jhu.thrax.hadoop.features.MapReduceFeature;
import edu.jhu.thrax.hadoop.features.FeatureFactory;

import edu.jhu.thrax.hadoop.output.OutputReducer;

public class OutputTool extends Configured implements Tool
{
    public int run(String [] argv) throws Exception
    {
        if (argv.length < 2) {
            System.err.println("usage: OutputTool <work dir> <f1 f2 ...>");
            return 1;
        }
        Configuration conf = getConf();
        Job job = new Job(conf, "thrax-collect");

        String thraxConf = conf.getRaw("thrax_work");
        if (thraxConf.endsWith(Path.SEPARATOR))
            thraxConf += "thrax.config";
        else
            thraxConf += Path.SEPARATOR + "thrax.config";
        ThraxConfig.configure(thraxConf);

        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(OutputReducer.class);

        job.setInputFormatClass(SequenceFileInputFormat.class);

        job.setMapOutputKeyClass(RuleWritable.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(RuleWritable.class);
        job.setOutputValueClass(NullWritable.class);

        if (!argv[0].endsWith(Path.SEPARATOR))
            argv[0] += Path.SEPARATOR;
        for (int i = 1; i < argv.length; i++) {
            if (FeatureFactory.get(argv[i]) instanceof MapReduceFeature) {
                FileInputFormat.addInputPath(job, new Path(argv[0] + argv[i]));
            }
{'fraction_non_alphanumeric': 0.11742591773551526, 'fraction_numerical': 0.0035382574082264487, 'mean_word_length': 3.559475806451613, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3946330', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1418, 'n_words': 215}