<commit_before>package net.gregorybringman.elementsreduce;

import net.gregorybringman.elementsreduce.types.ElementsMapWritable;
import net.gregorybringman.elementsreduce.types.ElementsStringArrayWritable;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * Runs the ElementsVersions map-reduce with an input data set of the format used in
 * {@link elements-data-table-1-Mayer.txt}.
 * 
 * @author <NAME>
 * 
 */
public class ElementsMapJob {

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        
        if (args.length != 2) {
            System.err.println("Usage: elementsmap <in> <out>");
            System.exit(2);
        }

        Job job = new Job(conf, "Elements Map");
        job.setNumReduceTasks(1);
        job.setJarByClass(ElementsMapJob.class);

        job.setMapperClass(ElementsVersionsMapper.class);
        job.setCombinerClass(ElementsVersionsMarkupReducer.class);
        job.setReducerClass(ElementsVersionsMarkupReducer.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(ElementsStringArrayWritable.class);

        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(ElementsMapWritable.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}<commit_msg>Added automatic output folder delete to Main job.<commit_after>package net.gregorybringman.elementsreduce;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import net.gregorybringman.elementsreduce.types.ElementsMapWritable;
import net.gregorybringman.elementsreduce.types.ElementsStringArrayWritable;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * Runs the ElementsVersions map-reduce with an input data set of the format used in
 * {@link elements-data-table-1-Mayer.txt}.
 * 
 * @author <NAME>
 * 
 */
public class ElementsMapJob {

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();

        if (args.length != 2) {
            System.err.println("Usage: elementsmap <in> <out>");
            System.exit(2);
        }

        deleteOutput(conf);
        
        Job job = new Job(conf, "Elements Map");
        job.setNumReduceTasks(1);
        job.setJarByClass(ElementsMapJob.class);

        job.setMapperClass(ElementsVersionsMapper.class);
        job.setCombinerClass(ElementsVersionsReducer.class);
        job.setReducerClass(ElementsVersionsReducer.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(ElementsStringArrayWritable.class);

        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(ElementsMapWritable.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

    private static void deleteOutput(Configuration conf) {
        Path outputPath = new Path("target/output");
        try {
            FileSystem  fs = FileSystem.get(new URI(outputPath.toString()), conf);
            if (fs.exists(outputPath)) {
                fs.delete(outputPath, true);
            }
            fs.close();
        }
        catch (IOException ioe) {
            throw new RuntimeException("File could not be deleted.", ioe);
        }
        catch (URISyntaxException use) {
            throw new RuntimeException("Invalid path / URI.", use);
        }
    }
}
{'fraction_non_alphanumeric': 0.10798569725864124, 'fraction_numerical': 0.0038140643623361145, 'mean_word_length': 3.2903885480572597, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2893351', 'n_tokens_mistral': 1237, 'n_tokens_neox': 1195, 'n_words': 220}