<commit_before>package edu.jhu.thrax.hadoop.jobs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import edu.jhu.thrax.hadoop.distributional.DistributionalContextCombiner;
import edu.jhu.thrax.hadoop.distributional.DistributionalContextMapper;
import edu.jhu.thrax.hadoop.distributional.DistributionalContextReducer;
import edu.jhu.thrax.hadoop.distributional.SignatureWritable;

public class DistributionalContextExtractionJob extends ThraxJob {
	
	public Job getJob(Configuration conf) throws IOException {
		Job job = new Job(conf, "distributional");
		job.setJarByClass(DistributionalContextMapper.class);
		job.setMapperClass(DistributionalContextMapper.class);
//		job.setCombinerClass(DistributionalContextCombiner.class);
		job.setReducerClass(DistributionalContextReducer.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(MapWritable.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(SignatureWritable.class);
		job.setOutputFormatClass(SequenceFileOutputFormat.class);

		int numReducers = conf.getInt("thrax.reducers", 4);
		job.setNumReduceTasks(numReducers);<commit_msg>Fixed reducer typing in job.<commit_after>package edu.jhu.thrax.hadoop.jobs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import edu.jhu.thrax.hadoop.distributional.DistributionalContextMapper;
import edu.jhu.thrax.hadoop.distributional.DistributionalContextReducer;
import edu.jhu.thrax.hadoop.distributional.SignatureWritable;

public class DistributionalContextExtractionJob extends ThraxJob {
	
	public Job getJob(Configuration conf) throws IOException {
		Job job = new Job(conf, "distributional");
		job.setJarByClass(DistributionalContextMapper.class);
		job.setMapperClass(DistributionalContextMapper.class);
//		job.setCombinerClass(DistributionalContextCombiner.class);
		job.setReducerClass(DistributionalContextReducer.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(MapWritable.class);
		job.setOutputKeyClass(SignatureWritable.class);
		job.setOutputValueClass(NullWritable.class);
		job.setOutputFormatClass(SequenceFileOutputFormat.class);

		int numReducers = conf.getInt("thrax.reducers", 4);
		job.setNumReduceTasks(numReducers);
{'fraction_non_alphanumeric': 0.10103024260551678, 'fraction_numerical': 0.0006646726487205051, 'mean_word_length': 7.113207547169812, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2144448', 'n_tokens_mistral': 1001, 'n_tokens_neox': 922, 'n_words': 119}