<commit_before>package fr.liglab.lcm.mapred;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import fr.liglab.lcm.mapred.writables.GIDandRebaseWritable;
import fr.liglab.lcm.mapred.writables.ItemAndSupportWritable;
import fr.liglab.lcm.mapred.writables.TransactionWritable;

/**
 * Driver methods for our 3 map-reduce jobs
 */
public class Driver {
	//////////////////// MANDATORY CONFIGURATION PROPERTIES ////////////////////
	
	public static final String KEY_INPUT    = "fr.liglab.lcm.input";
	public static final String KEY_OUTPUT   = "fr.liglab.lcm.output";
	public static final String KEY_MINSUP   = "fr.liglab.lcm.minsup";
	public static final String KEY_NBGROUPS = "fr.liglab.lcm.nbGroups";
	public static final String KEY_DO_TOP_K = "fr.liglab.lcm.topK";
	
	//////////////////// OPTIONAL CONFIGURATION PROPERTIES ////////////////////
	
	public static final String KEY_GROUPER_CLASS = "fr.liglab.lcm.class.grouper";
	public static final String KEY_MINING_ALGO = "fr.liglab.lcm.mapred.algo";
	public static final String KEY_SINGLE_GROUP_ID = "fr.liglab.lcm.single-group";
	public static final String KEY_DUMP_ON_HEAP_EXN = "fr.liglab.lcm.dump-path";
	
	
	//////////////////// INTERNAL CONFIGURATION PROPERTIES ////////////////////
	
	/**
	 * property key for item-counter-n-grouper output path
	 */
	static final String KEY_GROUPS_MAP = "fr.liglab.lcm.groupsMap";
	
	/**
	 * property key for mining output path
	 */
	static final String KEY_RAW_PATTERNS = "fr.liglab.lcm.rawpatterns";
	
	/**
	 * property key for aggregated patterns' output path
	 */
	static final String KEY_AGGREGATED_PATTERNS = "fr.liglab.lcm.aggregated";
	
	
	
	protected final Configuration conf;
	protected final String input;<commit_msg>all configuration properties now have a shorter key, starting with lcm.*<commit_after>package fr.liglab.lcm.mapred;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import fr.liglab.lcm.mapred.writables.GIDandRebaseWritable;
import fr.liglab.lcm.mapred.writables.ItemAndSupportWritable;
import fr.liglab.lcm.mapred.writables.TransactionWritable;

/**
 * Driver methods for our 3 map-reduce jobs
 */
public class Driver {
	//////////////////// MANDATORY CONFIGURATION PROPERTIES ////////////////////
	
	public static final String KEY_INPUT    = "lcm.input";
	public static final String KEY_OUTPUT   = "lcm.output";
	public static final String KEY_MINSUP   = "lcm.minsup";
	public static final String KEY_NBGROUPS = "lcm.nbGroups";
	public static final String KEY_DO_TOP_K = "lcm.topK";
	
	//////////////////// OPTIONAL CONFIGURATION PROPERTIES ////////////////////
	
	public static final String KEY_GROUPER_CLASS = "lcm.grouper";
	public static final String KEY_MINING_ALGO = "lcm.mapred.algo";
	public static final String KEY_SINGLE_GROUP_ID = "lcm.single-group";
	public static final String KEY_DUMP_ON_HEAP_EXN = "lcm.dump-path";
	
	
	//////////////////// INTERNAL CONFIGURATION PROPERTIES ////////////////////
	
	/**
	 * property key for item-counter-n-grouper output path
	 */
	static final String KEY_GROUPS_MAP = "lcm.groupsMap";
	
	/**
	 * property key for mining output path
	 */
	static final String KEY_RAW_PATTERNS = "lcm.rawpatterns";
	
	/**
	 * property key for aggregated patterns' output path
	 */
	static final String KEY_AGGREGATED_PATTERNS = "lcm.aggregated";
	
	
	
	protected final Configuration conf;
	protected final String input;
{'fraction_non_alphanumeric': 0.14418921992267456, 'fraction_numerical': 0.00045485558335228563, 'mean_word_length': 3.6052356020942407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6532470', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1356, 'n_words': 299}