<commit_before>package com.example.pagerank;

import java.util.Arrays;

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.example.pagerank.io.PageRankWritable;
import com.example.pagerank.mapreduce.BuildGraphMapper;

public class BuildGraphDriver extends Configured implements Tool {
	private static final Logger LOG = LoggerFactory
			.getLogger(BuildGraphDriver.class);

	private static final String INPUT = "input";
	private static final String OUTPUT = "output";
	private static final String SEED = "seed";
	private float seed = 0.5f;

	private String inputPath;

	private String outputPath;

	public static void main(String[] args) throws Exception {
		ToolRunner.run(new BuildGraphDriver(), args);
	}

	@Override
	public int run(String[] args) throws Exception {
		LOG.info("Building the Page Rank Graph");
		parseArgs(args);

		Configuration conf = getConf();
		conf.setFloat("seed", seed);

		Job job = Job.getInstance(conf, "Build Graph");
		job.setJobName(BuildGraphDriver.class.getSimpleName() + ":" + inputPath);
		job.setJarByClass(BuildGraphDriver.class);
		job.setNumReduceTasks(1);

		FileInputFormat.addInputPath(job, new Path(inputPath));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));

		job.setInputFormatClass(TextInputFormat.class);
		job.setOutputFormatClass(SequenceFileOutputFormat.class);

		job.setOutputKeyClass(PageRankWritable.class);
		job.setOutputValueClass(PageRankWritable.class);

		job.setMapOutputKeyClass(PageRankWritable.class);
		job.setMapOutputValueClass(PageRankWritable.class);

		job.setMapperClass(BuildGraphMapper.class);

		// Delete the output directory if it exists already.
		FileSystem.get(conf).delete(new Path(outputPath), true);

		job.waitForCompletion(true);

		return 0;
	}

	private void parseArgs(String[] args) {
		LOG.info("Parsing Arguments");
		Option inputOption = new Option("i", "input", true, "input path");
		Option outputOption = new Option("o", "output", true, "output path");
		Option seedOption = new Option("s", "seed", true, "initial page rank seed");

		Options options = new Options();
		options.addOption(outputOption);
		options.addOption(inputOption);
		options.addOption(seedOption);

		CommandLineParser parser = new BasicParser();
		CommandLine cmdline = null;

		try {
			cmdline = parser.parse(options, args);
		} catch (ParseException exp) {
			LOG.error("Error parsing command line: " + exp.getMessage(), exp);
			System.exit(-1);
		}<commit_msg>added total time to build driver<commit_after>package com.example.pagerank;

import java.util.Arrays;

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.example.pagerank.io.PageRankWritable;
import com.example.pagerank.mapreduce.BuildGraphMapper;

public class BuildGraphDriver extends Configured implements Tool {
	private static final Logger LOG = LoggerFactory
			.getLogger(BuildGraphDriver.class);

	private static final String INPUT = "input";
	private static final String OUTPUT = "output";
	private static final String SEED = "seed";
	private float seed = 0.5f;

	private String inputPath;

	private String outputPath;

	public static void main(String[] args) throws Exception {
		DateTime startDateTime = new DateTime();

		ToolRunner.run(new BuildGraphDriver(), args);
		DateTime endDateTime = new DateTime();
		LOG.info("All done!");
		Interval interval = new Interval(startDateTime, endDateTime);
		LOG.info("Total Time: " + interval.toDuration().getStandardSeconds());
	}

	@Override
	public int run(String[] args) throws Exception {
		LOG.info("Building the Page Rank Graph");
		parseArgs(args);

		Configuration conf = getConf();
		conf.setFloat("seed", seed);

		Job job = Job.getInstance(conf, "Build Graph");
		job.setJobName(BuildGraphDriver.class.getSimpleName() + ":" + inputPath);
		job.setJarByClass(BuildGraphDriver.class);
		job.setNumReduceTasks(1);

		FileInputFormat.addInputPath(job, new Path(inputPath));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));

		job.setInputFormatClass(TextInputFormat.class);
		job.setOutputFormatClass(SequenceFileOutputFormat.class);

		job.setOutputKeyClass(PageRankWritable.class);
		job.setOutputValueClass(PageRankWritable.class);

		job.setMapOutputKeyClass(PageRankWritable.class);
		job.setMapOutputValueClass(PageRankWritable.class);

		job.setMapperClass(BuildGraphMapper.class);

		// Delete the output directory if it exists already.
		FileSystem.get(conf).delete(new Path(outputPath), true);

		job.waitForCompletion(true);

		return 0;
	}

	private void parseArgs(String[] args) {
		LOG.info("Parsing Arguments");
		Option inputOption = new Option("i", "input", true, "input path");
		Option outputOption = new Option("o", "output", true, "output path");
		Option seedOption = new Option("s", "seed", true,
				"initial page rank seed");

		Options options = new Options();
		options.addOption(outputOption);
		options.addOption(inputOption);
		options.addOption(seedOption);

		CommandLineParser parser = new BasicParser();
		CommandLine cmdline = null;

		try {
			cmdline = parser.parse(options, args);
		} catch (ParseException exp) {
			LOG.error("Error parsing command line: " + exp.getMessage(), exp);
			System.exit(-1);
		}
{'fraction_non_alphanumeric': 0.11683453237410071, 'fraction_numerical': 0.002014388489208633, 'mean_word_length': 5.557547169811321, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6885878', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2066, 'n_words': 455}