<commit_before>package es.tid.bdp.profile.export.ps;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import es.tid.bdp.profile.dictionary.comscore.DistributedCacheDictionary;
import es.tid.bdp.profile.generated.data.ProfileProtocol.UserProfile;

/**
 * Export user profiles to mongodb
 *
 * @author sortega
 */
public class PSExporterJob extends Job {
    private static final String JOB_NAME = "PSExporterJob";

    public PSExporterJob(Configuration conf) throws IOException {
        super(conf, JOB_NAME);
        this.setJarByClass(PSExporterJob.class);
        this.setMapOutputKeyClass(Text.class);
        this.setMapOutputValueClass(UserProfile.class);
        this.setReducerClass(PSExporterReducer.class);
        this.setNumReduceTasks(1);
        this.setOutputKeyClass(NullWritable.class);
        this.setOutputValueClass(Text.class);
    }

    /**
     * Configure inputs and outputs.
     *
     * @param inputPath HDFS input path
     * @param outputPath HDFS output path
     *
     * @throws IOException
     */
    public void configure(Path inputPath, Path outputPath) throws IOException {
        this.setInputFormatClass(SequenceFileInputFormat.class);
        TextInputFormat.setInputPaths(this, inputPath);
        this.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(this, outputPath);<commit_msg>Bugfix: wrong job configuration<commit_after>package es.tid.bdp.profile.export.ps;

import java.io.IOException;

import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import es.tid.bdp.profile.dictionary.comscore.DistributedCacheDictionary;

/**
 * Export user profiles to mongodb
 *
 * @author sortega
 */
public class PSExporterJob extends Job {
    private static final String JOB_NAME = "PSExporterJob";

    public PSExporterJob(Configuration conf) throws IOException {
        super(conf, JOB_NAME);
        this.setJarByClass(PSExporterJob.class);
        this.setMapOutputKeyClass(Text.class);
        this.setMapOutputValueClass(ProtobufWritable.class);
        this.setReducerClass(PSExporterReducer.class);
        this.setNumReduceTasks(1);
        this.setOutputKeyClass(NullWritable.class);
        this.setOutputValueClass(Text.class);
    }

    /**
     * Configure inputs and outputs.
     *
     * @param inputPath HDFS input path
     * @param outputPath HDFS output path
     *
     * @throws IOException
     */
    public void configure(Path inputPath, Path outputPath) throws IOException {
        this.setInputFormatClass(SequenceFileInputFormat.class);
        TextInputFormat.setInputPaths(this, inputPath);
        this.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(this, outputPath);
{'fraction_non_alphanumeric': 0.09604357798165138, 'fraction_numerical': 0.0005733944954128441, 'mean_word_length': 4.027377521613833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2898469', 'n_tokens_mistral': 1030, 'n_tokens_neox': 1010, 'n_words': 174}