// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/io/WARCWritable.java
// public class WARCWritable
//         implements Writable
// {
// 
//     private WARCRecord record;
// 
//     /**
//      * Creates an empty writable (with a null record).
//      */
//     public WARCWritable()
//     {
//         this.record = null;
//     }
// 
//     /**
//      * Creates a writable wrapper around a given WARCRecord.
//      *
//      * @param record existing record
//      */
//     public WARCWritable(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Returns the record currently wrapped by this writable.
//      *
//      * @return current record
//      */
//     public WARCRecord getRecord()
//     {
//         return record;
//     }
// 
//     /**
//      * Updates the record held within this writable wrapper.
//      *
//      * @param record the record to be set
//      */
//     public void setRecord(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Appends the current record to a {@link DataOutput} stream.
//      */
//     @Override
//     public void write(DataOutput out)
//             throws IOException
//     {
//         if (record != null) {
//             record.write(out);
//         }
//     }
// 
//     /**
//      * Parses a {@link WARCRecord} out of a {@link DataInput} stream, and makes it the
//      * writable's current record.
//      *
//      * @throws IOException if the input is malformed
//      */
//     @Override
//     public void readFields(DataInput in)
//             throws IOException
//     {
//         record = new WARCRecord(in);
//     }
// }


import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.ConfigurationHelper;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.io.WARCWritable;
import de.tudarmstadt.ukp.dkpro.c4corpus.warc.io.WARCRecord;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
    public int run(String[] args)
            throws Exception
    {
        Job job = Job.getInstance(getConf());

        ConfigurationHelper
                .configureJob(job, LangLicenseStatistics.class, MapperClass.class,
                        ReducerClass.class,
                        args[0], args[1]);

        // intermediate data
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(MapWritable.class);

        // output data
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);

        return job.waitForCompletion(true) ? 0 : 1;
    }

    public static void main(String[] args)
            throws Exception
    {
        ToolRunner.run(new LangLicenseStatistics(), args);
    }

    public static class MapperClass
// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/io/WARCWritable.java
// public class WARCWritable
//         implements Writable
// {
// 
//     private WARCRecord record;
// 
//     /**
//      * Creates an empty writable (with a null record).
//      */
//     public WARCWritable()
//     {
//         this.record = null;
//     }
// 
//     /**
//      * Creates a writable wrapper around a given WARCRecord.
//      *
//      * @param record existing record
//      */
//     public WARCWritable(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Returns the record currently wrapped by this writable.
//      *
//      * @return current record
//      */
//     public WARCRecord getRecord()
//     {
//         return record;
//     }
// 
//     /**
//      * Updates the record held within this writable wrapper.
//      *
//      * @param record the record to be set
//      */
//     public void setRecord(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Appends the current record to a {@link DataOutput} stream.
//      */
//     @Override
//     public void write(DataOutput out)
//             throws IOException
//     {
//         if (record != null) {
//             record.write(out);
//         }
//     }
// 
//     /**
//      * Parses a {@link WARCRecord} out of a {@link DataInput} stream, and makes it the
//      * writable's current record.
//      *
//      * @throws IOException if the input is malformed
//      */
//     @Override
//     public void readFields(DataInput in)
//             throws IOException
//     {
//         record = new WARCRecord(in);
//     }
// }


// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/statistics/LangLicenseStatistics.java
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.ConfigurationHelper;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.io.WARCWritable;
import de.tudarmstadt.ukp.dkpro.c4corpus.warc.io.WARCRecord;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

    public int run(String[] args)
            throws Exception
    {
        Job job = Job.getInstance(getConf());

        ConfigurationHelper
                .configureJob(job, LangLicenseStatistics.class, MapperClass.class,
                        ReducerClass.class,
                        args[0], args[1]);

        // intermediate data
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(MapWritable.class);

        // output data
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);

        return job.waitForCompletion(true) ? 0 : 1;
    }

    public static void main(String[] args)
            throws Exception
    {
        ToolRunner.run(new LangLicenseStatistics(), args);
    }

    public static class MapperClass
            extends Mapper<LongWritable, WARCWritable, Text, MapWritable>