// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/io/WARCWritable.java
// public class WARCWritable
//         implements Writable
// {
// 
//     private WARCRecord record;
// 
//     /**
//      * Creates an empty writable (with a null record).
//      */
//     public WARCWritable()
//     {
//         this.record = null;
//     }
// 
//     /**
//      * Creates a writable wrapper around a given WARCRecord.
//      *
//      * @param record existing record
//      */
//     public WARCWritable(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Returns the record currently wrapped by this writable.
//      *
//      * @return current record
//      */
//     public WARCRecord getRecord()
//     {
//         return record;
//     }
// 
//     /**
//      * Updates the record held within this writable wrapper.
//      *
//      * @param record the record to be set
//      */
//     public void setRecord(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Appends the current record to a {@link DataOutput} stream.
//      */
//     @Override
//     public void write(DataOutput out)
//             throws IOException
//     {
//         if (record != null) {
//             record.write(out);
//         }
//     }
// 
//     /**
//      * Parses a {@link WARCRecord} out of a {@link DataInput} stream, and makes it the
//      * writable's current record.
//      *
//      * @throws IOException if the input is malformed
//      */
//     @Override
//     public void readFields(DataInput in)
//             throws IOException
//     {
//         record = new WARCRecord(in);
//     }
// }


import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.ConfigurationHelper;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.io.WARCWritable;
import de.tudarmstadt.ukp.dkpro.c4corpus.warc.io.WARCRecord;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/*
 * Copyright 2016
 * Ubiquitous Knowledge Processing (UKP) Lab
 * Technische Universität Darmstadt
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics;



/**
 * Collecting statistics over languages and licenses in processed Web corpus.
 *
 * @author Ivan Habernal
 */
public class LangLicenseStatistics
        extends Configured
        implements Tool
{

    private static final Text KEY_DOCUMENTS = new Text("documents");
    private static final Text KEY_TOKENS = new Text("tokens");

    private static final Pattern PATTERN = Pattern.compile("\\s+");

    /**
     * Returns an approximate number of words based on number of whitespace blocks.
     *
     * @param string string
     * @return word count
     */
// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/io/WARCWritable.java
// public class WARCWritable
//         implements Writable
// {
// 
//     private WARCRecord record;
// 
//     /**
//      * Creates an empty writable (with a null record).
//      */
//     public WARCWritable()
//     {
//         this.record = null;
//     }
// 
//     /**
//      * Creates a writable wrapper around a given WARCRecord.
//      *
//      * @param record existing record
//      */
//     public WARCWritable(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Returns the record currently wrapped by this writable.
//      *
//      * @return current record
//      */
//     public WARCRecord getRecord()
//     {
//         return record;
//     }
// 
//     /**
//      * Updates the record held within this writable wrapper.
//      *
//      * @param record the record to be set
//      */
//     public void setRecord(WARCRecord record)
//     {
//         this.record = record;
//     }
// 
//     /**
//      * Appends the current record to a {@link DataOutput} stream.
//      */
//     @Override
//     public void write(DataOutput out)
//             throws IOException
//     {
//         if (record != null) {
//             record.write(out);
//         }
//     }
// 
//     /**
//      * Parses a {@link WARCRecord} out of a {@link DataInput} stream, and makes it the
//      * writable's current record.
//      *
//      * @throws IOException if the input is malformed
//      */
//     @Override
//     public void readFields(DataInput in)
//             throws IOException
//     {
//         record = new WARCRecord(in);
//     }
// }


// Path: dkpro-c4corpus-hadoop/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/hadoop/statistics/LangLicenseStatistics.java
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.ConfigurationHelper;
import de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.io.WARCWritable;
import de.tudarmstadt.ukp.dkpro.c4corpus.warc.io.WARCRecord;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/*
 * Copyright 2016
 * Ubiquitous Knowledge Processing (UKP) Lab
 * Technische Universität Darmstadt
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics;



/**
 * Collecting statistics over languages and licenses in processed Web corpus.
 *
 * @author Ivan Habernal
 */
public class LangLicenseStatistics
        extends Configured
        implements Tool
{

    private static final Text KEY_DOCUMENTS = new Text("documents");
    private static final Text KEY_TOKENS = new Text("tokens");

    private static final Pattern PATTERN = Pattern.compile("\\s+");

    /**
     * Returns an approximate number of words based on number of whitespace blocks.
     *
     * @param string string
     * @return word count
     */
    public static int fastApproximateWordCount(String string)