<commit_before>/*
 * This file is part of CoAnSys project.
 * Copyright (c) 2012-2013 ICM-UW
 * 
 * CoAnSys is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * CoAnSys is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with CoAnSys. If not, see <http://www.gnu.org/licenses/>.
 */
package pl.edu.icm.coansys.statisticsgenerator.jobs;

import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Date;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pl.edu.icm.coansys.models.StatisticsProtos;
import pl.edu.icm.coansys.statisticsgenerator.mrtypes.SortedMapWritableComparable;
import pl.edu.icm.coansys.statisticsgenerator.operationcomponents.Partitioner;
import pl.edu.icm.coansys.statisticsgenerator.conf.StatGeneratorConfiguration;
import pl.edu.icm.coansys.statisticsgenerator.operationcomponents.StatisticCalculator;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class StatisticsGenerator implements Tool {

    private static Logger logger = LoggerFactory.getLogger(StatisticsGenerator.class);
    private Configuration conf;

    public static class StatisticsMap extends Mapper<Text, BytesWritable, SortedMapWritableComparable, BytesWritable> {

        private StatGeneratorConfiguration statGenConfiguration;

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            Configuration conf = context.getConfiguration();
            statGenConfiguration = new StatGeneratorConfiguration(conf);
        }

        @Override
        protected void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {

            StatisticsProtos.InputEntry inputEntry = StatisticsProtos.InputEntry.parseFrom(value.copyBytes());

            SortedMapWritableComparable outputKeyMap = new SortedMapWritableComparable();
            Map<String, Partitioner> partitioners = statGenConfiguration.getPartitioners();

            for (StatisticsProtos.KeyValue field : inputEntry.getFieldList()) {
                String fieldKey = field.getKey();
                if (partitioners.containsKey(fieldKey)) {
                    String[] partitions = partitioners.get(fieldKey).partition(field.getValue());
                    for (String partition : partitions) {
                        outputKeyMap.put(new Text(fieldKey), new Text(partition));
                    }
                }
            }

            context.write(outputKeyMap, value);
        }
    }

    public static class StatisticsReduce extends Reducer<SortedMapWritableComparable, BytesWritable, Text, BytesWritable> {

        private StatGeneratorConfiguration statGenConfiguration;

        @Override<commit_msg>Fixed error if partitioner returns more than 1 value<commit_after>/*
 * This file is part of CoAnSys project.
 * Copyright (c) 2012-2013 ICM-UW
 * 
 * CoAnSys is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * CoAnSys is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with CoAnSys. If not, see <http://www.gnu.org/licenses/>.
 */
package pl.edu.icm.coansys.statisticsgenerator.jobs;

import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pl.edu.icm.coansys.models.StatisticsProtos;
import pl.edu.icm.coansys.statisticsgenerator.mrtypes.SortedMapWritableComparable;
import pl.edu.icm.coansys.statisticsgenerator.operationcomponents.Partitioner;
import pl.edu.icm.coansys.statisticsgenerator.conf.StatGeneratorConfiguration;
import pl.edu.icm.coansys.statisticsgenerator.operationcomponents.StatisticCalculator;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class StatisticsGenerator implements Tool {

    private static Logger logger = LoggerFactory.getLogger(StatisticsGenerator.class);
    private Configuration conf;

    public static class StatisticsMap extends Mapper<Text, BytesWritable, SortedMapWritableComparable, BytesWritable> {

        private StatGeneratorConfiguration statGenConfiguration;

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            Configuration conf = context.getConfiguration();
            statGenConfiguration = new StatGeneratorConfiguration(conf);
        }

        @Override
        protected void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {

            StatisticsProtos.InputEntry inputEntry = StatisticsProtos.InputEntry.parseFrom(value.copyBytes());

            Set<SortedMapWritableComparable> outputKeyMaps = new HashSet<SortedMapWritableComparable>();
            outputKeyMaps.add(new SortedMapWritableComparable());

            Map<String, Partitioner> partitioners = statGenConfiguration.getPartitioners();

            for (StatisticsProtos.KeyValue field : inputEntry.getFieldList()) {
                String fieldKey = field.getKey();
                if (partitioners.containsKey(fieldKey)) {
                    String[] partitions = partitioners.get(fieldKey).partition(field.getValue());
                    // the number of output values should be multiplied by number of partitions (partitions.lenght).
                    // if 0, then return nothing...
                    if (partitions.length == 0) {
                        return;
                    }
                    for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
                        outputKeyMap.put(new Text(fieldKey), new Text(partitions[0]));
                    }
                    // if more than 1, then copy output values
                    for (int i = 1; i < partitions.length; i++) {
                        Text mapKey = new Text(fieldKey);
                        Text mapValue = new Text(partitions[i]);
                        Set<SortedMapWritableComparable> newMaps = new HashSet<SortedMapWritableComparable>();
                        for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
                            SortedMapWritableComparable newMap = new SortedMapWritableComparable(outputKeyMap);
                            newMap.put(mapKey, mapValue);
                            newMaps.add(newMap);
                        }
                        outputKeyMaps.addAll(newMaps);
                    }
                }
            }

            for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
                context.write(outputKeyMap, value);
            }
        }
    }

    public static class StatisticsReduce extends Reducer<SortedMapWritableComparable, BytesWritable, Text, BytesWritable> {

        private StatGeneratorConfiguration statGenConfiguration;

        @Override
{'fraction_non_alphanumeric': 0.08087248322147651, 'fraction_numerical': 0.003131991051454139, 'mean_word_length': 3.1624767225325883, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4676060', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2268, 'n_words': 600}