<commit_before>package com.mwzhang.java.cdf;

import org.apache.commons.math3.stat.Frequency;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.javatuples.Pair;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeSet;

/**
 * Created by Mingwei on 12/9/15.
 * <p>
 * CDF related static functionality
 * <p>
 * Generate CDF with an input of a list of Integers.
 */
public class Cdf {

    Frequency freq;
    DescriptiveStatistics stats;

    TreeSet<Double> listDoubles;
    TreeSet<Integer> listIntegers;

    public Cdf() {
        // do something
        this.freq = new Frequency();
        this.stats = new DescriptiveStatistics();
        this.listDoubles = new TreeSet<>();
        this.listIntegers = new TreeSet<>();
    }

    public static void main(String[] args) {

        // take CSV file and column id as input.
        if(args.length!=2){
            System.err.println("USAGE: java -jar javacdf.jar FILENAME COLUMN");
            return;
        }
        String filename = args[0];
        int col = Integer.valueOf(args[1]);

        Cdf cdf = new Cdf();

        // READING
        System.out.println("Reading CSV file...");
        try {
            FileReader fr = new FileReader(args[0]);
            BufferedReader reader = new BufferedReader(fr);

            String line;
            while((line=reader.readLine())!=null){
                String[] columns = line.split(",");
                if(columns.length<col){
                    System.err.println("ERR: file does not have enough columns");
                    return;
                }
                Double value = Double.valueOf(columns[col]);
                cdf.addFloat(value);
            }

            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("Outputting CSV file...");
        try {
            FileWriter fw = new FileWriter("output.csv");
            BufferedWriter writer = new BufferedWriter(fw);
            for(Pair<Double,Double> pair : cdf.getCdfFloat()){
                writer.write(String.format("%f,%f\n",pair.getValue0(),pair.getValue1()));
            }
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("Done.");

        /*
        Cdf cdf = new Cdf();
        cdf.addInt(1);
        cdf.addInt(2);
        cdf.addInt(3);
        cdf.addInt(4);
        cdf.addInt(5);
        System.out.println(cdf.getCdfInt().toString());

        cdf.reset();
        cdf.addFloat(0.1);
        cdf.addFloat(0.2);
        cdf.addFloat(0.3);
        cdf.addFloat(0.3);
        cdf.addFloat(0.3);
        cdf.addFloat(0.3);
        cdf.addFloat(0.4);
        cdf.addFloat(0.5);
        System.out.println(cdf.getCdfFloat().toString());
        */
    }

    public void addInt(Integer x) {
        freq.addValue(x);
        stats.addValue(x);
        listIntegers.add(x);
    }

    public void addFloat(double x) {
        freq.addValue(x);
        stats.addValue(x);
        listDoubles.add(x);
    }

    public Pair<List<Integer>, List<Double>> getCdfInt() {
        List<Double> percentiles = new ArrayList<>();
        List<Integer> values = new ArrayList<>();

        for (Integer n : listIntegers) {
            percentiles.add(freq.getCumPct(n));
            values.add(n);
        }

        return new Pair<>(values, percentiles);
    }

    public List<Pair<Double,Double>> getCdfFloat() {

        List<Pair<Double,Double>> results = new ArrayList<>();

        for (Double n : listDoubles) {
            Double p = freq.getCumPct(n);
            results.add(new Pair<> (n,p));
        }
        return results;
    }

    public void reset() {
        stats.clear();
        freq.clear();
    }
}
<commit_msg>bug fix and minor output revision<commit_after>package com.mwzhang.java.cdf;

import org.apache.commons.math3.stat.Frequency;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.javatuples.Pair;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeSet;

/**
 * Created by Mingwei on 12/9/15.
 * <p>
 * CDF related static functionality
 * <p>
 * Generate CDF with an input of a list of Integers.
 */
public class Cdf {

    Frequency freq;
    DescriptiveStatistics stats;

    TreeSet<Double> listDoubles;
    TreeSet<Integer> listIntegers;

    public Cdf() {
        // do something
        this.freq = new Frequency();
        this.stats = new DescriptiveStatistics();
        this.listDoubles = new TreeSet<>();
        this.listIntegers = new TreeSet<>();
    }

    public static void main(String[] args) {

        // take CSV file and column id as input.
        if(args.length!=2){
            System.err.println("USAGE: java -jar javacdf.jar FILENAME COLUMN");
            return;
        }
        String filename = args[0];
        int col = Integer.valueOf(args[1]);

        Cdf cdf = new Cdf();

        // READING
        System.out.println("Reading CSV file...");
        try {
            FileReader fr = new FileReader(args[0]);
            BufferedReader reader = new BufferedReader(fr);

            String line;
            while((line=reader.readLine())!=null){
                String[] columns = line.split(",");
                if(columns.length<col){
                    System.err.println("ERR: file does not have enough columns");
                    return;
                }
                Double value = Double.valueOf(columns[col - 1]);
                cdf.addFloat(value);
            }

            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("Outputting CSV file...");
        try {
            FileWriter fw = new FileWriter("output-cdf.csv");
            BufferedWriter writer = new BufferedWriter(fw);
            for(Pair<Double,Double> pair : cdf.getCdfFloat()){
                writer.write(String.format("%f,%f\n",pair.getValue0(),pair.getValue1()));
            }
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("Done.");

    }

    public void addInt(Integer x) {
        freq.addValue(x);
        stats.addValue(x);
        listIntegers.add(x);
    }

    public void addFloat(double x) {
        freq.addValue(x);
        stats.addValue(x);
        listDoubles.add(x);
    }

    public Pair<List<Integer>, List<Double>> getCdfInt() {
        List<Double> percentiles = new ArrayList<>();
        List<Integer> values = new ArrayList<>();

        for (Integer n : listIntegers) {
            percentiles.add(freq.getCumPct(n));
            values.add(n);
        }

        return new Pair<>(values, percentiles);
    }

    public List<Pair<Double,Double>> getCdfFloat() {

        List<Pair<Double,Double>> results = new ArrayList<>();

        for (Double n : listDoubles) {
            Double p = freq.getCumPct(n);
            results.add(new Pair<> (n,p));
        }
        return results;
    }

    public void reset() {
        stats.clear();
        freq.clear();
    }
}

{'fraction_non_alphanumeric': 0.13220620842572062, 'fraction_numerical': 0.0066518847006651885, 'mean_word_length': 1.896067415730337, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '242419', 'n_tokens_mistral': 2207, 'n_tokens_neox': 2098, 'n_words': 454}