<commit_before>package hex;

import java.util.Arrays;

import water.*;
import water.ValueArray.Column;
import water.util.Utils;

import com.google.common.base.Objects;
import com.google.gson.*;

public class Summary extends Iced {
  public static final int MAX_HIST_SZ = water.parser.Enum.MAX_ENUM_SIZE;
  public final static class ColSummary extends Iced {
    public transient Summary _summary;
    public final int _colId;
    final int NMAX = 5;
    public static final double [] DEFAULT_PERCENTILES = {0.01,0.05,0.10,0.25,0.33,0.50,0.66,0.75,0.90,0.95,0.99};
    final long [] _bins; // bins for histogram
    long _n;
    final double _start, _end, _binsz, _binszInv;
    double [] _min; // min N elements
    double [] _max; // max N elements
    private double [] _percentileValues;
    final double [] _percentiles;
    final boolean _enum;

    ColSummary(Summary s, int colId) {
      this(s,colId,null);
    }
    ColSummary(Summary s, int colId, double [] percentiles) {
      _summary = s;
      _colId = colId;
      Column c = s.ary()._cols[colId];
      _enum = c.isEnum();
      if(c.isFloat() || c.numDomainSize() > MAX_HIST_SZ){
        _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
        double binsz = 3.5 *  c._sigma/ Math.cbrt(c._n);
        int nbin = (int)((c._max - c._min) / binsz);
        long n = Math.max(c._n,1);
        double a = (c._max - c._min) / n;
        double b = Math.pow(10, Math.floor(Math.log10(a)));
        // selects among d, 5*d, and 10*d so that the number of
        // partitions go in [start, end] is closest to n
        if (a > 20*b/3)
           b *= 10;
        else if (a > 5*b/3)
           b *= 5;
        double start = b * Math.floor(c._min / b);
        double end = b * Math.ceil(c._max / b);
        n = (int)((end - start) / b);
        _bins = new long[nbin];
        _start = start;
        _binsz = binsz;
        _binszInv = 1.0/binsz;
        _end = start + nbin * binsz;
      } else {
        _start = c._min;
        _end = c._max;
        int sz = (int)c.numDomainSize();
        _bins = new long[sz];
        _binszInv = _binsz = 1.0;
        _percentiles = (c.isEnum())?null:Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
      }
      if(!_enum){
        _min = new double[NMAX];
        _max = new double[NMAX];
        Arrays.fill(_min, Double.POSITIVE_INFINITY);
        Arrays.fill(_max, Double.NEGATIVE_INFINITY);
      }
    }
    public final double [] percentiles(){return _percentiles;}

    private void computePercentiles(){
      _percentileValues = new double [_percentiles.length];
      int k = 0;
      long s = 0;
      double pval = Double.NEGATIVE_INFINITY;
      for(int j = 0; j < _percentiles.length; ++j){
        double s1 = _percentiles[j]*_n - s;
        long bc = 0;
        while(s1 > (bc = binCount(k))){
          s1 -= bc;
          s  += bc;
          k++;
        }
        _percentileValues[j] = pval =  Math.max(pval,_min[0] + k*_binsz) + s1/bc*_binsz;
      }
    }

    public double percentileValue(double threshold){
      if(_percentiles == null) throw new Error("Percentiles not available for enums!");
      int idx = Arrays.binarySearch(_percentiles, threshold);
      if(idx < 0) throw new Error("don't have requested percentile");
      if(_percentileValues == null)computePercentiles();
      return _percentileValues[idx];
    }

    void add(ColSummary other) {
      assert _bins.length == other._bins.length;
      assert Math.abs(_start - other._start) < 0.000001;
      assert Math.abs(_binszInv - other._binszInv) < 0.000000001;
      for (int i = 0; i < _bins.length; i++)
        _bins[i] += other._bins[i];
      if(_min != null){
        int j = 0, k = 0;
        double [] min = _min.clone();
        double [] max = _max.clone();
        for(int i = 0; i < _min.length; ++i){
          if(other._min[k] < _min[j]){
            min[i] = other._min[k++];
          } else if(_min[j] < other._min[k]){
            ++j;
          } else {
            ++j; ++k;
          }
        }
        j = k = 0;
        for(int i = 0; i < _max.length; ++i){
          if(other._max[k] > _max[j]){<commit_msg>Fixed reduce of Summary to update N (number of observation).<commit_after>package hex;

import java.util.Arrays;

import water.*;
import water.ValueArray.Column;
import water.util.Utils;

import com.google.common.base.Objects;
import com.google.gson.*;

public class Summary extends Iced {
  public static final int MAX_HIST_SZ = water.parser.Enum.MAX_ENUM_SIZE;
  public final static class ColSummary extends Iced {
    public transient Summary _summary;
    public final int _colId;
    final int NMAX = 5;
    public static final double [] DEFAULT_PERCENTILES = {0.01,0.05,0.10,0.25,0.33,0.50,0.66,0.75,0.90,0.95,0.99};
    final long [] _bins; // bins for histogram
    long _n;
    final double _start, _end, _binsz, _binszInv;
    double [] _min; // min N elements
    double [] _max; // max N elements
    private double [] _percentileValues;
    final double [] _percentiles;
    final boolean _enum;

    ColSummary(Summary s, int colId) {
      this(s,colId,null);
    }
    ColSummary(Summary s, int colId, double [] percentiles) {
      _summary = s;
      _colId = colId;
      Column c = s.ary()._cols[colId];
      _enum = c.isEnum();
      if(c.isFloat() || c.numDomainSize() > MAX_HIST_SZ){
        _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
        double binsz = 3.5 *  c._sigma/ Math.cbrt(c._n);
        int nbin = (int)((c._max - c._min) / binsz);
        long n = Math.max(c._n,1);
        double a = (c._max - c._min) / n;
        double b = Math.pow(10, Math.floor(Math.log10(a)));
        // selects among d, 5*d, and 10*d so that the number of
        // partitions go in [start, end] is closest to n
        if (a > 20*b/3)
           b *= 10;
        else if (a > 5*b/3)
           b *= 5;
        double start = b * Math.floor(c._min / b);
        double end = b * Math.ceil(c._max / b);
        n = (int)((end - start) / b);
        _bins = new long[nbin];
        _start = start;
        _binsz = binsz;
        _binszInv = 1.0/binsz;
        _end = start + nbin * binsz;
      } else {
        _start = c._min;
        _end = c._max;
        int sz = (int)c.numDomainSize();
        _bins = new long[sz];
        _binszInv = _binsz = 1.0;
        _percentiles = (c.isEnum())?null:Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
      }
      if(!_enum){
        _min = new double[NMAX];
        _max = new double[NMAX];
        Arrays.fill(_min, Double.POSITIVE_INFINITY);
        Arrays.fill(_max, Double.NEGATIVE_INFINITY);
      }
    }
    public final double [] percentiles(){return _percentiles;}

    private void computePercentiles(){
      _percentileValues = new double [_percentiles.length];
      int k = 0;
      long s = 0;
      double pval = Double.NEGATIVE_INFINITY;
      for(int j = 0; j < _percentiles.length; ++j){
        double s1 = _percentiles[j]*_n - s;
        long bc = 0;
        while(s1 > (bc = binCount(k))){
          s1 -= bc;
          s  += bc;
          k++;
        }
        _percentileValues[j] = pval =  Math.max(pval,_min[0] + k*_binsz) + s1/bc*_binsz;
      }
    }

    public double percentileValue(double threshold){
      if(_percentiles == null) throw new Error("Percentiles not available for enums!");
      int idx = Arrays.binarySearch(_percentiles, threshold);
      if(idx < 0) throw new Error("don't have requested percentile");
      if(_percentileValues == null)computePercentiles();
      return _percentileValues[idx];
    }

    void add(ColSummary other) {
      assert _bins.length == other._bins.length;
      assert Math.abs(_start - other._start) < 0.000001;
      assert Math.abs(_binszInv - other._binszInv) < 0.000000001;
      _n += other._n;
      for (int i = 0; i < _bins.length; i++)
        _bins[i] += other._bins[i];
      if(_min != null){
        int j = 0, k = 0;
        double [] min = _min.clone();
        double [] max = _max.clone();
        for(int i = 0; i < _min.length; ++i){
          if(other._min[k] < _min[j]){
            min[i] = other._min[k++];
          } else if(_min[j] < other._min[k]){
            ++j;
          } else {
            ++j; ++k;
          }
        }
        j = k = 0;
        for(int i = 0; i < _max.length; ++i){
          if(other._max[k] > _max[j]){
{'fraction_non_alphanumeric': 0.13833710003568456, 'fraction_numerical': 0.02117283216367313, 'mean_word_length': 2.1549718574108816, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4799096', 'n_tokens_mistral': 3041, 'n_tokens_neox': 2868, 'n_words': 724}