<commit_before>package hex;

import java.util.Arrays;

import water.*;
import water.ValueArray.Column;
import water.util.Utils;

import com.google.common.base.Objects;
import com.google.gson.*;

public class Summary extends Iced {
  public static final int MAX_HIST_SZ = water.parser.Enum.MAX_ENUM_SIZE;
  public final static class ColSummary extends Iced {
    public transient Summary _summary;
    public final int _colId;
    final int NMAX = 5;
    public static final double [] DEFAULT_PERCENTILES = {0.01,0.05,0.10,0.25,0.33,0.50,0.66,0.75,0.90,0.95,0.99};
    final long [] _bins; // bins for histogram
    long _n;
    final double _start, _end, _binsz, _binszInv;
    double [] _min; // min N elements
    double [] _max; // max N elements
    private double [] _percentileValues;
    final double [] _percentiles;
    final boolean _enum;

    ColSummary(Summary s, int colId) {
      this(s,colId,null);
    }
    ColSummary(Summary s, int colId, double [] percentiles) {
      _summary = s;
      _colId = colId;
      Column c = s.ary()._cols[colId];
      _enum = c.isEnum();
      final long n = Math.max(c._n,1);
      if(_enum){
        _percentiles = null;
        _binsz = _binszInv = 1;
        _bins = new long[(int)n];
        _start = 0;
        _end = n;
      } else {
        _min = new double[NMAX];
        _max = new double[NMAX];
        Arrays.fill(_min, Double.POSITIVE_INFINITY);
        Arrays.fill(_max, Double.NEGATIVE_INFINITY);
        if(c.isFloat() || c.numDomainSize() > MAX_HIST_SZ){
          _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);

          double a = (c._max - c._min) / n;
          double b = Math.pow(10, Math.floor(Math.log10(a)));
          // selects among d, 5*d, and 10*d so that the number of
          // partitions go in [start, end] is closest to n
          if (a > 20*b/3)
             b *= 10;
          else if (a > 5*b/3)
             b *= 5;
          double start = b * Math.floor(c._min / b);

          // guard against improper parse (date type) or zero c._sigma
          double binsz = Math.max(1e-4, 3.5 *  c._sigma/ Math.cbrt(c._n));
          // Pick smaller of two for number of bins to avoid blowup of longs
          int nbin = Math.max(Math.min(MAX_HIST_SZ,(int)((c._max - c._min) / binsz)),1);
          _bins = new long[nbin];

          _start = start;
          _binsz = binsz;
          _binszInv = 1.0/binsz;
          _end = start + nbin * binsz;
        } else {
          _start = c._min;
          _end = c._max;
          int sz = (int)c.numDomainSize();
          _bins = new long[sz];
          _binszInv = _binsz = 1.0;
          _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
        }
      }
    }
    public final double [] percentiles(){return _percentiles;}

    private void computePercentiles(){
      _percentileValues = new double [_percentiles.length];
      if( _bins.length == 0 ) return;
      int k = 0;
      long s = 0;
      double pval = Double.NEGATIVE_INFINITY;
      for(int j = 0; j < _percentiles.length; ++j){
        double s1 = _percentiles[j]*_n - s;
        long bc = 0;
        while(s1 > (bc = binCount(k))){
          s1 -= bc;
          s  += bc;
          k++;
        }
        _percentileValues[j] = pval =  Math.max(pval,_min[0] + k*_binsz) + s1/bc*_binsz;
      }
    }

    public double percentileValue(double threshold){
      if(_percentiles == null) throw new Error("Percentiles not available for enums!");
      int idx = Arrays.binarySearch(_percentiles, threshold);
      if(idx < 0) throw new Error("don't have requested percentile");
      if(_percentileValues == null)computePercentiles();
      return _percentileValues[idx];
    }

    void add(ColSummary other) {
      assert _bins.length == other._bins.length;
      assert Math.abs(_start - other._start) < 0.000001;
      assert Math.abs(_binszInv - other._binszInv) < 0.000000001;
      _n += other._n;
      for (int i = 0; i < _bins.length; i++)
        _bins[i] += other._bins[i];
      if(_min != null){
        int j = 0, k = 0;
        double [] min = _min.clone();
        double [] max = _max.clone();
        for(int i = 0; i < _min.length; ++i){
          if(other._min[k] < _min[j]){
            min[i] = other._min[k++];
          } else if(_min[j] < other._min[k]){
            ++j;
          } else {
            ++j; ++k;
          }
        }
        j = k = 0;
        for(int i = 0; i < _max.length; ++i){
          if(other._max[k] > _max[j]){
            max[i] = other._max[k++];
          } else if (_max[j] > other._max[k]){
            ++j;<commit_msg>Bugfix in the summary. Again constant column problem. Bin start came out as NaN which triggered assertion error later.<commit_after>package hex;

import java.util.Arrays;

import water.*;
import water.ValueArray.Column;
import water.util.Utils;

import com.google.common.base.Objects;
import com.google.gson.*;

public class Summary extends Iced {
  public static final int MAX_HIST_SZ = water.parser.Enum.MAX_ENUM_SIZE;
  public final static class ColSummary extends Iced {
    public transient Summary _summary;
    public final int _colId;
    final int NMAX = 5;
    public static final double [] DEFAULT_PERCENTILES = {0.01,0.05,0.10,0.25,0.33,0.50,0.66,0.75,0.90,0.95,0.99};
    final long [] _bins; // bins for histogram
    long _n;
    final double _start, _end, _binsz, _binszInv;
    double [] _min; // min N elements
    double [] _max; // max N elements
    private double [] _percentileValues;
    final double [] _percentiles;
    final boolean _enum;

    ColSummary(Summary s, int colId) {
      this(s,colId,null);
    }
    ColSummary(Summary s, int colId, double [] percentiles) {
      _summary = s;
      _colId = colId;
      Column c = s.ary()._cols[colId];
      _enum = c.isEnum();
      if(c._min == c._max){ // constant columns pecial case, not really any meaningfull data here, just don't blow up
        _start = c._min;
        _binsz = _binszInv = 1;
        _end = _start+1;
        _percentiles = null;
        _bins = new long[1];
        _min = new double[]{c._min};
        _max = new double[]{c._max};
      } else {
        final long n = Math.max(c._n,1);
        if(_enum){
          _percentiles = null;
          _binsz = _binszInv = 1;
          _bins = new long[(int)n];
          _start = 0;
          _end = n;
          _min = _max = null;
        } else {
          _min = new double[NMAX];
          _max = new double[NMAX];
          Arrays.fill(_min, Double.POSITIVE_INFINITY);
          Arrays.fill(_max, Double.NEGATIVE_INFINITY);
          if(c.isFloat() || c.numDomainSize() > MAX_HIST_SZ){
            double start, binsz;
            int nbin;
            double a = (c._max - c._min) / n;
            double b = Math.pow(10, Math.floor(Math.log10(a)));
            // selects among d, 5*d, and 10*d so that the number of
            // partitions go in [start, end] is closest to n
            if (a > 20*b/3)
               b *= 10;
            else if (a > 5*b/3)
               b *= 5;
            start = b * Math.floor(c._min / b);
            if(Double.isNaN(start))
              System.out.println("haha");
            // guard against improper parse (date type) or zero c._sigma
            binsz = Math.max(1e-4, 3.5 *  c._sigma/ Math.cbrt(c._n));
            // Pick smaller of two for number of bins to avoid blowup of longs
            nbin = Math.max(Math.min(MAX_HIST_SZ,(int)((c._max - c._min) / binsz)),1);
            _bins = new long[nbin];
            _start = start;
            _binsz = binsz;
            _binszInv = 1.0/binsz;
            _end = start + nbin * binsz;
            _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
          } else {
            _start = c._min;
            _end = c._max;
            int sz = (int)c.numDomainSize();
            _bins = new long[sz];
            _binszInv = _binsz = 1.0;
            _percentiles = Objects.firstNonNull(percentiles, DEFAULT_PERCENTILES);
          }
        }
      }
      assert !Double.isNaN(_start):"_start is NaN!";
    }
    public final double [] percentiles(){return _percentiles;}

    private void computePercentiles(){
      _percentileValues = new double [_percentiles.length];
      if( _bins.length == 0 ) return;
      int k = 0;
      long s = 0;
      double pval = Double.NEGATIVE_INFINITY;
      for(int j = 0; j < _percentiles.length; ++j){
        double s1 = _percentiles[j]*_n - s;
        long bc = 0;
        while(s1 > (bc = binCount(k))){
          s1 -= bc;
          s  += bc;
          k++;
        }
        _percentileValues[j] = pval =  Math.max(pval,_min[0] + k*_binsz) + s1/bc*_binsz;
      }
    }

    public double percentileValue(double threshold){
      if(_percentiles == null) throw new Error("Percentiles not available for enums!");
      int idx = Arrays.binarySearch(_percentiles, threshold);
      if(idx < 0) throw new Error("don't have requested percentile");
      if(_percentileValues == null)computePercentiles();
      return _percentileValues[idx];
    }

    void add(ColSummary other) {
      assert _bins.length == other._bins.length;
      assert Math.abs(_start - other._start) < 0.000001:"start - other._start = " + (_start - other._start);
      assert Math.abs(_binszInv - other._binszInv) < 0.000000001;
      _n += other._n;
      for (int i = 0; i < _bins.length; i++)
        _bins[i] += other._bins[i];
      if(_min != null){
        int j = 0, k = 0;
        double [] min = _min.clone();
        double [] max = _max.clone();
        for(int i = 0; i < _min.length; ++i){
          if(other._min[k] < _min[j]){
            min[i] = other._min[k++];
          } else if(_min[j] < other._min[k]){
            ++j;
          } else {
            ++j; ++k;
          }
        }
        j = k = 0;
        for(int i = 0; i < _max.length; ++i){
          if(other._max[k] > _max[j]){
            max[i] = other._max[k++];
          } else if (_max[j] > other._max[k]){
            ++j;
{'fraction_non_alphanumeric': 0.132026664013531, 'fraction_numerical': 0.019202069445826284, 'mean_word_length': 1.9713272243570794, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4987179', 'n_tokens_mistral': 3527, 'n_tokens_neox': 3315, 'n_words': 862}