<commit_before>package ai.h2o.automl;

import ai.h2o.automl.autocollect.AutoCollect;
import ai.h2o.automl.collectors.MetaCollector;
import ai.h2o.automl.guessers.ProblemTypeGuesser;
import ai.h2o.automl.tasks.DummyClassifier;
import ai.h2o.automl.tasks.DummyRegressor;
import ai.h2o.automl.tasks.VIF;
import hex.tree.DHistogram;
import water.*;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;
import water.util.ArrayUtils;
import water.util.AtomicUtils;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

/**
 * Cache common questions asked upon the frame.
 */
public class FrameMeta extends Iced {
  final String _datasetName;
  final Frame _fr;
  public int[] _numFeats;
  public int[] _intCols;
  public int[] _dblCols;
  public int[] _binaryCols;
  public int[] _intNotBinaryCols;
  private final int _response;
  private long _naCnt=-1;  // count of nas across whole frame
  private int _numFeat=-1; // count of numerical features
  private int _catFeat=-1; // count of categorical features
  private int _nclass=-1;  // number of classes if classification problem
  private double[][] _dummies=null; // dummy predictions
  public ColMeta[] _cols;

  public final static String[] METAVALUES = new String[]{
    "DatasetName", "NRow", "NCol", "LogNRow", "LogNCol", "NACount", "NAFraction",
    "NumberNumericFeat", "NumberCatFeat", "RatioNumericToCatFeat", "RatioCatToNumericFeat",
    "DatasetRatio", "LogDatasetRatio", "InverseDatasetRatio", "LogInverseDatasetRatio",
    "Classification", "DummyStratMSE", "DummyStratLogLoss", "DummyMostFreqMSE",
    "DummyMostFreqLogLoss", "DummyRandomMSE", "DummyRandomLogLoss", "DummyMedianMSE",
    "DummyMeanMSE", "NClass"};

  public static HashMap<String, Object> makeEmptyFrameMeta() {
    HashMap<String,Object> hm = new HashMap<>();
    for(String key: FrameMeta.METAVALUES) hm.put(key,null);
    return hm;
  }

  // Takes empty frame meta hashmap and fills in the metadata not requiring MRTask
  // TODO: make helper functions so that it's possible to iterate over METAVALUES only
  public void fillSimpleMeta(HashMap<String, Object> fm) {
    fm.put("DatasetName", _datasetName);
    fm.put("NRow", (double)_fr.numRows());
    fm.put("NCol", (double)_fr.numCols());
    fm.put("LogNRow", Math.log((double)fm.get("NRow")));
    fm.put("LogNCol", Math.log((double)fm.get("NCol")));
    fm.put("NACount", (double)naCount());
    fm.put("NAFraction", (double) naCount() / (double) (_fr.numCols() * _fr.numRows()));
    fm.put("NumberNumericFeat", (double)numberOfNumericFeatures());
    fm.put("NumberCatFeat", (double) numberOfCategoricalFeatures());
    fm.put("RatioNumericToCatFeat", Double.isInfinite((double) fm.get("NumberCatFeat"))     ? AutoCollect.SQLNAN : (double) fm.get("NumberNumericFeat") / (double) fm.get("NumberCatFeat"));
    fm.put("RatioCatToNumericFeat", Double.isInfinite((double) fm.get("NumberNumericFeat")) ? AutoCollect.SQLNAN : (double) fm.get("NumberCatFeat")     / (double) fm.get("NumberNumericFeat"));
    fm.put("DatasetRatio", (double) _fr.numCols() / (double) _fr.numRows());
    fm.put("LogDatasetRatio", Math.log((double) fm.get("DatasetRatio")));
    fm.put("InverseDatasetRatio", (double)_fr.numRows() / (double) _fr.numCols() );
    fm.put("LogInverseDatasetRatio", Math.log((double)fm.get("InverseDatasetRatio")));
    fm.put("Classification", _isClassification?1:0);
  }

  public void fillDummies(HashMap<String, Object> fm) {
    double[][] dummies = getDummies();
    fm.put("DummyStratMSE", _isClassification?dummies[0][0]: AutoCollect.SQLNAN);
    fm.put("DummyStratLogLoss", _isClassification?dummies[1][0]: AutoCollect.SQLNAN);
    fm.put("DummyMostFreqMSE", _isClassification?dummies[0][2]: AutoCollect.SQLNAN);
    fm.put("DummyMostFreqLogLoss", _isClassification?dummies[1][2]: AutoCollect.SQLNAN);
    fm.put("DummyRandomMSE", _isClassification?dummies[0][1]: AutoCollect.SQLNAN);
    fm.put("DummyRandomLogLoss", _isClassification?dummies[1][1]: AutoCollect.SQLNAN);
    fm.put("DummyMedianMSE", _isClassification? AutoCollect.SQLNAN:dummies[0][1]);
    fm.put("DummyMeanMSE", _isClassification? AutoCollect.SQLNAN:dummies[0][0]);
    fm.put("NClass", _nclass);
  }

  public long naCount() {
    if( _naCnt!=-1 ) return _naCnt;
    long cnt=0;
    for(Vec v: _fr.vecs()) cnt+=v.naCnt();
    return (_naCnt=cnt);
  }

  public int numberOfNumericFeatures() {
    if( _numFeat!=-1 ) return _numFeat;
    ArrayList<Integer> idxs = new ArrayList<>();
    ArrayList<Integer> intCols = new ArrayList<>();
    ArrayList<Integer> dblCols = new ArrayList<>();
    ArrayList<Integer> binCols = new ArrayList<>();
    ArrayList<Integer> intNotBinCols = new ArrayList<>();
    int cnt=0;
    int idx=0;
    for(Vec v: _fr.vecs()) {
      if( v.isNumeric() ) {
        cnt += 1;
        idxs.add(idx);
        if( v.isInt() ) intCols.add(idx);
        if( v.isBinary() ) binCols.add(idx);
        if( v.isInt() && !v.isBinary() ) intNotBinCols.add(idx);
        if( v.isNumeric() && !v.isInt() ) dblCols.add(idx);
      }
      idx++;
    }
    _numFeats = intListToA(idxs);
    _intCols  = intListToA(intCols);
    _dblCols  = intListToA(dblCols);
    _binaryCols  = intListToA(binCols);
    _intNotBinaryCols = intListToA(intNotBinCols);

    return (_numFeat=cnt);
  }

  private int[] intListToA(List<Integer> list) {
    int[] a=new int[0];
    if( list.size() >0 ) {
      a = new int[list.size()];
      for(int i=0;i<a.length;++i) a[i] = list.get(i);
    }
    return a;
  }

  public int numberOfCategoricalFeatures() {
    if( _catFeat!=-1 ) return _catFeat;
    int cnt=0;
    for(Vec v: _fr.vecs()) cnt+= v.isCategorical() ? 1 : 0;
    return (_catFeat=cnt);
  }

  public double[][] getDummies() {
    if( _dummies!=null ) return _dummies;
    DummyClassifier dc=null;
    _dummies = _isClassification
            ? DummyClassifier.getDummies(dc=new DummyClassifier(_fr.vec(_response), new String[]{"mse", "logloss"}), _fr.vec(_response), null)
            : DummyRegressor.getDummies(_fr.vec(_response), null, new String[]{"mse"});
    assert (_isClassification && dc!=null) || !_isClassification;
    _nclass = _isClassification?dc._nclass:1;
    return _dummies;<commit_msg>cache categorical column indexes<commit_after>package ai.h2o.automl;

import ai.h2o.automl.autocollect.AutoCollect;
import ai.h2o.automl.collectors.MetaCollector;
import ai.h2o.automl.guessers.ProblemTypeGuesser;
import ai.h2o.automl.tasks.DummyClassifier;
import ai.h2o.automl.tasks.DummyRegressor;
import ai.h2o.automl.tasks.VIF;
import hex.tree.DHistogram;
import water.*;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;
import water.util.ArrayUtils;
import water.util.AtomicUtils;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

/**
 * Cache common questions asked upon the frame.
 */
public class FrameMeta extends Iced {
  public final String _datasetName;
  public final Frame _fr;
  public int[] _catFeats;
  public int[] _numFeats;
  public int[] _intCols;
  public int[] _dblCols;
  public int[] _binaryCols;
  public int[] _intNotBinaryCols;
  private final int _response;
  private long _naCnt=-1;  // count of nas across whole frame
  private int _numFeat=-1; // count of numerical features
  private int _catFeat=-1; // count of categorical features
  private int _nclass=-1;  // number of classes if classification problem
  private double[][] _dummies=null; // dummy predictions
  public ColMeta[] _cols;

  public final static String[] METAVALUES = new String[]{
    "DatasetName", "NRow", "NCol", "LogNRow", "LogNCol", "NACount", "NAFraction",
    "NumberNumericFeat", "NumberCatFeat", "RatioNumericToCatFeat", "RatioCatToNumericFeat",
    "DatasetRatio", "LogDatasetRatio", "InverseDatasetRatio", "LogInverseDatasetRatio",
    "Classification", "DummyStratMSE", "DummyStratLogLoss", "DummyMostFreqMSE",
    "DummyMostFreqLogLoss", "DummyRandomMSE", "DummyRandomLogLoss", "DummyMedianMSE",
    "DummyMeanMSE", "NClass"};

  public static HashMap<String, Object> makeEmptyFrameMeta() {
    HashMap<String,Object> hm = new HashMap<>();
    for(String key: FrameMeta.METAVALUES) hm.put(key,null);
    return hm;
  }

  // Takes empty frame meta hashmap and fills in the metadata not requiring MRTask
  // TODO: make helper functions so that it's possible to iterate over METAVALUES only
  public void fillSimpleMeta(HashMap<String, Object> fm) {
    fm.put("DatasetName", _datasetName);
    fm.put("NRow", (double)_fr.numRows());
    fm.put("NCol", (double)_fr.numCols());
    fm.put("LogNRow", Math.log((double)fm.get("NRow")));
    fm.put("LogNCol", Math.log((double)fm.get("NCol")));
    fm.put("NACount", (double)naCount());
    fm.put("NAFraction", (double) naCount() / (double) (_fr.numCols() * _fr.numRows()));
    fm.put("NumberNumericFeat", (double)numberOfNumericFeatures());
    fm.put("NumberCatFeat", (double) numberOfCategoricalFeatures());
    fm.put("RatioNumericToCatFeat", Double.isInfinite((double) fm.get("NumberCatFeat"))     ? AutoCollect.SQLNAN : (double) fm.get("NumberNumericFeat") / (double) fm.get("NumberCatFeat"));
    fm.put("RatioCatToNumericFeat", Double.isInfinite((double) fm.get("NumberNumericFeat")) ? AutoCollect.SQLNAN : (double) fm.get("NumberCatFeat")     / (double) fm.get("NumberNumericFeat"));
    fm.put("DatasetRatio", (double) _fr.numCols() / (double) _fr.numRows());
    fm.put("LogDatasetRatio", Math.log((double) fm.get("DatasetRatio")));
    fm.put("InverseDatasetRatio", (double)_fr.numRows() / (double) _fr.numCols() );
    fm.put("LogInverseDatasetRatio", Math.log((double)fm.get("InverseDatasetRatio")));
    fm.put("Classification", _isClassification?1:0);
  }

  public void fillDummies(HashMap<String, Object> fm) {
    double[][] dummies = getDummies();
    fm.put("DummyStratMSE", _isClassification?dummies[0][0]: AutoCollect.SQLNAN);
    fm.put("DummyStratLogLoss", _isClassification?dummies[1][0]: AutoCollect.SQLNAN);
    fm.put("DummyMostFreqMSE", _isClassification?dummies[0][2]: AutoCollect.SQLNAN);
    fm.put("DummyMostFreqLogLoss", _isClassification?dummies[1][2]: AutoCollect.SQLNAN);
    fm.put("DummyRandomMSE", _isClassification?dummies[0][1]: AutoCollect.SQLNAN);
    fm.put("DummyRandomLogLoss", _isClassification?dummies[1][1]: AutoCollect.SQLNAN);
    fm.put("DummyMedianMSE", _isClassification? AutoCollect.SQLNAN:dummies[0][1]);
    fm.put("DummyMeanMSE", _isClassification? AutoCollect.SQLNAN:dummies[0][0]);
    fm.put("NClass", _nclass);
  }

  public long naCount() {
    if( _naCnt!=-1 ) return _naCnt;
    long cnt=0;
    for(Vec v: _fr.vecs()) cnt+=v.naCnt();
    return (_naCnt=cnt);
  }

  public int numberOfNumericFeatures() {
    if( _numFeat!=-1 ) return _numFeat;
    ArrayList<Integer> idxs = new ArrayList<>();
    ArrayList<Integer> intCols = new ArrayList<>();
    ArrayList<Integer> dblCols = new ArrayList<>();
    ArrayList<Integer> binCols = new ArrayList<>();
    ArrayList<Integer> intNotBinCols = new ArrayList<>();
    int cnt=0;
    int idx=0;
    for(Vec v: _fr.vecs()) {
      if( v.isNumeric() ) {
        cnt += 1;
        idxs.add(idx);
        if( v.isInt() ) intCols.add(idx);
        if( v.isBinary() ) binCols.add(idx);
        if( v.isInt() && !v.isBinary() ) intNotBinCols.add(idx);
        if( v.isNumeric() && !v.isInt() ) dblCols.add(idx);
      }
      idx++;
    }
    _numFeats = intListToA(idxs);
    _intCols  = intListToA(intCols);
    _dblCols  = intListToA(dblCols);
    _binaryCols  = intListToA(binCols);
    _intNotBinaryCols = intListToA(intNotBinCols);

    return (_numFeat=cnt);
  }

  private int[] intListToA(List<Integer> list) {
    int[] a=new int[0];
    if( list.size() >0 ) {
      a = new int[list.size()];
      for(int i=0;i<a.length;++i) a[i] = list.get(i);
    }
    return a;
  }

  public int numberOfCategoricalFeatures() {
    if( _catFeat!=-1 ) return _catFeat;
    ArrayList<Integer> idxs = new ArrayList<>();
    int cnt=0;
    int idx=0;
    for(Vec v: _fr.vecs())  {
      if( v.isCategorical() ) {
        cnt += 1;
        idxs.add(idx);
      }
      idx++;
    }
    _catFeats = intListToA(idxs);
    return (_catFeat=cnt);
  }

  public double[][] getDummies() {
    if( _dummies!=null ) return _dummies;
    DummyClassifier dc=null;
    _dummies = _isClassification
            ? DummyClassifier.getDummies(dc=new DummyClassifier(_fr.vec(_response), new String[]{"mse", "logloss"}), _fr.vec(_response), null)
            : DummyRegressor.getDummies(_fr.vec(_response), null, new String[]{"mse"});
    assert (_isClassification && dc!=null) || !_isClassification;
    _nclass = _isClassification?dc._nclass:1;
    return _dummies;
{'fraction_non_alphanumeric': 0.15564446536439946, 'fraction_numerical': 0.006746685494626187, 'mean_word_length': 4.403984739296312, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1558527', 'n_tokens_mistral': 4683, 'n_tokens_neox': 4597, 'n_words': 873}