<commit_before>package hex.singlenoderf;

import hex.singlenoderf.Data.Row;
import hex.singlenoderf.Tree.SplitNode.SplitInfo;
import jsr166y.CountedCompleter;
import jsr166y.RecursiveTask;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.util.Log;
import water.util.Log.Tag.Sys;
import water.util.Utils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

public class Tree extends H2OCountedCompleter {
  static public enum SelectStatType {ENTROPY, GINI};
  static public enum StatType { ENTROPY, GINI, MSE};

  /** Left and right seed initializer number for statistics */
  public static final long LTSS_INIT = 0xe779aef0a6fd0c16L;
  public static final long RTSS_INIT = 0x5e63c6377e5297a7L;
  /** Left and right seed initializer number for subtrees */
  public static final long RTS_INIT = 0xa7a34721109d3708L;
  public static final long LTS_INIT = 0x264ccf88cf4dec32L;
  /** If the number of rows is higher then given number, fork-join is used to build
   * subtrees, else subtrees are built sequentially
   */
  public static final int ROWS_FORK_TRESHOLD = 1<<11;

  final StatType _type;         // Flavor of split logic
  final Data _data;         // Data source
  final hex.singlenoderf.Sampling _sampler;      // Sampling strategy
  final int      _data_id;      // Data-subset identifier (so trees built on this subset are not validated on it)
  final int      _maxDepth;     // Tree-depth cutoff
  final int      _numSplitFeatures;  // Number of features to check at each splitting (~ split features)
  INode          _tree;         // Root of decision tree
  ThreadLocal<hex.singlenoderf.Statistic>[] _stats  = new ThreadLocal[2];
  final Job      _job;          // DRF job building this tree
  final long     _seed;         // Pseudo random seed: used to playback sampling
  int            _exclusiveSplitLimit;
  int            _verbose;
  final byte     _producerId;   // Id of node producing this tree
  final boolean  _regression; // If true, will build regression tree.

  /**
   * Constructor used to define the specs when building the tree from the top.
   */
  public Tree(final Job job, final Data data, byte producerId, int maxDepth, StatType stat,
              int numSplitFeatures, long seed, int treeId, int exclusiveSplitLimit,
              final hex.singlenoderf.Sampling sampler, int verbose, boolean regression) {
    _job              = job;
    _data             = data;
    _type             = stat;
    _data_id          = treeId;
    _maxDepth         = maxDepth-1;
    _numSplitFeatures = numSplitFeatures;
    _seed             = seed;
    _sampler          = sampler;
    _exclusiveSplitLimit = exclusiveSplitLimit;
    _verbose          = verbose;
    _producerId       = producerId;
    _regression       = regression;
  }

  // Oops, uncaught exception
  public boolean onExceptionalCompletion( Throwable ex, CountedCompleter _) {
    ex.printStackTrace();
    return true;
  }

  private hex.singlenoderf.Statistic getStatistic(int index, Data data, long seed, int exclusiveSplitLimit) {
    hex.singlenoderf.Statistic result = _stats[index].get();
    if (_type == StatType.MSE) {
      if(!_regression) {
        throw H2O.unimpl();
      }
      if (result == null) {
      result = new MSEStatistic(data, _numSplitFeatures, _seed, exclusiveSplitLimit);
      _stats[index].set(result);
      }
      result.forgetFeatures();
    } else {
    if( result==null ) {
      result  = _type == StatType.GINI ?
              new GiniStatistic(data,_numSplitFeatures, _seed, exclusiveSplitLimit) :
              new EntropyStatistic(data,_numSplitFeatures, _seed, exclusiveSplitLimit);
      _stats[index].set(result);
    }
    }
    result.forgetFeatures();   // All new features
    result.reset(data, seed, _regression);
    return result;
  }

  private StringBuffer computeStatistics() {
    StringBuffer sb = new StringBuffer();
    ArrayList<SplitInfo>[] stats = new ArrayList[_data.columns()];
    for (int i = 0; i < _data.columns()-1; i++) stats[i] = new ArrayList<SplitInfo>();
    _tree.computeStats(stats);
    for (int i = 0; i < _data.columns()-1; i++) {
      String colname = _data.colName(i);
      ArrayList<SplitInfo> colSplitStats = stats[i];
      Collections.sort(colSplitStats);
      int usage = 0;
      for (SplitInfo si : colSplitStats) {
        usage += si._used;
      }
      sb.append(colname).append(':').append(usage).append("x");
      for (SplitInfo si : colSplitStats) {
        sb.append(", <=").append(Utils.p2d(si.splitNode().split_value())).append('{').append(si.affectedLeaves()).append("}x"+si._used+" ");
      }

      sb.append('\n');
    }
    return sb;
  }

  // Actually build the tree
  @Override public void compute2() {
    if(Job.isRunning(_job.self())) {
      Timer timer    = new Timer();
      _stats[0]      = new ThreadLocal<hex.singlenoderf.Statistic>();
      _stats[1]      = new ThreadLocal<hex.singlenoderf.Statistic>();
      Data d = _sampler.sample(_data, _seed);
      hex.singlenoderf.Statistic left = getStatistic(0, d, _seed, _exclusiveSplitLimit);
      // calculate the split
      for( Row r : d ) left.addQ(r, _regression);
      if (!_regression)
        left.applyClassWeights();   // Weight the distributions
      hex.singlenoderf.Statistic.Split spl = left.split(d, false);
      if(spl.isLeafNode()) {
        if(_regression) {
          float av = d.computeAverage();
          _tree = new LeafNode(-1, d.rows(), av);
        } else {
          _tree =  new LeafNode(_data.unmapClass(spl._split), d.rows(),-1);
        }
      } else {
        _tree = new FJBuild (spl, d, 0, _seed).compute();
      }

      if (_verbose > 1)  Log.info(Sys.RANDF,computeStatistics().toString());
      _stats = null; // GC

      // Atomically improve the Model as well
      appendKey(_job.dest(),toKey());
      StringBuilder sb = new StringBuilder("[RF] Tree : ").append(_data_id+1);
      sb.append(" d=").append(_tree.depth()).append(" leaves=").append(_tree.leaves()).append(" done in ").append(timer).append('\n');
      Log.info(sb.toString());
//      Log.info(Sys.RANDF,_tree.toString(sb,  _verbose > 0 ? Integer.MAX_VALUE : 200).toString());
    }
    // Wait for completation
    tryComplete();
  }

  // Stupid static method to make a static anonymous inner class
  // which serializes "for free".
  static void appendKey(Key model, final Key tKey) {
    final int selfIdx = H2O.SELF.index();<commit_msg>print out the tree (not the tree stats)<commit_after>package hex.singlenoderf;

import hex.singlenoderf.Data.Row;
import hex.singlenoderf.Tree.SplitNode.SplitInfo;
import jsr166y.CountedCompleter;
import jsr166y.RecursiveTask;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.util.Log;
import water.util.Log.Tag.Sys;
import water.util.Utils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

public class Tree extends H2OCountedCompleter {
  static public enum SelectStatType {ENTROPY, GINI};
  static public enum StatType { ENTROPY, GINI, MSE};

  /** Left and right seed initializer number for statistics */
  public static final long LTSS_INIT = 0xe779aef0a6fd0c16L;
  public static final long RTSS_INIT = 0x5e63c6377e5297a7L;
  /** Left and right seed initializer number for subtrees */
  public static final long RTS_INIT = 0xa7a34721109d3708L;
  public static final long LTS_INIT = 0x264ccf88cf4dec32L;
  /** If the number of rows is higher then given number, fork-join is used to build
   * subtrees, else subtrees are built sequentially
   */
  public static final int ROWS_FORK_TRESHOLD = 1<<11;

  final StatType _type;         // Flavor of split logic
  final Data _data;         // Data source
  final hex.singlenoderf.Sampling _sampler;      // Sampling strategy
  final int      _data_id;      // Data-subset identifier (so trees built on this subset are not validated on it)
  final int      _maxDepth;     // Tree-depth cutoff
  final int      _numSplitFeatures;  // Number of features to check at each splitting (~ split features)
  INode          _tree;         // Root of decision tree
  ThreadLocal<hex.singlenoderf.Statistic>[] _stats  = new ThreadLocal[2];
  final Job      _job;          // DRF job building this tree
  final long     _seed;         // Pseudo random seed: used to playback sampling
  int            _exclusiveSplitLimit;
  int            _verbose;
  final byte     _producerId;   // Id of node producing this tree
  final boolean  _regression; // If true, will build regression tree.

  /**
   * Constructor used to define the specs when building the tree from the top.
   */
  public Tree(final Job job, final Data data, byte producerId, int maxDepth, StatType stat,
              int numSplitFeatures, long seed, int treeId, int exclusiveSplitLimit,
              final hex.singlenoderf.Sampling sampler, int verbose, boolean regression) {
    _job              = job;
    _data             = data;
    _type             = stat;
    _data_id          = treeId;
    _maxDepth         = maxDepth-1;
    _numSplitFeatures = numSplitFeatures;
    _seed             = seed;
    _sampler          = sampler;
    _exclusiveSplitLimit = exclusiveSplitLimit;
    _verbose          = verbose;
    _producerId       = producerId;
    _regression       = regression;
  }

  // Oops, uncaught exception
  public boolean onExceptionalCompletion( Throwable ex, CountedCompleter _) {
    ex.printStackTrace();
    return true;
  }

  private hex.singlenoderf.Statistic getStatistic(int index, Data data, long seed, int exclusiveSplitLimit) {
    hex.singlenoderf.Statistic result = _stats[index].get();
    if (_type == StatType.MSE) {
      if(!_regression) {
        throw H2O.unimpl();
      }
      if (result == null) {
      result = new MSEStatistic(data, _numSplitFeatures, _seed, exclusiveSplitLimit);
      _stats[index].set(result);
      }
      result.forgetFeatures();
    } else {
    if( result==null ) {
      result  = _type == StatType.GINI ?
              new GiniStatistic(data,_numSplitFeatures, _seed, exclusiveSplitLimit) :
              new EntropyStatistic(data,_numSplitFeatures, _seed, exclusiveSplitLimit);
      _stats[index].set(result);
    }
    }
    result.forgetFeatures();   // All new features
    result.reset(data, seed, _regression);
    return result;
  }

  private StringBuffer computeStatistics() {
    StringBuffer sb = new StringBuffer();
    ArrayList<SplitInfo>[] stats = new ArrayList[_data.columns()];
    for (int i = 0; i < _data.columns()-1; i++) stats[i] = new ArrayList<SplitInfo>();
    _tree.computeStats(stats);
    for (int i = 0; i < _data.columns()-1; i++) {
      String colname = _data.colName(i);
      ArrayList<SplitInfo> colSplitStats = stats[i];
      Collections.sort(colSplitStats);
      int usage = 0;
      for (SplitInfo si : colSplitStats) {
        usage += si._used;
      }
      sb.append(colname).append(':').append(usage).append("x");
      for (SplitInfo si : colSplitStats) {
        sb.append(", <=").append(Utils.p2d(si.splitNode().split_value())).append('{').append(si.affectedLeaves()).append("}x"+si._used+" ");
      }

      sb.append('\n');
    }
    return sb;
  }

  // Actually build the tree
  @Override public void compute2() {
    if(Job.isRunning(_job.self())) {
      Timer timer    = new Timer();
      _stats[0]      = new ThreadLocal<hex.singlenoderf.Statistic>();
      _stats[1]      = new ThreadLocal<hex.singlenoderf.Statistic>();
      Data d = _sampler.sample(_data, _seed);
      hex.singlenoderf.Statistic left = getStatistic(0, d, _seed, _exclusiveSplitLimit);
      // calculate the split
      for( Row r : d ) left.addQ(r, _regression);
      if (!_regression)
        left.applyClassWeights();   // Weight the distributions
      hex.singlenoderf.Statistic.Split spl = left.split(d, false);
      if(spl.isLeafNode()) {
        if(_regression) {
          float av = d.computeAverage();
          _tree = new LeafNode(-1, d.rows(), av);
        } else {
          _tree =  new LeafNode(_data.unmapClass(spl._split), d.rows(),-1);
        }
      } else {
        _tree = new FJBuild (spl, d, 0, _seed).compute();
      }

//      if (_verbose > 1)  Log.info(Sys.RANDF,computeStatistics().toString());
      _stats = null; // GC

      // Atomically improve the Model as well
      appendKey(_job.dest(),toKey());
      StringBuilder sb = new StringBuilder("[RF] Tree : ").append(_data_id+1);
      sb.append(" d=").append(_tree.depth()).append(" leaves=").append(_tree.leaves()).append(" done in ").append(timer).append('\n');
      Log.info(sb.toString());
      Log.info(Sys.RANDF,_tree.toString(sb,  _verbose > 0 ? Integer.MAX_VALUE : 200).toString());
    }
    // Wait for completation
    tryComplete();
  }

  // Stupid static method to make a static anonymous inner class
  // which serializes "for free".
  static void appendKey(Key model, final Key tKey) {
    final int selfIdx = H2O.SELF.index();
{'fraction_non_alphanumeric': 0.1126394620204799, 'fraction_numerical': 0.012226807274950329, 'mean_word_length': 2.8065735892961023, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '810631', 'n_tokens_mistral': 4120, 'n_tokens_neox': 3991, 'n_words': 1074}