<commit_before>package hex.rf;

import java.util.Arrays;
import java.util.Random;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.StringUtils;

import water.*;
import water.Job.Progress;
import water.api.Constants;
import water.util.*;
import water.util.Log.Tag.Sys;

import com.google.gson.*;

/**
 * A model is an ensemble of trees that can be serialized and that can be used
 * to classify data.
 */
public class RFModel extends Model implements Cloneable, Progress {
  /** Number of features these trees are built for */
  public int _features;
  /** Sampling strategy used for model */
  public Sampling.Strategy _samplingStrategy;
  /** Sampling rate used when building trees. */
  public float _sample;
  /** Strata sampling rate used for local-node strata-sampling */
  public float[] _strataSamples;
  /** Number of split features defined by user. */
  public int _splitFeatures;
  /** Number of computed split features per node - number of split features can differ for each node.
   * However, such difference would point to a problem with data distribution. */
  public int[] _nodesSplitFeatures;
  /** Number of keys the model expects to be built for it */
  public int _totalTrees;
  /** All the trees in the model */
  public Key[]     _tkeys;
  /** Local forests produced by nodes */
  public Key[][]   _localForests;
  /** Total time in seconds to produce model */
  public long      _time;

  public static final String KEY_PREFIX = "__RFModel_";

  /** A RandomForest Model
   * @param treeskey    a key of keys of trees
   * @param classes     the number of response classes
   * @param data        the dataset
   */
  public RFModel(Key selfKey, int[] cols, Key dataKey, Key[] tkeys, int features, Sampling.Strategy samplingStrategy, float sample, float[] strataSamples, int splitFeatures, int totalTrees) {
    super(selfKey, cols, dataKey);
    _features = features;
    _sample = sample;
    _splitFeatures = splitFeatures;
    _totalTrees = totalTrees;
    _tkeys = tkeys;
    _strataSamples = strataSamples;
    _samplingStrategy = samplingStrategy;
    _nodesSplitFeatures = new int[H2O.CLOUD.size()];
    _localForests       = new Key[H2O.CLOUD.size()][];
    for(int i=0;i<H2O.CLOUD.size();i++) _localForests[i] = new Key[0];
    for( Key tkey : _tkeys ) assert DKV.get(tkey)!=null;
  }

  public RFModel(Key selfKey, String [] colNames, String[] classNames, Key[] tkeys, int features, float sample) {
    super(selfKey,colNames,classNames);
    _features       = features;
    _sample         = sample;
    _splitFeatures  = features;
    _totalTrees     = tkeys.length;
    _tkeys          = tkeys;
    _samplingStrategy   = Sampling.Strategy.RANDOM;
    _nodesSplitFeatures = new int[H2O.CLOUD.size()];
    _localForests       = new Key[H2O.CLOUD.size()][];
    for(int i=0;i<H2O.CLOUD.size();i++) _localForests[i] = new Key[0];
    for( Key tkey : _tkeys ) assert DKV.get(tkey)!=null;
    assert classes() > 0;
  }

  /** Empty constructor for deserialization */
  public RFModel() {}

  @Override protected RFModel clone() {
    try {
      return (RFModel) super.clone();
    } catch( CloneNotSupportedException cne ) {
      throw Log.err(Sys.RANDF, "", H2O.unimpl());
    }
  }

  static public RFModel make(RFModel old, Key tkey) {
    RFModel m = old.clone();
    m._tkeys = Arrays.copyOf(old._tkeys,old._tkeys.length+1);
    m._tkeys[m._tkeys.length-1] = tkey;
    int idx = H2O.SELF.index();
    m._localForests[idx] = Arrays.copyOf(old._localForests[idx],old._localForests[idx].length+1);
    m._localForests[idx][m._localForests[idx].length-1] = tkey;
    return m;
  }

  // Make a random RF key
  public static final Key makeKey() {
    return Key.make(KEY_PREFIX + Key.make());
  }

  /** The number of trees in this model. */
  public int treeCount() { return _tkeys.length; }
  public int size()      { return _tkeys.length; }
  public int classes()   { ValueArray.Column C = response();  return (int)(C._max - C._min + 1); }

  @Override public float progress() {
    return size() / (float) _totalTrees;
  }

  public String name(int atree) {
    if( atree == -1 ) atree = size();
    assert atree <= size();
    return _selfKey.toString() + "[" + atree + "]";
  }

  /** Return the bits for a particular tree */
  public byte[] tree(int tree_id) {
    return DKV.get(_tkeys[tree_id]).memOrLoad();
  }

  /** Bad name, I know. But free all internal tree keys. */
  public void deleteKeys() {
    for( Key k : _tkeys )
      UKV.remove(k);
  }

  /**
   * Classify a row according to one particular tree.
   * @param tree_id  the number of the tree to use<commit_msg>Cache trees during RF scoring<commit_after>package hex.rf;

import java.util.Arrays;
import java.util.Random;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.StringUtils;

import water.*;
import water.Job.Progress;
import water.api.Constants;
import water.util.*;
import water.util.Log.Tag.Sys;

import com.google.gson.*;

/**
 * A model is an ensemble of trees that can be serialized and that can be used
 * to classify data.
 */
public class RFModel extends Model implements Cloneable, Progress {
  /** Number of features these trees are built for */
  public int _features;
  /** Sampling strategy used for model */
  public Sampling.Strategy _samplingStrategy;
  /** Sampling rate used when building trees. */
  public float _sample;
  /** Strata sampling rate used for local-node strata-sampling */
  public float[] _strataSamples;
  /** Number of split features defined by user. */
  public int _splitFeatures;
  /** Number of computed split features per node - number of split features can differ for each node.
   * However, such difference would point to a problem with data distribution. */
  public int[] _nodesSplitFeatures;
  /** Number of keys the model expects to be built for it */
  public int _totalTrees;
  /** All the trees in the model */
  public Key[]     _tkeys;
  /** Local forests produced by nodes */
  public Key[][]   _localForests;
  /** Total time in seconds to produce model */
  public long      _time;

  public transient byte[][] _trees; // The raw tree data, for faster classification passes

  public static final String KEY_PREFIX = "__RFModel_";

  /** A RandomForest Model
   * @param treeskey    a key of keys of trees
   * @param classes     the number of response classes
   * @param data        the dataset
   */
  public RFModel(Key selfKey, int[] cols, Key dataKey, Key[] tkeys, int features, Sampling.Strategy samplingStrategy, float sample, float[] strataSamples, int splitFeatures, int totalTrees) {
    super(selfKey, cols, dataKey);
    _features = features;
    _sample = sample;
    _splitFeatures = splitFeatures;
    _totalTrees = totalTrees;
    _tkeys = tkeys;
    _strataSamples = strataSamples;
    _samplingStrategy = samplingStrategy;
    _nodesSplitFeatures = new int[H2O.CLOUD.size()];
    _localForests       = new Key[H2O.CLOUD.size()][];
    for(int i=0;i<H2O.CLOUD.size();i++) _localForests[i] = new Key[0];
    for( Key tkey : _tkeys ) assert DKV.get(tkey)!=null;
  }

  public RFModel(Key selfKey, String [] colNames, String[] classNames, Key[] tkeys, int features, float sample) {
    super(selfKey,colNames,classNames);
    _features       = features;
    _sample         = sample;
    _splitFeatures  = features;
    _totalTrees     = tkeys.length;
    _tkeys          = tkeys;
    _samplingStrategy   = Sampling.Strategy.RANDOM;
    _nodesSplitFeatures = new int[H2O.CLOUD.size()];
    _localForests       = new Key[H2O.CLOUD.size()][];
    for(int i=0;i<H2O.CLOUD.size();i++) _localForests[i] = new Key[0];
    for( Key tkey : _tkeys ) assert DKV.get(tkey)!=null;
    assert classes() > 0;
  }

  /** Empty constructor for deserialization */
  public RFModel() {}

  @Override protected RFModel clone() {
    try {
      return (RFModel) super.clone();
    } catch( CloneNotSupportedException cne ) {
      throw Log.err(Sys.RANDF, "", H2O.unimpl());
    }
  }

  static public RFModel make(RFModel old, Key tkey) {
    RFModel m = old.clone();
    m._tkeys = Arrays.copyOf(old._tkeys,old._tkeys.length+1);
    m._tkeys[m._tkeys.length-1] = tkey;
    int idx = H2O.SELF.index();
    m._localForests[idx] = Arrays.copyOf(old._localForests[idx],old._localForests[idx].length+1);
    m._localForests[idx][m._localForests[idx].length-1] = tkey;
    return m;
  }

  // Make a random RF key
  public static final Key makeKey() {
    return Key.make(KEY_PREFIX + Key.make());
  }

  /** The number of trees in this model. */
  public int treeCount() { return _tkeys.length; }
  public int size()      { return _tkeys.length; }
  public int classes()   { ValueArray.Column C = response();  return (int)(C._max - C._min + 1); }

  @Override public float progress() {
    return size() / (float) _totalTrees;
  }

  public String name(int atree) {
    if( atree == -1 ) atree = size();
    assert atree <= size();
    return _selfKey.toString() + "[" + atree + "]";
  }

  /** Return the bits for a particular tree */
  public byte[] tree(int tree_id) {
    byte[][] ts = _trees;
    if( ts == null ) _trees = ts = new byte[tree_id+1][];
    if( tree_id >= ts.length ) _trees = ts = Arrays.copyOf(ts,tree_id+1);
    if( ts[tree_id] == null ) ts[tree_id] = DKV.get(_tkeys[tree_id]).memOrLoad();
    return ts[tree_id];
  }

  /** Bad name, I know. But free all internal tree keys. */
  public void deleteKeys() {
    for( Key k : _tkeys )
      UKV.remove(k);
  }

  /**
   * Classify a row according to one particular tree.
   * @param tree_id  the number of the tree to use
{'fraction_non_alphanumeric': 0.12156129969895152, 'fraction_numerical': 0.004567632097996471, 'mean_word_length': 3.210664335664336, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6324454', 'n_tokens_mistral': 3111, 'n_tokens_neox': 3066, 'n_words': 938}