<commit_before>package hex.singlenoderf;

import hex.FrameTask;
import jsr166y.ForkJoinTask;
import water.*;
import water.api.Constants;
import water.fvec.Frame;
import water.fvec.Vec;
import water.util.Log;
import water.util.Utils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;


public class SpeeDRF extends Job.ValidatedJob {

  @API(help = "Number of trees", filter = Default.class, json = true, lmin = 1, lmax = Integer.MAX_VALUE)
  public int num_trees   = 50;

  @API(help = "Number of features to randomly select at each split.", filter = Default.class, json = true, lmin = -1, lmax = Integer.MAX_VALUE)
  public int mtry = -1;

  @API(help = "Max Depth", filter = Default.class, json = true, lmin = 0, lmax = Integer.MAX_VALUE)
  public int max_depth = 20;

  @API(help = "Split Criterion Type", filter = Default.class, json=true)
  public Tree.StatType stat_type = Tree.StatType.ENTROPY;

  @API(help = "Class Weights (0.0,0.2,0.4,0.6,0.8,1.0)", filter = Default.class, displayName = "class weights", json = true)
  public double[] class_weights;

  @API(help = "Sampling Strategy", filter = Default.class, json = true)
  public Sampling.Strategy sampling_strategy = Sampling.Strategy.RANDOM;

  @API(help = "Strata Samples", filter = Default.class, json = true)
  int[] strata_samples;

  @API(help = "Sampling Rate at each split.", filter = Default.class, json  = true, dmin = 0, dmax = 1)
  public double sample = 0.67;

  @API(help = "OOBEE", filter = Default.class, json = true)
  public boolean oobee = false;

  public Key _modelKey = dest();

  /* Advanced settings */
  @API(help = "bin limit", filter = Default.class, json = true, lmin = 0, lmax = 65534)
  public int bin_limit = 1024;

  @API(help = "seed", filter = Default.class, json = true)
  public long seed = 784834182943470027L;

  @API(help = "Build trees in parallel", filter = Default.class, json = true)
  public boolean  parallel  = true;

  @API(help = "split limit")
  public int _exclusiveSplitLimit = 0;

//  @API(help = "iterative cm")
//  public  boolean  _iterativeCM = true;

  @API(help = "use non local data")
  public boolean _useNonLocalData = true;

  /** Return the query link to this page */
//  public static String link(Key k, String content) {
//    RString rs = new RString("<a href='RF.query?%key_param=%$key'>%content</a>");
//    rs.replace("key_param", DATA_KEY);
//    rs.replace("key", k.toString());
//    rs.replace("content", content);
//    return rs.toString();
//  }

  public DRFParams drfParams;

  @Override protected void queryArgumentValueSet(Argument arg, java.util.Properties inputArgs) {
    super.queryArgumentValueSet(arg, inputArgs);

    if (arg._name.equals("classification")) {
      arg._hideInQuery = true;
    }
    if (arg._name.equals("class_weights")) {
      if (source == null || response == null) {
        arg.disable("Requires source and response to be specified.");
      }
    }
    if (arg._name.equals("sampling_strategy")) {
      arg.setRefreshOnChange();
    }
    if (arg._name.equals("strata_samples")) {
      if (sampling_strategy != Sampling.Strategy.STRATIFIED_LOCAL) {
        arg.disable("No Strata for Random sampling.");
      }
    }
    if (arg._name.equals("class_weights")) {
      if (source == null || response == null) {
        arg.disable("Requires a dataset and response.");
      }
    }
  }

  @Override protected void execImpl() {
    SpeeDRFModel rf_model = initModel();
    buildForest(rf_model);
    cleanup();
    remove();
  }

  @Override protected Response redirect() {
    return SpeeDRFProgressPage.redirect(this, self(), dest());
  }

  public final void buildForest(SpeeDRFModel model) {
    try {
      source.read_lock(self());
      logStart();
      if (model == null) model = UKV.get(dest());
      model.write_lock(self());
      drfParams = DRFParams.create(model.fr.find(model.response), model.total_trees, model.depth, (int)model.fr.numRows(), model.bin_limit,
              model.statType, seed, parallel, model.weights, mtry, model.sampling_strategy, (float) sample, model.strata_samples, 1, _exclusiveSplitLimit, _useNonLocalData);
      DRFTask tsk = new DRFTask();
      tsk._job = Job.findJob(self());
      tsk._params = drfParams;
      tsk._rfmodel = model;
      tsk._drf = this;
      tsk.validateInputData();
      tsk.invokeOnAllNodes();
    }
    catch(JobCancelledException ex) {
      Log.info("Random Forest building was cancelled.");
    }
    catch(Exception ex) {
      ex.printStackTrace();
      throw new RuntimeException(ex);
    }
    finally {
      if (model != null) {
        model = UKV.get(dest());
        model.unlock(self());
      }
      source.unlock(self());
      emptyLTrash();
    }
  }

  public SpeeDRFModel initModel() {
    try {
      source.read_lock(self());
      float[] samps = new float[(int) (response.max() - response.min() + 1)];
      for (int i = 0; i < samps.length; ++i ) samps[i] = 67;
      if (strata_samples != null) {
        int[] _samples = new int[(int) (response.max() - response.min() + 1)];
        for (int i = 0; i < _samples.length; ++i ) _samples[i] = 67;
        if (class_weights == null) {
          strata_samples = _samples;
        }
        if(strata_samples.length > _samples.length) {
          System.arraycopy(_samples, 0, strata_samples, 0, _samples.length);
          strata_samples = _samples;
        }
        if(strata_samples.length < _samples.length) {
          System.arraycopy(strata_samples, 0, _samples, 0, strata_samples.length);
          strata_samples = _samples;
        }
        for (int i = 0; i < _samples.length; ++i) {
          samps[i] = (float)strata_samples[i];
        }
      }
      if (class_weights != null) {
        double[] weights = new double[(int) (response.max() - response.min() + 1)];
        for (int i = 0; i < weights.length; ++i ) weights[i] = 1.0;
        if (class_weights == null) {
          class_weights = weights;
        }
        if(class_weights.length > weights.length) {
          System.arraycopy(class_weights, 0, weights, 0, weights.length);
          class_weights = weights;
        }
        if(class_weights.length < weights.length) {
          System.arraycopy(class_weights, 0, weights, 0, class_weights.length);
          class_weights = weights;
        }
      }
      Frame train = FrameTask.DataInfo.prepareFrame(source, response, ignored_cols, false, false, false);
      Frame test = null;
      if (validation != null) {
        test = FrameTask.DataInfo.prepareFrame(validation, response, ignored_cols, false, false, false);
      }
      SpeeDRFModel model = new SpeeDRFModel(dest(), self(), source._key, train, response, new Key[0], seed);
      model.bin_limit = bin_limit;
      if (mtry == -1) {
        model.mtry = (int) Math.floor(Math.sqrt(source.numCols()));
      } else {
        model.mtry = mtry;
      }
      model.features = source.numCols();
      model.sampling_strategy = sampling_strategy;
      model.sample = (float) sample;
      model.weights = class_weights;
      model.time = 0;
      model.total_trees = num_trees;<commit_msg>pass in the response column of the validated frame to prepFrame<commit_after>package hex.singlenoderf;

import hex.FrameTask;
import jsr166y.ForkJoinTask;
import water.*;
import water.api.Constants;
import water.fvec.Frame;
import water.fvec.Vec;
import water.util.Log;
import water.util.Utils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;


public class SpeeDRF extends Job.ValidatedJob {

  @API(help = "Number of trees", filter = Default.class, json = true, lmin = 1, lmax = Integer.MAX_VALUE)
  public int num_trees   = 50;

  @API(help = "Number of features to randomly select at each split.", filter = Default.class, json = true, lmin = -1, lmax = Integer.MAX_VALUE)
  public int mtry = -1;

  @API(help = "Max Depth", filter = Default.class, json = true, lmin = 0, lmax = Integer.MAX_VALUE)
  public int max_depth = 50;

  @API(help = "Split Criterion Type", filter = Default.class, json=true)
  public Tree.StatType stat_type = Tree.StatType.ENTROPY;

  @API(help = "Class Weights (0.0,0.2,0.4,0.6,0.8,1.0)", filter = Default.class, displayName = "class weights", json = true)
  public double[] class_weights;

  @API(help = "Sampling Strategy", filter = Default.class, json = true)
  public Sampling.Strategy sampling_strategy = Sampling.Strategy.RANDOM;

  @API(help = "Strata Samples", filter = Default.class, json = true)
  int[] strata_samples;

  @API(help = "Sampling Rate at each split.", filter = Default.class, json  = true, dmin = 0, dmax = 1)
  public double sample = 0.67;

  @API(help = "OOBEE", filter = Default.class, json = true)
  public boolean oobee = false;

  public Key _modelKey = dest();

  /* Advanced settings */
  @API(help = "bin limit", filter = Default.class, json = true, lmin = 0, lmax = 65534)
  public int bin_limit = 1024;

  @API(help = "seed", filter = Default.class, json = true)
  public long seed = 784834182943470027L;

  @API(help = "Build trees in parallel", filter = Default.class, json = true)
  public boolean  parallel  = true;

  @API(help = "split limit")
  public int _exclusiveSplitLimit = 0;

//  @API(help = "iterative cm")
//  public  boolean  _iterativeCM = true;

  @API(help = "use non local data")
  public boolean _useNonLocalData = true;

  /** Return the query link to this page */
//  public static String link(Key k, String content) {
//    RString rs = new RString("<a href='RF.query?%key_param=%$key'>%content</a>");
//    rs.replace("key_param", DATA_KEY);
//    rs.replace("key", k.toString());
//    rs.replace("content", content);
//    return rs.toString();
//  }

  public DRFParams drfParams;

  @Override protected void queryArgumentValueSet(Argument arg, java.util.Properties inputArgs) {
    super.queryArgumentValueSet(arg, inputArgs);

    if (arg._name.equals("classification")) {
      arg._hideInQuery = true;
    }
    if (arg._name.equals("class_weights")) {
      if (source == null || response == null) {
        arg.disable("Requires source and response to be specified.");
      }
    }
    if (arg._name.equals("sampling_strategy")) {
      arg.setRefreshOnChange();
    }
    if (arg._name.equals("strata_samples")) {
      if (sampling_strategy != Sampling.Strategy.STRATIFIED_LOCAL) {
        arg.disable("No Strata for Random sampling.");
      }
    }
    if (arg._name.equals("class_weights")) {
      if (source == null || response == null) {
        arg.disable("Requires a dataset and response.");
      }
    }
  }

  @Override protected void execImpl() {
    SpeeDRFModel rf_model = initModel();
    buildForest(rf_model);
    cleanup();
    remove();
  }

  @Override protected Response redirect() {
    return SpeeDRFProgressPage.redirect(this, self(), dest());
  }

  public final void buildForest(SpeeDRFModel model) {
    try {
      source.read_lock(self());
      logStart();
      if (model == null) model = UKV.get(dest());
      model.write_lock(self());
      drfParams = DRFParams.create(model.fr.find(model.response), model.total_trees, model.depth, (int)model.fr.numRows(), model.bin_limit,
              model.statType, seed, parallel, model.weights, mtry, model.sampling_strategy, (float) sample, model.strata_samples, 1, _exclusiveSplitLimit, _useNonLocalData);
      DRFTask tsk = new DRFTask();
      tsk._job = Job.findJob(self());
      tsk._params = drfParams;
      tsk._rfmodel = model;
      tsk._drf = this;
      tsk.validateInputData();
      tsk.invokeOnAllNodes();
    }
    catch(JobCancelledException ex) {
      Log.info("Random Forest building was cancelled.");
    }
    catch(Exception ex) {
      ex.printStackTrace();
      throw new RuntimeException(ex);
    }
    finally {
      if (model != null) {
        model = UKV.get(dest());
        model.unlock(self());
      }
      source.unlock(self());
      emptyLTrash();
    }
  }

  public SpeeDRFModel initModel() {
    try {
      source.read_lock(self());
      float[] samps = new float[(int) (response.max() - response.min() + 1)];
      for (int i = 0; i < samps.length; ++i ) samps[i] = 67;
      if (strata_samples != null) {
        int[] _samples = new int[(int) (response.max() - response.min() + 1)];
        for (int i = 0; i < _samples.length; ++i ) _samples[i] = 67;
        if (class_weights == null) {
          strata_samples = _samples;
        }
        if(strata_samples.length > _samples.length) {
          System.arraycopy(_samples, 0, strata_samples, 0, _samples.length);
          strata_samples = _samples;
        }
        if(strata_samples.length < _samples.length) {
          System.arraycopy(strata_samples, 0, _samples, 0, strata_samples.length);
          strata_samples = _samples;
        }
        for (int i = 0; i < _samples.length; ++i) {
          samps[i] = (float)strata_samples[i];
        }
      }
      if (class_weights != null) {
        double[] weights = new double[(int) (response.max() - response.min() + 1)];
        for (int i = 0; i < weights.length; ++i ) weights[i] = 1.0;
        if (class_weights == null) {
          class_weights = weights;
        }
        if(class_weights.length > weights.length) {
          System.arraycopy(class_weights, 0, weights, 0, weights.length);
          class_weights = weights;
        }
        if(class_weights.length < weights.length) {
          System.arraycopy(class_weights, 0, weights, 0, class_weights.length);
          class_weights = weights;
        }
      }
      Frame train = FrameTask.DataInfo.prepareFrame(source, response, ignored_cols, false, false, false);
      Frame test = null;
      if (validation != null) {
        test = FrameTask.DataInfo.prepareFrame(validation, validation.vecs()[source.find(response)], ignored_cols, false, false, false);
      }
      SpeeDRFModel model = new SpeeDRFModel(dest(), self(), source._key, train, response, new Key[0], seed);
      model.bin_limit = bin_limit;
      if (mtry == -1) {
        model.mtry = (int) Math.floor(Math.sqrt(source.numCols()));
      } else {
        model.mtry = mtry;
      }
      model.features = source.numCols();
      model.sampling_strategy = sampling_strategy;
      model.sample = (float) sample;
      model.weights = class_weights;
      model.time = 0;
      model.total_trees = num_trees;
{'fraction_non_alphanumeric': 0.13766197575588687, 'fraction_numerical': 0.011425386651804375, 'mean_word_length': 2.9361118727721416, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2224678', 'n_tokens_mistral': 4735, 'n_tokens_neox': 4547, 'n_words': 1129}