<commit_before>package hex.grid;

import hex.Model;
import hex.ModelBuilder;
import hex.ModelParametersBuilderFactory;
import hex.ScoringInfo;
import hex.grid.HyperSpaceWalker.BaseWalker;
import jsr166y.CountedCompleter;
import water.*;
import water.exceptions.H2OConcurrentModificationException;
import water.exceptions.H2OIllegalArgumentException;
import water.fvec.Frame;
import water.util.Log;
import water.util.PojoUtils;

import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Map;

/**
 * Grid search job.
 *
 * This job represents a generic interface to launch "any" hyper space
 * search. It triggers sub-jobs for each point in hyper space.  It produces
 * <code>Grid</code> object which contains a list of build models. A triggered
 * model builder job can fail!
 *
 * Grid search is parametrized by hyper space walk strategy ({@link
 * hex.grid.HyperSpaceWalker} which defines how the space of hyper parameters
 * is traversed.
 *
 * The job is started by the <code>startGridSearch</code> method which create a new grid search, put
 * representation of Grid into distributed KV store, and for each parameter in hyper space of
 * possible parameters, it launches a separated model building job. The launch of jobs is sequential
 * and blocking. So after finish the last model, whole grid search job is done as well.
 *
 * By default, the grid search invokes cartezian grid search, but it can be
 * modified by passing explicit hyper space walk strategy via the
 * {@link #startGridSearch(Key, HyperSpaceWalker)} method.
 *
 * If any of forked jobs fails then the failure is ignored, and grid search
 * normally continue in traversing the hyper space.
 *
 * Typical usage from Java is:
 * <pre>{@code
 * // Create initial parameters and fill them by references to data
 * GBMModel.GBMParameters params = new GBMModel.GBMParameters();
 * params._train = fr._key;
 * params._response_column = "cylinders";
 *
 * // Define hyper-space to search
 * HashMap<String,Object[]> hyperParms = new HashMap<>();
 * hyperParms.put("_ntrees", new Integer[]{1, 2});
 * hyperParms.put("_distribution",new DistributionFamily[] {DistributionFamily.multinomial});
 * hyperParms.put("_max_depth",new Integer[]{1,2,5});
 * hyperParms.put("_learn_rate",new Float[]{0.01f,0.1f,0.3f});
 *
 * // Launch grid search job creating GBM models
 * GridSearch gridSearchJob = GridSearch.startGridSearch(params, hyperParms, GBM_MODEL_FACTORY);
 *
 * // Block till the end of the job and get result
 * Grid grid = gridSearchJob.get()
 *
 * // Get built models
 * Model[] models = grid.getModels()
 * }</pre>
 *
 * @see hex.grid.HyperSpaceWalker
 * @see #startGridSearch(Key, HyperSpaceWalker)
 */
public final class GridSearch<MP extends Model.Parameters> extends Keyed<GridSearch> {
  public final Key<Grid> _result;
  public final Job<Grid> _job;

  /** Walks hyper space and for each point produces model parameters. It is
   *  used only locally to fire new model builders.  */
  private final transient HyperSpaceWalker<MP, ?> _hyperSpaceWalker;

  private GridSearch(Key<Grid> gkey, HyperSpaceWalker<MP, ?> hyperSpaceWalker) {
    assert hyperSpaceWalker != null : "Grid search needs to know how to walk around hyper space!";
    _hyperSpaceWalker = hyperSpaceWalker;
    _result = gkey;
    String algoName = hyperSpaceWalker.getParams().algoName();
    _job = new Job<>(gkey, Grid.class.getName(), algoName + " Grid Search");
    // Note: do not validate parameters of created model builders here!
    // Leave it to launch time, and just mark the corresponding model builder job as failed.
  }

  Job<Grid> start() {
    final long gridSize = _hyperSpaceWalker.getMaxHyperSpaceSize();
    Log.info("Starting gridsearch: estimated size of search space = " + gridSize);
    // Create grid object and lock it
    // Creation is done here, since we would like make sure that after leaving
    // this function the grid object is in DKV and accessible.
    final Grid<MP> grid;
    Keyed keyed = DKV.getGet(_result);
    if (keyed != null) {
      if (! (keyed instanceof Grid))
        throw new H2OIllegalArgumentException("Name conflict: tried to create a Grid using the ID of a non-Grid object that's already in H2O: " + _job._result + "; it is a: " + keyed.getClass());
      grid = (Grid) keyed;
      grid.clearNonRelatedFailures();
      Frame specTrainFrame = _hyperSpaceWalker.getParams().train();
      Frame oldTrainFrame = grid.getTrainingFrame();
      if (oldTrainFrame != null && !specTrainFrame._key.equals(oldTrainFrame._key) ||
          oldTrainFrame != null && specTrainFrame.checksum() != oldTrainFrame.checksum())
        throw new H2OIllegalArgumentException("training_frame", "grid", "Cannot append new models to a grid with different training input");
      grid.write_lock(_job);
    } else {
      grid =
          new Grid<>(_result,
                     _hyperSpaceWalker.getParams(),
                     _hyperSpaceWalker.getHyperParamNames(),
                     _hyperSpaceWalker.getParametersBuilderFactory().getFieldNamingStrategy());
      grid.delete_and_lock(_job);
    }

    Model model = null;
    HyperSpaceWalker.HyperSpaceIterator<MP> it = _hyperSpaceWalker.iterator();
    long gridWork=0;
    if (gridSize > 0) {//if total grid space is known, walk it all and count up models to be built (not subject to time-based or converge-based early stopping)
      int count=0;
      while (it.hasNext(model) && (it.max_models() > 0 && count++ < it.max_models())) { //only walk the first max_models models, if specified
        try {
          Model.Parameters parms = it.nextModelParameters(model);
          gridWork += (parms._nfolds > 0 ? (parms._nfolds+1/*main model*/) : 1) *parms.progressUnits();
        } catch(Throwable ex) {
          //swallow invalid combinations
        }
      }
    } else {
      //TODO: Future totally unbounded search: need a time-based progress bar
      gridWork = Long.MAX_VALUE;
    }
    it.reset();

    // Install this as job functions
    return _job.start(new H2O.H2OCountedCompleter() {
      @Override public void compute2() {
        gridSearch(grid);
        tryComplete();
      }
      @Override
      public boolean onExceptionalCompletion(Throwable ex, CountedCompleter caller) {
        Log.warn("GridSearch job "+_job._description+" completed with exception: "+ex);
        return true;
      }
    }, gridWork, it.max_runtime_secs());
  }

  /**
   * Returns expected number of models in resulting Grid object.
   *
   * The number can differ from final number of models due to visiting duplicate points in hyper
   * space.
   *
   * @return expected number of models produced by this grid search
   */
  public long getModelCount() {
    return _hyperSpaceWalker.getMaxHyperSpaceSize();
  }

  /**
   * Invokes grid search based on specified hyper space walk strategy.
   *
   * It updates passed grid object in distributed store.
   *
   * @param grid grid object to save results; grid already locked
   */
  private void gridSearch(Grid<MP> grid) {
    Model model = null;
    // Prepare nice model key and override default key by appending model counter
    //String protoModelKey = _hyperSpaceWalker.getParams()._model_id == null
    //                       ? grid._key + "_model_"
    //                       : _hyperSpaceWalker.getParams()._model_id.toString() + H2O.calcNextUniqueModelId("") + "_";
    String protoModelKey = grid._key + "_model_";

    try {
      // Get iterator to traverse hyper space
      HyperSpaceWalker.HyperSpaceIterator<MP> it = _hyperSpaceWalker.iterator();
      // Number of traversed model parameters
      int counter = grid.getModelCount();
      while (it.hasNext(model)) {
        if (_job.stop_requested()) throw new Job.JobCancelledException();  // Handle end-user cancel request
        double max_runtime_secs = it.max_runtime_secs();

        double time_remaining_secs = Double.MAX_VALUE;
        if (max_runtime_secs > 0) {
          time_remaining_secs = it.time_remaining_secs();
          if (time_remaining_secs < 0) {
            Log.info("Grid max_runtime_secs of " + max_runtime_secs + " secs has expired; stopping early.");
            throw new Job.JobCancelledException();
          }
        }

        MP params;
        try {
          // Get parameters for next model
          params = it.nextModelParameters(model);

          // Sequential model building, should never propagate
          // exception up, just mark combination of model parameters as wrong

          // Do we need to limit the model build time?
          if (max_runtime_secs > 0) {
            Log.info("Grid time is limited to: " + max_runtime_secs + " for grid: " + grid._key + ". Remaining time is: " + time_remaining_secs);
            if (params._max_runtime_secs == 0) { // unlimited
              params._max_runtime_secs = time_remaining_secs;
              Log.info("Due to the grid time limit, changing model max runtime to: " + params._max_runtime_secs + " secs.");
            } else {
              double was = params._max_runtime_secs;
              params._max_runtime_secs = Math.min(params._max_runtime_secs, time_remaining_secs);
              Log.info("Due to the grid time limit, changing model max runtime from: " + was + " secs to: " + params._max_runtime_secs + " secs.");
            }
          }

          try {
            ScoringInfo scoringInfo = new ScoringInfo();
            scoringInfo.time_stamp_ms = System.currentTimeMillis();

            //// build the model!
            model = buildModel(params, grid, ++counter, protoModelKey);
            if (model != null) {
              model.fillScoringInfo(scoringInfo);
              grid.setScoringInfos(ScoringInfo.prependScoringInfo(scoringInfo, grid.getScoringInfos()));
              ScoringInfo.sort(grid.getScoringInfos(), _hyperSpaceWalker.search_criteria().stopping_metric()); // Currently AUTO for Cartesian and user-specified for RandomDiscrete
            }
          } catch (RuntimeException e) { // Catch everything
            if (!Job.isCancelledException(e)) {
              StringWriter sw = new StringWriter();
              PrintWriter pw = new PrintWriter(sw);
              e.printStackTrace(pw);
              Log.warn("Grid search: model builder for parameters " + params + " failed! Exception: ", e, sw.toString());
            }
            
            grid.appendFailedModelParameters(model != null ? model._key : null, params, e);
          }
        } catch (IllegalArgumentException e) {
          Log.warn("Grid search: construction of model parameters failed! Exception: ", e);
          // Model parameters cannot be constructed for some reason
          it.modelFailed(model);
          Object[] rawParams = it.getCurrentRawParameters();
          grid.appendFailedModelParameters(model._key, rawParams, e);
        } finally {
          // Update progress by 1 increment<commit_msg>Categorize failures with no model built in all cases as not model related.<commit_after>package hex.grid;

import hex.Model;
import hex.ModelBuilder;
import hex.ModelParametersBuilderFactory;
import hex.ScoringInfo;
import hex.grid.HyperSpaceWalker.BaseWalker;
import jsr166y.CountedCompleter;
import water.*;
import water.exceptions.H2OConcurrentModificationException;
import water.exceptions.H2OIllegalArgumentException;
import water.fvec.Frame;
import water.util.Log;
import water.util.PojoUtils;

import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Map;

/**
 * Grid search job.
 *
 * This job represents a generic interface to launch "any" hyper space
 * search. It triggers sub-jobs for each point in hyper space.  It produces
 * <code>Grid</code> object which contains a list of build models. A triggered
 * model builder job can fail!
 *
 * Grid search is parametrized by hyper space walk strategy ({@link
 * hex.grid.HyperSpaceWalker} which defines how the space of hyper parameters
 * is traversed.
 *
 * The job is started by the <code>startGridSearch</code> method which create a new grid search, put
 * representation of Grid into distributed KV store, and for each parameter in hyper space of
 * possible parameters, it launches a separated model building job. The launch of jobs is sequential
 * and blocking. So after finish the last model, whole grid search job is done as well.
 *
 * By default, the grid search invokes cartezian grid search, but it can be
 * modified by passing explicit hyper space walk strategy via the
 * {@link #startGridSearch(Key, HyperSpaceWalker)} method.
 *
 * If any of forked jobs fails then the failure is ignored, and grid search
 * normally continue in traversing the hyper space.
 *
 * Typical usage from Java is:
 * <pre>{@code
 * // Create initial parameters and fill them by references to data
 * GBMModel.GBMParameters params = new GBMModel.GBMParameters();
 * params._train = fr._key;
 * params._response_column = "cylinders";
 *
 * // Define hyper-space to search
 * HashMap<String,Object[]> hyperParms = new HashMap<>();
 * hyperParms.put("_ntrees", new Integer[]{1, 2});
 * hyperParms.put("_distribution",new DistributionFamily[] {DistributionFamily.multinomial});
 * hyperParms.put("_max_depth",new Integer[]{1,2,5});
 * hyperParms.put("_learn_rate",new Float[]{0.01f,0.1f,0.3f});
 *
 * // Launch grid search job creating GBM models
 * GridSearch gridSearchJob = GridSearch.startGridSearch(params, hyperParms, GBM_MODEL_FACTORY);
 *
 * // Block till the end of the job and get result
 * Grid grid = gridSearchJob.get()
 *
 * // Get built models
 * Model[] models = grid.getModels()
 * }</pre>
 *
 * @see hex.grid.HyperSpaceWalker
 * @see #startGridSearch(Key, HyperSpaceWalker)
 */
public final class GridSearch<MP extends Model.Parameters> extends Keyed<GridSearch> {
  public final Key<Grid> _result;
  public final Job<Grid> _job;

  /** Walks hyper space and for each point produces model parameters. It is
   *  used only locally to fire new model builders.  */
  private final transient HyperSpaceWalker<MP, ?> _hyperSpaceWalker;

  private GridSearch(Key<Grid> gkey, HyperSpaceWalker<MP, ?> hyperSpaceWalker) {
    assert hyperSpaceWalker != null : "Grid search needs to know how to walk around hyper space!";
    _hyperSpaceWalker = hyperSpaceWalker;
    _result = gkey;
    String algoName = hyperSpaceWalker.getParams().algoName();
    _job = new Job<>(gkey, Grid.class.getName(), algoName + " Grid Search");
    // Note: do not validate parameters of created model builders here!
    // Leave it to launch time, and just mark the corresponding model builder job as failed.
  }

  Job<Grid> start() {
    final long gridSize = _hyperSpaceWalker.getMaxHyperSpaceSize();
    Log.info("Starting gridsearch: estimated size of search space = " + gridSize);
    // Create grid object and lock it
    // Creation is done here, since we would like make sure that after leaving
    // this function the grid object is in DKV and accessible.
    final Grid<MP> grid;
    Keyed keyed = DKV.getGet(_result);
    if (keyed != null) {
      if (! (keyed instanceof Grid))
        throw new H2OIllegalArgumentException("Name conflict: tried to create a Grid using the ID of a non-Grid object that's already in H2O: " + _job._result + "; it is a: " + keyed.getClass());
      grid = (Grid) keyed;
      grid.clearNonRelatedFailures();
      Frame specTrainFrame = _hyperSpaceWalker.getParams().train();
      Frame oldTrainFrame = grid.getTrainingFrame();
      if (oldTrainFrame != null && !specTrainFrame._key.equals(oldTrainFrame._key) ||
          oldTrainFrame != null && specTrainFrame.checksum() != oldTrainFrame.checksum())
        throw new H2OIllegalArgumentException("training_frame", "grid", "Cannot append new models to a grid with different training input");
      grid.write_lock(_job);
    } else {
      grid =
          new Grid<>(_result,
                     _hyperSpaceWalker.getParams(),
                     _hyperSpaceWalker.getHyperParamNames(),
                     _hyperSpaceWalker.getParametersBuilderFactory().getFieldNamingStrategy());
      grid.delete_and_lock(_job);
    }

    Model model = null;
    HyperSpaceWalker.HyperSpaceIterator<MP> it = _hyperSpaceWalker.iterator();
    long gridWork=0;
    if (gridSize > 0) {//if total grid space is known, walk it all and count up models to be built (not subject to time-based or converge-based early stopping)
      int count=0;
      while (it.hasNext(model) && (it.max_models() > 0 && count++ < it.max_models())) { //only walk the first max_models models, if specified
        try {
          Model.Parameters parms = it.nextModelParameters(model);
          gridWork += (parms._nfolds > 0 ? (parms._nfolds+1/*main model*/) : 1) *parms.progressUnits();
        } catch(Throwable ex) {
          //swallow invalid combinations
        }
      }
    } else {
      //TODO: Future totally unbounded search: need a time-based progress bar
      gridWork = Long.MAX_VALUE;
    }
    it.reset();

    // Install this as job functions
    return _job.start(new H2O.H2OCountedCompleter() {
      @Override public void compute2() {
        gridSearch(grid);
        tryComplete();
      }
      @Override
      public boolean onExceptionalCompletion(Throwable ex, CountedCompleter caller) {
        Log.warn("GridSearch job "+_job._description+" completed with exception: "+ex);
        return true;
      }
    }, gridWork, it.max_runtime_secs());
  }

  /**
   * Returns expected number of models in resulting Grid object.
   *
   * The number can differ from final number of models due to visiting duplicate points in hyper
   * space.
   *
   * @return expected number of models produced by this grid search
   */
  public long getModelCount() {
    return _hyperSpaceWalker.getMaxHyperSpaceSize();
  }

  /**
   * Invokes grid search based on specified hyper space walk strategy.
   *
   * It updates passed grid object in distributed store.
   *
   * @param grid grid object to save results; grid already locked
   */
  private void gridSearch(Grid<MP> grid) {
    Model model = null;
    // Prepare nice model key and override default key by appending model counter
    //String protoModelKey = _hyperSpaceWalker.getParams()._model_id == null
    //                       ? grid._key + "_model_"
    //                       : _hyperSpaceWalker.getParams()._model_id.toString() + H2O.calcNextUniqueModelId("") + "_";
    String protoModelKey = grid._key + "_model_";

    try {
      // Get iterator to traverse hyper space
      HyperSpaceWalker.HyperSpaceIterator<MP> it = _hyperSpaceWalker.iterator();
      // Number of traversed model parameters
      int counter = grid.getModelCount();
      while (it.hasNext(model)) {
        if (_job.stop_requested()) throw new Job.JobCancelledException();  // Handle end-user cancel request
        double max_runtime_secs = it.max_runtime_secs();

        double time_remaining_secs = Double.MAX_VALUE;
        if (max_runtime_secs > 0) {
          time_remaining_secs = it.time_remaining_secs();
          if (time_remaining_secs < 0) {
            Log.info("Grid max_runtime_secs of " + max_runtime_secs + " secs has expired; stopping early.");
            throw new Job.JobCancelledException();
          }
        }

        MP params;
        try {
          // Get parameters for next model
          params = it.nextModelParameters(model);

          // Sequential model building, should never propagate
          // exception up, just mark combination of model parameters as wrong

          // Do we need to limit the model build time?
          if (max_runtime_secs > 0) {
            Log.info("Grid time is limited to: " + max_runtime_secs + " for grid: " + grid._key + ". Remaining time is: " + time_remaining_secs);
            if (params._max_runtime_secs == 0) { // unlimited
              params._max_runtime_secs = time_remaining_secs;
              Log.info("Due to the grid time limit, changing model max runtime to: " + params._max_runtime_secs + " secs.");
            } else {
              double was = params._max_runtime_secs;
              params._max_runtime_secs = Math.min(params._max_runtime_secs, time_remaining_secs);
              Log.info("Due to the grid time limit, changing model max runtime from: " + was + " secs to: " + params._max_runtime_secs + " secs.");
            }
          }

          try {
            ScoringInfo scoringInfo = new ScoringInfo();
            scoringInfo.time_stamp_ms = System.currentTimeMillis();

            //// build the model!
            model = buildModel(params, grid, ++counter, protoModelKey);
            if (model != null) {
              model.fillScoringInfo(scoringInfo);
              grid.setScoringInfos(ScoringInfo.prependScoringInfo(scoringInfo, grid.getScoringInfos()));
              ScoringInfo.sort(grid.getScoringInfos(), _hyperSpaceWalker.search_criteria().stopping_metric()); // Currently AUTO for Cartesian and user-specified for RandomDiscrete
            }
          } catch (RuntimeException e) { // Catch everything
            if (!Job.isCancelledException(e)) {
              StringWriter sw = new StringWriter();
              PrintWriter pw = new PrintWriter(sw);
              e.printStackTrace(pw);
              Log.warn("Grid search: model builder for parameters " + params + " failed! Exception: ", e, sw.toString());
            }
            
            grid.appendFailedModelParameters(model != null ? model._key : null, params, e);
          }
        } catch (IllegalArgumentException e) {
          Log.warn("Grid search: construction of model parameters failed! Exception: ", e);
          // Model parameters cannot be constructed for some reason
          it.modelFailed(model);
          Object[] rawParams = it.getCurrentRawParameters();
          grid.appendFailedModelParameters(model != null ? model._key : null, rawParams, e);
        } finally {
          // Update progress by 1 increment
{'fraction_non_alphanumeric': 0.09769539078156313, 'fraction_numerical': 0.003279285844416105, 'mean_word_length': 2.9690889370932756, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6532855', 'n_tokens_mistral': 6167, 'n_tokens_neox': 5935, 'n_words': 1946}