<commit_before>package hex.glm;

import hex.DataInfo;
import hex.Model;
import hex.SupervisedModelBuilder;
import hex.glm.GLMModel.FinalizeAndUnlockTsk;
import hex.glm.GLMModel.GLMOutput;
import hex.glm.GLMModel.GLMParameters.Family;
import hex.glm.GLMModel.GLMParameters;
import hex.glm.GLMModel.GLMParameters.Link;
import hex.glm.GLMModel.GLMParameters.Solver;
import hex.glm.GLMTask.*;
import hex.gram.Gram;
import hex.gram.Gram.Cholesky;
import hex.gram.Gram.NonSPDMatrixException;
import hex.optimization.ADMM;
import hex.optimization.ADMM.ProximalSolver;
import hex.optimization.L_BFGS;
import hex.optimization.L_BFGS.*;
import hex.schemas.GLMV3;
import hex.schemas.ModelBuilderSchema;
import jsr166y.CountedCompleter;
import water.*;
import water.H2O.H2OCallback;
import water.exceptions.H2OModelBuilderIllegalArgumentException;
import water.fvec.*;
import water.H2O.H2OCountedCompleter;
import water.parser.ValueString;
import water.util.*;
import water.util.MRUtils.ParallelTasks;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Created by tomasnykodym on 8/27/14.
 *
 * Generalized linear model implementation.
 */
public class GLM extends SupervisedModelBuilder<GLMModel,GLMModel.GLMParameters,GLMModel.GLMOutput> {
  static final double LINE_SEARCH_STEP = .5;
  static final int NUM_LINE_SEARCH_STEPS = 16;
  @Override
  public Model.ModelCategory[] can_build() {
    return new Model.ModelCategory[]{
            Model.ModelCategory.Regression,
            Model.ModelCategory.Binomial,
    };
  }

  @Override
  protected void checkMemoryFootPrint() {
    if (_parms._solver == Solver.IRLSM) {
      long p = _train.degreesOfFreedom() - _train.lastVec().cardinality();
      HeartBeat hb = H2O.CLOUD._memary[H2O.SELF.index()]._heartbeat;
      long mem_usage = hb._cpus_allowed * p * p * 8/*doubles*/; //one gram per core
      long max_mem = hb.get_max_mem();
      if (mem_usage > max_mem) {
        String msg = "Gram matrices (one per thread) won't fit in the driver node's memory ("
                + PrettyPrint.bytes(mem_usage) + " > " + PrettyPrint.bytes(max_mem)
                + ") - try reducing the number of columns and/or the number of categorical factors.";
        error("_train", msg);
        cancel(msg);
      }
    }
  }

  public GLM(Key dest, String desc, GLMModel.GLMParameters parms) { super(dest, desc, parms); init(false); }
  public GLM(GLMModel.GLMParameters parms) { super("GLM", parms); init(false); }

  static class TooManyPredictorsException extends RuntimeException {}

  private BetaConstraint _bc = new BetaConstraint();
  DataInfo _dinfo;
  private Vec _rowFilter;
  private transient GLMTaskInfo [] _tInfos;
  private int _lambdaId;
  private transient DataInfo _validDinfo;
  private transient ArrayList<Integer> _scoring_iters = new ArrayList<>();
  // time per iteration in ms
  private transient ArrayList<Integer> _scoring_times = new ArrayList<>();
  private transient ArrayList<Double> _likelihoods = new ArrayList<>();
  private transient ArrayList<Double> _objectives = new ArrayList<>();

  long _t0 = System.currentTimeMillis();

  public void addScoringHistory(int i, double l, double o) {
    long t = System.currentTimeMillis();
    _scoring_times.add((int)(_t0-t));
    _scoring_iters.add(i);
    _likelihoods.add(l);
    _objectives.add(o);
    _t0 = t;
  }

  private transient double _iceptAdjust = 0;

  @Override public void init(boolean expensive) {
    _t0 = System.currentTimeMillis();
    super.init(expensive);
    hide("_score_each_iteration", "Not used by GLM.");
    hide("_balance_classes", "Not applicable since class balancing is not required for GLM.");
    hide("_max_after_balance_size", "Not applicable since class balancing is not required for GLM.");
    hide("_class_sampling_factors", "Not applicable since class balancing is not required for GLM.");
    _parms.validate(this);
    if (expensive) {
      if(_parms._lambda_search || !_parms._intercept)
        _parms._use_all_factor_levels= true;
      if(_parms._max_active_predictors == -1)
        _parms._max_active_predictors = _parms._solver == Solver.IRLSM ?6000:100000000;
      if (_parms._link == Link.family_default)
        _parms._link = _parms._family.defaultLink;
      _dinfo = new DataInfo(Key.make(), _train, _valid, 1, _parms._use_all_factor_levels || _parms._lambda_search, _parms._standardize ? DataInfo.TransformType.STANDARDIZE : DataInfo.TransformType.NONE, DataInfo.TransformType.NONE, true, false);
      DKV.put(_dinfo._key, _dinfo);
      // handle BetaConstraints if I got them
      double[] betaStart = null;
      double[] betaGiven = null;
      double[] betaLB = null;
      double[] betaUB = null;
      double[] rho = null;
      if (_parms._beta_constraints != null) {
        Frame beta_constraints = _parms._beta_constraints.get();
        Vec v = beta_constraints.vec("names");
        String[] dom;
        int[] map;
        if (v.isString()) {
          dom = new String[(int) v.length()];
          map = new int[dom.length];<commit_msg>PUBDEV-536: Detailed admission control for GLM.<commit_after>package hex.glm;

import hex.DataInfo;
import hex.Model;
import hex.SupervisedModelBuilder;
import hex.glm.GLMModel.FinalizeAndUnlockTsk;
import hex.glm.GLMModel.GLMOutput;
import hex.glm.GLMModel.GLMParameters.Family;
import hex.glm.GLMModel.GLMParameters;
import hex.glm.GLMModel.GLMParameters.Link;
import hex.glm.GLMModel.GLMParameters.Solver;
import hex.glm.GLMTask.*;
import hex.gram.Gram;
import hex.gram.Gram.Cholesky;
import hex.gram.Gram.NonSPDMatrixException;
import hex.optimization.ADMM;
import hex.optimization.ADMM.ProximalSolver;
import hex.optimization.L_BFGS;
import hex.optimization.L_BFGS.*;
import hex.schemas.GLMV3;
import hex.schemas.ModelBuilderSchema;
import jsr166y.CountedCompleter;
import water.*;
import water.H2O.H2OCallback;
import water.exceptions.H2OModelBuilderIllegalArgumentException;
import water.fvec.*;
import water.H2O.H2OCountedCompleter;
import water.parser.ValueString;
import water.util.*;
import water.util.MRUtils.ParallelTasks;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Created by tomasnykodym on 8/27/14.
 *
 * Generalized linear model implementation.
 */
public class GLM extends SupervisedModelBuilder<GLMModel,GLMModel.GLMParameters,GLMModel.GLMOutput> {
  static final double LINE_SEARCH_STEP = .5;
  static final int NUM_LINE_SEARCH_STEPS = 16;
  @Override
  public Model.ModelCategory[] can_build() {
    return new Model.ModelCategory[]{
            Model.ModelCategory.Regression,
            Model.ModelCategory.Binomial,
    };
  }

  protected void checkMemoryFootPrint(DataInfo dinfo) {
    if (_parms._solver == Solver.IRLSM && !_parms._lambda_search) {
      HeartBeat hb = H2O.CLOUD._memary[H2O.SELF.index()]._heartbeat;
      double p = dinfo.fullN() - dinfo.largestCat();
      long mem_usage = (long)(hb._cpus_allowed * (p*p + dinfo.largestCat()) * 8/*doubles*/ * Math.log((double)_train.lastVec().nChunks())/Math.log(2.)); //one gram per core
      long max_mem = hb.get_max_mem();
      if (mem_usage > max_mem) {
        String msg = "Gram matrices (one per thread) won't fit in the driver node's memory ("
                + PrettyPrint.bytes(mem_usage) + " > " + PrettyPrint.bytes(max_mem)
                + ") - try reducing the number of columns and/or the number of categorical factors.";
        error("_train", msg);
        cancel(msg);
      }
    }
  }

  public GLM(Key dest, String desc, GLMModel.GLMParameters parms) { super(dest, desc, parms); init(false); }
  public GLM(GLMModel.GLMParameters parms) { super("GLM", parms); init(false); }

  static class TooManyPredictorsException extends RuntimeException {}

  private BetaConstraint _bc = new BetaConstraint();
  DataInfo _dinfo;
  private Vec _rowFilter;
  private transient GLMTaskInfo [] _tInfos;
  private int _lambdaId;
  private transient DataInfo _validDinfo;
  private transient ArrayList<Integer> _scoring_iters = new ArrayList<>();
  // time per iteration in ms
  private transient ArrayList<Integer> _scoring_times = new ArrayList<>();
  private transient ArrayList<Double> _likelihoods = new ArrayList<>();
  private transient ArrayList<Double> _objectives = new ArrayList<>();

  long _t0 = System.currentTimeMillis();

  public void addScoringHistory(int i, double l, double o) {
    long t = System.currentTimeMillis();
    _scoring_times.add((int)(_t0-t));
    _scoring_iters.add(i);
    _likelihoods.add(l);
    _objectives.add(o);
    _t0 = t;
  }

  private transient double _iceptAdjust = 0;

  @Override public void init(boolean expensive) {
    _t0 = System.currentTimeMillis();
    super.init(expensive);
    hide("_score_each_iteration", "Not used by GLM.");
    hide("_balance_classes", "Not applicable since class balancing is not required for GLM.");
    hide("_max_after_balance_size", "Not applicable since class balancing is not required for GLM.");
    hide("_class_sampling_factors", "Not applicable since class balancing is not required for GLM.");
    _parms.validate(this);
    if (expensive) {
      if(_parms._lambda_search || !_parms._intercept)
        _parms._use_all_factor_levels= true;
      if(_parms._max_active_predictors == -1)
        _parms._max_active_predictors = _parms._solver == Solver.IRLSM ?6000:100000000;
      if (_parms._link == Link.family_default)
        _parms._link = _parms._family.defaultLink;
      _dinfo = new DataInfo(Key.make(), _train, _valid, 1, _parms._use_all_factor_levels || _parms._lambda_search, _parms._standardize ? DataInfo.TransformType.STANDARDIZE : DataInfo.TransformType.NONE, DataInfo.TransformType.NONE, true, false);
      DKV.put(_dinfo._key, _dinfo);
      checkMemoryFootPrint(_dinfo);
      // handle BetaConstraints if I got them
      double[] betaStart = null;
      double[] betaGiven = null;
      double[] betaLB = null;
      double[] betaUB = null;
      double[] rho = null;
      if (_parms._beta_constraints != null) {
        Frame beta_constraints = _parms._beta_constraints.get();
        Vec v = beta_constraints.vec("names");
        String[] dom;
        int[] map;
        if (v.isString()) {
          dom = new String[(int) v.length()];
          map = new int[dom.length];
{'fraction_non_alphanumeric': 0.11226551226551226, 'fraction_numerical': 0.00808080808080808, 'mean_word_length': 3.94106463878327, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2752527', 'n_tokens_mistral': 3442, 'n_tokens_neox': 3305, 'n_words': 767}