<commit_before>package hex.glm;

import hex.FrameTask.DataInfo;
import hex.glm.GLMParams.CaseMode;
import hex.glm.GLMParams.Family;
import hex.glm.GLMValidation.GLMXValidation;

import java.util.HashMap;

import water.*;
import water.H2O.H2OCountedCompleter;
import water.api.DocGen;
import water.api.Request.API;
import water.fvec.Chunk;
import water.util.Utils;

public class GLMModel extends Model implements Comparable<GLMModel> {
  static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
  static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.

  @API(help="mean of response in the training dataset")
  final double     ymu;

  @API(help="predicate applied to the response column to turn it into 0/1")
  final CaseMode  _caseMode;

  @API(help="value used to co compare agains using case-predicate to turn the response into 0/1")
  final double _caseVal;

  @API(help="Input data info")
  DataInfo data_info;

  @API(help="warnings")
  final String []  warnings;
  @API(help="Decision threshold.")
  final double     threshold;
  @API(help="glm params")
  final GLMParams  glm;
  @API(help="beta epsilon - stop iterating when beta diff is below this threshold.")
  final double     beta_eps;
  @API(help="regularization parameter driving proportion of L1/L2 penalty.")
  final double     alpha;

  @API(help="column names including expanded categorical values")
  public String [] coefficients_names;

  @API(help="index of lambda giving best results")
  int best_lambda_idx;

  public double auc(){
    if(glm.family == Family.binomial && submodels != null && submodels[best_lambda_idx].validation != null)
      return submodels[best_lambda_idx].validation.auc;
    return -1;
  }
  public double aic(){
    if(submodels != null && submodels[best_lambda_idx].validation != null)
      return submodels[best_lambda_idx].validation.aic;
    return Double.MAX_VALUE;
  }
  public double devExplained(){
    if(submodels == null || submodels[best_lambda_idx].validation == null)
      return 0;
    GLMValidation val = submodels[best_lambda_idx].validation;
    return 1.0 - val.residual_deviance/val.null_deviance;
  }

  @Override public GLMModel clone(){
    GLMModel res = (GLMModel)super.clone();
    res.submodels = submodels.clone();
    return res;
  }

  @Override
  public int compareTo(GLMModel m){
//    assert m._dataKey.equals(_dataKey);
    assert m.glm.family == glm.family;
    assert m.glm.link == glm.link;
    switch(glm.family){
      case binomial: // compare by AUC, higher is better
        return (int)(1e6*(m.auc()-auc()));
      case gamma: // compare by percentage of explained deviance, higher is better
        return (int)(100*(m.devExplained()-devExplained()));
      default: // compare by AICs by default, lower is better
        return (int)(100*(aic()- m.aic()));
    }
  }
  @API(help="Overall run time")
  long run_time;
  @API(help="computation started at")
  long start_time;

  static class Submodel extends Iced {
    static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
    static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.
    @API(help="number of iterations computed.")
    final int        iteration;
    @API(help="running time of the algo in ms.")
    final long       run_time;
    @API(help="Validation")
    GLMValidation validation;
    @API(help="Beta vector containing model coefficients.") double []  beta;
    @API(help="Beta vector containing normalized coefficients (coefficients obtained on normalized data).") double []  norm_beta;

    final int rank;
    final int [] idxs;

    public Submodel(double [] beta, double [] norm_beta, long run_time, int iteration){
      this.beta = beta;
      this.norm_beta = norm_beta;
      this.run_time = run_time;
      this.iteration = iteration;
      int r = 0;
      if(beta != null){
        final double [] b = norm_beta != null?norm_beta:beta;
        // grab the indeces of non-zero coefficients<commit_msg>Added Submodel.idxs to the json output.<commit_after>package hex.glm;

import hex.FrameTask.DataInfo;
import hex.glm.GLMParams.CaseMode;
import hex.glm.GLMParams.Family;
import hex.glm.GLMValidation.GLMXValidation;

import java.util.HashMap;

import water.*;
import water.H2O.H2OCountedCompleter;
import water.api.DocGen;
import water.api.Request.API;
import water.fvec.Chunk;
import water.util.Utils;

public class GLMModel extends Model implements Comparable<GLMModel> {
  static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
  static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.

  @API(help="mean of response in the training dataset")
  final double     ymu;

  @API(help="predicate applied to the response column to turn it into 0/1")
  final CaseMode  _caseMode;

  @API(help="value used to co compare agains using case-predicate to turn the response into 0/1")
  final double _caseVal;

  @API(help="Input data info")
  DataInfo data_info;

  @API(help="warnings")
  final String []  warnings;
  @API(help="Decision threshold.")
  final double     threshold;
  @API(help="glm params")
  final GLMParams  glm;
  @API(help="beta epsilon - stop iterating when beta diff is below this threshold.")
  final double     beta_eps;
  @API(help="regularization parameter driving proportion of L1/L2 penalty.")
  final double     alpha;

  @API(help="column names including expanded categorical values")
  public String [] coefficients_names;

  @API(help="index of lambda giving best results")
  int best_lambda_idx;

  public double auc(){
    if(glm.family == Family.binomial && submodels != null && submodels[best_lambda_idx].validation != null)
      return submodels[best_lambda_idx].validation.auc;
    return -1;
  }
  public double aic(){
    if(submodels != null && submodels[best_lambda_idx].validation != null)
      return submodels[best_lambda_idx].validation.aic;
    return Double.MAX_VALUE;
  }
  public double devExplained(){
    if(submodels == null || submodels[best_lambda_idx].validation == null)
      return 0;
    GLMValidation val = submodels[best_lambda_idx].validation;
    return 1.0 - val.residual_deviance/val.null_deviance;
  }

  @Override public GLMModel clone(){
    GLMModel res = (GLMModel)super.clone();
    res.submodels = submodels.clone();
    return res;
  }

  @Override
  public int compareTo(GLMModel m){
//    assert m._dataKey.equals(_dataKey);
    assert m.glm.family == glm.family;
    assert m.glm.link == glm.link;
    switch(glm.family){
      case binomial: // compare by AUC, higher is better
        return (int)(1e6*(m.auc()-auc()));
      case gamma: // compare by percentage of explained deviance, higher is better
        return (int)(100*(m.devExplained()-devExplained()));
      default: // compare by AICs by default, lower is better
        return (int)(100*(aic()- m.aic()));
    }
  }
  @API(help="Overall run time")
  long run_time;
  @API(help="computation started at")
  long start_time;

  static class Submodel extends Iced {
    static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
    static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.
    @API(help="number of iterations computed.")
    final int        iteration;
    @API(help="running time of the algo in ms.")
    final long       run_time;
    @API(help="Validation")
    GLMValidation validation;
    @API(help="Beta vector containing model coefficients.") double []  beta;
    @API(help="Beta vector containing normalized coefficients (coefficients obtained on normalized data).") double []  norm_beta;

    final int rank;

    @API(help="Indexes to the coefficient_names array containing names (and order) of the non-zero coefficients in this model.")
    final int [] idxs;

    public Submodel(double [] beta, double [] norm_beta, long run_time, int iteration){
      this.beta = beta;
      this.norm_beta = norm_beta;
      this.run_time = run_time;
      this.iteration = iteration;
      int r = 0;
      if(beta != null){
        final double [] b = norm_beta != null?norm_beta:beta;
        // grab the indeces of non-zero coefficients
{'fraction_non_alphanumeric': 0.11048780487804878, 'fraction_numerical': 0.005609756097560975, 'mean_word_length': 3.599551318003365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4920781', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2482, 'n_words': 748}