<commit_before>package hex.glm;

import hex.FrameTask;
import hex.glm.GLMParams.Family;
import hex.gram.Gram;

import java.util.ArrayList;

import water.H2O.H2OCountedCompleter;
import water.*;
import water.util.Utils;

/**
 * Contains all GLM related tasks.
 *
 * @author tomasnykodym
 *
 */

public abstract class GLMTask<T extends GLMTask<T>> extends FrameTask<T> {
  final protected GLMParams _glm;
  public GLMTask(Job job, DataInfo dinfo, GLMParams glm){this(job,dinfo,glm,null);}
  public GLMTask(Job job, DataInfo dinfo, GLMParams glm,H2OCountedCompleter cmp){super(job,dinfo,cmp);_glm = glm;}

  //helper function to compute eta - i.e. beta * row
  protected final double computeEta(final int ncats, final int [] cats, final double [] nums, final double [] beta){
    double res = 0;
    for(int i = 0; i < ncats; ++i)res += beta[cats[i]];
    final int numStart = _dinfo.numStart();
    for(int i = 0; i < nums.length; ++i)res += nums[i]*beta[numStart+i];
    res += beta[beta.length-1]; // intercept
    return res;
  }
  /**
   * Helper task to compute precise mean of response and number of observations.
   * (We skip rows with NAs, so we can't use Vec's mean in general.
   *
   * @author tomasnykodym
   *
   */
  static class YMUTask extends FrameTask<YMUTask>{
    private long   _nobs;
    protected double _ymu;
    public double _ymin = Double.POSITIVE_INFINITY;
    public double _ymax = Double.NEGATIVE_INFINITY;
    public YMUTask(Job job, DataInfo dinfo) {this(job,dinfo,null);}
    public YMUTask(Job job, DataInfo dinfo, H2OCountedCompleter cmp) {
      super(job,dinfo,cmp);
    }
    @Override protected void processRow(long gid, double[] nums, int ncats, int[] cats, double [] responses) {
      double response = responses[0];
      _ymu += response;
      if(response < _ymin)_ymin = response;
      if(response > _ymax)_ymax = response;
      ++_nobs;
    }
    @Override public void reduce(YMUTask t){
      _ymu = _ymu*((double)_nobs/(_nobs+t._nobs)) + t._ymu*t._nobs/(_nobs+t._nobs);
      _nobs += t._nobs;
      _ymax = Math.max(_ymax,t._ymax);
      _ymin = Math.min(_ymin,t._ymin);
    }
    @Override protected void chunkDone(){_ymu /= _nobs;}
    public double ymu(){return _ymu;}
    public long nobs(){return _nobs;}
  }
  /**
   * Task to compute Lambda Max for the given dataset.<commit_msg>Bugfix in GLM2: task to compute response mean was not expecting empty chunks (which can occur e.g.  as a result of filter operation in exec2) which lead to division by zeros and NaN as result.<commit_after>package hex.glm;

import hex.FrameTask;
import hex.glm.GLMParams.Family;
import hex.gram.Gram;

import java.util.ArrayList;

import water.H2O.H2OCountedCompleter;
import water.*;
import water.util.Utils;

/**
 * Contains all GLM related tasks.
 *
 * @author tomasnykodym
 *
 */

public abstract class GLMTask<T extends GLMTask<T>> extends FrameTask<T> {
  final protected GLMParams _glm;
  public GLMTask(Job job, DataInfo dinfo, GLMParams glm){this(job,dinfo,glm,null);}
  public GLMTask(Job job, DataInfo dinfo, GLMParams glm,H2OCountedCompleter cmp){super(job,dinfo,cmp);_glm = glm;}

  //helper function to compute eta - i.e. beta * row
  protected final double computeEta(final int ncats, final int [] cats, final double [] nums, final double [] beta){
    double res = 0;
    for(int i = 0; i < ncats; ++i)res += beta[cats[i]];
    final int numStart = _dinfo.numStart();
    for(int i = 0; i < nums.length; ++i)res += nums[i]*beta[numStart+i];
    res += beta[beta.length-1]; // intercept
    return res;
  }
  /**
   * Helper task to compute precise mean of response and number of observations.
   * (We skip rows with NAs, so we can't use Vec's mean in general.
   *
   * @author tomasnykodym
   *
   */
  static class YMUTask extends FrameTask<YMUTask>{
    private long   _nobs;
    protected double _ymu;
    public double _ymin = Double.POSITIVE_INFINITY;
    public double _ymax = Double.NEGATIVE_INFINITY;
    public YMUTask(Job job, DataInfo dinfo) {this(job,dinfo,null);}
    public YMUTask(Job job, DataInfo dinfo, H2OCountedCompleter cmp) {
      super(job,dinfo,cmp);
    }
    @Override protected void processRow(long gid, double[] nums, int ncats, int[] cats, double [] responses) {
      double response = responses[0];
      _ymu += response;
      if(response < _ymin)_ymin = response;
      if(response > _ymax)_ymax = response;
      ++_nobs;
    }
    @Override public void reduce(YMUTask t){
      if(t._nobs != 0){
        if(_nobs == 0){
          _ymu = t._ymu;
          _nobs = t._nobs;
          _ymin = t._ymin;
          _ymax = t._ymax;
        } else {
          _ymu = _ymu*((double)_nobs/(_nobs+t._nobs)) + t._ymu*t._nobs/(_nobs+t._nobs);
          _nobs += t._nobs;
          _ymax = Math.max(_ymax,t._ymax);
          _ymin = Math.min(_ymin,t._ymin);
        }
      }
    }
    @Override protected void chunkDone(){_ymu /= _nobs;}
    public double ymu(){return _ymu;}
    public long nobs(){return _nobs;}
  }
  /**
   * Task to compute Lambda Max for the given dataset.
{'fraction_non_alphanumeric': 0.12198497429814156, 'fraction_numerical': 0.004349545274812179, 'mean_word_length': 3.247691015952981, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2868602', 'n_tokens_mistral': 1833, 'n_tokens_neox': 1787, 'n_words': 488}