<commit_before>package hex.glm;

import hex.DataInfo;
import hex.DataInfo.TransformType;
import hex.ModelMetrics;
import hex.ModelMetricsBinomialGLM;
import hex.ModelMetricsRegressionGLM;
import hex.deeplearning.DeepLearningModel.DeepLearningParameters.MissingValuesHandling;
import hex.glm.GLMModel.GLMParameters.Link;
import hex.glm.GLMModel.GLMParameters.Solver;
import hex.glm.GLMModel.GLMWeightsFun;
import hex.glm.GLMTask.*;
import org.junit.*;

import hex.glm.GLMModel.GLMParameters;
import hex.glm.GLMModel.GLMParameters.Family;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.exceptions.H2OModelBuilderIllegalArgumentException;
import water.fvec.*;
import water.parser.BufferedString;
import water.parser.ParseDataset;
import water.util.ArrayUtils;

import java.util.*;
import java.util.concurrent.ExecutionException;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class GLMTest  extends TestUtil {

  @BeforeClass public static void setup() { stall_till_cloudsize(1); }

  protected static void testScoring(GLMModel m, Frame fr) {
    Scope.enter();
    // standard predictions
    Frame fr2 = new Frame(fr);
    Frame preds = Scope.track(m.score(fr2));
    m.adaptTestForTrain(fr2,true,false);
    fr2.remove(fr2.numCols()-1); // remove response
    int p = m._output._dinfo._cats + m._output._dinfo._nums;
    int p2 = fr2.numCols() - (m._output._dinfo._weights?1:0)- (m._output._dinfo._offset?1:0);
    assert p == p2: p + " != " + p2;
    fr2.add(preds.names(),preds.vecs());
    // test score0
    new TestScore0(m,m._output._dinfo._weights,m._output._dinfo._offset).doAll(fr2);
    // test pojo
    if((!m._output._dinfo._weights && !m._output._dinfo._offset))
      Assert.assertTrue(m.testJavaScoring(fr,preds,1e-15));
    Scope.exit();
  }


  // class to test score0 since score0 is now not being called by the standard bulk scoring
  public static class TestScore0 extends MRTask {
    final GLMModel _m;
    final boolean _weights;
    final boolean _offset;

    public TestScore0(GLMModel m, boolean w, boolean o) {_m = m; _weights = w; _offset = o;}

    @Override public void map(Chunk [] chks) {
      int nout = _m._parms._family == Family.multinomial?_m._output.nclasses()+1:_m._parms._family == Family.binomial?3:1;
      Chunk [] outputs = Arrays.copyOfRange(chks,chks.length-nout,chks.length);
      chks = Arrays.copyOf(chks,chks.length-nout);
      Chunk off = new C0DChunk(0,chks[0]._len);
      Chunk w = new C0DChunk(1,chks[0]._len);
      double [] tmp = new double[_m._output._dinfo._cats + _m._output._dinfo._nums];
      double [] predictions = new double [nout];
      if(_offset) {
        off = chks[chks.length - 1];
        chks = Arrays.copyOf(chks, chks.length - 1);
      }
      if(_weights) {
        w = chks[chks.length - 1];
        chks = Arrays.copyOf(chks, chks.length - 1);
      }
      if(_weights || _offset) {
        for (int i = 0; i < chks[0]._len; ++i) {
          _m.score0(chks, w.atd(i), off.atd(i),i, tmp, predictions);
          for (int j = 0; j < predictions.length; ++j)
            assertEquals("mismatch at row " + (i + chks[0].start()) + ", p = " + j + ": " + outputs[j].atd(i) + " != " + predictions[j], outputs[j].atd(i), predictions[j], 1e-6);
        }
      } else {
        for (int i = 0; i < chks[0]._len; ++i) {
          _m.score0(chks, i, tmp, predictions);
          for (int j = 0; j < predictions.length; ++j)
            assertEquals("mismatch at row " + (i + chks[0].start()) + ", p = " + j + ": " + outputs[j].atd(i) + " != " + predictions[j] + ", preds = " + Arrays.toString(predictions), outputs[j].atd(i), predictions[j], 1e-6);
        }
      }
    }
  }
  //------------------- simple tests on synthetic data------------------------------------
  @Test
  public void testGaussianRegression() throws InterruptedException, ExecutionException {
    Key raw = Key.make("gaussian_test_data_raw");
    Key parsed = Key.make("gaussian_test_data_parsed");
    GLMModel model = null;
    Frame fr = null, res = null;
    try {
      // make data so that the expected coefficients is icept = col[0] = 1.0
      FVecTest.makeByteVec(raw, "x,y\n0,0\n1,0.1\n2,0.2\n3,0.3\n4,0.4\n5,0.5\n6,0.6\n7,0.7\n8,0.8\n9,0.9");
      fr = ParseDataset.parse(parsed, raw);
      GLMParameters params = new GLMParameters(Family.gaussian);
      params._train = fr._key;
      // params._response = 1;
      params._response_column = fr._names[1];
      params._lambda = new double[]{0};
//      params._standardize= false;
      model = new GLM(params).trainModel().get();
      HashMap<String, Double> coefs = model.coefficients();
      assertEquals(0.0, coefs.get("Intercept"), 1e-4);
      assertEquals(0.1, coefs.get("x"), 1e-4);
      testScoring(model,fr);
    } finally {
      if (fr != null) fr.remove();
      if (res != null) res.remove();
      if (model != null) model.remove();
    }
  }

  /**
   * Test Poisson regression on simple and small synthetic dataset.
   * Equation is: y = exp(x+1);
   */
  @Test
  public void testPoissonRegression() throws InterruptedException, ExecutionException {
    Key raw = Key.make("poisson_test_data_raw");
    Key parsed = Key.make("poisson_test_data_parsed");

    GLMModel model = null;
    Frame fr = null, res = null;
    try {
      // make data so that the expected coefficients is icept = col[0] = 1.0
      FVecTest.makeByteVec(raw, "x,y\n0,2\n1,4\n2,8\n3,16\n4,32\n5,64\n6,128\n7,256");
      fr = ParseDataset.parse(parsed, raw);
      Vec v = fr.vec(0);
      System.out.println(v.min() + ", " + v.max() + ", mean = " + v.mean());
      GLMParameters params = new GLMParameters(Family.poisson);
      params._train = fr._key;
      // params._response = 1;
      params._response_column = fr._names[1];
      params._lambda = new double[]{0};
      params._standardize = false;
      model = new GLM(params).trainModel().get();
      for (double c : model.beta())
        assertEquals(Math.log(2), c, 1e-2); // only 1e-2 precision cause the perfect solution is too perfect -> will trigger grid search
      testScoring(model,fr);
      model.delete();
      fr.delete();

      // Test 2, example from http://www.biostat.umn.edu/~dipankar/bmtry711.11/lecture_13.pdf
      FVecTest.makeByteVec(raw, "x,y\n1,0\n2,1\n3,2\n4,3\n5,1\n6,4\n7,9\n8,18\n9,23\n10,31\n11,20\n12,25\n13,37\n14,45\n150,7.193936e+16\n");
      fr = ParseDataset.parse(parsed, raw);
      GLMParameters params2 = new GLMParameters(Family.poisson);
      params2._train = fr._key;
      // params2._response = 1;
      params2._response_column = fr._names[1];
      params2._lambda = new double[]{0};
      params2._standardize = true;
      params2._beta_epsilon = 1e-5;
      model = new GLM(params2, Key.make("poisson_test_2")).trainModel().get();
      assertEquals(0.3396, model.beta()[1], 1e-1);
      assertEquals(0.2565, model.beta()[0], 1e-1);
      // test scoring
      testScoring(model,fr);
    } finally {
      if (fr != null) fr.delete();
      if (res != null) res.delete();
      if (model != null) model.delete();
    }
  }


  /**
   * Test Gamma regression on simple and small synthetic dataset.<commit_msg>Yet another model key removal from GLMTest.<commit_after>package hex.glm;

import hex.DataInfo;
import hex.DataInfo.TransformType;
import hex.ModelMetrics;
import hex.ModelMetricsBinomialGLM;
import hex.ModelMetricsRegressionGLM;
import hex.deeplearning.DeepLearningModel.DeepLearningParameters.MissingValuesHandling;
import hex.glm.GLMModel.GLMParameters.Link;
import hex.glm.GLMModel.GLMParameters.Solver;
import hex.glm.GLMModel.GLMWeightsFun;
import hex.glm.GLMTask.*;
import org.junit.*;

import hex.glm.GLMModel.GLMParameters;
import hex.glm.GLMModel.GLMParameters.Family;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.exceptions.H2OModelBuilderIllegalArgumentException;
import water.fvec.*;
import water.parser.BufferedString;
import water.parser.ParseDataset;
import water.util.ArrayUtils;

import java.util.*;
import java.util.concurrent.ExecutionException;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class GLMTest  extends TestUtil {

  @BeforeClass public static void setup() { stall_till_cloudsize(1); }

  protected static void testScoring(GLMModel m, Frame fr) {
    Scope.enter();
    // standard predictions
    Frame fr2 = new Frame(fr);
    Frame preds = Scope.track(m.score(fr2));
    m.adaptTestForTrain(fr2,true,false);
    fr2.remove(fr2.numCols()-1); // remove response
    int p = m._output._dinfo._cats + m._output._dinfo._nums;
    int p2 = fr2.numCols() - (m._output._dinfo._weights?1:0)- (m._output._dinfo._offset?1:0);
    assert p == p2: p + " != " + p2;
    fr2.add(preds.names(),preds.vecs());
    // test score0
    new TestScore0(m,m._output._dinfo._weights,m._output._dinfo._offset).doAll(fr2);
    // test pojo
    if((!m._output._dinfo._weights && !m._output._dinfo._offset))
      Assert.assertTrue(m.testJavaScoring(fr,preds,1e-15));
    Scope.exit();
  }


  // class to test score0 since score0 is now not being called by the standard bulk scoring
  public static class TestScore0 extends MRTask {
    final GLMModel _m;
    final boolean _weights;
    final boolean _offset;

    public TestScore0(GLMModel m, boolean w, boolean o) {_m = m; _weights = w; _offset = o;}

    @Override public void map(Chunk [] chks) {
      int nout = _m._parms._family == Family.multinomial?_m._output.nclasses()+1:_m._parms._family == Family.binomial?3:1;
      Chunk [] outputs = Arrays.copyOfRange(chks,chks.length-nout,chks.length);
      chks = Arrays.copyOf(chks,chks.length-nout);
      Chunk off = new C0DChunk(0,chks[0]._len);
      Chunk w = new C0DChunk(1,chks[0]._len);
      double [] tmp = new double[_m._output._dinfo._cats + _m._output._dinfo._nums];
      double [] predictions = new double [nout];
      if(_offset) {
        off = chks[chks.length - 1];
        chks = Arrays.copyOf(chks, chks.length - 1);
      }
      if(_weights) {
        w = chks[chks.length - 1];
        chks = Arrays.copyOf(chks, chks.length - 1);
      }
      if(_weights || _offset) {
        for (int i = 0; i < chks[0]._len; ++i) {
          _m.score0(chks, w.atd(i), off.atd(i),i, tmp, predictions);
          for (int j = 0; j < predictions.length; ++j)
            assertEquals("mismatch at row " + (i + chks[0].start()) + ", p = " + j + ": " + outputs[j].atd(i) + " != " + predictions[j], outputs[j].atd(i), predictions[j], 1e-6);
        }
      } else {
        for (int i = 0; i < chks[0]._len; ++i) {
          _m.score0(chks, i, tmp, predictions);
          for (int j = 0; j < predictions.length; ++j)
            assertEquals("mismatch at row " + (i + chks[0].start()) + ", p = " + j + ": " + outputs[j].atd(i) + " != " + predictions[j] + ", preds = " + Arrays.toString(predictions), outputs[j].atd(i), predictions[j], 1e-6);
        }
      }
    }
  }
  //------------------- simple tests on synthetic data------------------------------------
  @Test
  public void testGaussianRegression() throws InterruptedException, ExecutionException {
    Key raw = Key.make("gaussian_test_data_raw");
    Key parsed = Key.make("gaussian_test_data_parsed");
    GLMModel model = null;
    Frame fr = null, res = null;
    try {
      // make data so that the expected coefficients is icept = col[0] = 1.0
      FVecTest.makeByteVec(raw, "x,y\n0,0\n1,0.1\n2,0.2\n3,0.3\n4,0.4\n5,0.5\n6,0.6\n7,0.7\n8,0.8\n9,0.9");
      fr = ParseDataset.parse(parsed, raw);
      GLMParameters params = new GLMParameters(Family.gaussian);
      params._train = fr._key;
      // params._response = 1;
      params._response_column = fr._names[1];
      params._lambda = new double[]{0};
//      params._standardize= false;
      model = new GLM(params).trainModel().get();
      HashMap<String, Double> coefs = model.coefficients();
      assertEquals(0.0, coefs.get("Intercept"), 1e-4);
      assertEquals(0.1, coefs.get("x"), 1e-4);
      testScoring(model,fr);
    } finally {
      if (fr != null) fr.remove();
      if (res != null) res.remove();
      if (model != null) model.remove();
    }
  }

  /**
   * Test Poisson regression on simple and small synthetic dataset.
   * Equation is: y = exp(x+1);
   */
  @Test
  public void testPoissonRegression() throws InterruptedException, ExecutionException {
    Key raw = Key.make("poisson_test_data_raw");
    Key parsed = Key.make("poisson_test_data_parsed");

    GLMModel model = null;
    Frame fr = null, res = null;
    try {
      // make data so that the expected coefficients is icept = col[0] = 1.0
      FVecTest.makeByteVec(raw, "x,y\n0,2\n1,4\n2,8\n3,16\n4,32\n5,64\n6,128\n7,256");
      fr = ParseDataset.parse(parsed, raw);
      Vec v = fr.vec(0);
      System.out.println(v.min() + ", " + v.max() + ", mean = " + v.mean());
      GLMParameters params = new GLMParameters(Family.poisson);
      params._train = fr._key;
      // params._response = 1;
      params._response_column = fr._names[1];
      params._lambda = new double[]{0};
      params._standardize = false;
      model = new GLM(params).trainModel().get();
      for (double c : model.beta())
        assertEquals(Math.log(2), c, 1e-2); // only 1e-2 precision cause the perfect solution is too perfect -> will trigger grid search
      testScoring(model,fr);
      model.delete();
      fr.delete();

      // Test 2, example from http://www.biostat.umn.edu/~dipankar/bmtry711.11/lecture_13.pdf
      FVecTest.makeByteVec(raw, "x,y\n1,0\n2,1\n3,2\n4,3\n5,1\n6,4\n7,9\n8,18\n9,23\n10,31\n11,20\n12,25\n13,37\n14,45\n150,7.193936e+16\n");
      fr = ParseDataset.parse(parsed, raw);
      GLMParameters params2 = new GLMParameters(Family.poisson);
      params2._train = fr._key;
      // params2._response = 1;
      params2._response_column = fr._names[1];
      params2._lambda = new double[]{0};
      params2._standardize = true;
      params2._beta_epsilon = 1e-5;
      model = new GLM(params2).trainModel().get();
      assertEquals(0.3396, model.beta()[1], 1e-1);
      assertEquals(0.2565, model.beta()[0], 1e-1);
      // test scoring
      testScoring(model,fr);
    } finally {
      if (fr != null) fr.delete();
      if (res != null) res.delete();
      if (model != null) model.delete();
    }
  }


  /**
   * Test Gamma regression on simple and small synthetic dataset.
{'fraction_non_alphanumeric': 0.15857359635811835, 'fraction_numerical': 0.03096978893640502, 'mean_word_length': 2.9788693743139407, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6015085', 'n_tokens_mistral': 5261, 'n_tokens_neox': 5069, 'n_words': 1040}