<commit_before>package io.citrine.lolo.trees

import io.citrine.lolo.TestUtils
import org.junit.Test

/**
  * Created by maxhutch on 11/29/16.
  */
@Test
class RegressionTreeTest {

  /**
    * Test a simple tree with only real inputs
    */
  @Test
  def testSimpleTree(): Unit = {
    val csv = TestUtils.readCsv("double_example.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val DT = DTLearner.train(trainingData).getModel()
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }
  }

  /**
    * Test a larger case and time it as a benchmark guideline
    */
  @Test
  def longerTest(): Unit = {
    val csv = TestUtils.readCsv("large_example.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val N = 100
    val start = System.nanoTime()
    val DTMeta = DTLearner.train(trainingData)
    val DT = DTMeta.getModel()
    (0 until N).map(i => DTLearner.train(trainingData).getModel())
    val duration = (System.nanoTime() - start) / 1.0e9

    println(s"Training large case took ${duration / N} s")

    /* We should be able to memorize the inputs */
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }

    /* The first feature should be the most important */
    val importances = DTMeta.getFeatureImportance()
    println(importances.toList)
    assert(importances(0) == importances.max)
  }

  /**
    * Train with a categorical variable
    */
  @Test
  def testCategorical(): Unit = {
    val csv = TestUtils.readCsv("large_example_with_cat.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val N = 100
    val start = System.nanoTime()
    val DTMeta = DTLearner.train(trainingData)
    val DT = DTMeta.getModel()
    (0 until N).map(i => DTLearner.train(trainingData).getModel())
    val duration = (System.nanoTime() - start) / 1.0e9

    println(s"Training large case took ${duration / N} s")

    /* We should be able to memorize the inputs */
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }

    /* The first feature should be the most important */
    val importances = DTMeta.getFeatureImportance()
    assert(importances(0) == importances.max)
  }
}

/** Companion driver */
object RegressionTreeTest {
  /**
    * Test driver
    *
    * @param argv args
    */
  def main(argv: Array[String]): Unit = {
    new RegressionTreeTest().testSimpleTree()
    new RegressionTreeTest().longerTest()
    new RegressionTreeTest().testCategorical()
  }
}<commit_msg>Adding a linear leaf test.<commit_after>package io.citrine.lolo.trees

import io.citrine.lolo.TestUtils
import io.citrine.lolo.linear.LinearRegressionLearner
import org.junit.Test

/**
  * Created by maxhutch on 11/29/16.
  */
@Test
class RegressionTreeTest {

  /**
    * Test a simple tree with only real inputs
    */
  @Test
  def testSimpleTree(): Unit = {
    val csv = TestUtils.readCsv("double_example.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val DT = DTLearner.train(trainingData).getModel()
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }
  }

  /**
    * Test a larger case and time it as a benchmark guideline
    */
  @Test
  def longerTest(): Unit = {
    val csv = TestUtils.readCsv("large_example.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val N = 100
    val start = System.nanoTime()
    val DTMeta = DTLearner.train(trainingData)
    val DT = DTMeta.getModel()
    (0 until N).map(i => DTLearner.train(trainingData).getModel())
    val duration = (System.nanoTime() - start) / 1.0e9

    println(s"Training large case took ${duration / N} s")

    /* We should be able to memorize the inputs */
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }

    /* The first feature should be the most important */
    val importances = DTMeta.getFeatureImportance()
    println(importances.toList)
    assert(importances(0) == importances.max)
  }

  /**
    * Train with a categorical variable
    */
  @Test
  def testCategorical(): Unit = {
    val csv = TestUtils.readCsv("large_example_with_cat.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner()
    val N = 100
    val start = System.nanoTime()
    val DTMeta = DTLearner.train(trainingData)
    val DT = DTMeta.getModel()
    (0 until N).map(i => DTLearner.train(trainingData).getModel())
    val duration = (System.nanoTime() - start) / 1.0e9

    println(s"Training large case took ${duration / N} s")

    /* We should be able to memorize the inputs */
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }

    /* The first feature should be the most important */
    val importances = DTMeta.getFeatureImportance()
    assert(importances(0) == importances.max)
  }

  /**
    * Train with linear leaves.  This case is under-constriained, so we should hit the results exactly
    */
  @Test
  def testLinearLeaves(): Unit = {
    val csv = TestUtils.readCsv("large_example_with_cat.csv")
    val trainingData = csv.map(vec => (vec.init, vec.last.asInstanceOf[Double]))
    val DTLearner = new RegressionTreeLearner(
      minLeafInstances = 2,
      leafLearner = Some(new LinearRegressionLearner(regParam = 0.0))
    )
    val DTMeta = DTLearner.train(trainingData)
    val DT = DTMeta.getModel()

    /* We should be able to memorize the inputs */
    trainingData.foreach { case (x, y) =>
      assert(Math.abs(y - DT.predict(x)) < 1.0e-9)
    }

    /* The first feature should be the most important */
    val importances = DTMeta.getFeatureImportance()
    assert(importances(0) == importances.max)
  }
}

/** Companion driver */
object RegressionTreeTest {
  /**
    * Test driver
    *
    * @param argv args
    */
  def main(argv: Array[String]): Unit = {
    new RegressionTreeTest().testSimpleTree()
    new RegressionTreeTest().longerTest()
    new RegressionTreeTest().testCategorical()
    new RegressionTreeTest().testLinearLeaves()
  }
}
{'fraction_non_alphanumeric': 0.1288644015916743, 'fraction_numerical': 0.010560146923783287, 'mean_word_length': 3.059006211180124, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4668664', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2156, 'n_words': 535}