<commit_before>/*
 * Copyright (c) 2017 Scaleborn UG, www.scaleborn.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.scaleborn.linereg;

import static org.junit.Assert.assertEquals;

import java.util.ArrayList;
import java.util.List;
import org.scaleborn.linereg.evaluation.DerivationEquation;
import org.scaleborn.linereg.evaluation.DerivationEquationBuilder;
import org.scaleborn.linereg.evaluation.SlopeCoefficients;
import org.scaleborn.linereg.evaluation.commons.CommonsMathSolver;
import org.scaleborn.linereg.sampling.exact.ExactModelSamplingFactory;
import org.scaleborn.linereg.sampling.exact.ExactSamplingContext;
import org.scaleborn.linereg.statistics.Statistics;
import org.scaleborn.linereg.statistics.Statistics.DefaultStatistics;
import org.scaleborn.linereg.statistics.StatsModel;
import org.scaleborn.linereg.statistics.StatsSampling;
import org.scaleborn.linereg.statistics.StatsSampling.StatsSamplingProxy;

/**
 * Created by mbok on 19.03.17.
 */
public class TestModels {

  /**
   * Test model for verification.
   */
  public static class TestModel {

    private int buckets;
    private int featureCount;
    final double[][] observations;
    private Statistics expectedStatistics;
    private double[] expectedSlopeCoefficients;
    private StatsSampling statsSampling;
    private DerivationEquation equation;

    public TestModel(final int buckets, final int featureCount, final double[][] observations,
        double[] expectedSlopeCoefficients,
        Statistics statistics) {
      this.buckets = buckets;
      this.featureCount = featureCount;
      this.observations = observations;
      this.expectedStatistics = statistics;
      this.expectedSlopeCoefficients = expectedSlopeCoefficients;

      sample();
    }

    private StatsSampling<?> createSampling() {
      ExactModelSamplingFactory f = new ExactModelSamplingFactory();
      ExactSamplingContext samplingContext = f.createContext(featureCount);
      return new StatsSamplingProxy<>(samplingContext, f
          .createResponseVarianceTermSampling(samplingContext), f
          .createCoefficientLinearTermSampling(samplingContext), f
          .createCoefficientSquareTermSampling(samplingContext));
    }

    @SuppressWarnings("unchecked")
    private void sample() {
      List<StatsSampling<?>> bucketSamplings = new ArrayList<>(buckets);
      for (int i = 0; i < buckets; i++) {
        bucketSamplings.add(createSampling());
      }
      int b = 0;
      for (int i = 0; i < observations.length; i++) {
        bucketSamplings.get(b).sample(observations[i], observations[i][observations[i].length - 1]);
        b++;
        if (b >= buckets) {
          b = 0;
        }
      }
      statsSampling = createSampling();
      for (int i = 0; i < buckets; i++) {
        statsSampling.merge(bucketSamplings.get(i));
      }
      equation = new DerivationEquationBuilder()
          .buildDerivationEquation(statsSampling);
    }

    public double[] getExpectedSlopeCoefficients() {
      return expectedSlopeCoefficients;
    }

    public Statistics getExpectedStatistics() {
      return expectedStatistics;
    }

    public DerivationEquation getEquation() {
      return equation;
    }

    public StatsModel evaluateModel() {
      SlopeCoefficients coefficients = new CommonsMathSolver().solveCoefficients(equation);
      return new StatsModel(statsSampling, coefficients);
    }

    public void assertCoefficients(double[] givenCoefficients, double delta) {
      assertEquals("Count of evaluated coefficients not equal",
          expectedSlopeCoefficients.length, givenCoefficients.length);
      for (int i = 0; i < expectedSlopeCoefficients.length; i++) {
        assertEquals("Evaluated coefficient " + i + " not eqal",
            expectedSlopeCoefficients[i], givenCoefficients[i], delta);
      }
    }

    public void assertStatistics(final Statistics statistics) {
      assertEquals("RSS not equal", expectedStatistics.getRss(), statistics.getRss(), 0.0001d);
      assertEquals("MSE not equal", expectedStatistics.getMse(), statistics.getMse(), 0.0001d);
    }
  }

  public static TestModel SIMPLE_MODEL_1 = new TestModel(2, 1, new double[][]{
      new double[]{-2, 5},<commit_msg>Fix test e<commit_after>/*
 * Copyright (c) 2017 Scaleborn UG, www.scaleborn.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.scaleborn.linereg;

import static org.junit.Assert.assertEquals;

import java.util.ArrayList;
import java.util.List;
import org.scaleborn.linereg.evaluation.DerivationEquation;
import org.scaleborn.linereg.evaluation.DerivationEquationBuilder;
import org.scaleborn.linereg.evaluation.SlopeCoefficients;
import org.scaleborn.linereg.evaluation.commons.CommonsMathSolver;
import org.scaleborn.linereg.sampling.exact.ExactModelSamplingFactory;
import org.scaleborn.linereg.sampling.exact.ExactSamplingContext;
import org.scaleborn.linereg.statistics.Statistics;
import org.scaleborn.linereg.statistics.Statistics.DefaultStatistics;
import org.scaleborn.linereg.statistics.StatsModel;
import org.scaleborn.linereg.statistics.StatsSampling;
import org.scaleborn.linereg.statistics.StatsSampling.StatsSamplingProxy;

/**
 * Created by mbok on 19.03.17.
 */
public class TestModels {

  /**
   * Test model for verification.
   */
  public static class TestModel {

    private final int buckets;
    private final int featureCount;
    final double[][] observations;
    private final Statistics expectedStatistics;
    private final double[] expectedSlopeCoefficients;
    private StatsSampling statsSampling;
    private DerivationEquation equation;

    public TestModel(final int buckets, final int featureCount, final double[][] observations,
        final double[] expectedSlopeCoefficients,
        final Statistics statistics) {
      this.buckets = buckets;
      this.featureCount = featureCount;
      this.observations = observations;
      this.expectedStatistics = statistics;
      this.expectedSlopeCoefficients = expectedSlopeCoefficients;

      sample();
    }

    private StatsSampling<?> createSampling() {
      final ExactModelSamplingFactory f = new ExactModelSamplingFactory();
      final ExactSamplingContext samplingContext = f.createContext(this.featureCount);
      return new StatsSamplingProxy(samplingContext, f
          .createResponseVarianceTermSampling(samplingContext), f
          .createCoefficientLinearTermSampling(samplingContext), f
          .createCoefficientSquareTermSampling(samplingContext));
    }

    private void sample() {
      final List<StatsSampling<?>> bucketSamplings = new ArrayList<>(this.buckets);
      for (int i = 0; i < this.buckets; i++) {
        bucketSamplings.add(createSampling());
      }
      int b = 0;
      for (int i = 0; i < this.observations.length; i++) {
        bucketSamplings.get(b).sample(
            this.observations[i], this.observations[i][this.observations[i].length - 1]);
        b++;
        if (b >= this.buckets) {
          b = 0;
        }
      }
      this.statsSampling = createSampling();
      for (int i = 0; i < this.buckets; i++) {
        this.statsSampling.merge(bucketSamplings.get(i));
      }
      this.equation = new DerivationEquationBuilder()
          .buildDerivationEquation(this.statsSampling);
    }

    public double[] getExpectedSlopeCoefficients() {
      return this.expectedSlopeCoefficients;
    }

    public Statistics getExpectedStatistics() {
      return this.expectedStatistics;
    }

    public DerivationEquation getEquation() {
      return this.equation;
    }

    public StatsModel evaluateModel() {
      final SlopeCoefficients coefficients = new CommonsMathSolver().solveCoefficients(
          this.equation);
      return new StatsModel(this.statsSampling, coefficients);
    }

    public void assertCoefficients(final double[] givenCoefficients, final double delta) {
      assertEquals("Count of evaluated coefficients not equal",
          this.expectedSlopeCoefficients.length, givenCoefficients.length);
      for (int i = 0; i < this.expectedSlopeCoefficients.length; i++) {
        assertEquals("Evaluated coefficient " + i + " not eqal",
            this.expectedSlopeCoefficients[i], givenCoefficients[i], delta);
      }
    }

    public void assertStatistics(final Statistics statistics) {
      assertEquals("RSS not equal", this.expectedStatistics.getRss(), statistics.getRss(), 0.0001d);
      assertEquals("MSE not equal", this.expectedStatistics.getMse(), statistics.getMse(), 0.0001d);
    }
  }

  public static TestModel SIMPLE_MODEL_1 = new TestModel(2, 1, new double[][]{
      new double[]{-2, 5},
{'fraction_non_alphanumeric': 0.09469657810400083, 'fraction_numerical': 0.007443399152279541, 'mean_word_length': 3.7121285923039453, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4037987', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2603, 'n_words': 674}