<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import unittest
import random
import math

from telemetry.util import statistics


def Relax(samples, iterations=10):
  """Lloyd relaxation in 1D.

  Keeps the position of the first and last sample.
  """
  for _ in xrange(0, iterations):
    voronoi_boundaries = []
    for i in xrange(1, len(samples)):
      voronoi_boundaries.append((samples[i] + samples[i-1]) * 0.5)

    relaxed_samples = []
    relaxed_samples.append(samples[0])
    for i in xrange(1, len(samples)-1):
      relaxed_samples.append(
          (voronoi_boundaries[i-1] + voronoi_boundaries[i]) * 0.5)
    relaxed_samples.append(samples[-1])
    samples = relaxed_samples
  return samples

def CreateRandomSamples(num_samples):
  samples = []
  position = 0.0
  samples.append(position)
  for _ in xrange(1, num_samples):
    position += random.random()
    samples.append(position)
  return samples

class StatisticsUnitTest(unittest.TestCase):

  def testNormalizeSamples(self):
    samples = []
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [])
    self.assertEquals(scale, 1.0)

    samples = [0.0, 0.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [0.5, 0.5])
    self.assertEquals(scale, 1.0)

    samples = [0.0, 1.0/3.0, 2.0/3.0, 1.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0])
    self.assertEquals(scale, 0.75)

    samples = [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, samples)
    self.assertEquals(scale, 1.0)

  def testDiscrepancyRandom(self):
    """Tests NormalizeSamples and Discrepancy with random samples.

    Generates 10 sets of 10 random samples, computes the discrepancy,
    relaxes the samples using Llloyd's algorithm in 1D, and computes the
    discrepancy of the relaxed samples. Discrepancy of the relaxed samples
    must be less than or equal to the discrepancy of the original samples.
    """
    random.seed(1234567)
    for _ in xrange(0, 10):
      samples = CreateRandomSamples(10)
      samples = statistics.NormalizeSamples(samples)[0]
      d = statistics.Discrepancy(samples)
      relaxed_samples = Relax(samples)
      d_relaxed = statistics.Discrepancy(relaxed_samples)
      self.assertTrue(d_relaxed <= d)

  def testDiscrepancyAnalytic(self):
    """Computes discrepancy for sample sets with known statistics."""
    samples = []
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.0)

    samples = [0.5]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.5)

    samples = [0.0, 1.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 1.0)

    samples = [0.5, 0.5, 0.5]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 1.0)

    samples = [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.25)

    samples = [0.0, 1.0/3.0, 2.0/3.0, 1.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.5)

    samples = statistics.NormalizeSamples(samples)[0]<commit_msg>telemetry: add additional discrepancy test cases<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import unittest
import random
import math

from telemetry.util import statistics


def Relax(samples, iterations=10):
  """Lloyd relaxation in 1D.

  Keeps the position of the first and last sample.
  """
  for _ in xrange(0, iterations):
    voronoi_boundaries = []
    for i in xrange(1, len(samples)):
      voronoi_boundaries.append((samples[i] + samples[i-1]) * 0.5)

    relaxed_samples = []
    relaxed_samples.append(samples[0])
    for i in xrange(1, len(samples)-1):
      relaxed_samples.append(
          (voronoi_boundaries[i-1] + voronoi_boundaries[i]) * 0.5)
    relaxed_samples.append(samples[-1])
    samples = relaxed_samples
  return samples

def CreateRandomSamples(num_samples):
  samples = []
  position = 0.0
  samples.append(position)
  for _ in xrange(1, num_samples):
    position += random.random()
    samples.append(position)
  return samples

class StatisticsUnitTest(unittest.TestCase):

  def testNormalizeSamples(self):
    samples = []
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [])
    self.assertEquals(scale, 1.0)

    samples = [0.0, 0.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [0.5, 0.5])
    self.assertEquals(scale, 1.0)

    samples = [0.0, 1.0/3.0, 2.0/3.0, 1.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0])
    self.assertEquals(scale, 0.75)

    samples = [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0]
    normalized_samples, scale = statistics.NormalizeSamples(samples)
    self.assertEquals(normalized_samples, samples)
    self.assertEquals(scale, 1.0)

  def testDiscrepancyRandom(self):
    """Tests NormalizeSamples and Discrepancy with random samples.

    Generates 10 sets of 10 random samples, computes the discrepancy,
    relaxes the samples using Llloyd's algorithm in 1D, and computes the
    discrepancy of the relaxed samples. Discrepancy of the relaxed samples
    must be less than or equal to the discrepancy of the original samples.
    """
    random.seed(1234567)
    for _ in xrange(0, 10):
      samples = CreateRandomSamples(10)
      samples = statistics.NormalizeSamples(samples)[0]
      d = statistics.Discrepancy(samples)
      relaxed_samples = Relax(samples)
      d_relaxed = statistics.Discrepancy(relaxed_samples)
      self.assertTrue(d_relaxed <= d)

  def testDiscrepancyAnalytic(self):
    """Computes discrepancy for sample sets with known statistics."""
    samples = []
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.0)

    samples = [0.5]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.5)

    samples = [0.0, 1.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 1.0)

    samples = [0.5, 0.5, 0.5]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 1.0)

    samples = [1.0/8.0, 3.0/8.0, 5.0/8.0, 7.0/8.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.25)

    samples = [1.0/8.0, 5.0/8.0, 5.0/8.0, 7.0/8.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.5)

    samples = [1.0/8.0, 3.0/8.0, 5.0/8.0, 5.0/8.0, 7.0/8.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.4)

    samples = [0.0, 1.0/3.0, 2.0/3.0, 1.0]
    d = statistics.Discrepancy(samples)
    self.assertEquals(d, 0.5)

    samples = statistics.NormalizeSamples(samples)[0]
{'fraction_non_alphanumeric': 0.1220979020979021, 'fraction_numerical': 0.04895104895104895, 'mean_word_length': 3.408754623921085, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '411795', 'n_tokens_mistral': 2616, 'n_tokens_neox': 2402, 'n_words': 569}