<commit_before>#!/usr/bin/python
import unittest


def is_zero(val, abs_tol=1e-3):
    return abs(val) < abs_tol


def is_within(a, lo, hi):
    return lo <= a and a <= hi


def percentile(lst, factor):
    n = len(lst)
    if n < 1:
        return 0
    lst = sorted(lst)
    k = (n-1) * factor
    m = int(k)
    if m == k:
        return lst[m]
    t = (factor - 1.0 * m / (n-1)) * (n-1)
    return (1.0-t)*lst[m] + t*lst[m+1]


def median(lst):
    return percentile(lst, 0.5)


def interpolate_linear(x, y, new_x):
    n = len(x)
    new_n = len(new_x)
    if n < 2 or new_n < 1:
        return []
    if min(new_x) < min(x):
        raise ValueError('A value in x_new is below the interpolation')
    if max(x) < max(new_x):
        raise ValueError('A value in x_new is above the interpolation')
    new_y = [None] * new_n
    j = 0
    for i in xrange(1, n):
        if j >= new_n:
            break
        dx = x[i] - x[i-1]
        dy = y[i] - y[i-1]
        if dx == 0:
            w = 0
        else:
            w = 1.0 * dy / dx
        while j < new_n and is_within(new_x[j], x[i-1], x[i]):
            dxx = new_x[j] - x[i-1]
            dyy = w * dxx
            new_y[j] = y[i-1] + dyy
            j += 1
    return new_y


def interpolate_point(segment_start, segment_end, p):
    return (1.0 - p) * segment_start + p * segment_end


def linspace(lo, hi, step):
    sz = int((hi - lo) / step)
    if sz == 0:
        return []
    return [interpolate_point(lo, hi, 1.0*i/sz) for i in xrange(0, sz+1)]


class MyTest(unittest.TestCase):

    def test_is_zero(self):
        self.assertEqual(is_zero(1e-7, abs_tol=1e-3), True)
        self.assertEqual(is_zero(1e-3, abs_tol=1e-7), False)

    def test_percentile(self):
        self.assertAlmostEqual(percentile([], 0), 0)
        self.assertAlmostEqual(percentile([], 0.3), 0)
        self.assertAlmostEqual(percentile([], 0.5), 0)
        self.assertAlmostEqual(percentile([], 0.7), 0)
        self.assertAlmostEqual(percentile([], 1.0), 0)

        self.assertAlmostEqual(percentile([1], 0), 1)<commit_msg>add interpolate_linear_evenly<commit_after>#!/usr/bin/python
import unittest


def is_zero(val, abs_tol=1e-3):
    return abs(val) < abs_tol


def is_within(a, lo, hi):
    return lo <= a and a <= hi


def percentile(lst, factor):
    n = len(lst)
    if n < 1:
        return 0
    lst = sorted(lst)
    k = (n-1) * factor
    m = int(k)
    if m == k:
        return lst[m]
    t = (factor - 1.0 * m / (n-1)) * (n-1)
    return (1.0-t)*lst[m] + t*lst[m+1]


def median(lst):
    return percentile(lst, 0.5)


def interpolate_linear(x, y, new_x):
    n = len(x)
    new_n = len(new_x)
    if n < 2 or new_n < 1:
        return []
    if min(new_x) < min(x):
        raise ValueError('A value in x_new is below the interpolation')
    if max(x) < max(new_x):
        raise ValueError('A value in x_new is above the interpolation')
    new_y = [None] * new_n
    j = 0
    for i in xrange(1, n):
        if j >= new_n:
            break
        dx = x[i] - x[i-1]
        dy = y[i] - y[i-1]
        if dx == 0:
            w = 0
        else:
            w = 1.0 * dy / dx
        while j < new_n and is_within(new_x[j], x[i-1], x[i]):
            dxx = new_x[j] - x[i-1]
            dyy = w * dxx
            new_y[j] = y[i-1] + dyy
            j += 1
    return new_y


def interpolate_linear_evenly(x, y, steps=None, step=None):
    if steps is None and step is None:
        raise ValueError('Bad arguments: steps, step')        
    lo = min(x)
    hi = max(x)
    step = float(hi - lo) / steps if steps is not None else step
    new_x = linspace(lo, hi, step)
    ys = interpolate_linear(x, y, new_x)
    return zip(new_x, ys)


def interpolate_point(segment_start, segment_end, p):
    return (1.0 - p) * segment_start + p * segment_end


def linspace(lo, hi, step):
    sz = int((hi - lo) / step)
    if sz == 0:
        return []
    return [interpolate_point(lo, hi, 1.0*i/sz) for i in xrange(0, sz+1)]


class MyTest(unittest.TestCase):

    def test_is_zero(self):
        self.assertEqual(is_zero(1e-7, abs_tol=1e-3), True)
        self.assertEqual(is_zero(1e-3, abs_tol=1e-7), False)

    def test_percentile(self):
        self.assertAlmostEqual(percentile([], 0), 0)
        self.assertAlmostEqual(percentile([], 0.3), 0)
        self.assertAlmostEqual(percentile([], 0.5), 0)
        self.assertAlmostEqual(percentile([], 0.7), 0)
        self.assertAlmostEqual(percentile([], 1.0), 0)

        self.assertAlmostEqual(percentile([1], 0), 1)
{'fraction_non_alphanumeric': 0.14802559011692037, 'fraction_numerical': 0.026472534745201854, 'mean_word_length': 2.1464260929909784, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7419948', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1740, 'n_words': 454}