<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import random
import pytest

from asv.step_detect import (solve_potts, solve_potts_autogamma, solve_potts_approx,
                             detect_regressions, golden_search, median, rolling_median_dev,
                             L1Dist, detect_steps)
from asv import step_detect


try:
    import numpy as np
    HAVE_NUMPY = True
except ImportError:
    HAVE_NUMPY = False

try:
    from asv import _rangemedian
    HAVE_RANGEMEDIAN = True
except ImportError:
    HAVE_RANGEMEDIAN = False


@pytest.fixture(params=[
    "python",
    pytest.param("rangemedian", marks=pytest.mark.skipif(not HAVE_RANGEMEDIAN, reason="compiled asv._rangemedian required"))
])
def use_rangemedian(request):
    if request.param == "rangemedian":
        assert isinstance(step_detect.get_mu_dist([0], [1]), _rangemedian.RangeMedian)
        return True
    else:
        step_detect._rangemedian = None

        def restore():
            if HAVE_RANGEMEDIAN:
                step_detect._rangemedian = _rangemedian
        request.addfinalizer(restore)

        assert isinstance(step_detect.get_mu_dist([0], [1]), L1Dist)
        return False


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_solve_potts(use_rangemedian):
    try:
        np.random.seed(1234)
    except NameError:
        # work around a bug in old pypy/pytest
        pytest.skip("test needs numpy")
        return

    # Easy case, exact solver
    y = [1, 1, 1, 2, 2, 2, 3, 3, 3]
    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=0.1)
    assert right == [3, 6, 9]
    assert np.allclose(values, [1, 2, 3], atol=0)

    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=8.0)
    assert right == [9]
    assert np.allclose(values, [2], atol=0)

    # l1 norm
    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=0.1)
    assert right == [3, 6, 9]
    assert np.allclose(values, [1, 2, 3], atol=0)

    # Bigger case, exact solver
    t = np.arange(100)
    y0 = (+ 0.4 * (t >= 5)
          + 0.9 * (t >= 10)
          - 0.2 * (t >= 20)
          + 0.2 * (t >= 50)
          + 1.1 * (t >= 70))
    y = y0 + 0.05 * np.random.rand(y0.size)
    right, values, dists = solve_potts(y.tolist(), w=[1]*len(y), gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    # Bigger case, approximative solver
    right, values, dists = solve_potts_approx(y.tolist(), w=[1]*len(y), gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    # Larger case
    t = np.arange(3000)
    y0 = (+ 0.4 * (t >= 10)
          + 0.9 * (t >= 30)
          - 0.2 * (t >= 200)
          + 0.2 * (t >= 600)
          + 1.1 * (t >= 2500)
          - 0.5 * (t >= 2990))

    # Small amount of noise shouldn't disturb step finding
    y = y0 + 0.05 * np.random.randn(y0.size)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [10, 30, 200, 600, 2500, 2990, 3000]

    # Large noise should prevent finding any steps
    y = y0 + 5.0 * np.random.randn(y0.size)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [3000]

    # The routine shouldn't choke on datasets with 10k points.
    # Appending noisy data to weakly noisy data should retain the
    # steps in the former
    y = y0 + 0.025 * np.random.rand(y.size)
    ypad = 0.025 * np.random.randn(10000 - 3000)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist() + ypad.tolist(),
                                                        w=[1]*(len(y)+len(ypad)))
    assert right == [10, 30, 200, 600, 2500, 2990, 3000, 10000]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_autocorrelated():
    # Check that a low-amplitude cosine signal is not interpreted as
    # multiple steps
    j = np.arange(1000)
    y = 0.2 * np.cos(j/100.0) + 1.0 * (j >= 500)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [500, 1000]


def test_zero_variance():
    # Should not choke on this data
    y = [1.0]*1000
    right, values, dists, gamma = solve_potts_autogamma(y, w=[1]*len(y))
    assert right == [1000]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_weighted():
    t = np.arange(100)
    y0 = (+ 0.4 * (t >= 5)
          + 0.9 * (t >= 10)
          - 0.2 * (t >= 20)
          + 0.2 * (t >= 50)
          + 1.1 * (t >= 70))
    y = y0 + 0.05 * np.random.rand(y0.size)

    y = y.tolist()
    w = [1]*len(y)

    y[15] = 2
    right, values, dists = solve_potts(y, w=w, gamma=0.1)
    assert right == [5, 10, 15, 16, 20, 50, 70, 100]

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 15, 16, 20, 50, 70]

    w[15] = 0.1
    right, values, dists = solve_potts(y, w=w, gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 20, 50, 70]

    # Missing weights and data should be handled properly
    y[35] = None
    w[23] = None

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 20, 50, 70]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_detect_regressions(use_rangemedian):
    try:
        np.random.seed(1234)
    except NameError:
        # work around a bug in old pypy/pytest
        pytest.skip("test needs numpy")
        return

    for seed in [1234, 5678, 8901, 2345]:
        np.random.seed(seed)
        t = np.arange(4000)
        y = 0.7 * np.random.rand(t.size)

        y -= 0.3 * (t >= 1000)
        y += 2.0 * (t >= 3234 + (seed % 123))
        y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
        y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))

        y = y.tolist()<commit_msg>TST: fix missing random seed + remove old backcompat stuff<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import random
import pytest

from asv.step_detect import (solve_potts, solve_potts_autogamma, solve_potts_approx,
                             detect_regressions, golden_search, median, rolling_median_dev,
                             L1Dist, detect_steps)
from asv import step_detect


try:
    import numpy as np
    np.random.seed
    HAVE_NUMPY = True
except (ImportError, NameError):
    HAVE_NUMPY = False

try:
    from asv import _rangemedian
    HAVE_RANGEMEDIAN = True
except ImportError:
    HAVE_RANGEMEDIAN = False


@pytest.fixture(params=[
    "python",
    pytest.param("rangemedian", marks=pytest.mark.skipif(not HAVE_RANGEMEDIAN, reason="compiled asv._rangemedian required"))
])
def use_rangemedian(request):
    if request.param == "rangemedian":
        assert isinstance(step_detect.get_mu_dist([0], [1]), _rangemedian.RangeMedian)
        return True
    else:
        step_detect._rangemedian = None

        def restore():
            if HAVE_RANGEMEDIAN:
                step_detect._rangemedian = _rangemedian
        request.addfinalizer(restore)

        assert isinstance(step_detect.get_mu_dist([0], [1]), L1Dist)
        return False


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_solve_potts(use_rangemedian):
    np.random.seed(1234)

    # Easy case, exact solver
    y = [1, 1, 1, 2, 2, 2, 3, 3, 3]
    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=0.1)
    assert right == [3, 6, 9]
    assert np.allclose(values, [1, 2, 3], atol=0)

    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=8.0)
    assert right == [9]
    assert np.allclose(values, [2], atol=0)

    # l1 norm
    right, values, dists = solve_potts(y, w=[1]*len(y), gamma=0.1)
    assert right == [3, 6, 9]
    assert np.allclose(values, [1, 2, 3], atol=0)

    # Bigger case, exact solver
    t = np.arange(100)
    y0 = (+ 0.4 * (t >= 5)
          + 0.9 * (t >= 10)
          - 0.2 * (t >= 20)
          + 0.2 * (t >= 50)
          + 1.1 * (t >= 70))
    y = y0 + 0.05 * np.random.rand(y0.size)
    right, values, dists = solve_potts(y.tolist(), w=[1]*len(y), gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    # Bigger case, approximative solver
    right, values, dists = solve_potts_approx(y.tolist(), w=[1]*len(y), gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    # Larger case
    t = np.arange(3000)
    y0 = (+ 0.4 * (t >= 10)
          + 0.9 * (t >= 30)
          - 0.2 * (t >= 200)
          + 0.2 * (t >= 600)
          + 1.1 * (t >= 2500)
          - 0.5 * (t >= 2990))

    # Small amount of noise shouldn't disturb step finding
    y = y0 + 0.05 * np.random.randn(y0.size)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [10, 30, 200, 600, 2500, 2990, 3000]

    # Large noise should prevent finding any steps
    y = y0 + 5.0 * np.random.randn(y0.size)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [3000]

    # The routine shouldn't choke on datasets with 10k points.
    # Appending noisy data to weakly noisy data should retain the
    # steps in the former
    y = y0 + 0.025 * np.random.rand(y.size)
    ypad = 0.025 * np.random.randn(10000 - 3000)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist() + ypad.tolist(),
                                                        w=[1]*(len(y)+len(ypad)))
    assert right == [10, 30, 200, 600, 2500, 2990, 3000, 10000]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_autocorrelated():
    # Check that a low-amplitude cosine signal is not interpreted as
    # multiple steps
    j = np.arange(1000)
    y = 0.2 * np.cos(j/100.0) + 1.0 * (j >= 500)
    right, values, dists, gamma = solve_potts_autogamma(y.tolist(), w=[1]*len(y))
    assert right == [500, 1000]


def test_zero_variance():
    # Should not choke on this data
    y = [1.0]*1000
    right, values, dists, gamma = solve_potts_autogamma(y, w=[1]*len(y))
    assert right == [1000]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_weighted():
    np.random.seed(1234)

    t = np.arange(100)
    y0 = (+ 0.4 * (t >= 5)
          + 0.9 * (t >= 10)
          - 0.2 * (t >= 20)
          + 0.2 * (t >= 50)
          + 1.1 * (t >= 70))
    y = y0 + 0.05 * np.random.rand(y0.size)

    y = y.tolist()
    w = [1]*len(y)

    y[15] = 2
    right, values, dists = solve_potts(y, w=w, gamma=0.1)
    assert right == [5, 10, 15, 16, 20, 50, 70, 100]

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 15, 16, 20, 50, 70]

    w[15] = 0.1
    right, values, dists = solve_potts(y, w=w, gamma=0.1)
    assert right == [5, 10, 20, 50, 70, 100]

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 20, 50, 70]

    # Missing weights and data should be handled properly
    y[35] = None
    w[23] = None

    steps = detect_steps(y, w=w)
    steps_pos = [s[0] for s in steps]
    assert steps_pos == [0, 5, 10, 20, 50, 70]


@pytest.mark.skipif(not HAVE_NUMPY, reason="test needs numpy")
def test_detect_regressions(use_rangemedian):
    np.random.seed(1234)

    for seed in [1234, 5678, 8901, 2345]:
        np.random.seed(seed)
        t = np.arange(4000)
        y = 0.7 * np.random.rand(t.size)

        y -= 0.3 * (t >= 1000)
        y += 2.0 * (t >= 3234 + (seed % 123))
        y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
        y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))

        y = y.tolist()
{'fraction_non_alphanumeric': 0.14998739601714142, 'fraction_numerical': 0.07327115368456433, 'mean_word_length': 2.431949250288351, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6685597', 'n_tokens_mistral': 5138, 'n_tokens_neox': 4413, 'n_words': 1198}