<commit_before>"""
Tests common to all generators.
"""
from __future__ import division

import collections
import itertools
import math

# 99% values of the chi-squared statistic used in the goodness-of-fit tests
# below, indexed by degrees of freedom. Values calculated using
# scipy.stats.chi2(dof).ppf(0.99)
chisq_99percentile = {
    3: 11.344866730144371,
    4: 13.276704135987622,
    12: 26.21696730553585,
    23: 41.63839811885848,
    31: 52.19139483319192,
}


class TestCommon(object):
    def test_creation_without_seed(self):
        gen1 = self.gen_class()
        gen2 = self.gen_class()

        sample1 = [gen1.random() for _ in range(10)]
        sample2 = [gen2.random() for _ in range(10)]

        # Possible in theory for these two samples to be identical; vanishingly
        # unlikely in practice.
        self.assertNotEqual(sample1, sample2)

    def test_sequence_default(self):
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345)
        self.assertEqual(gen1.getstate(), gen2.getstate())

    def test_independent_sequences(self):
        # Crude statistical test for lack of correlation. If X and Y are
        # independent and uniform on [0, 1], then (X - 0.5) * (Y - 0.5) has
        # mean 0 and standard deviation 1/12. By the central limit theorem, we
        # expect the average V of N such independent values to be roughly
        # normally distributed with mean 0 and standard deviation 1 /
        # (12*sqrt(N)). So we expect |V| to be at most 1 / (4*sqrt(N)) with
        # over 99% probability.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        N = 10000
        xs = [gen1.random() for _ in range(N)]
        ys = [gen2.random() for _ in range(N)]
        v = sum((x - 0.5) * (y - 0.5) for x, y in zip(xs, ys)) / N
        # Check we're within 3 standard deviations of the mean.
        self.assertLess(abs(v), 0.25/math.sqrt(N))

    def test_no_shared_state(self):
        # Get samples first from gen1, then from gen2.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample1_1 = [gen1.random() for _ in range(10)]
        sample2_1 = [gen2.random() for _ in range(10)]

        # Now in the opposite order: from gen2, then from gen1.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample2_2 = [gen2.random() for _ in range(10)]
        sample1_2 = [gen1.random() for _ in range(10)]

        # Now interleaved.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample1_3 = []
        sample2_3 = []
        for _ in range(10):
            sample1_3.append(gen1.random())
            sample2_3.append(gen2.random())

        # Results should be the same in all cases.
        self.assertEqual(sample1_1, sample1_2)
        self.assertEqual(sample1_1, sample1_3)
        self.assertEqual(sample2_1, sample2_2)
        self.assertEqual(sample2_1, sample2_3)

    def test_restore_state_from_different_generator(self):
        state = self.gen.getstate()

        bad_version = 'pcgrandom.BOGUS'
        bad_state = (bad_version,) + state[1:]
        with self.assertRaises(ValueError):
            self.gen.setstate(bad_state)

    def test_save_and_restore_state(self):
        # Generate some values.
        [self.gen.random() for _ in range(10)]

        # Save the state, generate some more.
        state = self.gen.getstate()
        samples2 = [self.gen.random() for _ in range(10)]

        # Restore the state, check we get the same samples.
        self.gen.setstate(state)
        samples3 = [self.gen.random() for _ in range(10)]
        self.assertEqual(samples2, samples3)

    def test_jumpahead(self):
        # Generate samples, each sample consuming exactly one output
        # from the core generator.
        original_state = self.gen.getstate()
        samples = [self.gen._next_output() for _ in range(1000)]

        # Rewind, check we can produce the exact same samples.
        self.gen.jumpahead(-1000)
        same_again = [self.gen._next_output() for _ in range(1000)]
        self.assertEqual(samples, same_again)

        # Now jump around randomly within the collection of samples,
        # and check we can reproduce them.
        positions = [self.gen.randrange(1000) for _ in range(1000)]

        self.gen.setstate(original_state)
        current_pos = 0
        for next_pos in positions:
            self.gen.jumpahead(next_pos - current_pos)
            sample = self.gen._next_output()
            self.assertEqual(sample, samples[next_pos])
            current_pos = next_pos + 1

    def test_jumpahead_zero(self):
        # Corner case: jumpahead(0) should work.
        state = self.gen.getstate()
        self.gen.jumpahead(0)
        self.assertEqual(self.gen.getstate(), state)

    def test_state_preserves_gauss(self):
        # Test a state with gauss_next = None
        state = self.gen.getstate()
        samples1 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.gen.setstate(state)
        samples2 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.assertEqual(samples1, samples2)

        # ... and a state with gauss_next non-None.
        state = self.gen.getstate()
        samples1 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.gen.setstate(state)
        samples2 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.assertEqual(samples1, samples2)

    def test_seed_resets_gauss_state(self):
        self.gen.seed(2143)
        self.assertIsNone(self.gen.gauss_next)
        x1 = self.gen.gauss(0.0, 1.0)
        self.assertIsNotNone(self.gen.gauss_next)

        self.gen.seed(2143)
        self.assertIsNone(self.gen.gauss_next)
        x2 = self.gen.gauss(0.0, 1.0)
        self.assertIsNotNone(self.gen.gauss_next)

        self.assertEqual(x1, x2)

    def test_getrandbits(self):
        k = 5
        samples = [self.gen.getrandbits(k) for _ in range(10000)]
        self.check_uniform(range(2**k), samples)

    def test_getrandbits_large(self):
        k = 101
        nsamples = 10000
        samples = [self.gen.getrandbits(k) for _ in range(nsamples)]

        # Count number of times individual bits have appeared.
        counts = {}
        for bitpos in range(k):
            bit = 2**bitpos
            counts[bitpos] = sum(1 for sample in samples if (sample & bit))

        # Assuming that each bit is "fair", each count roughly follows a normal
        # distribution with mean 0.5*nsamples and standard deviation
        # 0.5*sqrt(nsamples). We'll call a count bad if it's more than 3
        # standard deviations from the mean.

        bad_counts = 0
        for count in counts.values():
            if abs(counts[0] - 0.5*nsamples) > 1.5*math.sqrt(nsamples):
                bad_counts += 1

        # There's about a 1 in 370 chance of any one count being bad,
        # and the counts should be independent. To be safe, we allow<commit_msg>FIX: bad test for bad counts.<commit_after>"""
Tests common to all generators.
"""
from __future__ import division

import collections
import itertools
import math

# 99% values of the chi-squared statistic used in the goodness-of-fit tests
# below, indexed by degrees of freedom. Values calculated using
# scipy.stats.chi2(dof).ppf(0.99)
chisq_99percentile = {
    3: 11.344866730144371,
    4: 13.276704135987622,
    12: 26.21696730553585,
    23: 41.63839811885848,
    31: 52.19139483319192,
}


class TestCommon(object):
    def test_creation_without_seed(self):
        gen1 = self.gen_class()
        gen2 = self.gen_class()

        sample1 = [gen1.random() for _ in range(10)]
        sample2 = [gen2.random() for _ in range(10)]

        # Possible in theory for these two samples to be identical; vanishingly
        # unlikely in practice.
        self.assertNotEqual(sample1, sample2)

    def test_sequence_default(self):
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345)
        self.assertEqual(gen1.getstate(), gen2.getstate())

    def test_independent_sequences(self):
        # Crude statistical test for lack of correlation. If X and Y are
        # independent and uniform on [0, 1], then (X - 0.5) * (Y - 0.5) has
        # mean 0 and standard deviation 1/12. By the central limit theorem, we
        # expect the average V of N such independent values to be roughly
        # normally distributed with mean 0 and standard deviation 1 /
        # (12*sqrt(N)). So we expect |V| to be at most 1 / (4*sqrt(N)) with
        # over 99% probability.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        N = 10000
        xs = [gen1.random() for _ in range(N)]
        ys = [gen2.random() for _ in range(N)]
        v = sum((x - 0.5) * (y - 0.5) for x, y in zip(xs, ys)) / N
        # Check we're within 3 standard deviations of the mean.
        self.assertLess(abs(v), 0.25/math.sqrt(N))

    def test_no_shared_state(self):
        # Get samples first from gen1, then from gen2.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample1_1 = [gen1.random() for _ in range(10)]
        sample2_1 = [gen2.random() for _ in range(10)]

        # Now in the opposite order: from gen2, then from gen1.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample2_2 = [gen2.random() for _ in range(10)]
        sample1_2 = [gen1.random() for _ in range(10)]

        # Now interleaved.
        gen1 = self.gen_class(seed=12345, sequence=0)
        gen2 = self.gen_class(seed=12345, sequence=1)
        sample1_3 = []
        sample2_3 = []
        for _ in range(10):
            sample1_3.append(gen1.random())
            sample2_3.append(gen2.random())

        # Results should be the same in all cases.
        self.assertEqual(sample1_1, sample1_2)
        self.assertEqual(sample1_1, sample1_3)
        self.assertEqual(sample2_1, sample2_2)
        self.assertEqual(sample2_1, sample2_3)

    def test_restore_state_from_different_generator(self):
        state = self.gen.getstate()

        bad_version = 'pcgrandom.BOGUS'
        bad_state = (bad_version,) + state[1:]
        with self.assertRaises(ValueError):
            self.gen.setstate(bad_state)

    def test_save_and_restore_state(self):
        # Generate some values.
        [self.gen.random() for _ in range(10)]

        # Save the state, generate some more.
        state = self.gen.getstate()
        samples2 = [self.gen.random() for _ in range(10)]

        # Restore the state, check we get the same samples.
        self.gen.setstate(state)
        samples3 = [self.gen.random() for _ in range(10)]
        self.assertEqual(samples2, samples3)

    def test_jumpahead(self):
        # Generate samples, each sample consuming exactly one output
        # from the core generator.
        original_state = self.gen.getstate()
        samples = [self.gen._next_output() for _ in range(1000)]

        # Rewind, check we can produce the exact same samples.
        self.gen.jumpahead(-1000)
        same_again = [self.gen._next_output() for _ in range(1000)]
        self.assertEqual(samples, same_again)

        # Now jump around randomly within the collection of samples,
        # and check we can reproduce them.
        positions = [self.gen.randrange(1000) for _ in range(1000)]

        self.gen.setstate(original_state)
        current_pos = 0
        for next_pos in positions:
            self.gen.jumpahead(next_pos - current_pos)
            sample = self.gen._next_output()
            self.assertEqual(sample, samples[next_pos])
            current_pos = next_pos + 1

    def test_jumpahead_zero(self):
        # Corner case: jumpahead(0) should work.
        state = self.gen.getstate()
        self.gen.jumpahead(0)
        self.assertEqual(self.gen.getstate(), state)

    def test_state_preserves_gauss(self):
        # Test a state with gauss_next = None
        state = self.gen.getstate()
        samples1 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.gen.setstate(state)
        samples2 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.assertEqual(samples1, samples2)

        # ... and a state with gauss_next non-None.
        state = self.gen.getstate()
        samples1 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.gen.setstate(state)
        samples2 = [self.gen.gauss(0.0, 1.0) for _ in range(5)]
        self.assertEqual(samples1, samples2)

    def test_seed_resets_gauss_state(self):
        self.gen.seed(2143)
        self.assertIsNone(self.gen.gauss_next)
        x1 = self.gen.gauss(0.0, 1.0)
        self.assertIsNotNone(self.gen.gauss_next)

        self.gen.seed(2143)
        self.assertIsNone(self.gen.gauss_next)
        x2 = self.gen.gauss(0.0, 1.0)
        self.assertIsNotNone(self.gen.gauss_next)

        self.assertEqual(x1, x2)

    def test_getrandbits(self):
        k = 5
        samples = [self.gen.getrandbits(k) for _ in range(10000)]
        self.check_uniform(range(2**k), samples)

    def test_getrandbits_large(self):
        k = 101
        nsamples = 10000
        samples = [self.gen.getrandbits(k) for _ in range(nsamples)]

        # Count number of times individual bits have appeared.
        counts = {}
        for bitpos in range(k):
            bit = 2**bitpos
            counts[bitpos] = sum(1 for sample in samples if (sample & bit))

        # Assuming that each bit is "fair", each count roughly follows a normal
        # distribution with mean 0.5*nsamples and standard deviation
        # 0.5*sqrt(nsamples). We'll call a count bad if it's more than 3
        # standard deviations from the mean.

        bad_counts = 0
        for count in counts.values():
            if abs(count - 0.5*nsamples) > 1.5*math.sqrt(nsamples):
                bad_counts += 1

        # There's about a 1 in 370 chance of any one count being bad,
        # and the counts should be independent. To be safe, we allow
{'fraction_non_alphanumeric': 0.10569788604227916, 'fraction_numerical': 0.053408931821363574, 'mean_word_length': 2.405721096543504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3925159', 'n_tokens_mistral': 4999, 'n_tokens_neox': 4488, 'n_words': 1218}