<commit_before>import numpy as np
from itertools import chain
from pymaptools.containers import clusters_to_labels
from lsh_hdc.ranking import RocCurve, LiftCurve, \
    aul_score_from_clusters, aul_score_from_labels
from nose.tools import assert_almost_equal
from pymaptools.sample import discrete_sample, random_seed


def simulate_predictions(n=100, seed=None):
    """simulate classifier predictions for data size of n
    """
    if seed is None:
        seed = random_seed()
    np.random.seed(seed % (2 ** 32))
    probas = [np.random.random() for _ in xrange(n)]
    classes = [discrete_sample({0: (1 - p), 1: p}) for p in probas]
    return classes, probas


def simulate_clustering(galpha=2, gbeta=10, nclusters=20, pos_ratio=0.2, err_pos=0.1,
                        err_neg=0.02):

    csizes = map(int, np.random.gamma(galpha, gbeta, nclusters))
    npos = sum(csizes)
    nneg = int(npos * ((1.0 - pos_ratio) / pos_ratio))
    clusters = []
    for csize in csizes:
        cluster = np.random.choice(2, csize, p=[err_pos, 1.0 - err_pos])
        clusters.append(list(cluster))
    negs = np.random.choice(2, nneg, p=[1.0 - err_neg, err_neg])
    clusters.extend([[x] for x in negs])
    return clusters


def test_aul_simulated():
    """Two different implementations of aul_score should return same numbers
    """

    # test lots of small examples
    for _ in xrange(100):
        clusters = simulate_clustering(galpha=1, gbeta=2, nclusters=4,
                                       pos_ratio=0.5)
        lc = LiftCurve.from_clusters(clusters)
        expected_score = lc.aul_score(plot=True)[0]
        actual_score = lc.aul_score(plot=False)
        assert_almost_equal(expected_score, actual_score, 4)

    # test a few large ones too
    for _ in xrange(10):
        clusters = simulate_clustering()
        lc = LiftCurve.from_clusters(clusters)
        expected_score = lc.aul_score(plot=True)[0]
        actual_score = lc.aul_score(plot=False)
        assert_almost_equal(expected_score, actual_score, 4)<commit_msg>older numpy doesn't have random.choice<commit_after>import numpy as np
from itertools import chain
from pymaptools.containers import clusters_to_labels
from lsh_hdc.ranking import RocCurve, LiftCurve, \
    aul_score_from_clusters, aul_score_from_labels
from nose.tools import assert_almost_equal
from pymaptools.sample import discrete_sample, random_seed


def simulate_predictions(n=100, seed=None):
    """simulate classifier predictions for data size of n
    """
    if seed is None:
        seed = random_seed()
    np.random.seed(seed % (2 ** 32))
    probas = [np.random.random() for _ in xrange(n)]
    classes = [discrete_sample({0: (1 - p), 1: p}) for p in probas]
    return classes, probas


def simulate_clustering(galpha=2, gbeta=10, nclusters=20, pos_ratio=0.2, err_pos=0.1,
                        err_neg=0.02):

    csizes = map(int, np.random.gamma(galpha, gbeta, nclusters))
    npos = sum(csizes)
    nneg = int(npos * ((1.0 - pos_ratio) / pos_ratio))
    clusters = []
    for csize in csizes:
        #cluster = np.random.choice(2, csize, p=[err_pos, 1.0 - err_pos])
        cluster = [discrete_sample({0: err_pos, 1: 1.0 - err_pos}) for _ in xrange(csize)]
        clusters.append(list(cluster))
    #negs = np.random.choice(2, nneg, p=[1.0 - err_neg, err_neg])
    negs = [discrete_sample({0: 1.0 - err_neg, 1: err_neg}) for _ in xrange(nneg)]
    clusters.extend([[x] for x in negs])
    return clusters


def test_aul_simulated():
    """Two different implementations of aul_score should return same numbers
    """

    # test lots of small examples
    for _ in xrange(100):
        clusters = simulate_clustering(galpha=1, gbeta=2, nclusters=4,
                                       pos_ratio=0.5)
        lc = LiftCurve.from_clusters(clusters)
        expected_score = lc.aul_score(plot=True)[0]
        actual_score = lc.aul_score(plot=False)
        assert_almost_equal(expected_score, actual_score, 4)

    # test a few large ones too
    for _ in xrange(10):
        clusters = simulate_clustering()
        lc = LiftCurve.from_clusters(clusters)
        expected_score = lc.aul_score(plot=True)[0]
        actual_score = lc.aul_score(plot=False)
        assert_almost_equal(expected_score, actual_score, 4)
{'fraction_non_alphanumeric': 0.10730114299043621, 'fraction_numerical': 0.021926755306741312, 'mean_word_length': 3.138996138996139, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5286742', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1445, 'n_words': 347}