<commit_before>from __future__ import division

from nose.tools import assert_almost_equal

from iterutils import powerset

from dit.algorithms import binding_information as B, tse_complexity as TSE
from dit.example_dists import n_mod_m
from dit.math.misc import combinations as nCk

# test based on Olbrich's talk
def test_tse1():
    for i, j in zip(range(3, 6), range(2, 5)):
        d = n_mod_m(i, j)
        indices = [ [k] for k in range(i) ]
        tse = TSE(d)
        x = 1/2 * sum(B(d, rv)/nCk(i, len(rv)) for rv in powerset(indices))
        assert_almost_equal(tse, x)<commit_msg>add another tse test<commit_after>from __future__ import division

from nose.tools import assert_almost_equal

from iterutils import powerset

from dit import Distribution as D
from dit.algorithms import binding_information as B, tse_complexity as TSE
from dit.example_dists import n_mod_m
from dit.math.misc import combinations as nCk

# test based on Olbrich's talk
def test_tse1():
    for i, j in zip(range(3, 6), range(2, 5)):
        d = n_mod_m(i, j)
        indices = [ [k] for k in range(i) ]
        tse = TSE(d)
        x = 1/2 * sum(B(d, rv)/nCk(i, len(rv)) for rv in powerset(indices))
        yield assert_almost_equal, tse, x

def test_tse2():
    for n in xrange(2, 7):
        d = D(['0'*n, '1'*n], [1/2, 1/2])
        tse = TSE(d)
        yield assert_almost_equal, tse, (n-1)/2
{'fraction_non_alphanumeric': 0.10727141828653708, 'fraction_numerical': 0.017998560115190784, 'mean_word_length': 3.276923076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3375538', 'n_tokens_mistral': 522, 'n_tokens_neox': 505, 'n_words': 164}