<commit_before>
def test_answer():
    from maud.window_func import _weight_boxcar
    from numpy import array, absolute
    r = array([-3, -2.1, -1, 0, 1, 2.6, 3, 100])
    l = 5
    w = _weight_boxcar(r,l)
    a = array([ 0.,  1.,  1.,  1.,  1.,  0.,  0.,  0.])
    assert (w == a).all()
<commit_msg>adding test_boxcar but not ready yet<commit_after>from numpy import array, absolute
from numpy.random import random

from maud.window_func import _weight_boxcar
#from maud.cwindow_func import _weight_boxcar as c_weight_boxcar
#from maud.cwindow_func_scalar import _weight_boxcar_scalar

# Gotta adjust the coeficcients to the boxcar window !!!

def test_knownanswer():
    r = array([-3, -2, -1, 0, 1, 2, 3, 100])
    l = 5
    w = _weight_boxcar(r,l)
    d = w - array([ 0., 0.0954915, 0.6545085, 1., 0.6545085, 0.0954915, 0., 0.])
    assert absolute(d).max() < 1e-8


#def test_PxC(N=50):
#    for n in range(N):
#        r = 5*(2*random(10)-1)
#        l = 10*random()
#        w = _weight_boxcar(r,l)
#        cw = c_weight_boxcar(r,l)
#        assert type(w) == type(cw)
#        assert absolute(w - cw).max() < 1e-10


#def test_cython_scalar():
#    R = array([-3, -2, -1, 0, 1, 2, 3, 100])
#    W = array([ 0., 0.0954915, 0.6545085, 1., 0.6545085, 0.0954915, 0., 0.])
#    l = 5
#    for r, w in zip(R, W):
#        w2 = _weight_boxcar_scalar(r, l)
#        d = w - w2
#        assert absolute(d) < 1e-8

def out_of_window():
    r = 5*(2*random(10)-1)
    l = 10*random()
    w = _weight_boxcar(r,l)
#    cw = c_weight_boxcar(r,l)
    ind = r>l/2
    assert (w[ind]==0).all()
    assert (cw[ind]==0).all()

# Question: _weight_boxcar(ma.masked_all(3), 5) should return a masked array?
# _weight_boxcar(ma.masked_all(4),5)
{'fraction_non_alphanumeric': 0.16964800923254472, 'fraction_numerical': 0.08482400461627236, 'mean_word_length': 2.761388286334056, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '879425', 'n_tokens_mistral': 822, 'n_tokens_neox': 732, 'n_words': 188}