<commit_before>from __future__ import division, print_function

import pytest
import pickle
import io
import numpy as np
import random
from numpy.testing import assert_allclose

from thinc.nn import NeuralNet
from thinc.eg import Example

np.random.seed(2)


def test_create():
    model = NeuralNet((4, 8, 3))

    assert model.nr_in == 4
    assert model.nr_out == 3
    assert model.nr_layer == 3
    assert model.widths == (4, 8, 3)
    assert model.mem is not None

    with pytest.raises(AttributeError) as excinfo:
        model.mem = None


def test_fwd_bias():
    model = NeuralNet((2, 2), rho=0.0)
    
    assert model.nr_weight == 10
    model.weights = [1.0] * model.nr_weight

    eg = model.predict_dense([0, 0])
    assert_allclose(eg.scores, [0.5, 0.5])

    # Set bias for class 0
    syn = [1.,1.,1.,1.]
    bias = [100000.,1.]
    gamma = [0,0]
    beta = [0,0]
    model.weights = syn + bias + gamma + beta
    assert model.weights == list(syn + bias + gamma + beta)
    eg = model.predict_dense([0, 0])
    assert_allclose(eg.scores, [1.0, 0.0])

    # Set bias for class 1
    model.weights = syn + [1.,10000.0] + gamma + beta
    eg = model.predict_dense([0,0])
    assert_allclose(eg.scores, [0.0, 1.0])

    # Set bias for both
    model.weights = syn + [10000.0,10000.0] + gamma + beta
    eg = model.predict_dense([0,0])
    assert_allclose(eg.scores, [0.5, 0.5])


#def test_fwd_linear():
#    model = NeuralNet((2,2), rho=0.0)
#    assert model.nr_out == 2
#    assert model.widths == (2, 2)
#
#    syn = [1.,0.,0.,1.]
#    bias = [0.,0.]
#    gamma = [0.,0.]
#    beta = [0.,0.]
#    model.weights = syn+bias+gamma+beta
#
#    ff = [0,0]
#    tf = [1,0]
#    ft = [0,1]
#    tt = [1,1]
#    eg = model(ff)
#
#    assert_allclose(eg.scores, [0.5, 0.5])
#
#    eg = model(ft)
#    
#    assert_allclose(eg.scores, [ 0.26894142,  0.73105858])
#    assert_allclose([sum(eg.scores)], [1.0])
#
#    eg = model(tf)
#    assert_allclose(eg.scores, [0.73105858, 0.26894142])
#    assert_allclose(sum(eg.scores), [1.0])
#
#    eg = model(tt)
#    assert_allclose(eg.scores, [0.5, 0.5])
#
#
#def test_xor_manual():
#    model = NeuralNet((2,2,2), rho=0.0)
#    assert model.nr_out == 2
#    assert model.widths == (2, 2, 2)
#
#    # Make a network that detects X-or
#    # It should output 0 if inputs are 0,0 or 1,1 and 1 if inputs are 0,1 or 1,0
#    # A linear model can't do this!
#    # 
#    # What we do is create two intermediate predictors, for 0,1 and 1,0
#    # These predictors rely on a bias towards 0. The non-linearity is essential
#    # Then our output layer can detect either of these signals firing
#    #
#    # 0,0 --> neither fire
#    # 0,1 --> A0 fires
#    # 1,0 --> A1 fires
#    # 1,1 --> neither fire
#    #
#
#    model.weights = np.asarray([
#                [4.0, -10.0],   # A.0*in.0, A.0*in.1
#                [-10.0, 5.0], # A.1*in.0, A.1*in.1
#                [0.0, 0.0],     # A.0 bias, A.1 bias
#                [1.0, 1.0],     # A.0 gamma, A.1 gamma
#                [0.0, 0.0],     # A.0 beta, A.1 beta
#                [-10.0, -10.0],  # out.0*A.0, out.0*A.1
#                [10.0, 10.0],   # out.1*A.0, out.1*A.1
#                [10.0, -10.0],   # out.0 bias, out.1 bias
#                [1.0, 1.0],     # out.0 gamma, out.1 gamma
#                [0.0, 0.0],     # out.0 beta, out.1 beta
#            ]).flatten()
#
#    ff = [0,0]
#    tf = [1,0]
#    ft = [0,1]
#    tt = [1,1]
#
#    eg = model(ff)
#    assert eg.scores[0] > 0.99
# 
#    eg = model(tt)
#    assert eg.scores[0] > 0.99
#    
#    eg = model(tf)
#    assert eg.scores[1] > 0.99
#
#    eg = model(ft)
#    assert eg.scores[1] > 0.99
# 
#
#@pytest.fixture
#def xor_data():
#    ff = np.asarray([0.,0.], dtype='f')
#    tf = np.asarray([1.,0.], dtype='f')
#    ft = np.asarray([0.,1.], dtype='f')
#    tt = np.asarray([1.,1.], dtype='f')
#    costs0 = np.asarray([0., 1.], dtype='f')
#    costs1 = np.asarray([1., 0.], dtype='f')
#    return [(ff, 0, costs0), (tf, 1, costs1), (ft, 1, costs1), (tt, 0, costs0)]
#
#
#@pytest.fixture
#def or_data():
#    ff = np.asarray([0.,0.], dtype='f')
#    tf = np.asarray([1.,0.], dtype='f')
#    ft = np.asarray([0.,1.], dtype='f')
#    tt = np.asarray([1.,1.], dtype='f')
#    costs0 = np.asarray([0., 1.], dtype='f')
#    costs1 = np.asarray([1., 0.], dtype='f')
#    return [(ff, 0, costs0), (tf, 1, costs1), (ft, 1, costs1), (tt, 1, costs1)]
#
#
#def test_learn_linear(or_data):
#    '''Test that a linear model can learn OR.'''
#    # Need high eta on this sort of toy problem, or learning takes forever!
#    model = NeuralNet((2, 2), rho=0.0, eta=0.1, eps=1e-4)
#
#    assert model.nr_in == 2
#    assert model.nr_out == 2
#    assert model.nr_layer == 2
#    
#    # It takes about this many iterations, with the settings above.
#    for _ in range(50):
#        for feats, label, costs in or_data:
#            batch = model.train([feats], [costs])
#        random.shuffle(or_data)
#    acc = 0.0
#    for features, label, costs in or_data:
#        eg = model(features)<commit_msg>* Forward tests passing<commit_after>from __future__ import division, print_function

import pytest
import pickle
import io
import numpy as np
import random
from numpy.testing import assert_allclose

from thinc.nn import NeuralNet
from thinc.eg import Example

np.random.seed(2)


def test_create():
    model = NeuralNet((4, 8, 3))

    assert model.nr_in == 4
    assert model.nr_out == 3
    assert model.nr_layer == 3
    assert model.widths == (4, 8, 3)
    assert model.mem is not None

    with pytest.raises(AttributeError) as excinfo:
        model.mem = None


def test_fwd_bias():
    model = NeuralNet((2, 2), rho=0.0)
    
    assert model.nr_weight == 10
    model.weights = [1.0] * model.nr_weight

    eg = model.predict_dense([0, 0])
    assert_allclose(eg.scores, [0.5, 0.5])

    # Set bias for class 0
    syn = [1.,1.,1.,1.]
    bias = [100000.,1.]
    gamma = [0,0]
    beta = [0,0]
    model.weights = syn + bias + gamma + beta
    assert model.weights == list(syn + bias + gamma + beta)
    eg = model.predict_dense([0, 0])
    assert_allclose(eg.scores, [1.0, 0.0])

    # Set bias for class 1
    model.weights = syn + [1.,10000.0] + gamma + beta
    eg = model.predict_dense([0,0])
    assert_allclose(eg.scores, [0.0, 1.0])

    # Set bias for both
    model.weights = syn + [10000.0,10000.0] + gamma + beta
    eg = model.predict_dense([0,0])
    assert_allclose(eg.scores, [0.5, 0.5])


def test_fwd_linear():
    model = NeuralNet((2,2), rho=0.0)
    assert model.nr_out == 2
    assert model.widths == (2, 2)

    syn = [1.,0.,0.,1.]
    bias = [0.,0.]
    gamma = [0.,0.]
    beta = [0.,0.]
    model.weights = syn+bias+gamma+beta

    ff = [0,0]
    tf = [1,0]
    ft = [0,1]
    tt = [1,1]
    eg = model.predict_dense(ff)

    assert_allclose(eg.scores, [0.5, 0.5])

    eg = model.predict_dense(ft)
    
    assert_allclose(eg.scores, [ 0.26894142,  0.73105858])
    assert_allclose([sum(eg.scores)], [1.0])

    eg = model.predict_dense(tf)
    assert_allclose(eg.scores, [0.73105858, 0.26894142])
    assert_allclose(sum(eg.scores), [1.0])

    eg = model.predict_dense(tt)
    assert_allclose(eg.scores, [0.5, 0.5])


def test_xor_manual():
    model = NeuralNet((2,2,2), rho=0.0)
    assert model.nr_out == 2
    assert model.widths == (2, 2, 2)

    # Make a network that detects X-or
    # It should output 0 if inputs are 0,0 or 1,1 and 1 if inputs are 0,1 or 1,0
    # A linear model can't do this!
    # 
    # What we do is create two intermediate predictors, for 0,1 and 1,0
    # These predictors rely on a bias towards 0. The non-linearity is essential
    # Then our output layer can detect either of these signals firing
    #
    # 0,0 --> neither fire
    # 0,1 --> A0 fires
    # 1,0 --> A1 fires
    # 1,1 --> neither fire
    #

    model.weights = np.asarray([
                [4.0, -10.0],   # A.0*in.0, A.0*in.1
                [-10.0, 5.0], # A.1*in.0, A.1*in.1
                [0.0, 0.0],     # A.0 bias, A.1 bias
                [1.0, 1.0],     # A.0 gamma, A.1 gamma
                [0.0, 0.0],     # A.0 beta, A.1 beta
                [-10.0, -10.0],  # out.0*A.0, out.0*A.1
                [10.0, 10.0],   # out.1*A.0, out.1*A.1
                [10.0, -10.0],   # out.0 bias, out.1 bias
                [1.0, 1.0],     # out.0 gamma, out.1 gamma
                [0.0, 0.0],     # out.0 beta, out.1 beta
            ]).flatten()

    ff = [0,0]
    tf = [1,0]
    ft = [0,1]
    tt = [1,1]

    eg = model.predict_dense(ff)
    assert eg.scores[0] > 0.99
 
    eg = model.predict_dense(tt)
    assert eg.scores[0] > 0.99
    
    eg = model.predict_dense(tf)
    assert eg.scores[1] > 0.99

    eg = model.predict_dense(ft)
    assert eg.scores[1] > 0.99
 

@pytest.fixture
def xor_data():
    ff = np.asarray([0.,0.], dtype='f')
    tf = np.asarray([1.,0.], dtype='f')
    ft = np.asarray([0.,1.], dtype='f')
    tt = np.asarray([1.,1.], dtype='f')
    costs0 = np.asarray([0., 1.], dtype='f')
    costs1 = np.asarray([1., 0.], dtype='f')
    return [(ff, 0, costs0), (tf, 1, costs1), (ft, 1, costs1), (tt, 0, costs0)]


@pytest.fixture
def or_data():
    ff = np.asarray([0.,0.], dtype='f')
    tf = np.asarray([1.,0.], dtype='f')
    ft = np.asarray([0.,1.], dtype='f')
    tt = np.asarray([1.,1.], dtype='f')
    costs0 = np.asarray([0., 1.], dtype='f')
    costs1 = np.asarray([1., 0.], dtype='f')
    return [(ff, 0, costs0), (tf, 1, costs1), (ft, 1, costs1), (tt, 1, costs1)]


#def test_learn_linear(or_data):
#    '''Test that a linear model can learn OR.'''
#    # Need high eta on this sort of toy problem, or learning takes forever!
#    model = NeuralNet((2, 2), rho=0.0, eta=0.1, eps=1e-4)
#
#    assert model.nr_in == 2
#    assert model.nr_out == 2
#    assert model.nr_layer == 2
#    
#    # It takes about this many iterations, with the settings above.
#    for _ in range(50):
#        for feats, label, costs in or_data:
#            batch = model.train([feats], [costs])
#        random.shuffle(or_data)
#    acc = 0.0
#    for features, label, costs in or_data:
#        eg = model(features)
{'fraction_non_alphanumeric': 0.181163708086785, 'fraction_numerical': 0.06785009861932939, 'mean_word_length': 2.260771704180064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4299736', 'n_tokens_mistral': 4413, 'n_tokens_neox': 4027, 'n_words': 1018}