<commit_before>'''
Created on Feb 24, 2013

@author: jasonrudy
'''
import pickle
import copy
import os
from .testing_utils import (if_statsmodels, if_pandas, if_patsy,
                            if_environ_has, assert_list_almost_equal_value,
                            assert_list_almost_equal,
                            if_sklearn_version_greater_than_or_equal_to)
from nose.tools import (assert_equal, assert_true, assert_almost_equal,
                        assert_list_equal, assert_raises, assert_not_equal)
import numpy
from scipy.sparse import csr_matrix
from pyearth._types import BOOL
from pyearth._basis import (Basis, ConstantBasisFunction,
                            HingeBasisFunction, LinearBasisFunction)
from pyearth import Earth

numpy.random.seed(0)

basis = Basis(10)
constant = ConstantBasisFunction()
basis.append(constant)
bf1 = HingeBasisFunction(constant, 0.1, 10, 1, False, 'x1')
bf2 = HingeBasisFunction(constant, 0.1, 10, 1, True, 'x1')
bf3 = LinearBasisFunction(bf1, 2, 'x2')
basis.append(bf1)
basis.append(bf2)
basis.append(bf3)
X = numpy.random.normal(size=(100, 10))
missing = numpy.zeros_like(X, dtype=BOOL)
B = numpy.empty(shape=(100, 4), dtype=numpy.float64)
basis.transform(X, missing, B)
beta = numpy.random.normal(size=4)
y = numpy.empty(shape=100, dtype=numpy.float64)
y[:] = numpy.dot(B, beta) + numpy.random.normal(size=100)
default_params = {"penalty": 1}


@if_sklearn_version_greater_than_or_equal_to('0.17.2')
def test_check_estimator():
    import sklearn.utils.estimator_checks
    sklearn.utils.estimator_checks.MULTI_OUTPUT.append('Earth')
    sklearn.utils.estimator_checks.check_estimator(Earth)


def test_get_params():
    assert_equal(
        Earth().get_params(), {'penalty': None, 'min_search_points': None,
                               'endspan_alpha': None, 'check_every': None,
                               'max_terms': None, 'max_degree': None,
                               'minspan_alpha': None, 'thresh': None,
                               'zero_tol': None,
                               'minspan': None, 'endspan': None,
                               'allow_linear': None,
                               'use_fast': None, 'fast_K': None,
                               'fast_h': None, 'smooth': None,
                               'enable_pruning': True,
                               'allow_missing': False,
                               'feature_importance_type': None,
                               'verbose': False})
    assert_equal(
        Earth(
            max_degree=3).get_params(), {'penalty': None,
                                         'min_search_points': None,
                                         'endspan_alpha': None,
                                         'check_every': None,
                                         'max_terms': None, 'max_degree': 3,
                                         'minspan_alpha': None,
                                         'thresh': None, 'zero_tol': None,
                                         'minspan': None,
                                         'endspan': None,
                                         'allow_linear': None,
                                         'use_fast': None,
                                         'fast_K': None, 'fast_h': None,
                                         'smooth': None,
                                         'enable_pruning': True,
                                         'allow_missing': False,
                                         'feature_importance_type': None,
                                         'verbose': False})


@if_statsmodels
def test_linear_fit():
    from statsmodels.regression.linear_model import GLS, OLS

    earth = Earth(**default_params)
    earth.fit(X, y)
    earth.linear_fit(X, y)
    soln = OLS(y, earth.transform(X)).fit().params
    assert_almost_equal(numpy.mean((earth.coef_ - soln) ** 2), 0.0)

    sample_weight = 1.0 / (numpy.random.normal(size=y.shape) ** 2)
    earth.fit(X, y)
    earth.linear_fit(X, y, sample_weight)
    soln = GLS(y, earth.transform(
        X), 1.0 / sample_weight).fit().params
    assert_almost_equal(numpy.mean((earth.coef_ - soln) ** 2), 0.0)


def test_sample_weight():
    group = numpy.random.binomial(1, .5, size=1000) == 1
    sample_weight = 1 / (group * 100 + 1.0)
    x = numpy.random.uniform(-10, 10, size=1000)
    y = numpy.abs(x)
    y[group] = numpy.abs(x[group] - 5)
    y += numpy.random.normal(0, 1, size=1000)
    model = Earth().fit(x[:, numpy.newaxis], y, sample_weight=sample_weight)

    # Check that the model fits better for the more heavily weighted group
    assert_true(model.score(x[group], y[group]) < model.score(
        x[numpy.logical_not(group)], y[numpy.logical_not(group)]))

    # Make sure that the score function gives the same answer as the trace
    pruning_trace = model.pruning_trace()
    rsq_trace = pruning_trace.rsq(model.pruning_trace().get_selected())
    assert_almost_equal(model.score(x, y, sample_weight=sample_weight),
                        rsq_trace)

    # Uncomment below to see what this test situation looks like
#     from matplotlib import pyplot
#     print model.summary()
#     print model.score(x,y,sample_weight = sample_weight)
#     pyplot.figure()
#     pyplot.plot(x,y,'b.')
#     pyplot.plot(x,model.predict(x),'r.')
#     pyplot.show()


def test_output_weight():
    x = numpy.random.uniform(-1, 1, size=(1000, 1))
    y = (numpy.dot(x, numpy.random.normal(0, 1, size=(1, 10)))) ** 5 + 1
    y = (y - y.mean(axis=0)) / y.std(axis=0)
    group = numpy.array([1] * 5 + [0] * 5)
    output_weight = numpy.array([1] * 5 + [2] * 5, dtype=float)
    model = Earth().fit(x, y, output_weight=output_weight)

    # Check that the model fits at least better
    # the more heavily weighted group
    mse = ((model.predict(x) - y)**2).mean(axis=0)
    group1_mean = mse[group].mean()
    group2_mean = mse[numpy.logical_not(group)].mean()
    assert_true(group1_mean > group2_mean or
                round(abs(group1_mean - group2_mean), 7) == 0)


def test_missing_data():
    earth = Earth(allow_missing=True, **default_params)
    missing_ = numpy.random.binomial(1, .05, X.shape).astype(bool)
    X_ = X.copy()
    X_[missing_] = None
    earth.fit(X_, y)
    res = str(earth.score(X_, y))
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress_missing_data.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .03)


def test_fit():
    earth = Earth(**default_params)
    earth.fit(X, y)
    res = str(earth.rsq_)
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .03)


def test_smooth():
    model = Earth(penalty=1, smooth=True)
    model.fit(X, y)
    res = str(model.rsq_)
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress_smooth.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .03)


def test_linvars():
    earth = Earth(**default_params)
    earth.fit(X, y, linvars=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    res = str(earth.trace()) + '\n' + earth.summary()
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_linvars_regress.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()

    assert_equal(res, prev)


def test_linvars_coefs():
    nb_vars = 11
    coefs = numpy.random.uniform(size=(nb_vars,))
    X = numpy.random.uniform(size=(100, nb_vars))
    bias = 1
    y = numpy.dot(X, coefs[:, numpy.newaxis]) + bias
    earth = Earth(max_terms=nb_vars * 2,
                  max_degree=1,
                  enable_pruning=False,
                  check_every=1,
                  thresh=0,
                  minspan=1,
                  endspan=1).fit(X, y, linvars=range(nb_vars))<commit_msg>increase tolerance of some test cases to account for platform differences<commit_after>'''
Created on Feb 24, 2013

@author: jasonrudy
'''
import pickle
import copy
import os
from .testing_utils import (if_statsmodels, if_pandas, if_patsy,
                            if_environ_has, assert_list_almost_equal_value,
                            assert_list_almost_equal,
                            if_sklearn_version_greater_than_or_equal_to)
from nose.tools import (assert_equal, assert_true, assert_almost_equal,
                        assert_list_equal, assert_raises, assert_not_equal)
import numpy
from scipy.sparse import csr_matrix
from pyearth._types import BOOL
from pyearth._basis import (Basis, ConstantBasisFunction,
                            HingeBasisFunction, LinearBasisFunction)
from pyearth import Earth

numpy.random.seed(0)

basis = Basis(10)
constant = ConstantBasisFunction()
basis.append(constant)
bf1 = HingeBasisFunction(constant, 0.1, 10, 1, False, 'x1')
bf2 = HingeBasisFunction(constant, 0.1, 10, 1, True, 'x1')
bf3 = LinearBasisFunction(bf1, 2, 'x2')
basis.append(bf1)
basis.append(bf2)
basis.append(bf3)
X = numpy.random.normal(size=(100, 10))
missing = numpy.zeros_like(X, dtype=BOOL)
B = numpy.empty(shape=(100, 4), dtype=numpy.float64)
basis.transform(X, missing, B)
beta = numpy.random.normal(size=4)
y = numpy.empty(shape=100, dtype=numpy.float64)
y[:] = numpy.dot(B, beta) + numpy.random.normal(size=100)
default_params = {"penalty": 1}


@if_sklearn_version_greater_than_or_equal_to('0.17.2')
def test_check_estimator():
    import sklearn.utils.estimator_checks
    sklearn.utils.estimator_checks.MULTI_OUTPUT.append('Earth')
    sklearn.utils.estimator_checks.check_estimator(Earth)


def test_get_params():
    assert_equal(
        Earth().get_params(), {'penalty': None, 'min_search_points': None,
                               'endspan_alpha': None, 'check_every': None,
                               'max_terms': None, 'max_degree': None,
                               'minspan_alpha': None, 'thresh': None,
                               'zero_tol': None,
                               'minspan': None, 'endspan': None,
                               'allow_linear': None,
                               'use_fast': None, 'fast_K': None,
                               'fast_h': None, 'smooth': None,
                               'enable_pruning': True,
                               'allow_missing': False,
                               'feature_importance_type': None,
                               'verbose': False})
    assert_equal(
        Earth(
            max_degree=3).get_params(), {'penalty': None,
                                         'min_search_points': None,
                                         'endspan_alpha': None,
                                         'check_every': None,
                                         'max_terms': None, 'max_degree': 3,
                                         'minspan_alpha': None,
                                         'thresh': None, 'zero_tol': None,
                                         'minspan': None,
                                         'endspan': None,
                                         'allow_linear': None,
                                         'use_fast': None,
                                         'fast_K': None, 'fast_h': None,
                                         'smooth': None,
                                         'enable_pruning': True,
                                         'allow_missing': False,
                                         'feature_importance_type': None,
                                         'verbose': False})


@if_statsmodels
def test_linear_fit():
    from statsmodels.regression.linear_model import GLS, OLS

    earth = Earth(**default_params)
    earth.fit(X, y)
    earth.linear_fit(X, y)
    soln = OLS(y, earth.transform(X)).fit().params
    assert_almost_equal(numpy.mean((earth.coef_ - soln) ** 2), 0.0)

    sample_weight = 1.0 / (numpy.random.normal(size=y.shape) ** 2)
    earth.fit(X, y)
    earth.linear_fit(X, y, sample_weight)
    soln = GLS(y, earth.transform(
        X), 1.0 / sample_weight).fit().params
    assert_almost_equal(numpy.mean((earth.coef_ - soln) ** 2), 0.0)


def test_sample_weight():
    group = numpy.random.binomial(1, .5, size=1000) == 1
    sample_weight = 1 / (group * 100 + 1.0)
    x = numpy.random.uniform(-10, 10, size=1000)
    y = numpy.abs(x)
    y[group] = numpy.abs(x[group] - 5)
    y += numpy.random.normal(0, 1, size=1000)
    model = Earth().fit(x[:, numpy.newaxis], y, sample_weight=sample_weight)

    # Check that the model fits better for the more heavily weighted group
    assert_true(model.score(x[group], y[group]) < model.score(
        x[numpy.logical_not(group)], y[numpy.logical_not(group)]))

    # Make sure that the score function gives the same answer as the trace
    pruning_trace = model.pruning_trace()
    rsq_trace = pruning_trace.rsq(model.pruning_trace().get_selected())
    assert_almost_equal(model.score(x, y, sample_weight=sample_weight),
                        rsq_trace)

    # Uncomment below to see what this test situation looks like
#     from matplotlib import pyplot
#     print model.summary()
#     print model.score(x,y,sample_weight = sample_weight)
#     pyplot.figure()
#     pyplot.plot(x,y,'b.')
#     pyplot.plot(x,model.predict(x),'r.')
#     pyplot.show()


def test_output_weight():
    x = numpy.random.uniform(-1, 1, size=(1000, 1))
    y = (numpy.dot(x, numpy.random.normal(0, 1, size=(1, 10)))) ** 5 + 1
    y = (y - y.mean(axis=0)) / y.std(axis=0)
    group = numpy.array([1] * 5 + [0] * 5)
    output_weight = numpy.array([1] * 5 + [2] * 5, dtype=float)
    model = Earth().fit(x, y, output_weight=output_weight)

    # Check that the model fits at least better
    # the more heavily weighted group
    mse = ((model.predict(x) - y)**2).mean(axis=0)
    group1_mean = mse[group].mean()
    group2_mean = mse[numpy.logical_not(group)].mean()
    assert_true(group1_mean > group2_mean or
                round(abs(group1_mean - group2_mean), 7) == 0)


def test_missing_data():
    earth = Earth(allow_missing=True, **default_params)
    missing_ = numpy.random.binomial(1, .05, X.shape).astype(bool)
    X_ = X.copy()
    X_[missing_] = None
    earth.fit(X_, y)
    res = str(earth.score(X_, y))
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress_missing_data.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .03)


def test_fit():
    earth = Earth(**default_params)
    earth.fit(X, y)
    res = str(earth.rsq_)
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .05)


def test_smooth():
    model = Earth(penalty=1, smooth=True)
    model.fit(X, y)
    res = str(model.rsq_)
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_regress_smooth.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()
    assert_true(abs(float(res) - float(prev)) < .05)


def test_linvars():
    earth = Earth(**default_params)
    earth.fit(X, y, linvars=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    res = str(earth.trace()) + '\n' + earth.summary()
    filename = os.path.join(os.path.dirname(__file__),
                            'earth_linvars_regress.txt')
#     with open(filename, 'w') as fl:
#         fl.write(res)
    with open(filename, 'r') as fl:
        prev = fl.read()

    assert_equal(res, prev)


def test_linvars_coefs():
    nb_vars = 11
    coefs = numpy.random.uniform(size=(nb_vars,))
    X = numpy.random.uniform(size=(100, nb_vars))
    bias = 1
    y = numpy.dot(X, coefs[:, numpy.newaxis]) + bias
    earth = Earth(max_terms=nb_vars * 2,
                  max_degree=1,
                  enable_pruning=False,
                  check_every=1,
                  thresh=0,
                  minspan=1,
                  endspan=1).fit(X, y, linvars=range(nb_vars))
{'fraction_non_alphanumeric': 0.1289627945926546, 'fraction_numerical': 0.01938030864935997, 'mean_word_length': 2.034852060265021, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3365455', 'n_tokens_mistral': 5571, 'n_tokens_neox': 5235, 'n_words': 1107}