<commit_before>from __future__ import division
from pymer4.models import Lmer, Lm, Lm2
from pymer4.utils import get_resource_path
import pandas as pd
import numpy as np
from scipy.special import logit
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'  # Recent versions of rpy2 sometimes cause the python kernel to die when running R code; this handles that


def test_gaussian_lm2():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lm2('DV ~ IV3 + IV2', group='Group', data=df)
    model.fit(summarize=False)
    assert model.coefs.shape == (3, 8)
    estimates = np.array([16.11554138, -1.38425772, 0.59547697])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)
    assert model.fixef.shape == (47, 3)

    # Test bootstrapping and permutation tests
    model.fit(permute=500, conf_int='boot', n_boot=500, summarize=False)
    assert model.ci_type == 'boot (500)'
    assert model.sig_type == 'permutation (500)'


def test_gaussian_lm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lm('DV ~ IV1 + IV3', data=df)
    model.fit(summarize=False)

    assert model.coefs.shape == (3, 8)
    estimates = np.array([42.24840439, 0.24114414, -3.34057784])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)

    # Test robust SE against statsmodels
    standard_se = np.array([6.83783939, 0.30393886, 3.70656475])
    assert np.allclose(model.coefs['SE'], standard_se, atol=.001)

    hc0_se = np.array([7.16661817, 0.31713064, 3.81918182])
    model.fit(robust='hc0', summarize=False)
    assert np.allclose(model.coefs['SE'], hc0_se, atol=.001)

    hc1_se = np.array([7.1857547, 0.31797745, 3.82937992])
    # hc1 is the default
    model.fit(robust=True, summarize=False)
    assert np.allclose(model.coefs['SE'], hc1_se, atol=.001)

    hc2_se = np.array([7.19555971, 0.31842139, 3.83386854])
    model.fit(robust='hc1', summarize=False)
    assert np.allclose(model.coefs['SE'], hc2_se, atol=.001)

    hc3_se = np.array([7.22466699, 0.31971942, 3.84863701])
    model.fit(robust='hc3', summarize=False)
    assert np.allclose(model.coefs['SE'], hc3_se, atol=.001)

    hac_lag1_se = np.array([8.20858448, 0.39184764, 3.60205873])
    model.fit(robust='hac', summarize=False)
    assert np.allclose(model.coefs['SE'], hac_lag1_se, atol=.001)

    # Test bootstrapping
    model.fit(summarize=False, conf_int='boot')
    assert model.ci_type == 'boot (500)'

    # Test permutation
    model.fit(summarize=False, permute=500)
    assert model.sig_type == 'permutation (500)'


def test_gaussian_lmm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV ~ IV3 + IV2 + (IV2|Group) + (1|IV3)', data=df)
    model.fit(summarize=False)

    assert model.coefs.shape == (3, 8)
    estimates = np.array([12.04334602, -1.52947016, 0.67768509])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)

    assert isinstance(model.fixef, list)
    assert model.fixef[0].shape == (47, 3)
    assert model.fixef[1].shape == (3, 3)

    assert isinstance(model.ranef, list)
    assert model.ranef[0].shape == (47, 2)
    assert model.ranef[1].shape == (3, 1)

    assert model.ranef_corr.shape == (1, 3)
    assert model.ranef_var.shape == (4, 3)

    assert np.allclose(
        model.coefs.loc[:, 'Estimate'], model.fixef[0].mean(), atol=.01)

    # Test prediction
    assert np.allclose(model.predict(
        model.data, use_rfx=True), model.data.fits)


def test_post_hoc():
    np.random.seed(1)
    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV ~ IV1*IV3*DV_l + (IV1|Group)', data=df, family='gaussian')
    model.fit(factors={
        'IV3': ['0.5', '1.0', '1.5'],
        'DV_l': ['0', '1']
    }, summarize=False)

    marginal, contrasts = model.post_hoc(
        marginal_vars='IV3', p_adjust='dunnet')
    assert marginal.shape[0] == 3
    assert contrasts.shape[0] == 3

    marginal, contrasts = model.post_hoc(
        marginal_vars=['IV3', 'DV_l'])
    assert marginal.shape[0] == 6
    assert contrasts.shape[0] == 15


def test_logistic_lmm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV_l ~ IV1+ (IV1|Group)', data=df, family='binomial')<commit_msg>add smoketest for simulate method<commit_after>from __future__ import division
from pymer4.models import Lmer, Lm, Lm2
from pymer4.utils import get_resource_path
import pandas as pd
import numpy as np
from scipy.special import logit
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'  # Recent versions of rpy2 sometimes cause the python kernel to die when running R code; this handles that


def test_gaussian_lm2():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lm2('DV ~ IV3 + IV2', group='Group', data=df)
    model.fit(summarize=False)
    assert model.coefs.shape == (3, 8)
    estimates = np.array([16.11554138, -1.38425772, 0.59547697])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)
    assert model.fixef.shape == (47, 3)

    # Test bootstrapping and permutation tests
    model.fit(permute=500, conf_int='boot', n_boot=500, summarize=False)
    assert model.ci_type == 'boot (500)'
    assert model.sig_type == 'permutation (500)'


def test_gaussian_lm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lm('DV ~ IV1 + IV3', data=df)
    model.fit(summarize=False)

    assert model.coefs.shape == (3, 8)
    estimates = np.array([42.24840439, 0.24114414, -3.34057784])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)

    # Test robust SE against statsmodels
    standard_se = np.array([6.83783939, 0.30393886, 3.70656475])
    assert np.allclose(model.coefs['SE'], standard_se, atol=.001)

    hc0_se = np.array([7.16661817, 0.31713064, 3.81918182])
    model.fit(robust='hc0', summarize=False)
    assert np.allclose(model.coefs['SE'], hc0_se, atol=.001)

    hc1_se = np.array([7.1857547, 0.31797745, 3.82937992])
    # hc1 is the default
    model.fit(robust=True, summarize=False)
    assert np.allclose(model.coefs['SE'], hc1_se, atol=.001)

    hc2_se = np.array([7.19555971, 0.31842139, 3.83386854])
    model.fit(robust='hc1', summarize=False)
    assert np.allclose(model.coefs['SE'], hc2_se, atol=.001)

    hc3_se = np.array([7.22466699, 0.31971942, 3.84863701])
    model.fit(robust='hc3', summarize=False)
    assert np.allclose(model.coefs['SE'], hc3_se, atol=.001)

    hac_lag1_se = np.array([8.20858448, 0.39184764, 3.60205873])
    model.fit(robust='hac', summarize=False)
    assert np.allclose(model.coefs['SE'], hac_lag1_se, atol=.001)

    # Test bootstrapping
    model.fit(summarize=False, conf_int='boot')
    assert model.ci_type == 'boot (500)'

    # Test permutation
    model.fit(summarize=False, permute=500)
    assert model.sig_type == 'permutation (500)'


def test_gaussian_lmm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV ~ IV3 + IV2 + (IV2|Group) + (1|IV3)', data=df)
    model.fit(summarize=False)

    assert model.coefs.shape == (3, 8)
    estimates = np.array([12.04334602, -1.52947016, 0.67768509])
    assert np.allclose(model.coefs['Estimate'], estimates, atol=.001)

    assert isinstance(model.fixef, list)
    assert model.fixef[0].shape == (47, 3)
    assert model.fixef[1].shape == (3, 3)

    assert isinstance(model.ranef, list)
    assert model.ranef[0].shape == (47, 2)
    assert model.ranef[1].shape == (3, 1)

    assert model.ranef_corr.shape == (1, 3)
    assert model.ranef_var.shape == (4, 3)

    assert np.allclose(
        model.coefs.loc[:, 'Estimate'], model.fixef[0].mean(), atol=.01)

    # Test prediction
    assert np.allclose(model.predict(
        model.data, use_rfx=True), model.data.fits)

    # Smoketest for simulate
    model.simulate(2)

    model.simulat(2, use_rfx=True)

def test_post_hoc():
    np.random.seed(1)
    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV ~ IV1*IV3*DV_l + (IV1|Group)', data=df, family='gaussian')
    model.fit(factors={
        'IV3': ['0.5', '1.0', '1.5'],
        'DV_l': ['0', '1']
    }, summarize=False)

    marginal, contrasts = model.post_hoc(
        marginal_vars='IV3', p_adjust='dunnet')
    assert marginal.shape[0] == 3
    assert contrasts.shape[0] == 3

    marginal, contrasts = model.post_hoc(
        marginal_vars=['IV3', 'DV_l'])
    assert marginal.shape[0] == 6
    assert contrasts.shape[0] == 15


def test_logistic_lmm():

    df = pd.read_csv(os.path.join(get_resource_path(), 'sample_data.csv'))
    model = Lmer('DV_l ~ IV1+ (IV1|Group)', data=df, family='binomial')
{'fraction_non_alphanumeric': 0.16001364721937905, 'fraction_numerical': 0.08620493574434208, 'mean_word_length': 3.8239166209544706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272474', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3330, 'n_words': 651}