<jupyter_start><jupyter_text>Load data<jupyter_code>results_study1 = pd.read_csv(os.path.join(base_dir,'Data/Study1/ComputationalModel/Results_study1.csv'),
                         index_col=0)
results_study1.loc[results_study1['baseMult']==4,'set'] = 'Study 1, x2-x4-x6'
results_study1.loc[results_study1['baseMult']==6,'set'] = 'Study 1, x4-x6-x8'
results_study2 = pd.read_csv(os.path.join(base_dir,'Data/Study2/ComputationalModel/Results_study2.csv'),
                         index_col=0)
results_study2['set'] = 'Study 2'
results_all = results_study1[['set','sub','model','SSE','AIC','BIC']].append(
    results_study2[['set','sub','model','SSE','AIC','BIC']])
results_all.head()<jupyter_output><empty_output><jupyter_text>Exclude near perfect fit and other exclusions<jupyter_code>excludeSub = np.array(results_all.loc[results_all['SSE']<10,'sub'].unique())
excludeSub
exclude1 = np.array(pd.read_csv(os.path.join(base_dir,'Data/Study1/HMTG/exclude.csv'),index_col=None,header=None).T)[0]
excludeSub = np.hstack([excludeSub,exclude1])
exclude2 = np.array(pd.read_csv(os.path.join(base_dir,'Data/Study2/HMTG/exclude.csv'),index_col=0,header=0).T)[0]
excludeSub = np.hstack([excludeSub,exclude2])
excludeSub = np.unique(excludeSub)
excludeSub
results_all = results_all.loc[~results_all['sub'].isin(excludeSub)]
results_all.head()<jupyter_output><empty_output><jupyter_text>Quick check<jupyter_code>for dataset in np.unique(results_all['set']):
    for model in ['GA','IA','MP_ppSOE']:
        mean_AIC = results_all.loc[(results_all['set']==dataset) & (results_all['model']==model),'AIC'].mean()
        print('%s, model %s: mean AIC = %.2f'%(dataset,model,mean_AIC))
    print('')
for dataset in np.unique(results_all['set']):
    dat_IA = results_all.loc[(results_all['set']==dataset) & (results_all['model']=='IA'),'AIC']
    dat_MP_ppSOE = results_all.loc[(results_all['set']==dataset) & (results_all['model']=='MP_ppSOE'),'AIC']
    print(scipy.stats.ttest_rel(dat_IA,dat_MP_ppSOE))
    print('')<jupyter_output>Ttest_relResult(statistic=2.6456910922749097, pvalue=0.009572267417278029)

Ttest_relResult(statistic=1.3088901986547687, pvalue=0.19379662927047084)

Ttest_relResult(statistic=3.308567994275955, pvalue=0.0017069091133856554)<jupyter_text>Plot model fits<jupyter_code># Design
sns.set_style('darkgrid', {"axes.facecolor": ".95"})
sns.set_palette('tab10')
sns.set_context('talk')
# Data
measures = ['AIC']
measureNames = ['AIC']
models = ['GR','GA','IA','MP_ppSOE']
modelNames = ['GR','GA','IA','MS']
fig,axes = plt.subplots(nrows = len(measures), ncols = 3,figsize=[12,4],sharey=False)
stats = pd.DataFrame(columns=['set','measure','n','t','p'])
ystarts = [400,450,350]
for m,measure in enumerate(measures):
    for i,dataset in enumerate(np.unique(results_all['set'])):
        ax = axes[i]
        sns.barplot(data=results_all.loc[(results_all['set']==dataset) & (results_all['model'].isin(models))],
                    x='model',y=measure,alpha=.3,errwidth=0,ax=ax)
        sns.swarmplot(data=results_all.loc[(results_all['set']==dataset) & (results_all['model'].isin(models))],
                    x='model',y=measure,alpha=1,s=3,ax=ax,zorder = 1)
        sns.barplot(data=results_all.loc[(results_all['set']==dataset) & (results_all['model'].isin(models))],
                    x='model',y=measure,errwidth=2,capsize=0.1,errcolor='k',alpha=0,ax=ax,zorder = 10)
        ax.set(title='%s'%(dataset),
                  xlabel='Model',ylabel=measureNames[m],xticklabels=modelNames)
        IA = results_all.loc[(results_all['set']==dataset) & (results_all['model']=='IA'),measure]
        MP = results_all.loc[(results_all['set']==dataset) & (results_all['model']=='MP_ppSOE'),measure]
        out = scipy.stats.ttest_rel(IA,MP,nan_policy='omit')
        stats = stats.append(pd.DataFrame([[dataset,measure,len(IA),out[0],out[1]]],columns=stats.columns)).reset_index(drop=True)
        ystart = ystarts[i]
        FigureTools.add_sig_markers(ax, relationships = [[2,3,out[1]]], ystart = ystart)#, markerType = 'pval')
plt.suptitle('Model comparisons',y=1.03)
plt.tight_layout()
FigureTools.add_subplot_letter(axes[0])
FigureTools.add_subplot_letter(axes[2],'B')
plt.savefig(base_dir + '/Results/Figure2.pdf',bbox_inches='tight')
stats<jupyter_output><empty_output><jupyter_text>Test mean AIC/BIC across contexts<jupyter_code>measure = 'AIC'
sets = ['Study 1, x2-x4-x6','Study 1, x4-x6-x8']
IA = results_all.loc[(results_all['set'].isin(sets)) & (results_all['model']=='IA'),
                     ['sub','set',measure]].groupby('sub').mean()
MP = results_all.loc[(results_all['set'].isin(sets)) & (results_all['model']=='MP_ppSOE'),
                     ['sub','set',measure]].groupby('sub').mean()
# IA
out = scipy.stats.ttest_rel(IA,MP,nan_policy='omit')
print(len(IA))
print(out)
sns.set_style('whitegrid');
(IA-MP).plot(kind='hist',bins=50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1991014907085971, 'fraction_numerical': 0.03982029814171942, 'mean_word_length': 6.376506024096385, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '615461', 'license': 'MIT', 'n_tokens_mistral': 2009, 'n_tokens_neox': 1846, 'n_words': 181, 'path': 'Code/1c.Model_comparison_(figure_2).ipynb', 'repo_name': 'jeroenvanbaar/ReciprocityMotives', 'size': '4897'}