<jupyter_start><jupyter_text>Supplementary Results 6 part 2: compare models for study 2  Load model fits<jupyter_code>def load_model_fits(date_string, features,
                    niter = 10, gaze = False, model_string = 'FeatureRL', fit_to = 'joint'):
    
    print(date_string, features)
    
    model_fits = pd.read_csv(base_dir + 
         '/Data/Model_results/' +
         '%s_%s_%s_fitto-%s_gaze-%s_niter-%i.csv'%(
             model_string,features,date_string, fit_to, gaze, niter),
           index_col = None, dtype={'sub_ID':str, 'comb':str, 'fit_to':str, 'sub_ind':int})
    display(model_fits.head())
    print(model_fits.shape)
    return model_fits<jupyter_output><empty_output><jupyter_text>FeatureRL with motives (groups 1 and 2)<jupyter_code>date_string = '2020-03-31'
features = 'CoGrRiNa'
model_fits = load_model_fits(date_string, features)<jupyter_output>2020-03-31 CoGrRiNa<jupyter_text>Group 1: Extract full model fits (comb 1111)<jupyter_code>group_1 = model_fits.query('comb == "1111"').copy().reset_index(drop=True)
group_1['model_descr'] = 'All Motives'
display(group_1.head())
group_1.shape<jupyter_output><empty_output><jupyter_text>Group 2: Tag best model per subject<jupyter_code>model_fits['best_model'] = False
for sub_ID in model_fits['sub_ID'].unique():
    sub_dat = model_fits.query('sub_ID == @sub_ID').sort_values(by='BIC',ascending=True).copy()
    best_model = sub_dat.iloc[0]
    best_BIC = best_model['BIC']
    model_fits.loc[(model_fits['sub_ID']==sub_ID) & (model_fits['BIC']==best_BIC),'best_model'] = True
group_2 = model_fits.query('best_model == True').copy().reset_index(drop=True)
group_2['model_descr'] = 'Best Motives'
display(group_2.head())
group_2.shape<jupyter_output><empty_output><jupyter_text>Game Types + intercept (group 3)<jupyter_code>date_string = '2020-03-31'
features = 'CoHgShSgPd'
model_fits = load_model_fits(date_string, features)
group_3 = model_fits.copy()
group_3['model_descr'] = 'Game Types'
display(group_3.head())
group_3.shape<jupyter_output><empty_output><jupyter_text>Games + intercept (group 4)<jupyter_code>date_string = '2020-03-31'
features = 'CoST'
model_fits = load_model_fits(date_string, features)
group_4 = (model_fits
           .copy().reset_index(drop=True))
group_4['model_descr'] = 'Games'
display(group_4.head())
group_4.shape<jupyter_output><empty_output><jupyter_text>Compare<jupyter_code>all_model_fits = group_1.append(group_2).append(group_3).append(group_4)
all_model_fits.columns<jupyter_output><empty_output><jupyter_text>Remove outliers<jupyter_code>BICs = all_model_fits['BIC'].values
plt.hist(BICs, 100);
iqr = (all_model_fits['BIC'].describe()['75%'] - 
       all_model_fits['BIC'].describe()['25%'])
accepted_range = [all_model_fits['BIC'].describe()['25%'] - 3*iqr,
                  all_model_fits['BIC'].describe()['75%'] + 3*iqr]
min_BIC,max_BIC = accepted_range
print(accepted_range)
plt.plot([min_BIC,min_BIC],[0,50],'r:')
plt.plot([max_BIC,max_BIC],[0,50],'r:')
outlier_subjects = all_model_fits.query(
    '(BIC < @min_BIC | BIC > @max_BIC)')[
    ['sub_ind','sub_ID']].drop_duplicates()
outlier_sub_inds = outlier_subjects['sub_ind'].unique()
outlier_subjects
all_model_fits_cleaned = all_model_fits.loc[
    ~all_model_fits['sub_ID'].isin(outlier_subjects.sub_ID),:].copy()
all_model_fits_cleaned[['SSE','LL','BIC']].describe()
all_model_fits_cleaned['model_descr'].unique()<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>sns.set_context('paper')
fig,ax = plt.subplots(1,1,figsize = [6,4])
bar_order = ['Games', 'Game Types', 'All Motives', 'Best Motives']
# bar_order = ['Games', 'Game Types']
xticklabels = ['\n'.join(a.split(' ')) for a in bar_order]

sns.barplot(data = all_model_fits_cleaned, x = 'model_descr', y = 'BIC', ax = ax, alpha = .5,
            order = bar_order, errcolor = 'k', capsize = .1, errwidth = 1)
sns.stripplot(data = all_model_fits_cleaned, x = 'model_descr', y = 'BIC', ax = ax, alpha = .5,
            order = bar_order)
# FigureTools.add_sig_markers(ax,relationships=[[2,3,0],[1,2,0]], distScale=.5, ystart = 30)
ax.set(ylabel = 'BIC', xlabel = 'Features in model',
      xticklabels = xticklabels);
# plt.savefig('/Users/jeroen/Desktop/modelcomp.pdf',transparent=True, bbox_inches = 'tight')<jupyter_output><empty_output><jupyter_text>Stats<jupyter_code>def compare_models(name1,name2, plot_fig = True):
    testdat = all_model_fits_cleaned.query('model_descr == @name1 or model_descr == @name2')
    if plot_fig:
        fig,ax = plt.subplots(1,2,figsize=[10,3])
        sns.barplot(data = testdat, x = 'model_descr', y = 'BIC', ax = ax[0], order = [name1,name2], alpha = .5)
        sns.swarmplot(data = testdat, x = 'model_descr', y = 'BIC', ax = ax[0], order = [name1,name2], alpha = .5)
    testdat = testdat.pivot(index = 'sub_ID', columns = 'model_descr', values = 'BIC').dropna()
    diffs = testdat[name1] - testdat[name2]
    if plot_fig:
        ax[1].hist(diffs)
        ax[1].set(xlabel = '%s - %s'%(name1,name2))
    if scipy.stats.normaltest(diffs)[1] < 0.05:
        stats = scipy.stats.wilcoxon(testdat[name1], testdat[name2])
        test_type = 'nonparametric'
    else:
        stats = scipy.stats.ttest_rel(testdat[name1], testdat[name2])
        test_type = 'parametric'
#     print('mean dBIC = %.2f, %i wins %i losses %i ties, %s stat = %.4f, p = %.4f'%(
#             np.mean(diffs), sum(diffs<0), sum(diffs>0), sum(diffs==0), test_type, stats[0], stats[1]))
    coh_d = (np.mean(diffs))/np.std(diffs)
    print('mean dBIC = %.2f +- %.2f, %i wins %i losses, %s stat = %.4f, p = %.4f, d = %.2f'%(
            np.mean(diffs), np.std(diffs), sum(diffs<0), sum(diffs>0), test_type, stats[0], stats[1], coh_d))
    return stats, diffs
model_list = bar_order
for name1 in model_list:
    for name2 in model_list:
        if name1 is not name2:
            print('%s minus %s: '%(name1,name2))
            stats,diffs = compare_models(name1,name2, plot_fig=False)
            print('')<jupyter_output>Games minus Game Types: 
mean dBIC = 75.13 +- 29.02, 0 wins 50 losses, nonparametric stat = 0.0000, p = 0.0000, d = 2.59

Games minus All Motives: 
mean dBIC = 90.71 +- 38.89, 0 wins 50 losses, nonparametric stat = 0.0000, p = 0.0000, d = 2.33

Games minus Best Motives: 
mean dBIC = 102.41 +- 41.11, 0 wins 50 losses, nonparametric stat = 0.0000, p = 0.0000, d = 2.49

Game Types minus Games: 
mean dBIC = -75.13 +- 29.02, 50 wins 0 losses, nonparametric stat = 0.0000, p = 0.0000, d = -2.59

Game Types minus All Motives: 
mean dBIC = 15.58 +- 13.89, 3 wins 47 losses, nonparametric stat = 13.0000, p = 0.0000, d = 1.12

Game Types minus Best Motives: 
mean dBIC = 27.28 +- 18.13, 0 wins 50 losses, nonparametric stat = 0.0000, p = 0.0000, d = 1.50

All Motives minus Games: 
mean dBIC = -90.71 +- 38.89, 50 wins 0 losses, nonparametric stat = 0.0000, p = 0.0000, d = -2.33

All Motives minus Game Types: 
mean dBIC = -15.58 +- 13.89, 47 wins 3 losses, nonparametric stat = 13.0000, p = 0.0000, d =[...]<jupyter_text>BIC with stat bars<jupyter_code>sns.set_context('talk')
fig,ax = plt.subplots(1,1,figsize = [6,5])
# bar_order = ['S,T pairs', 'Game Types', 'All Motives', 'Best Motives']
xticklabels = ['\n'.join(a.split(' ')) for a in bar_order]
sns.barplot(data = all_model_fits_cleaned, x = 'model_descr', y = 'BIC', ax = ax, alpha = .5,
            order = bar_order, errcolor = 'k', capsize = .1, errwidth = 1)
FigureTools.add_sig_markers(ax,relationships=[[2,3,0],[1,2,0],[1,3,0]], distScale = .5, ystart = 30)
ax.set(ylabel = 'BIC', xlabel = 'Features in model',
      xticklabels = xticklabels);<jupyter_output><empty_output><jupyter_text>Save best model per subject for use in eye tracking analysis<jupyter_code>group_2.to_csv(base_dir + '/Data/Model_results/Best_motives_per_participant.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1641143440530883, 'fraction_numerical': 0.0479836651352731, 'mean_word_length': 4.724616508400292, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '410408', 'license': 'MIT', 'n_tokens_mistral': 3381, 'n_tokens_neox': 3055, 'n_words': 588, 'path': 'Study2_EyeTracking/Analysis_scripts/2.Modeling/1.Compare_models_and_individual_differences.ipynb', 'repo_name': 'jeroenvanbaar/NHB_motives_structure', 'size': '7836'}