<jupyter_start><jupyter_text>Statistical Inference on Per-class Metrics  Imports<jupyter_code>import pandas as pd
import numpy as np
import scipy as sp
from scipy.linalg import eigh
import matplotlib.pyplot as plt
import matplotlib as matplotlib
import random as random
import seaborn as sns # gives a better presentation for matplotlib
sns.set()
from scipy import stats
from scipy.stats import f_oneway
from statsmodels.stats.multicomp import pairwise_tukeyhsd<jupyter_output><empty_output><jupyter_text>Loading Data<jupyter_code>df = pd.read_table('Results/Results.txt',decimal=',')
df = df
df=df[df['split']!='train']
#df=df[df['split']=='train']<jupyter_output><empty_output><jupyter_text>Reporting Per-class Metrics<jupyter_code>df=df.reset_index()
df['class'] = df.reset_index()['index'].apply(lambda x: x%10)
df['category'] = df['class'].apply(lambda x: x<6)
for name in ['F1','Precision','Recall']:
    df['LS_'+name] = df[name]+df[name+'_diff']
df_mean = df.groupby('class').mean()
df_std = df.groupby('class').std()
pd.concat([df_mean,df_std]).to_csv('explainability/Per_class_summary.csv',sep='\t')
for name in ['F1','Precision','Recall']:
    df_mean['p_'+name] = df_mean[name+'_diff']/df_mean[name]*100
    df['p_'+name] = df[name+'_diff']/df[name]*100
df_mean<jupyter_output><empty_output><jupyter_text>Statistic Tests  T-Tests on the Difference within Categories<jupyter_code>def cohen_d(x1,x2):
    n1 = len(x1)
    n2 = len(x2)
    t,p = stats.ttest_ind(x1,x2)
    d = t * np.sqrt(1./n1+1./n2)
    print("tvalue = {:.2f}, pvalue = {:.2f}, cohen's d = {:.2f}".format(t,p,d))
cohen_d(df[~df.category]['LS_Precision'],df[df.category]['LS_Precision'])
cohen_d(df[~df.category]['LS_Recall'],df[df.category]['LS_Recall'])
cohen_d(df[~df.category]['LS_F1'],df[df.category]['LS_F1'])
stats.ttest_ind(df[~df.category]['p_Precision'],df[df.category]['p_Precision'])
stats.ttest_ind(df[~df.category]['p_Recall'],df[df.category]['p_Recall'])<jupyter_output><empty_output><jupyter_text>One-way ANOVA<jupyter_code>f_oneway(df[df['class']==0]['LS_F1'], df[df['class']==1]['LS_F1'], df[df['class']==2]['LS_F1'], 
         df[df['class']==3]['LS_F1'], df[df['class']==4]['LS_F1'], df[df['class']==5]['LS_F1'])
f_oneway(df[df['class']==6]['LS_F1'], df[df['class']==7]['LS_F1'], df[df['class']==8]['LS_F1'], 
         df[df['class']==9]['LS_F1'])
f_oneway(df[df['class']==0]['p_F1'], df[df['class']==1]['p_F1'], df[df['class']==2]['p_F1'], 
         df[df['class']==3]['p_F1'], df[df['class']==4]['p_F1'], df[df['class']==5]['p_F1'])
f_oneway(df[df['class']==6]['p_F1'], df[df['class']==7]['p_F1'], df[df['class']==8]['p_F1'], 
         df[df['class']==9]['p_F1'])<jupyter_output><empty_output><jupyter_text>Post-hoc Comparison<jupyter_code>print( pairwise_tukeyhsd(df[df.category]['LS_F1'], df[df.category]['class']))
print( pairwise_tukeyhsd(df[~df.category]['LS_F1'], df[~df.category]['class']))
print( pairwise_tukeyhsd(df[df.category]['LS_Precision'], df[df.category]['class']))
print( pairwise_tukeyhsd(df[~df.category]['LS_Precision'], df[~df.category]['class']))
print( pairwise_tukeyhsd(df[df.category]['LS_Recall'], df[df.category]['class']))
print( pairwise_tukeyhsd(df[~df.category]['LS_Recall'], df[~df.category]['class']))
print( pairwise_tukeyhsd(df['p_F1'], df['class']))<jupyter_output>Multiple Comparison of Means - Tukey HSD, FWER=0.05 
=====================================================
group1 group2 meandiff p-adj   lower    upper  reject
-----------------------------------------------------
     0      1  -1.3449    0.9   -8.008  5.3182  False
     0      2  -1.8337    0.9  -8.4968  4.8294  False
     0      3  -2.9597    0.9  -9.6228  3.7034  False
     0      4  -1.5308    0.9  -8.1939  5.1322  False
     0      5   2.6349    0.9  -4.0282   9.298  False
     0      6  -1.8329    0.9   -8.496  4.8302  False
     0      7  -2.4992    0.9  -9.1623  4.1639  False
     0      8   0.9539    0.9  -5.7092   7.617  False
     0      9  -1.1709    0.9   -7.834  5.4922  False
     1      2  -0.4888    0.9  -7.1519  6.1743  False
     1      3  -1.6149    0.9   -8.278  5.0482  False
     1      4   -0.186    0.9  -6.8491  6.4771  False
     1      5   3.9798 0.6249  -2.6833 10.6429  False
     1      6   -0.488    0.9  -7.1511  6.1751  False
     1      7  -1.1543    0.[...]<jupyter_text>Data Association<jupyter_code>def get_prior():
    prior = pd.read_csv(args.prior_csv,sep=';',names=names, skiprows=1)
    return(prior)
args = Namespace(
    prior_csv = 'Data/Coappearance_matrix.csv',
)
names = ['C1','C2','C3','C4','C5','C6','N7','N8','N9','N10']
prior = get_prior()
prior
mask = np.zeros_like(prior)
mask[np.triu_indices_from(mask)] = True
with sns.axes_style("white"):
    f, ax = plt.subplots(figsize=(8, 8))
    ax = sns.heatmap(prior, annot=True, center=0, square=True, vmax=500, vmin=0, cmap="RdBu",fmt='d',cbar=False)
    label_x = ax.get_xticklabels()
    plt.setp(label_x, horizontalalignment='right')
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22271179651512116, 'fraction_numerical': 0.07310234328059283, 'mean_word_length': 4.329775880469584, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '152888', 'license': 'MIT', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2068, 'n_words': 315, 'path': 'Statistic_Test.ipynb', 'repo_name': 'zzbn12345/WHOSe_Heritage', 'size': '4993'}