<jupyter_start><jupyter_text>Preparing and Analysing Expert Evaluation  Imports<jupyter_code>import pandas as pd
import numpy as np
import scipy as sp
from scipy.linalg import eigh
import matplotlib.pyplot as plt
import matplotlib as matplotlib
import networkx as nx
from networkx.algorithms import bipartite
from argparse import Namespace
import random as random
import seaborn as sns # gives a better presentation for matplotlib
sns.set()<jupyter_output><empty_output><jupyter_text>Load in the Data<jupyter_code>models = ['ulmfit', 'bert']
files = ['error_analysis', 'venice_des_pred', 'social_media_pred']
categories = ['justification', 'description', 'social_media']
classes = ['Criteria i', 'Criteria ii', 'Criteria iii', 'Criteria iv', 'Criteria v', 'Criteria vi', 
              'Criteria vii', 'Criteria viii', 'Criteria ix', 'Criteria x', 'Others']
cul = set(classes[:6])
data = {}
for m in models:
    for i in range(len(files)):
        file = files[i] + '.csv'
        cat = categories[i]
        if i==2:
            temp = pd.read_csv('Results/'+m +'/'+file,decimal='\t')
        else:
            temp = pd.read_csv('Results/'+m + '/'+file)
        temp_dict = pd.DataFrame.to_dict(temp)
        if cat in data:
            data[cat][m] = temp_dict
        else:
            data[cat] = {}
            data[cat][m] = temp_dict
sm_df = pd.DataFrame.from_dict(data['social_media']['ulmfit'])[['data','origin','ids']]<jupyter_output><empty_output><jupyter_text>Preparing Human Evaluation Dataset  Checking the Consistency of BERT and ULMFiT predictions for Social Media Data<jupyter_code>for m in models:
    for i in range(1,4):
        sm_df[m+'_max_'+str(i)+'_val'] = pd.Series(data['social_media'][m]['max_'+str(i)+'_val']).astype(float)
        sm_df[m+'_max_'+str(i)+'_col'] = pd.Series(data['social_media'][m]['max_'+str(i)+'_col'])
    sm_df[m+'_max_1'] = pd.Series(data['social_media'][m]['max_1']).astype(float)
    sm_df[m+'_max_3'] = pd.Series(data['social_media'][m]['max_3']).astype(float)
sm_df['same_1'] = (sm_df['bert_max_1_col']==sm_df['ulmfit_max_1_col'])
sm_df['same_1'].value_counts()
def IoU(line):
    a = set()
    b = set()
    for i in range(1,4):
        a.add(line['bert_max_'+str(i)+'_col'])
        b.add(line['ulmfit_max_'+str(i)+'_col'])
    I = a.intersection(b)
    O = a.union(b)
    return len(I)/len(O)
iou = []
for i in range(len(sm_df)):
    iou.append(IoU(sm_df.iloc[i]))
sm_df['same_3'] = iou
sm_df['same_3'].value_counts()<jupyter_output><empty_output><jupyter_text>Checking the Consistency of BERT and ULMFiT predictions for SOUV Data in Venice<jupyter_code>vn_df = pd.DataFrame.from_dict(data['description']['ulmfit'])[['data','id','source']]
for m in models:
    for i in range(1,4):
        vn_df[m+'_max_'+str(i)+'_val'] = pd.Series(data['description'][m]['max_'+str(i)+'_val']).astype(float)
        vn_df[m+'_max_'+str(i)+'_col'] = pd.Series(data['description'][m]['max_'+str(i)+'_col'])
    vn_df[m+'_max_1'] = pd.Series(data['description'][m]['max_1']).astype(float)
    vn_df[m+'_max_3'] = pd.Series(data['description'][m]['max_3']).astype(float)
vn_df['same_1'] = (vn_df['bert_max_1_col']==vn_df['ulmfit_max_1_col'])
vn_df['same_1'].value_counts()
iou = []
for i in range(len(vn_df)):
    iou.append(IoU(vn_df.iloc[i]))
vn_df['same_3'] = iou
vn_df['same_3'].value_counts()<jupyter_output><empty_output><jupyter_text>Creating a New Social Media Dataset that is consistent<jupyter_code>new_sm = sm_df[sm_df['same_3']>=0.5][sm_df['ulmfit_max_3']>0.8][sm_df['bert_max_3']>0.8]
(len(new_sm['ids'].unique()), len(sm_df['ids'].unique())/4196, len(new_sm['ids'].unique())/4196)
new_sm<jupyter_output><empty_output><jupyter_text>Plotting the Distributions of different Data Sources<jupyter_code>import seaborn as sns
sns.set(style="white")
plt.figure()
sm_df[sm_df['same_3']>=0.5]['bert_max_1'].hist(alpha=.7, bins=10)
sm_df[sm_df['same_3']>=0.5]['ulmfit_max_1'].hist(alpha=.7, bins=10)
import seaborn as sns
sns.set(style="white")
plt.figure()
sm_df[sm_df['same_3']>=0.5]['bert_max_3'].hist(alpha=.7, bins=10)
sm_df[sm_df['same_3']>=0.5]['ulmfit_max_3'].hist(alpha=.7, bins=10)
import seaborn as sns
sns.set(style="white")
plt.figure(figsize=(8,8))
plt.scatter(sm_df[sm_df['same_3']>=0.5]['bert_max_3'],sm_df[sm_df['same_3']>=0.5]['ulmfit_max_3'])
plt.scatter(sm_df[sm_df['same_3']>=0.5]['bert_max_1'],sm_df[sm_df['same_3']>=0.5]['ulmfit_max_1'])
import seaborn as sns
sns.set(style="white")
plt.figure(figsize=(8,8))
plt.scatter(new_sm['bert_max_3'],new_sm['ulmfit_max_3'])
plt.scatter(new_sm['bert_max_1'],new_sm['ulmfit_max_1'])
import seaborn as sns
sns.set(style="white")
plt.figure(figsize=(8,8))
plt.scatter(vn_df['bert_max_3'],vn_df['ulmfit_max_3'])
plt.scatter(vn_df['bert_max_1'],vn_df['ulmfit_max_1'])<jupyter_output><empty_output><jupyter_text>Creating a new Dataset about SOUV in Venice<jupyter_code>vn_df[vn_df.same_3>=0.5]['source'].value_counts()
vn_df['source'].value_counts()
new_vn = vn_df[vn_df.same_3>=0.5]
new_vn['sm'] = False
new_vn<jupyter_output>/opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy<jupyter_text>Constructing the Dataset for Expert Evaluation<jupyter_code>rating = new_sm.sample(n=30, random_state=42)
rating['sm'] = True
rating
sns.set(style="white")
plt.figure(figsize=(8,8))
plt.scatter(rating['bert_max_3'],rating['ulmfit_max_3'])
plt.scatter(rating['bert_max_1'],rating['ulmfit_max_1'])
rates = pd.concat([new_vn,rating])
rates<jupyter_output><empty_output><jupyter_text>Generating the Positive and Negative Classes for Evaluation<jupyter_code>def gen_pos(line):
    c = dict()
    for i in range(1,4):
        a = (line['bert_max_'+str(i)+'_col'], line['bert_max_'+str(i)+'_val'])
        b = (line['ulmfit_max_'+str(i)+'_col'], line['ulmfit_max_'+str(i)+'_val'])
        if a[0]==b[0]:
            if len(c)<3:
                c[a[0]] = max(a[1],b[1])
        elif a[1] >= b[1]:
            if a[0] not in c and len(c)<3:
                c[a[0]] = a[1]
            if b[0] not in c and len(c)<3:
                c[b[0]] = b[1]
        else:
            if b[0] not in c and len(c)<3:
                c[b[0]] = b[1]
            if a[0] not in c and len(c)<3:
                c[a[0]] = a[1]
                          
    #return [(key,val) for key,val in c.items()]
    return list(c)
def gen_neg(line):
    d = set()
    for i in range(1,4):
        a = (line['bert_max_'+str(i)+'_col'], line['bert_max_'+str(i)+'_val'])
        b = (line['ulmfit_max_'+str(i)+'_col'], line['ulmfit_max_'+str(i)+'_val'])
        
        d.add(a[0])
        d.add(b[0])
    
    return random.sample(cul.difference(d),1)
gen_pos(rates.iloc[1])
gen_neg(rates.iloc[1])
pos = []
neg = []
for i in range(len(rates)):
    pos.append(gen_pos(rates.iloc[i]))
    neg.append(gen_neg(rates.iloc[i]))
rates['pos'] = pos
rates['neg'] = neg<jupyter_output><empty_output><jupyter_text>Saving the Dataset for Human Evaluation<jupyter_code>rates.to_csv('human_rates.csv',sep='\t')<jupyter_output><empty_output><jupyter_text>Analysing Human Evaluation Dataset  Pre-processing Questionnaire Data<jupyter_code>experts = pd.read_csv('Data/Questionnaire/OUV_Venice_expert_May+9%2C+2021_05.19.csv',header=1).drop(0).T.reset_index()
experts
experts.iloc[16:26]<jupyter_output><empty_output><jupyter_text>The Rating Dataset<jupyter_code>experts_rates = experts.iloc[26:254].drop([134, 135, 136, 137])
experts_rates<jupyter_output><empty_output><jupyter_text>Re-structure the model ratings for fitting the Orders<jupyter_code>rates_new = rates.drop([30,31,311])
rates_new['neg'].loc._setitem_with_indexer(16,['Criteria i'])
rates_new=rates_new.append(rates_new.loc[220],ignore_index=True).drop(44)
rates_new<jupyter_output><empty_output><jupyter_text>Extracting Confidence Scores from Both Models<jupyter_code>bert_scores = {}
ulmfit_scores = {}
for i in range(len(rates_new)):
    now = rates_new.iloc[i]
    bert_scores[i] = {}
    ulmfit_scores[i] = {}
    for j in [1,2,3]:
        bert_scores[i][now['bert_max_'+str(j)+'_col']] = now['bert_max_'+str(j)+'_val']
        ulmfit_scores[i][now['ulmfit_max_'+str(j)+'_col']] = now['ulmfit_max_'+str(j)+'_val']
bert_scores
ulmfit_scores<jupyter_output><empty_output><jupyter_text>Building up the Complete Rating Dataset<jupyter_code>cl = []
pos = []
top1 = []
same1 = []
same3 = []
bs = []
us = []
for i in range(len(experts_rates)):
    num = experts_rates.iloc[i]['index'].split('Criterion (')[1].split(')')[0]
    tp = rates_new['pos'].iloc[int(i/4)][0][9:]
    ng = rates_new['neg'].iloc[int(i/4)][0][9:]
    cl.append(num)
    pos.append(num!=ng)
    top1.append(num==tp)
    same1.append(rates_new['same_1'].iloc[int(i/4)])
    same3.append(rates_new['same_3'].iloc[int(i/4)])
    try:
        bs.append(bert_scores[int(i/4)]['Criteria '+num])
    except:
        bs.append(0)
    try:
        us.append(ulmfit_scores[int(i/4)]['Criteria '+num])
    except:
        us.append(0)
experts_rates['class'] = cl
experts_rates['pos'] = pos
experts_rates['top1'] = top1
experts_rates['same1'] = same1
experts_rates['same3'] = same3
experts_rates['source'] = 0
experts_rates['source'].loc[:85] = 'justification'
experts_rates['source'].loc[86:133] = 'synthesis'
experts_rates['source'].loc[134:] = 'social_media'
experts_rates['bert'] = bs
experts_rates['ulmfit'] = us
experts_rates['score'] = (experts_rates['bert']+experts_rates['ulmfit'])/2
experts_rates['exp'] = experts_rates[[1,2,3,4,5,6,7,8]].astype(int).mean(axis=1)
experts_rates<jupyter_output><empty_output><jupyter_text>Inspecting the Behaviour on Top-1 predictions<jupyter_code>experts_rates[experts_rates.top1]<jupyter_output><empty_output><jupyter_text>Saving The Dataset<jupyter_code>experts_rates.to_csv('expert_rates_full.csv',sep='\t')<jupyter_output><empty_output><jupyter_text>Correlation between Expert Rating and Model Score<jupyter_code>from scipy import stats
stats.pearsonr(experts_rates[experts_rates.top1]['score'], experts_rates[experts_rates.top1]['exp'])
stats.pearsonr(experts_rates['score'], experts_rates['exp'])<jupyter_output><empty_output><jupyter_text>Flatenning Data for Overall Distributions<jupyter_code>data = []
for source in experts_rates['source'].unique():
    for a in [True,False]:
        for b in [True, False]:
            now = experts_rates[experts_rates.top1==a][experts_rates.pos==b][experts_rates.source==source][[1,2,3,4,5,6,7,8]].astype(int)
            if not len(now)==0:
                data.append(np.array(now).flatten())<jupyter_output><empty_output><jupyter_text>Box Plots and Violin Plots to Show Distributions<jupyter_code>fig, axs = plt.subplots(nrows=1,ncols=1,figsize=(12, 6))
axs.boxplot(data)
axs.set_title('Default', fontsize=10)
fig, axs = plt.subplots(nrows=1,ncols=1,figsize=(12, 6))
axs.violinplot(data)
axs.set_title('Default', fontsize=10)
def adjacent_values(vals, q1, q3):
    upper_adjacent_value = q3 + (q3 - q1) * 1.5
    upper_adjacent_value = np.clip(upper_adjacent_value, q3, vals[-1])

    lower_adjacent_value = q1 - (q3 - q1) * 1.5
    lower_adjacent_value = np.clip(lower_adjacent_value, vals[0], q1)
    return lower_adjacent_value, upper_adjacent_value


def set_axis_style(ax, labels, ylabels = None):
    ax.xaxis.set_tick_params(direction='out')
    ax.xaxis.set_ticks_position('bottom')
    ax.set_xticks(np.arange(1, len(labels) + 1))
    ax.set_xticklabels(labels,fontsize=21)
    if ylabels:
        ax.set_yticks(np.arange(1, len(ylabels) + 1))
        ax.set_yticklabels(ylabels,fontsize=21)
    ax.set_xlim(0.25, len(labels) + 0.75)
    ax.set_ylim(0.75, 5.9)
    ax.tick_params(axis='y', labelsize=18)
    #ax.set_xlabel('Sample name',fontsize=18)
   
    
fig, ax2 = plt.subplots(nrows=1, ncols=1, figsize=(15, 4.5))
pos = [1,2,3,5,6,7,9,10,11]

#ax1.set_title('Default violin plot')
#ax1.set_ylabel('Observed values')
#ax1.violinplot(data)

#ax2.set_title('Customized violin plot')
parts = ax2.violinplot(
        data, pos, showmeans=False, showmedians=False,
        showextrema=False)

colors = ['#D43F3A', 'forestgreen', 'purple']

for i in range(len(parts['bodies'])):
    pc = parts['bodies'][i]
    pc.set_facecolor(colors[i%3])
    pc.set_edgecolor('black')
    pc.set_alpha(0.75)

quartile1, medians, quartile3 = ([],[],[])
for i in range(len(data)):
    quartile1.append(np.percentile(data[i], 25))
    medians.append(np.percentile(data[i], 50))
    quartile3.append(np.percentile(data[i], 75))
    
whiskers = np.array([
    adjacent_values(sorted_array, q1, q3)
    for sorted_array, q1, q3 in zip(data, quartile1, quartile3)])
whiskers_min, whiskers_max = whiskers[:, 0], whiskers[:, 1]

inds = np.arange(1, len(medians) + 1)
ax2.scatter(pos, medians, marker='o', color='white', s=45, zorder=3,edgecolor='k')
ax2.vlines(pos, quartile1, quartile3, color='k', linestyle='-', lw=3)
ax2.vlines(pos, whiskers_min, whiskers_max, color='k', linestyle='-', lw=1)

# set style for the axes
labels = ['', 'Justification for Criteria', '','', '', 'Brief Synthesis', '','','', 'Social Media', '']
ylabels = ['1-make no sense', '2-make little sense', '3-not sure', '4-make some sense', '5-make much sense']
for ax in [ax1, ax2]:
    set_axis_style(ax, labels, ylabels)

ax.legend([parts["bodies"][0], parts["bodies"][1], parts["bodies"][2]],
          ['top-1 predictions', 'top-3 predictions','negative classes'], loc='upper center', 
          ncol=3, fontsize=18, frameon=False)    

plt.subplots_adjust(bottom=0.15, wspace=0.05)
plt.show()
[len(d) for d in data]
[np.percentile(d, 50) for d in data]<jupyter_output><empty_output><jupyter_text>Non-parametric Tests for the Distributions  Kruskal-Wallis H Tests<jupyter_code>stats.kruskal(data[0],data[1],data[2])
stats.kruskal(data[3],data[4],data[5])
stats.kruskal(data[6],data[7],data[8])
stats.kruskal(np.concatenate([data[6],data[7],data[8]]),
              np.concatenate([data[3],data[4],data[5]]),np.concatenate([data[0],data[1],data[2]]))
stats.kruskal(np.concatenate([data[0],data[3],data[6]]),
              np.concatenate([data[1],data[4],data[7]]),np.concatenate([data[2],data[5],data[8]]))<jupyter_output><empty_output><jupyter_text>Mann-Whitney U Tests<jupyter_code>for i in range(9):
    for j in range(9):
        if i<j and ((j-i==2 and i not in [1,2,4,5]) or (j-i==1 and i not in [2,5])):
            print(i,j)
            print(stats.mannwhitneyu(data[i],data[j]))<jupyter_output>0 1
MannwhitneyuResult(statistic=8157.0, pvalue=2.2413514188783913e-12)
0 2
MannwhitneyuResult(statistic=3161.0, pvalue=4.448213041617074e-15)
1 2
MannwhitneyuResult(statistic=12638.0, pvalue=0.02553981634675701)
3 4
MannwhitneyuResult(statistic=6256.0, pvalue=2.303516340132966e-06)
3 5
MannwhitneyuResult(statistic=2401.5, pvalue=1.5338799854926273e-09)
4 5
MannwhitneyuResult(statistic=7603.5, pvalue=0.005720740227056704)
6 7
MannwhitneyuResult(statistic=40629.0, pvalue=1.3115907989947074e-08)
6 8
MannwhitneyuResult(statistic=13784.5, pvalue=1.988954931232016e-23)
7 8
MannwhitneyuResult(statistic=39284.5, pvalue=4.574464592358076e-11)
{'fraction_non_alphanumeric': 0.18177649679025285, 'fraction_numerical': 0.0442159046246561, 'mean_word_length': 5.569277108433735, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 101, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '57968', 'license': 'MIT', 'n_tokens_mistral': 6581, 'n_tokens_neox': 6182, 'n_words': 922, 'path': 'Human_Study_Analysis.ipynb', 'repo_name': 'zzbn12345/WHOSe_Heritage', 'size': '15266'}