<jupyter_start><jupyter_text>Accuracy Rate of Each relation<jupyter_code>def correctness(df):
  correct_dict = {}
  wrong_dict = {}
  
  for index, row in df.iterrows():
    
    true_label = row['true_label']
    predict_label = row['predict_label']
    relation = row['relation']
    
    if true_label == predict_label:
      if relation in correct_dict:
        correct_dict[relation] += 1
      else:
        correct_dict[relation] = 1
    else:
      if relation in wrong_dict:
        wrong_dict[relation] += 1
      else:
        wrong_dict[relation] = 1
        
  correctness_dict = {}
  for k in relation_descriptions:
    if k not in correct_dict:
      correct_dict[k] = 0
    if k not in wrong_dict:
      wrong_dict[k] = 0
      
    if (correct_dict[k] + wrong_dict[k]) == 0:
      cor = 0
    else:
      cor = correct_dict[k] / (correct_dict[k] + wrong_dict[k])
    
    correctness_dict[k] = cor
    
  return correctness_dict
# connect
correct_connect_5 = correctness(connect_5)
correct_connect_5_list = sorted([correct_connect_5[r] for r in correct_connect_5])

correct_connect_10 = correctness(connect_10)
correct_connect_10_list = sorted([correct_connect_10[r] for r in correct_connect_10])

correct_connect_15 = correctness(connect_15)
correct_connect_15_list = sorted([correct_connect_15[r] for r in correct_connect_15])

# inference
correct_inference_5 = correctness(inference_5)
correct_inference_5_list = sorted([correct_inference_5[r] for r in correct_inference_5])

correct_inference_10 = correctness(inference_10)
correct_inference_10_list = sorted([correct_inference_10[r] for r in correct_inference_10])

correct_inference_15 = correctness(inference_15)
correct_inference_15_list = sorted([correct_inference_15[r] for r in correct_inference_15])

# mlp
correct_mlp_5 = correctness(mlp_5)
correct_mlp_5_list = sorted([correct_mlp_5[r] for r in correct_mlp_5])

correct_mlp_10 = correctness(mlp_10)
correct_mlp_10_list = sorted([correct_mlp_10[r] for r in correct_mlp_10])

correct_mlp_15 = correctness(mlp_15)
correct_mlp_15_list = sorted([correct_mlp_15[r] for r in correct_mlp_15])


# pooling
correct_pooling_5 = correctness(pooling_5)
correct_pooling_5_list = sorted([correct_pooling_5[r] for r in correct_pooling_5])

correct_pooling_10 = correctness(pooling_10)
correct_pooling_10_list = sorted([correct_pooling_10[r] for r in correct_pooling_10])

correct_pooling_15 = correctness(pooling_15)
correct_pooling_15_list = sorted([correct_pooling_15[r] for r in correct_pooling_15])
import matplotlib.pyplot as plt
import seaborn as sns

fig, axs = plt.subplots(4, 3, figsize=(20,19))
sns.set()

sns.set_style("darkgrid", {"axes.facecolor": ".9"})
axs[0, 0].set_title('CCIM_5', fontsize=18)
# axs[0, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
axs[0, 0].set_ylabel('Density', fontsize=18)

axs[1, 0].set_title('ICIM_5', fontsize=18)
# axs[1, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
axs[1, 0].set_ylabel('Density', fontsize=18)

axs[2, 0].set_title('PCIM_5', fontsize=18)
# axs[2, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
axs[2, 0].set_ylabel('Density', fontsize=18)

axs[3, 0].set_title('MCIM_5', fontsize=18)
axs[3, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
axs[3, 0].set_ylabel('Density', fontsize=18)

axs[0, 1].set_title('CCIM_10', fontsize=18)
# axs[0, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[0, 1].set_ylabel('Frequency', fontsize=18)

axs[1, 1].set_title('ICIM_10', fontsize=18)
# axs[1, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[1, 1].set_ylabel('Frequency', fontsize=18)

axs[2, 1].set_title('PCIM_10', fontsize=18)
# axs[2, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[2, 1].set_ylabel('Frequency', fontsize=18)

axs[3, 1].set_title('MCIM_10', fontsize=18)
axs[3, 1].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[3, 1].set_ylabel('Frequency', fontsize=18)

axs[0, 2].set_title('CCIM_15', fontsize=18)
# axs[0, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[0, 1].set_ylabel('Frequency', fontsize=18)

axs[1, 2].set_title('ICIM_15', fontsize=18)
# axs[1, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[1, 1].set_ylabel('Frequency', fontsize=18)

axs[2, 2].set_title('PCIM_15', fontsize=18)
# axs[2, 0].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[2, 1].set_ylabel('Frequency', fontsize=18)

axs[3, 2].set_title('MCIM_15', fontsize=18)
axs[3, 2].set_xlabel('Accuracy Rate (%)', fontsize=18)
# axs[3, 1].set_ylabel('Frequency', fontsize=18)

sns.distplot([c * 100 for c in correct_connect_5_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[0, 0])
sns.distplot([c * 100 for c in correct_inference_5_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[1, 0])
sns.distplot([c * 100 for c in correct_pooling_5_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[2, 0])
sns.distplot([c * 100 for c in correct_mlp_5_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[3, 0])


sns.distplot([c * 100 for c in correct_connect_10_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[0, 1])
sns.distplot([c * 100 for c in correct_inference_10_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[1, 1])
sns.distplot([c * 100 for c in correct_pooling_10_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[2, 1])
sns.distplot([c * 100 for c in correct_mlp_10_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[3, 1])


sns.distplot([c * 100 for c in correct_connect_15_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[0, 2])
sns.distplot([c * 100 for c in correct_inference_15_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[1, 2])
sns.distplot([c * 100 for c in correct_pooling_15_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[2, 2])
sns.distplot([c * 100 for c in correct_mlp_15_list], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=axs[3, 2])

matplotlib.rc('ytick', labelsize=15)
matplotlib.rc('xtick', labelsize=15)<jupyter_output><empty_output><jupyter_text>ANOVA<jupyter_code>correct_connect_5_df = pd.DataFrame({'Model': ['CCIM'] * len(correct_connect_5_list), 'Description Number': [5] * len(correct_connect_5_list), 'Accuracy' : [correct_connect_5[r] for r in correct_connect_5], 'Relation': list(correct_connect_5.keys())})
correct_connect_10_df = pd.DataFrame({'Model': ['CCIM'] * len(correct_connect_10_list), 'Description Number': [10] * len(correct_connect_10_list), 'Accuracy' : [correct_connect_10[r] for r in correct_connect_10], 'Relation': list(correct_connect_10.keys())})
correct_connect_15_df = pd.DataFrame({'Model': ['CCIM'] * len(correct_connect_15_list), 'Description Number': [15] * len(correct_connect_15_list), 'Accuracy' : [correct_connect_15[r] for r in correct_connect_15], 'Relation': list(correct_connect_15.keys())})

correct_inference_5_df = pd.DataFrame({'Model': ['ICIM'] * len(correct_inference_5_list), 'Description Number': [5] * len(correct_inference_5_list), 'Accuracy' : [correct_inference_5[r] for r in correct_inference_5], 'Relation': list(correct_inference_5.keys())})
correct_inference_10_df = pd.DataFrame({'Model': ['ICIM'] * len(correct_inference_10_list), 'Description Number': [10] * len(correct_inference_10_list), 'Accuracy' : [correct_inference_10[r] for r in correct_inference_10], 'Relation': list(correct_inference_10.keys())})
correct_inference_15_df = pd.DataFrame({'Model': ['ICIM'] * len(correct_inference_15_list), 'Description Number': [15] * len(correct_inference_15_list), 'Accuracy' : [correct_inference_15[r] for r in correct_inference_15], 'Relation': list(correct_inference_15.keys())})

correct_pooling_5_df = pd.DataFrame({'Model': ['PCIM'] * len(correct_pooling_5_list), 'Description Number': [5] * len(correct_pooling_5_list), 'Accuracy' : [correct_pooling_5[r] for r in correct_pooling_5], 'Relation': list(correct_pooling_5.keys())})
correct_pooling_10_df = pd.DataFrame({'Model': ['PCIM'] * len(correct_pooling_10_list), 'Description Number': [10] * len(correct_pooling_10_list), 'Accuracy' : [correct_pooling_10[r] for r in correct_pooling_10], 'Relation': list(correct_pooling_10.keys())})
correct_pooling_15_df = pd.DataFrame({'Model': ['PCIM'] * len(correct_pooling_15_list), 'Description Number': [15] * len(correct_pooling_15_list), 'Accuracy' : [correct_pooling_15[r] for r in correct_pooling_15], 'Relation': list(correct_pooling_15.keys())})

correct_mlp_5_df = pd.DataFrame({'Model': ['MCIM'] * len(correct_mlp_5_list), 'Description Number': [5] * len(correct_mlp_5_list), 'Accuracy' : [correct_mlp_5[r] for r in correct_mlp_5], 'Relation': list(correct_mlp_5.keys())})
correct_mlp_10_df = pd.DataFrame({'Model': ['MCIM'] * len(correct_mlp_10_list), 'Description Number': [10] * len(correct_mlp_10_list), 'Accuracy' : [correct_mlp_10[r] for r in correct_mlp_10], 'Relation': list(correct_mlp_10.keys())})
correct_mlp_15_df = pd.DataFrame({'Model': ['MCIM'] * len(correct_mlp_15_list), 'Description Number': [15] * len(correct_mlp_15_list), 'Accuracy' : [correct_mlp_15[r] for r in correct_mlp_15], 'Relation': list(correct_mlp_15.keys())})



total_df = pd.concat([correct_connect_5_df, correct_connect_10_df, correct_connect_15_df, correct_inference_5_df, correct_inference_10_df, correct_inference_15_df, 
           correct_pooling_5_df, correct_pooling_10_df, correct_pooling_15_df, correct_mlp_5_df, correct_mlp_10_df, correct_mlp_15_df], ignore_index=True)
import matplotlib.pyplot as plt
import seaborn as sns

sns.set(style="whitegrid", font_scale=1.5)
ax = sns.boxplot(x="Model", y="Accuracy", data=total_df, palette="Blues")
fig, axs = plt.subplots(1, 3, figsize=(18,4))
axs[0].set_title('5 Descriptions', fontsize=18)
axs[1].set_title('10 Descriptions', fontsize=18)
axs[2].set_title('15 Descriptions', fontsize=18)
sns.boxplot(x="Model", y="Accuracy", data=total_df[total_df['Description Number'] == 5], palette="Blues", ax=axs[0])
sns.boxplot(x="Model", y="Accuracy", data=total_df[total_df['Description Number'] == 10], palette="Blues", ax=axs[1])
sns.boxplot(x="Model", y="Accuracy", data=total_df[total_df['Description Number'] == 15], palette="Blues", ax=axs[2])
import scipy.stats as stats
# 总的

stats.f_oneway(total_df[total_df['Model'] == 'CCIM']['Accuracy'], total_df[total_df['Model'] == 'ICIM']['Accuracy'], total_df[total_df['Model'] == 'PCIM']['Accuracy'], total_df[total_df['Model'] == 'MCIM']['Accuracy'])
stats.f_oneway(total_df[total_df['Model'] == 'CCIM'][total_df['Description Number'] == 5]['Accuracy'], total_df[total_df['Model'] == 'ICIM'][total_df['Description Number'] == 5]['Accuracy'], 
               total_df[total_df['Model'] == 'PCIM'][total_df['Description Number'] == 5]['Accuracy'], total_df[total_df['Model'] == 'MCIM'][total_df['Description Number'] == 5]['Accuracy'])
stats.f_oneway(total_df[total_df['Model'] == 'CCIM'][total_df['Description Number'] == 10]['Accuracy'], total_df[total_df['Model'] == 'ICIM'][total_df['Description Number'] == 10]['Accuracy'], 
               total_df[total_df['Model'] == 'PCIM'][total_df['Description Number'] == 10]['Accuracy'], total_df[total_df['Model'] == 'MCIM'][total_df['Description Number'] == 10]['Accuracy'])
stats.f_oneway(total_df[total_df['Model'] == 'CCIM'][total_df['Description Number'] == 15]['Accuracy'], total_df[total_df['Model'] == 'ICIM'][total_df['Description Number'] == 15]['Accuracy'], 
               total_df[total_df['Model'] == 'PCIM'][total_df['Description Number'] == 15]['Accuracy'], total_df[total_df['Model'] == 'MCIM'][total_df['Description Number'] == 15]['Accuracy'])
ax = sns.boxplot(x="Description Number", y="Accuracy", data=total_df, palette="Blues")
fig, axs = plt.subplots(1, 4, figsize=(23,4))
axs[0].set_title('CCIM', fontsize=18)
axs[1].set_title('ICIM', fontsize=18)
axs[2].set_title('PCIM', fontsize=18)
axs[3].set_title('MCIM', fontsize=18)
sns.boxplot(x="Description Number", y="Accuracy", data=total_df[total_df['Model'] == 'CCIM'], palette="Blues", ax=axs[0])
sns.boxplot(x="Description Number", y="Accuracy", data=total_df[total_df['Model'] == 'ICIM'], palette="Blues", ax=axs[1])
sns.boxplot(x="Description Number", y="Accuracy", data=total_df[total_df['Model'] == 'PCIM'], palette="Blues", ax=axs[2])
sns.boxplot(x="Description Number", y="Accuracy", data=total_df[total_df['Model'] == 'MCIM'], palette="Blues", ax=axs[3])
import scipy.stats as stats

stats.f_oneway(total_df[total_df['Description Number'] == 5]['Accuracy'], total_df[total_df['Description Number'] == 10]['Accuracy'], total_df[total_df['Description Number'] == 15]['Accuracy'])

stats.f_oneway(total_df[total_df['Description Number'] == 5][total_df['Model'] == 'CCIM']['Accuracy'], total_df[total_df['Description Number'] == 10][total_df['Model'] == 'CCIM']['Accuracy'], 
               total_df[total_df['Description Number'] == 15][total_df['Model'] == 'CCIM']['Accuracy'])
stats.f_oneway(total_df[total_df['Description Number'] == 5][total_df['Model'] == 'ICIM']['Accuracy'], total_df[total_df['Description Number'] == 10][total_df['Model'] == 'ICIM']['Accuracy'], 
               total_df[total_df['Description Number'] == 15][total_df['Model'] == 'ICIM']['Accuracy'])
stats.f_oneway(total_df[total_df['Description Number'] == 5][total_df['Model'] == 'PCIM']['Accuracy'], total_df[total_df['Description Number'] == 10][total_df['Model'] == 'PCIM']['Accuracy'], 
               total_df[total_df['Description Number'] == 15][total_df['Model'] == 'PCIM']['Accuracy'])
stats.f_oneway(total_df[total_df['Description Number'] == 5][total_df['Model'] == 'MCIM']['Accuracy'], total_df[total_df['Description Number'] == 10][total_df['Model'] == 'MCIM']['Accuracy'], 
               total_df[total_df['Description Number'] == 15][total_df['Model'] == 'MCIM']['Accuracy'])

import matplotlib.pyplot as plt
import seaborn as sns
fig, ax = plt.subplots(1, 1)
# fig, ax = plt.subplots(1, 1, figsize=(20,19))
# sns.set()

sns.set_style("darkgrid", {"axes.facecolor": ".9"})
# ax.set_title('CCIM_5', fontsize=18)
ax.set_xlabel('Accuracy Rate (%)', fontsize=18)
ax.set_ylabel('Density', fontsize=18)


sns.distplot([c * 100 for c in total_df['Accuracy']], hist=True, kde=True, 
             bins=int(30), color = '#2b90d9',
             hist_kws={'edgecolor':'blue'}, kde_kws={'linewidth': 4, "label": "KDE"}, ax=ax)


# matplotlib.rc('ytick', labelsize=15)
# matplotlib.rc('xtick', labelsize=15)
dict_list = [correct_connect_5, correct_connect_10, correct_connect_15, correct_inference_5, correct_inference_10, correct_inference_15, correct_mlp_5, correct_mlp_10, correct_mlp_15, correct_pooling_5, correct_pooling_10, correct_pooling_15]
total_dict = {r : 0 for r in relation_descriptions}
for d in dict_list:
  for d_k in d:
    total_dict[d_k] += d[d_k]
total_dict_mean = {a : total_dict[a]/len(dict_list) for a in total_dict}
relation_order = []
for elem in sorted(total_dict_mean.items(),  key=lambda x: x[1]) :
  relation_order.append(elem[0])
len(relation_order)
a = [total_df[total_df['Relation'] == a]['Accuracy'] for a in relation_descriptions]
stats.f_oneway(a[0], a[1], a[2])
stats.f_oneway(*a)
plt.figure(figsize=(6, 18))

ax = sns.boxplot(x="Accuracy", y="Relation", data=total_df, palette="Set2", order=relation_order[:60])
plt.figure(figsize=(6, 18))

ax = sns.boxplot(x="Accuracy", y="Relation", data=total_df, palette="Set2", order=relation_order[60:])
len(relation_descriptions['found in taxon']), len(relation_descriptions['IUCN conservation status']), \
len(relation_descriptions['publication date']), len(relation_descriptions['license']), len(relation_descriptions['place of burial'])
len(relation_descriptions['native language']), len(relation_descriptions['site of astronomical discovery']),\
len(relation_descriptions['director']), len(relation_descriptions['discoverer or inventor']), len(relation_descriptions['drafted by'])
relation_descriptions['publication date']
relation_descriptions['native language']
relation_length = {r : [0, 0] for r in relation_descriptions}

for i in range(10):
  current_train = '/content/gdrive/My Drive/master_thesis/data/uwre/train.' + str(i)
  current_dev = '/content/gdrive/My Drive/master_thesis/data/uwre/dev.' + str(i)
  current_test = '/content/gdrive/My Drive/master_thesis/data/uwre/test.' + str(i)
  
  for path in [current_train, current_dev, current_test]:
    print(path)
    with open(path) as f:
        for line in f:
            loaded_example = json.loads(line)
            relation = loaded_example["relation"]
            sentence = loaded_example["sentence"]
            the_len = len(sentence.split())
            
            relation_length[relation][0] += 1
            relation_length[relation][1] += the_len
relation_length['native language']
relation_avelen = {r : (relation_length[r][1] / relation_length[r][0]) for r in relation_length}
order_avelen = [relation_avelen[r] for r in relation_order]
plt.plot(order_avelen)
plt.ylabel('Sentence Instance Lenegth')
plt.xlabel('Relation')
plt.show()
relation_order
import numpy as np
import re
import random
import json
import collections
import pickle
import os
import importlib
# train_mnli_path = the_path + "data/multinli_0.9/multinli_0.9_train.jsonl"
# dev_match_path = the_path + "data/multinli_0.9/multinli_0.9_dev_matched.jsonl"
# dev_mismatch_path = the_path + "data/multinli_0.9/multinli_0.9_dev_mismatched.jsonl"

train_uwre_path = "/content/gdrive/My Drive/master_thesis/data/uwre/train."
dev_uwre_path = "/content/gdrive/My Drive/master_thesis/data/uwre/dev."
test_uwre_path = "/content/gdrive/My Drive/master_thesis/data/uwre/test."

def load_uwre_data(path):
    """
    Load UWRE data.
    "uwre" is set to "genre". 
    """
    relation_dict = {}
    positive_relation = {}
    negative_relation = {}
    
    entail_dict = {}
    entail_list = []
    
    with open(path) as f:
        for line in f:
            loaded_example = json.loads(line)
            
            entail = loaded_example["gold_label"]
            relation = loaded_example["relation"]
            
            if relation not in relation_dict:
                relation_dict[relation] = 1
            else:
                relation_dict[relation] += 1
                
            if entail not in entail_dict:
                entail_dict[entail] = 1
            else:
                entail_dict[entail] += 1
                
            if entail == 'entailment':
              entail_list.append(0)
              if relation in positive_relation:
                positive_relation[relation] += 1
              else:
                positive_relation[relation] = 1
            
            elif entail == 'neutral':
              entail_list.append(1)
              if relation in negative_relation:
                negative_relation[relation] += 1
              else:
                negative_relation[relation] = 1
            else:
              print("illegal entail label:", entail)
    return relation_dict, entail_dict, entail_list, positive_relation, negative_relation

train_relation_dicts = []
train_entail_dicts = []
train_entail_list = []

dev_relation_dicts = []
dev_entail_dicts = []
dev_entail_list = []

test_relation_dicts = []
test_entail_dicts = []
test_entail_list = []

train_positive = []
train_negative = []
dev_positive = []
dev_negative = []
test_positive = []
test_negative = []

for i in range(10):
  current_train = train_uwre_path + str(i)
  current_dev = dev_uwre_path + str(i)
  current_test = test_uwre_path + str(i)
  
  train_relation, train_entail, train_ent, train_p, train_n = load_uwre_data(current_train)
  dev_relation, dev_entail, dev_ent, dev_p, dev_n = load_uwre_data(current_dev)
  test_relation, test_entail, test_ent, test_p, test_n = load_uwre_data(current_test)
  
  train_positive.append(train_p)
  train_negative.append(train_n)
  dev_positive.append(dev_p)
  dev_negative.append(dev_n)
  test_positive.append(test_p)
  test_negative.append(test_n)
  
  train_relation_dicts.append(train_relation)
  train_entail_dicts.append(train_entail)
  train_entail_list.extend(train_ent)
  
  dev_relation_dicts.append(dev_relation)
  dev_entail_dicts.append(dev_entail)
  dev_entail_list.extend(dev_ent)
  
  test_relation_dicts.append(test_relation)
  test_entail_dicts.append(test_entail)
  test_entail_list.extend(test_ent)
def cal_off(positive, negative):
  offset = {}

  for i in range(len(positive)):
    tp = positive[i]
    tn = negative[i]
    for counter, relation in enumerate(tp):
      if relation in offset:
        offset[relation].append((tn[relation] - tp[relation])/tn[relation])
      else:
        offset[relation] = [(tn[relation] - tp[relation])/tn[relation]]
     
  offset_ave = {}
  
  for r in offset.keys():
    offset_ave[r] = sum(offset[r])/len(offset[r])
    
  return offset_ave
        
train_off = cal_off(train_positive, train_negative)
dev_off = cal_off(dev_positive, dev_negative)
test_off = cal_off(test_positive, test_negative)

train_off_list = [train_off[r] for r in relation_order]
def ent_ave(pos_list, neg_list):
  ent = {}
  neu = {}
  for d_list in pos_list:
    for d in d_list:
      for k in d:
        if k in ent:
          ent[k] += d[k]
        else:
          ent[k] = d[k]
  for d_list in neg_list:
    for d in d_list:
      for k in d:
        if k in neu:
          neu[k] += d[k]
        else:
          neu[k] = d[k]
    
    off_dict = {k : (neu[k]-ent[k])/(neu[k]+ent[k]) for k in neu}
    
  return off_dict
    
# offset_dict = ent_ave([train_positive, dev_positive, test_positive], [train_negative, dev_negative, test_negative])
offset_dict = ent_ave([train_positive], [train_negative])

order_off = [offset_dict[r] for r in relation_order]

import matplotlib.ticker as ticker
from matplotlib.ticker import FuncFormatter
import matplotlib

# plt.plot([100 * i for i in order_off])
fig, ax = plt.subplots(1, figsize=(10,5))
matplotlib.rc('ytick', labelsize=18)
ax.plot([100 * i for i in order_off])
ax.yaxis.set_major_formatter(ticker.PercentFormatter())
ax.set_title('Difference rate of Positive and Negative', fontsize=18)
ax.set_xlabel('Relation Index', fontsize=18)
ax.set_ylabel('Different Rate', fontsize=18)

plt.show()
balanced = []
unbalanced = []

for r in offset_dict:
  if offset_dict[r] < 0.03:
    balanced.append(r)
  else:
    unbalanced.append(r)
balanced_acc = [total_dict_mean[r] for r in balanced]
unbalanced_acc = [total_dict_mean[r] for r in unbalanced]
len(balanced_acc)
stats.f_oneway(balanced_acc, unbalanced_acc)
value = balanced_acc + unbalanced_acc
ind = ['balanced'] * len(balanced_acc) + ['unbalanced'] * len(unbalanced_acc)
bal_df = pd.DataFrame()
bal_df['Accuracy'] = value
bal_df['Balance Status'] = ind
ax = sns.boxplot(x="Balance Status", y="Accuracy", data=bal_df, palette="Blues")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1615660440293529, 'fraction_numerical': 0.034356237491661105, 'mean_word_length': 5.306863002892453, 'pattern_counts': {'":': 15, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '371094', 'license': 'MIT', 'n_tokens_mistral': 9347, 'n_tokens_neox': 8718, 'n_words': 1578, 'path': 'data_process/result_plot/Predict_Analysis.ipynb', 'repo_name': 'tonghaining/zero_shot_relation_extraction', 'size': '23984'}