<jupyter_start><jupyter_text>Global Threshold Analysis (RE)<jupyter_code>import numpy as np
import sys, os
sys.path.append(os.path.relpath("../../SciREX_yurun/scirex/predictors"))
sys.path.append(os.path.relpath("../../SciREX_yurun/"))
sys.path.append(os.path.relpath("../../SciREX_yurun/scirex/evaluation_scripts/"))
sys.path.append(os.path.relpath("scirex/evaluation_scripts/"))
from predict_n_ary_relations import *
from scirex_relation_evaluate import *
import argparse
import pandas as pd
import matplotlib.pyplot as plt
thresholds = np.arange(0,1, 0.1)
test_outputs_folder = '../../SciREX_yurun/test_outputs/'
archive_folder = '../../SciREX/outputs_baseline_try/pwc_outputs/experiment_scirex_full/main'
span_file = test_outputs_folder + 'ner_predictions.jsonl'
cluster_file = test_outputs_folder + 'salient_clusters_predictions_using_gold.jsonl'
output_file_base = test_outputs_folder + 'relations_predictions_gold_salient_clusters_{0:.1f}.jsonl'
cuda_device = 0
'''
Generate the predictions,
YORO 
YOU ONLY RUN ONCE :)
'''
# for relation_threshold in thresholds:
#     output_file = output_file_base.format(relation_threshold)
# #     print(output_file)
#     predict(archive_folder, span_file, cluster_file, output_file, cuda_device, relation_threshold)
#     print(output_file)
class Args():
    gold_file='../../SciREX_yurun/scirex_dataset/release_data/test.jsonl'
    ner_file='../../SciREX_yurun/test_outputs/ner_predictions.jsonl'
    clusters_file='../../SciREX_yurun/test_outputs/salient_clusters_predictions_using_gold.jsonl'
    relations_file=''

args=Args()
metrics_list = []
outputs_list = []
for relation_threshold in thresholds:
    output_file = output_file_base.format(relation_threshold)
    outputs_list.append(output_file)
    args.relations_file = output_file
    print("\n\n\n\n=====================Thr: %.1f ====================="%relation_threshold)
    all_metrics = main(args)
    metrics_list.append(all_metrics)

# 2-ary-relation
def n_ary_acc(n, item='f1'):
    n_ary_f1s = []
    for idx, relation_threshold in enumerate(thresholds):
        n_ary_f1s.append(metrics_list[idx][n].describe().loc['mean'][[item]].values[0])
    return n_ary_f1s
two_ary_F1s = n_ary_acc(2, item='f1')
four_ary_F1s = n_ary_acc(4, item='f1')

two_ary_ps = n_ary_acc(2, item='p')
four_ary_ps = n_ary_acc(4, item='p')

two_ary_rs = n_ary_acc(2, item='r')
four_ary_rs = n_ary_acc(4, item='r')<jupyter_output><empty_output><jupyter_text>Relation prediction F1 score v.s. threshold<jupyter_code>two_ary_F1s = n_ary_acc(2)
plt.plot(thresholds, two_ary_F1s, marker='o')
plt.xlim(-0.1,1.0)
plt.ylim(0,1)
plt.title('2-ary relation prediction F1 v.s. Global Threshold')
plt.xlabel('Global Threshold')
plt.ylabel('F-1')
np.mean(four_ary_ps)
np.mean(two_ary_ps)
np.mean(four_ary_rs)
np.mean(two_ary_rs)
four_ary_F1s = n_ary_F1s(4)
plt.plot(thresholds, four_ary_F1s, marker='+', color='y', label='4-ary F1')
plt.plot(thresholds, two_ary_F1s, marker='x', color='y', label='2-ary F1')

# plt.plot(thresholds, four_ary_ps, marker='+', color='b', label='4-ary precision')
# plt.plot(thresholds, two_ary_ps, marker='x', color='b', label='2-ary precision')

# plt.plot(thresholds, four_ary_rs, marker='+', color='r', label='4-ary recall')
# plt.plot(thresholds, two_ary_rs, marker='x', color='r', label='2-ary recall')
plt.xlim(-0.1,1.0)
plt.ylim(0,1)
plt.title('Relation Extraction F-1 v.s. Global Threshold')
plt.xlabel('Global Threshold')
plt.ylabel('F1')
plt.legend()<jupyter_output><empty_output><jupyter_text>Number of zero predictions v.s. threshold<jupyter_code>num_zero = []
outputs_list = [] 
for idx, relation_threshold in enumerate(thresholds):
    output_file = output_file_base.format(relation_threshold)
    outputs = pd.read_json(path_or_buf=output_file, lines=True)
    outputs_list.append(outputs)
    num_zero.append(len(outputs[outputs["predicted_relations"].apply(len) == 0]))
# two_ary_F1s = n_ary_F1s(2)

plt.plot(thresholds, num_zero, marker='o')
plt.xlim(-0.1,1.0)
plt.ylim(-1,30)
plt.title('# zero predictions v.s. Global Threshold')
plt.xlabel('Global Threshold')
plt.ylabel('# zero predictions')
from matplotlib.ticker import MaxNLocator
label_fontsize = 18
fig, axs = plt.subplots(1, 2, figsize=(14, 6))
axs[0].plot(thresholds, num_zero, marker='o')
axs[0].set_ylabel('# zero predicted relation docs', fontsize=label_fontsize)
axs[0].set_xlabel('Global threshold', fontsize=label_fontsize)
axs[0].tick_params(axis='x', labelsize=label_fontsize)
axs[0].tick_params(axis='y', labelsize=label_fontsize)
axs[1].plot(thresholds, four_ary_F1s, marker='x', )
axs[0].yaxis.set_major_locator(MaxNLocator(integer=True))
axs[0].set_ylim(0,)
# axs[1].plot(thresholds, two_ary_F1s, marker='+', label='2-ary')
axs[1].legend()
axs[1].set_ylim(0,1)
axs[1].set_ylabel('F-1', fontsize=label_fontsize)
axs[1].set_xlabel('Global threshold', fontsize=label_fontsize)
axs[1].tick_params(axis='x', labelsize=label_fontsize)
axs[1].tick_params(axis='y', labelsize=label_fontsize)
axs[0].style.use("ggplot")
axs[1].style.use("ggplot")<jupyter_output>2021-05-13 17:39:24,923:WARNING:No handles with labels found to put in legend.<jupyter_text>All of the zero predictions are from the same set of docs for all thresholds<jupyter_code># All the 23 emtpy lists are from same docs
zero_relation_docs = set(outputs_list[0][outputs_list[0]["predicted_relations"].apply(len) == 0].doc_id)
nonzero_relation_docs = set(outputs_list[0][outputs_list[0]["predicted_relations"].apply(len) != 0].doc_id)
outputs_list[0]<jupyter_output><empty_output><jupyter_text>NER predictions: zero relation prediction docs' NER predictions<jupyter_code>ners = pd.read_json(path_or_buf='../../SciREX_yurun/test_outputs/ner_predictions.jsonl', lines=True)
ners
def get_num_entities(ners, relation_docs):
    num_entities = dict()
    num_entities['Method'] = []
    num_entities['Task'] = []
    num_entities['Material'] = []
    num_entities['Metric'] = []

    for doc in relation_docs:
        ners_doc = ners.loc[ners.doc_id == doc].ner
        ner_df = pd.DataFrame(list(ners_doc)[0], columns=['start_idx', 'end_idx', 'type'])
        ner_df_group = ner_df.groupby(['type']).count()['start_idx']
        num_entities['Method'].append(ner_df_group['Method'])
        num_entities['Task'].append(ner_df_group['Task'])
        num_entities['Material'].append(ner_df_group['Material'])
        num_entities['Metric'].append(ner_df_group['Metric'])
    return num_entities
zero_relation_num_entities = get_num_entities(ners, zero_relation_docs)
nonzero_relation_num_entities = get_num_entities(ners, nonzero_relation_docs)<jupyter_output><empty_output><jupyter_text>There isn't any doc with 0 ner prediction on any type<jupyter_code>pd.DataFrame.from_dict(zero_relation_num_entities)
pd.DataFrame.from_dict(nonzero_relation_num_entities)
def distribution_4_types_entities(num_entities):
    plt.hist(num_entities['Method'], label='Method', bins=15, alpha=0.5)
    plt.hist(num_entities['Task'], label='Task', bins=15, alpha=0.5)
    plt.hist(num_entities['Metric'], label='Metric', bins=15, alpha=0.5)
    plt.hist(num_entities['Material'], label='Material', bins=15, alpha=0.5)
    plt.legend()
    plt.title('Distribution of # ner predictions')<jupyter_output><empty_output><jupyter_text>zero relation docs and posititve  relations docs' ner distribution looks similar<jupyter_code># zero relation docs' ners
distribution_4_types_entities(zero_relation_num_entities)
# > 0 relation docs' ners
distribution_4_types_entities(nonzero_relation_num_entities)<jupyter_output><empty_output><jupyter_text>Problem could be in the relation inference scores! => either 0 or ~1  Look at the errors that the model made when it's over confident<jupyter_code>test_gt = pd.read_json(path_or_buf='../../SciREX_yurun/scirex_dataset/release_data/test.jsonl', lines=True)
train_gt = pd.read_json(path_or_buf='../../SciREX_yurun/scirex_dataset/release_data/train.jsonl', lines=True)
train_gt.ner
test_gt.ner
test_gt[['doc_id', 'n_ary_relations']] # Ground truth
outputs # predicted results with threshold 0.9
re_pred_gt = outputs.merge(test_gt[['doc_id', 'n_ary_relations']], on='doc_id', how='inner')<jupyter_output><empty_output><jupyter_text>Failure cases14318685b5959b51d0f1e3db34643eb2855dc6d9 Image_net v.s. Image_net_detection0c47cad9729c38d9db1f75491b1ee4bd883a5d4e prediction is Random conbination44078d0daed8b13114cffb15b368acc467f96351 score is percentage38cc89399dd6f5aaab1654f27ab3c9eeade12a36 score is float<jupyter_code>doc_id = '160563abbd75265b19afc8b4169bab9e1eb33d97'
# {"Material": "dataset", "Metric": "metric", "Task": "task", "Method": "model_name"}
list(re_pred_gt.loc[re_pred_gt.doc_id == doc_id]['predicted_relations'])
list(re_pred_gt.loc[re_pred_gt.doc_id == doc_id]['n_ary_relations'])<jupyter_output><empty_output><jupyter_text>NER prediction is not a problem<jupyter_code>test_gt.ner.apply(len).hist(alpha=0.5, density=1)
train_gt.ner.apply(len).hist(alpha=0.5, density=1)
ners = test_gt
relation_docs = ['0a6c36de8726b6feaab586046ddc1d1a008f44f9']
ner_df
def get_num_entities(ners, relation_docs):
    num_entities = dict()
    num_entities['Method'] = []
    num_entities['Task'] = []
    num_entities['Material'] = []
    num_entities['Metric'] = []
    types = ['Method', 'Task', 'Material', 'Metric']
    for doc in relation_docs:
        ners_doc = ners.loc[ners.doc_id == doc].ner
        ner_df = pd.DataFrame(list(ners_doc)[0], columns=['start_idx', 'end_idx', 'type'])
        ner_df_group = ner_df.groupby(['type']).count()['start_idx']

        for type_name in types:  
            if type_name in list(ner_df_group.keys()):
                num_entities[type_name].append(ner_df_group[type_name])
            else:
                num_entities[type_name].append(0)
    return num_entities
num_entities_test_df = pd.DataFrame.from_dict(get_num_entities(test_gt, test_gt.doc_id.unique()))
num_entities_train_df = pd.DataFrame.from_dict(get_num_entities(train_gt, train_gt.doc_id.unique()))
num_entities_train_df<jupyter_output><empty_output><jupyter_text>Check training and testing dataset distributions<jupyter_code>num_entities_test_df.Method.hist(density=1, alpha=0.5)
num_entities_train_df.Method.hist(density=1, alpha=0.5)
num_entities_test_df.Task.hist(density=1, alpha=0.5)
num_entities_train_df.Task.hist(density=1, alpha=0.5)
num_entities_test_df.Material.hist(density=1, alpha=0.5)
num_entities_train_df.Material.hist(density=1, alpha=0.5)
num_entities_test_df.Metric.hist(density=1, alpha=0.5)
num_entities_train_df.Metric.hist(density=1, alpha=0.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1471227440234338, 'fraction_numerical': 0.030993102144949446, 'mean_word_length': 6.898507462686567, 'pattern_counts': {'":': 4, '<': 44, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '122885', 'license': 'Apache-2.0', 'n_tokens_mistral': 4139, 'n_tokens_neox': 3999, 'n_words': 578, 'path': 'notebooks/GlobalThresholdAnalysis.ipynb', 'repo_name': 'prabh27/SciREX', 'size': '10583'}