<jupyter_start><jupyter_text>Error analysis for the type of examples that NER blinding get wrong compared to the baseline<jupyter_code>%load_ext autoreload
%autoreload
import os
from sys import path
import re
import pandas as pd
path.append('../../..')
import numpy as np
from scipy.stats import ttest_rel
from relation_extraction.data.converters.converter_ddi import relation_dict
output_path = '/scratch/geeticka/relation-extraction-result/ddi-analyze/'
def res(path): return os.path.join(output_path, path)
original_sentences_path = os.path.join('/scratch/geeticka/relation-extraction-result/ddi-analyze/test_original.txt')
ner_blinding_sentences_path = os.path.join('/crimea/geeticka/data/relation_extraction/ddi/pre-processed/ner_blinding/test_ner_blinding.txt')
relation_dict
def read_answers_line(line):
    linenum, relation = line.strip().split()
    return linenum, relation
def asstring(list_of_strings):
    return " ".join(list_of_strings)
def read_sentence_and_entities(line):
    line = line.strip().split()
    sentence = line[5:]
    relation = relation_dict[int(line[0])]
    entity1_idx = (int(line[1]), int(line[2]))
    entity2_idx = (int(line[3]), int(line[4]))
    entity1 = sentence[entity1_idx[0] : entity1_idx[1] + 1]
    entity2 = sentence[entity2_idx[0] : entity2_idx[1] + 1]
    
    return relation, asstring(entity1), asstring(entity2), asstring(sentence)
needed_linenum_and_relation = {}
with open(res("answers_for_dev-ner-blinding.txt")) as textfile1, open(res('answers_for_dev-baseline.txt')) as textfile2, \
open(res('answers_for_dev-baseline_gold.txt')) as textfile3: 
    for x, y, z in zip(textfile1, textfile2, textfile3):
        linenum, ner_blinding_relation = read_answers_line(x)
        _, baseline_relation = read_answers_line(y)
        _, gold_relation = read_answers_line(z)
        if baseline_relation == gold_relation and ner_blinding_relation != gold_relation: # punct digit making mistakes
            needed_linenum_and_relation[int(linenum) - 1] = (baseline_relation, ner_blinding_relation)
len(list(needed_linenum_and_relation.keys()))
len(list(needed_linenum_and_relation.keys()))/len(open(res('answers_for_dev-baseline_gold.txt')).readlines()) * 100<jupyter_output><empty_output><jupyter_text>Remember that baseline is correct here so the first relation listed is the correct one, unlike in the i2b2 notebook where the order is reversed because baseline is the incorrect one there<jupyter_code>entities_blinded_per_sentence = []
interaction = 0; mechanism = 0; advice = 0; effect = 0; none = 0
print('We print the baseline first and then the ner blinded version. Gold relation corresponds to baseline\n\n')
curr_linenum = 0
with open(original_sentences_path) as original_sentences, open(ner_blinding_sentences_path) as ner_blinding_sentences:
    for x, y in zip(original_sentences, ner_blinding_sentences):
        needed_linenums = list(needed_linenum_and_relation.keys())
        if curr_linenum in needed_linenums:
            _, e1_b, e2_b, s_b = read_sentence_and_entities(x.strip())
            _, e1_c, e2_c, s_c = read_sentence_and_entities(y.strip())
            r_b, r_c = needed_linenum_and_relation[curr_linenum]
            entities_blinded_per_sentence.append(s_c.split().count('ENTITY'))
            if r_b == 'int': interaction += 1
            elif r_b == 'mechanism': mechanism += 1
            elif r_b == 'advise': advice += 1
            elif r_b == 'effect': effect += 1
            elif r_b == 'none': none += 1
            print('Predicted Relation: \t {0}, {1} \nEntities: \t {2}, {3} \t {4}, {5} \nSentences: \n\t{6} \n\t {7}'.format(
                    r_b, r_c, e1_b, e1_c, e2_b, e2_c, s_b, s_c))
            print('\n')
        curr_linenum += 1
print(interaction, mechanism, advice, effect, none)
interaction + mechanism + advice + effect + none
(np.mean(entities_blinded_per_sentence), np.max(entities_blinded_per_sentence), np.min(entities_blinded_per_sentence),
np.std(entities_blinded_per_sentence), np.median(entities_blinded_per_sentence))
num_sentences_where_entities_blinded = 0 # count the number of sentences where the numbers were blinded
for numbers in entities_blinded_per_sentence:
    if numbers > 0:
        num_sentences_where_entities_blinded += 1
num_sentences_where_entities_blinded<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10941475826972011, 'fraction_numerical': 0.015035854730511219, 'mean_word_length': 4.689473684210526, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '463689', 'license': 'MIT', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1444, 'n_words': 316, 'path': 'notebooks/error-analysis/Preprocessing-related/ddi-Test-Error-Analysis-NER-blinding.ipynb', 'repo_name': 'geetickachauhan/relation-extraction', 'size': '4323'}