<jupyter_start><jupyter_text>Find Reasons for Errors<jupyter_code>import sys
import os
import re
import pickle
import warnings
import numpy as np
import pandas as pd

working_dir = '../src/models'
sys.path.append(os.path.abspath(working_dir))
from use_evaluation_model_rotate import load_data
warnings.filterwarnings('ignore')
# paths etc.
model_output_path = '../models/RotatE_FB15k-237_3/output/'
model_path = '../models/RotatE_FB15k-237_3/'
data_path = '../data/processed/FB15k-237'
train_triples, valid_triples, test_triples = load_data(data_path)

# dictionaries to translate IDs into normal words (and vice versa)
path = os.path.join(data_path, 'id_to_relation.pickle')
id2relation = pickle.load(open(path, 'rb'))

# load prediction_table_head.pkl
path = os.path.join(model_output_path, 'prediction_table_head.pkl')
prediction_table_head = pd.read_pickle(path)
prediction_table_head['rank of true head'] = pd.to_numeric(prediction_table_head['rank of true head'], downcast='integer')
s = []
p = []
o = []
topics = []

for _, triple_str in prediction_table_head['triple'].iteritems():
    triple = re.split(',', triple_str[1:-1])
    s.append(int(triple[0].strip()))
    p.append(int(triple[1].strip()))
    o.append(int(triple[2].strip()))
    topics.append(id2relation[int(triple[1].strip())].split('/')[1])

prediction_table_h = pd.DataFrame(columns=['s', 'p', 'o', 'rank'])
prediction_table_h['s'] = s
prediction_table_h['p'] = p
prediction_table_h['o'] = o
prediction_table_h['rank'] = prediction_table_head['rank of true head']
prediction_table_h['topic'] = topics

# load prediction_table_tail.pkl
path = os.path.join(model_output_path, 'prediction_table_tail.pkl')
prediction_table_tail = pd.read_pickle(path)
prediction_table_tail['rank of true tail'] = pd.to_numeric(prediction_table_tail['rank of true tail'], downcast='integer')

s = []
p = []
o = []
topics = []
for _, triple_str in prediction_table_tail['triple'].iteritems():
    triple = re.split(',', triple_str[1:-1])
    s.append(int(triple[0].strip()))
    p.append(int(triple[1].strip()))
    o.append(int(triple[2].strip()))
    topics.append(id2relation[int(triple[1].strip())].split('/')[1])

prediction_table_t = pd.DataFrame(columns=['s', 'p', 'o', 'rank'])
prediction_table_t['s'] = s
prediction_table_t['p'] = p
prediction_table_t['o'] = o
prediction_table_t['rank'] = prediction_table_tail['rank of true tail']
prediction_table_t['topic'] = topics

# average position per head entity
path = os.path.join(model_output_path, 'avg_position_table_head.pkl')
avg_position_table_head = pd.read_pickle(path)

# average position per tail entity
path = os.path.join(model_output_path, 'avg_position_table_tail.pkl')
avg_position_table_tail = pd.read_pickle(path)
# dictionaries to translate IDs into normal words (and vice versa)
path = os.path.join(data_path, 'id_to_relation.pickle')
id2relation = pickle.load(open(path, 'rb'))

path = os.path.join(data_path, 'relation_to_id.pickle')
relation2id = pickle.load(open(path, 'rb'))

path = os.path.join(data_path, 'id_to_entity.pickle')
id2entity = pickle.load(open(path, 'rb'))

path = os.path.join(data_path, 'entity_to_id.pickle')
entity2id = pickle.load(open(path, 'rb'))

path = os.path.join(data_path[:-10], 'mid2name.pkl')
mid2name = pickle.load(open(path, 'rb'))  # only strings!

name2mid = {name: mid for mid, name in mid2name.items()}
all_entities_train = set()
all_relations_train = set()

all_entities_valid = set()
all_relations_valid = set()

all_entities_test = set()
all_relations_test = set()

for (s, p, o) in train_triples:
    all_entities_train.add(s)
    all_relations_train.add(p)
    all_entities_train.add(o)

for (s, p, o) in valid_triples:
    all_entities_valid.add(s)
    all_relations_valid.add(p)
    all_entities_valid.add(o)

for (s, p, o) in test_triples:
    all_entities_test.add(s)
    all_relations_test.add(p)
    all_entities_test.add(o)
test_entities_not_in_train = np.setdiff1d(list(all_entities_test), list(all_entities_train))
print('{} entities in the test data never occured in the train data'.format(len(test_entities_not_in_train)))

valid_entities_not_in_train = np.setdiff1d(list(all_entities_valid), list(all_entities_train))
print('{} entities in the valid data never occured in the train data\n'.format(len(valid_entities_not_in_train)))

mask1 = avg_position_table_head['entity_id'].isin(test_entities_not_in_train)
mask2 = avg_position_table_tail['entity_id'].isin(test_entities_not_in_train)
mask3 = prediction_table_h['s'].isin(test_entities_not_in_train)
mask4 = prediction_table_t['o'].isin(test_entities_not_in_train)

rank_per_head = prediction_table_h[mask3][['s', 'rank']]
rank_per_head.columns = ['entity_id', 'rank']

rank_per_tail = prediction_table_t[mask4][['o', 'rank']]
rank_per_tail.columns = ['entity_id', 'rank']

merged_df = pd.merge(avg_position_table_head[mask1], avg_position_table_tail[mask2].round(), on='entity_id', how='outer')
merged_df = pd.merge(merged_df, rank_per_head[mask3], on='entity_id', how='outer')
merged_df = pd.merge(merged_df, rank_per_tail[mask4], on='entity_id', how='outer')

entity_names = []
for entity_id in merged_df['entity_id']:
    entity_names.append(mid2name[id2entity[entity_id]])
merged_df['entity_id'] = entity_names

merged_df.columns = ['entity_name', 'avg pos head', 'avg pos tail', 'avg rank head', 'avg rank tail']
merged_df['avg pos head'] = pd.to_numeric(merged_df['avg pos head'].round(), downcast='integer')
merged_df['avg pos tail'] = pd.to_numeric(merged_df['avg pos tail'], downcast='integer')
merged_df['avg rank head'] = pd.to_numeric(merged_df['avg rank head'], downcast='integer')
merged_df['avg rank tail'] = pd.to_numeric(merged_df['avg rank tail'], downcast='integer')
merged_df = merged_df.groupby(['entity_name', 'avg pos head', 'avg pos tail']).mean().round()
merged_df.fillna('UNK', inplace=True)  # UNK = unknown



print('\naverage position of this entities and rank when this entity should be predicted:\n')
display(merged_df)

avg_avg_pos_head = avg_position_table_head[mask1]['average position'].mean()
print('\naverage average position of this entities in the ranking (head prediction): ' + str(round(avg_avg_pos_head, 2)))

avg_avg_pos_tail = avg_position_table_tail[mask2]['average position'].mean()
print('average average position of this entities in the ranking (tail prediction): ' + str(round(avg_avg_pos_tail,2)))

avg_rank_head = rank_per_head[mask3]['rank'].mean()
print('average rank of triples containing this entities (head prediction): ' + str(avg_rank_head))

avg_rank_tail = rank_per_tail[mask4]['rank'].mean()
print('average rank of triples containing this entities (tail prediction): ' + str(avg_rank_tail))
test_relations_not_in_train = np.setdiff1d(list(all_relations_test), list(all_relations_train))
print('{} relations in the test data never occured in the train data'.format(len(test_relations_not_in_train)))

valid_relations_not_in_train = np.setdiff1d(list(all_relations_valid), list(all_relations_train))
print('{} relations in the valid data never occured in the train data\n'.format(len(valid_relations_not_in_train)))
rank_per_head = prediction_table_h[['s', 'rank']].groupby('s')['rank'].mean().sort_values(ascending=False)

# check whether they occured in train<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14624777061325284, 'fraction_numerical': 0.009054740019207023, 'mean_word_length': 7.246606334841629, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '172184', 'license': 'MIT', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2601, 'n_words': 499, 'path': 'notebooks/find_error_reasons_rotate.ipynb', 'repo_name': 'wang-yuhao/Practical-Big-Data-Science-ADL-AI', 'size': '7289'}