<jupyter_start><jupyter_text>Vector comparison<jupyter_code>import json

bert_rep = json.load(open('final_project/relations_representations_middle.json'))
bert_rep.keys()
bert_rep['adjoins'].keys()
from sklearn.metrics.pairwise import cosine_distances

#dist = [cosine_distances(bert_rep['adjoins'][i], bert_rep['adjoins'][j]) for i in bert_rep['adjoins'].keys() for j in bert_rep['adjoins'].keys() 
# if i != j]

dist2 = [cosine_distances(bert_rep['adjoins'][i], bert_rep['author'][j]) for i in bert_rep['adjoins'].keys() for j in bert_rep['author'].keys()]
from sklearn.metrics.pairwise import cosine_distances, euclidean_distances, manhattan_distances
import numpy as np

def _distance_within(bert_representations, relation, size_sample, distances):
    np.random.seed(42)
    relations = list(bert_representations[relation].keys())
    no_examples = len(relations)
    no_samples_relation = int(no_examples * size_sample)
    indices_sample = np.random.randint(0, no_examples, no_samples_relation)
    keys_in_sample = [relations[i] for i in indices_sample]
    relations_to_consider = {relation_example: bert_representations[relation][relation_example]
                             for relation_example in keys_in_sample}
    distance = [distances(relations_to_consider[i], relations_to_consider[j]) for i in keys_in_sample
                for j in keys_in_sample if i != j]
    return distance

def calculate_distances_within(bert_representations, size_sample, distances):
    return {relation: _distance_within(bert_representations, relation, size_sample, distances)
           for relation in bert_representations.keys()}

def _distance_between(bert_representations, relation1, relation2, size_sample, distances):
    relations_examples1 = list(bert_representations[relation1].keys())
    relations_examples2 = list(bert_representations[relation2].keys())
    no_examples1 = len(relations_examples1)
    no_examples2 = len(relations_examples2)
    no_samples_relation1 = int(no_examples1 * size_sample)
    no_samples_relation2 = int(no_examples2 * size_sample)
    indices_sample1 = np.random.randint(0, no_examples1, no_samples_relation1)
    indices_sample2 = np.random.randint(0, no_examples2, no_samples_relation2)
    keys_in_sample1 = [relations_examples1[i] for i in indices_sample1]
    keys_in_sample2 = [relations_examples2[i] for i in indices_sample2]
    relations_to_consider1 = {relation_example: bert_representations[relation1][relation_example]
                             for relation_example in keys_in_sample1}
    relations_to_consider2 = {relation_example: bert_representations[relation2][relation_example]
                             for relation_example in keys_in_sample2}
    distance = [distances(relations_to_consider1[i], relations_to_consider2[j]) 
                for i in keys_in_sample1 for j in keys_in_sample2]
    return distance#np.mean(distance)

def calculate_distances_between(bert_representations, size_sample, distances):
    relations = list(bert_representations.keys())
    combination_relations = [(i, j) for i in relations for j in relations if i != j]
    return {str((relation1, relation2)): _distance_between(bert_representations, relation1, relation2, size_sample, distances)
           for relation1, relation2 in combination_relations}

def mean_distance(distances_dict):
    return {key : np.mean(list(map(lambda x: x[0][0], distances_dict[key]))) for key in distances_dict}
distances_within = calculate_distances_within(bert_rep, 0.1, manhattan_distances)
distances_between = calculate_distances_between(bert_rep, 0.1, manhattan_distances)
keys_na = [(key, entity) for key in bert_rep for entity in bert_rep[key] if 
           type(bert_rep[key][entity]) != type([])]

bert_rep['is_a'].pop("('Staphylococcus_aureus', 'Bacteria')")
relations = list(bert_rep.keys())
combination_relations = [(i, j) for i in relations for j in relations if i != j]
distances_absolute = {str((relation1, relation2)): distances_between[str((relation1, relation2))] - distances_within[relation2]
                         for relation1, relation2 in combination_relations}
distances_absolute.keys()
def to_list(lst_of_arrays:list):
    return list(map(lambda x:x[0][0], lst_of_arrays))
distances = []
rel = []
for relation in distances_within.keys():
    dist = to_list(distances_within[relation])
    distances.extend(dist)
    rel.extend([relation] * len(dist))
import pandas as pd
from plotnine import *
%matplotlib inline

#type_ =  np.array(['adjoins']*len(n))
df = pd.DataFrame({'Distance': distances, 'Relations': rel})
(ggplot(df)
     + geom_boxplot(aes('Relations', 'Distance'))
) + theme(axis_text_x  = element_text(angle = 90, hjust = 1))
import ast

rel = list(map(lambda x : ast.literal_eval(x)[0], distances_between.keys()))
mean_distances_between = list(map(lambda x:distances_between[x], distances_between.keys()))
distances = []
rel = []
for relation in distances_between.keys():
    dist = to_list(distances_between[relation])
    distances.extend(dist)
    rel.extend([relation] * len(dist))
len(distances)
df_between = pd.DataFrame({'Distance': distances, 'Relations': rel})
(ggplot(df_between)
     + geom_boxplot(aes('Relations', 'Distance'))
) + theme(axis_text_x  = element_text(angle = 90, hjust = 1))
np.mean(mean_distances_between)
mean_distances_within = list(map(lambda x:distances_within[x][0][0][0], distances_within.keys()))
np.mean(mean_distances_within)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11227106227106227, 'fraction_numerical': 0.01336996336996337, 'mean_word_length': 5.659756097560976, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '558170', 'license': 'Apache-2.0', 'n_tokens_mistral': 1810, 'n_tokens_neox': 1718, 'n_words': 353, 'path': 'final_project_experiments-ROBERTa.ipynb', 'repo_name': 'julioadl/cs224u', 'size': '5460'}