<jupyter_start><jupyter_text>Correlation of Embedding Distances<jupyter_code>def calculate_embedding_distances(vec_np, meta_df):
    distances = squareform(pdist(vec_np)) 
    all_words = [
        str(row['name'])
        for _,row in sorted(meta_df.iterrows(), key=lambda x: x[0])
    ]
    relevant_words = set([word for word in all_words if not (word.endswith('_hidden') or word.endswith('_base'))])
    return pd.DataFrame(distances, columns=all_words, index=all_words) \
        .drop(labels=[x for x in all_words if x not in relevant_words], axis=0) \
        .drop(labels=[x for x in all_words if x not in relevant_words], axis=1)
local_mlflow_dir = '../gsim01/mlruns/' + experiment.experiment_id + '/'
distances_df = None
suffixes = []

for run in tqdm(runs, desc='Calculating embedding distances per run'):
    run_id = run.info.run_id
    if run_id not in set(relevant_run_df['info_run_id']):
        print('Ignoring run {}'.format(run_id))
        continue

    run_mlflow_dir = Path(local_mlflow_dir + run_id)
    if not run_mlflow_dir.is_dir():
        print('Run {} is not in local MlFlow dir'.format(run_id))
        continue

    run_vec_path = Path(local_mlflow_dir + run_id + '/artifacts/vecs.tsv')
    run_meta_path = Path(local_mlflow_dir + run_id + '/artifacts/meta.tsv')
    if not run_vec_path.exists() or not run_meta_path.exists():
        print('No vecs.tsv file for run {} in local MlFlow dir'.format(run_id))
        continue

    vec_np = np.genfromtxt(fname=run_vec_path, delimiter="\t") # shape: (num_features, embedding_dim)
    meta_df = pd.read_csv(run_meta_path, sep='\t', header=None, names=['name'])

    suffix = '_' + run.data.tags['model_type'] + '_' + run.data.params['ModelConfigbase_feature_embeddings_trainable'] + '_' + run_id
    distance_df = calculate_embedding_distances(vec_np, meta_df)
    if distances_df is None:
        distances_df = distance_df.copy()
        print('Initialized distances df with length {}'.format(len(distances_df)))
    
    distances_df = pd.merge(distances_df, distance_df, left_index=True, right_index=True, how='inner', suffixes=('',suffix))
    print('Merged prediction output df, new length {} (dropna: {}, columns: {})'.format(
        len(distances_df), len(distances_df.dropna()), len(distances_df.columns)
    ))
    suffixes.append(suffix)

distances_df.head()
suffix_mapping = {x:"_".join(x.split("_")[0:len(x.split("_"))-1]) for x in suffixes}
for suffix in suffix_mapping:
    same_mapped_suffixes = [s for s in suffix_mapping if suffix_mapping[s] == suffix_mapping[suffix]]
    if len(same_mapped_suffixes) > 1:
        for idx in range(len(same_mapped_suffixes)):
            suffix_mapping[same_mapped_suffixes[idx]] = suffix_mapping[same_mapped_suffixes[idx]] + '_' + str(idx)

suffix_mapping 
distances_df_renamed = distances_df.rename(columns={
    pre+suf:pre+suffix_mapping[suf] for suf in suffix_mapping for pre in set(distances_df.index)
})
print(len(distances_df.columns))
print(len(distances_df_renamed.columns))
distances_df_renamed.head()
all_suffixes = list(set(suffix_mapping.values()))
all_labels = set(distances_df_renamed.index)
correlation_summary_dfs = []
for label in tqdm(all_labels, desc="Adding correlation summaries per label"):
    relevant_columns = [x for x in distances_df_renamed.columns if x.startswith(label)]
    for method in ["spearman", "pearson", "kendall"]:
        correlation_df = distances_df_renamed[relevant_columns].corr(method=method)
        for s1_idx in range(len(all_suffixes)):
            s1 = all_suffixes[s1_idx]
            for s2_idx in range(s1_idx+1, len(all_suffixes)):
                s2 = all_suffixes[s2_idx]
                correlation_summary_dfs.append(
                    pd.DataFrame.from_dict({
                        's1': [s1, s2], 
                        's2': [s2, s1], 
                        's1<>s2': ["{}<>{}".format(s1, s2),"{}<>{}".format(s2, s1)], 
                        'label': [label, label], 
                        'correlation': [correlation_df.loc[label+s1, label+s2],correlation_df.loc[label+s2, label+s1]],
                        'method': [method, method],
                    })
                )

correlation_summary_df = pd.concat(correlation_summary_dfs, ignore_index=True)
correlation_summary_df.head()
#fig = px.box(correlation_summary_df, facet_col="s1", facet_row="s2", x="method", y="correlation", hover_data=["label"]) 
#fig.show()
#sns.catplot(data=correlation_summary_df, x="method", y="correlation", row="s1", col="s2", kind='box', height=5, aspect=1)
#plt.show()
grouped_df = correlation_summary_df.groupby(by=["method", "s1", "s2"], as_index=False).mean()
grouped_df = grouped_df[grouped_df["method"] == "pearson"].pivot(index='s1', columns='s2', values='correlation').fillna(1)

plt.figure(figsize=(20,20))
#ax = sns.heatmap(data=grouped_df, vmin=-1, vmax=1, center=0, cmap='Blues', square=True)
ax = sns.heatmap(data=grouped_df, vmax=1, cmap='Blues', square=True, annot=True)
ax.figure.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14311199207135777, 'fraction_numerical': 0.010109018830525273, 'mean_word_length': 4.046, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '221798', 'license': 'MIT', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1735, 'n_words': 325, 'path': 'notebooks/embedding_comparison.ipynb', 'repo_name': 'lenaWitterauf/Domain-Guided-Monitoring', 'size': '5045'}