<jupyter_start><jupyter_text>Word2Vec result visualizer<jupyter_code>import pandas as pd
!pip install plotly
import plotly.express as px
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np<jupyter_output>Requirement already satisfied: plotly in /opt/venv/lib/python3.7/site-packages (4.10.0)
Requirement already satisfied: retrying>=1.3.3 in /opt/venv/lib/python3.7/site-packages (from plotly) (1.3.3)
Requirement already satisfied: six in /opt/venv/lib/python3.7/site-packages (from plotly) (1.15.0)
[33mWARNING: You are using pip version 20.1.1; however, version 20.2.3 is available.
You should consider upgrading via the '/opt/venv/bin/python -m pip install --upgrade pip' command.[0m<jupyter_text>Evaluate<jupyter_code>df = pd.read_csv("../output/all_relatedness_table_v2.csv")
df.columns<jupyter_output><empty_output><jupyter_text>Select and prepare results<jupyter_code># read and extract result files
## ANALOGY
analogy_result = pd.read_csv("../output/all_analogy_result.csv")
analogy_result.loc[:, 'Unnamed: 0'] = analogy_result['Unnamed: 0'].apply(lambda x: x.replace("_word2vec_mc=10_", "word2vec_"))
analogy_result.rename(columns={'Unnamed: 0': 'model_name'}, inplace=True)
analogy_to_keep = ['model_name', 'google_analogy_cos3mul_score', 'bats_analogy_cos3mul_score']
analogy_result = analogy_result.loc[:, analogy_to_keep]

## ASSOCIATIONS
association_result = pd.read_csv("../output/all_association_result_v2.csv")
association_result.rename(columns={'col': 'model_name'}, inplace=True)

## SIMILARITY
relatedness_result = pd.read_csv("../output/all_relatedness_table_v2.csv")
relatedness_result.loc[:, 'Unnamed: 0'] = relatedness_result['Unnamed: 0'].apply(lambda x: x.replace("word2vec_mc=10_", "word2vec_"))
relatedness_result.rename(columns={'Unnamed: 0': 'model_name'}, inplace=True)
# to_keep = ['model_name','EN-WS-353-ALL', 'EN-MEN-TR-3k', 'EN-YP-130', 'EN-SIMLEX-999', 'EN-VERB-143', 'EN-SimVerb-3500']
relatedness_to_keep = ['model_name','EN-WS-353-SIM', 'EN-RG-65', 'EN-SIMLEX-999', 'EN-VERB-143', 'EN-SimVerb-3500']
relatedness_result = relatedness_result.loc[:, relatedness_to_keep]

## COMBINE
final_result = pd.merge(analogy_result, association_result, on='model_name')
final_result = pd.merge(final_result, relatedness_result, on='model_name')
final_result[['algorithm', 'dimension', 'window', 'method', 'model_type']] = final_result['model_name'].str.split("_", expand=True)
final_result.loc[:, 'dimension'] = final_result.loc[:, 'dimension'].apply(lambda x: x.replace("size=", ""))
final_result.loc[:, 'dimension'] = final_result.loc[:, 'dimension'].astype(int)
final_result.sort_values(by=['dimension'], inplace=True)
# final_result.loc[:, 'window'] = final_result.loc[:, 'window'].apply(lambda x: x.replace("window=", ""))
# final_result.loc[:, 'window'] = final_result.loc[:, 'window'].astype(int)
final_result.loc[:, 'method'] = final_result.loc[:, 'method'].apply(lambda x: x.replace("sg=", ""))
final_result.loc[:, 'method'] = final_result.loc[:, 'method'].map({'0':'cbow', '1':'sg'})
# add aggregated scores
final_result.loc[:, 'final_analogy_score'] = final_result.loc[:, analogy_to_keep[1:]].mean(axis=1)
final_result.loc[:, 'final_association_hit_rate'] = final_result.loc[:, ['swow8500_hit_rate', 'eat_hit_rate']].mean(axis=1) 
final_result.loc[:, 'final_association_coverage'] = final_result.loc[:, ['swow8500_avg_coverage', 'eat_avg_coverage']].mean(axis=1) 
final_result.loc[:, 'final_relatedness_score'] = final_result.loc[:, relatedness_to_keep[1:]].mean(axis=1) 
# final_result.loc[:, 'final_association_coverage'] = final_result['final_association_coverage'].fillna(0)
final_result = final_result.fillna(0)<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code>final_result.head()
# final_result.columns
px.line(final_result, x="dimension", y="final_analogy_score", color="window", facet_row="method", facet_col="model_type", title="(C) Analogy result").update_traces(mode="lines+markers")
px.line(final_result, x="dimension", y="final_relatedness_score", color="window", facet_row="method", facet_col="model_type", title="Similarity result").update_traces(mode="lines+markers")
asso_result = pd.melt(final_result, id_vars=['model_name', 'dimension', 'window', 'method', 'model_type'], value_vars=["final_association_hit_rate", "final_association_coverage"])
# asso_result[['algorithm', 'dimension', 'window', 'method', 'model_type']] = asso_result['model_name'].str.split("_", expand=True)
asso_result.rename(columns={'variable':'score'}, inplace=True)
asso_result['score'].replace({"final_association_hit_rate": 'hit rate', "final_association_coverage": "coverage"}, inplace=True)
asso_result.loc[:, 'score, window'] = asso_result['score'] + " @ "+ asso_result['window'].astype(str)
px.scatter(asso_result, x="dimension", y="value", color="score, window", facet_row="method", facet_col="model_type", title="(B) Association result (hit rate & coverage)").update_traces(mode="lines+markers")
# asso_result = pd.melt(final_result, id_vars=['model_name', 'dimension', 'window', 'method', 'model_type'], value_vars=["final_association_hit_rate", "final_association_coverage"])
# # asso_result[['algorithm', 'dimension', 'window', 'method', 'model_type']] = asso_result['model_name'].str.split("_", expand=True)
# asso_result.rename(columns={'variable':'score'}, inplace=True)
# asso_result['score'].replace({"final_association_hit_rate": 'hit rate', "final_association_coverage": "coverage"}, inplace=True)
# asso_result.loc[:, 'score, window'] = asso_result['score'] + " @ "+ asso_result['window'].astype(str)
px.scatter(final_result, x="dimension", y="final_association_coverage", color="window", facet_row="method", facet_col="model_type", title="(B) Association result (hit rate & coverage)").update_traces(mode="lines+markers")<jupyter_output><empty_output><jupyter_text>Add table<jupyter_code># final_result[['model_name', 'google_analogy_cos3mul_score',
#        'bats_analogy_cos3mul_score', 'swow8500_hit_rate',
#        'swow8500_avg_coverage', 'eat_hit_rate', 'eat_avg_coverage',
#        'EN-WS-353-ALL', 'EN-MEN-TR-3k', 'EN-YP-130', 'EN-SIMLEX-999',
#        'EN-VERB-143', 'EN-SimVerb-3500']]
final_table = final_result.copy()
final_table.loc[:, 'window'] = final_table.loc[:, 'window'].apply(lambda x: x.replace("window=", ""))
final_table.loc[:, 'window'] = final_table.loc[:, 'window'].astype(int)
final_table.sort_values(by=['method', 'window', 'dimension', 'model_type'], inplace=True)
final_table = final_table[['dimension', 'window', 'method', 'model_type', 'google_analogy_cos3mul_score',
       'bats_analogy_cos3mul_score', 'swow8500_hit_rate',
       'swow8500_avg_coverage', 'eat_hit_rate', 'eat_avg_coverage',
       'EN-WS-353-ALL', 'EN-MEN-TR-3k', 'EN-YP-130', 'EN-SIMLEX-999',
       'EN-VERB-143', 'EN-SimVerb-3500']]
final_table.columns = ['dimension', 'window', 'method', 'model-type', 'google-analogy',
       'bats-analogy', 'swow8500-hit-rate',
       'swow8500-avg-coverage', 'eat-hit-rate', 'eat-avg-coverage',
       'EN-WS-353-ALL', 'EN-MEN-TR-3k', 'EN-YP-130', 'EN-SIMLEX-999',
       'EN-VERB-143', 'EN-SimVerb-3500']
final_table = final_table.round(2)
# print(final_table.to_latex(index=False))
# final_table.round(2)
final_result.<jupyter_output><empty_output><jupyter_text>Result summary table<jupyter_code>final_result.columns
def agg_scores(x):
    y = x.values
    mean, maxx, avg_change = np.mean(y), max(y), np.mean([y[1]-y[0], y[2]-y[1]])
    # return f"{mean:.3f} ({avg_change:+.2f})"
    return f"{maxx:.3f} ({mean:.3f})"

def get_pivot_score(val):
    final_result['window'] = final_result['window'].apply(lambda x: x.replace("window=", ""))
    pivot = pd.pivot_table(final_result, values=val, index=['method', 'window'],
                        columns=['model_type'], aggfunc=agg_scores)
    tuples = [(val,) + x for x in pivot.index]
    pivot.index = pd.MultiIndex.from_tuples(tuples, names=['type','method', 'window'])
    return pivot
pivot_results = pd.concat([get_pivot_score('final_relatedness_score'), 
                            get_pivot_score('final_association_coverage'), 
                            get_pivot_score('final_analogy_score')])
print(pivot_results.to_latex(index=True))
# pivot_results.to_csv('../output/word2vec_pivot_result_v2.csv')
pivot_results<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1848699196739, 'fraction_numerical': 0.023018822683131518, 'mean_word_length': 7.375502008032129, 'pattern_counts': {'":': 4, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '462025', 'license': 'MIT', 'n_tokens_mistral': 3156, 'n_tokens_neox': 3035, 'n_words': 439, 'path': 'notebooks/word2vec_results.ipynb', 'repo_name': 'imohitmayank/word-embedding-models-eval', 'size': '8341'}