<jupyter_start><jupyter_text>Table 2: Main evaluation<jupyter_code>results_1850 = evaluate_results(Path(f'results_1850'),avg=avg)
results_1920 = evaluate_results(Path(f'results_1920'),avg=avg)
results_2000 = evaluate_results(Path(f'results_2000'),avg=avg)
df_1850 = pd.DataFrame.from_dict(results_1850, orient='index', columns=['precision','recall','fscore','preds'])
df_1920 = pd.DataFrame.from_dict(results_1920, orient='index', columns=['precision','recall','fscore','preds'])
df_2000 = pd.DataFrame.from_dict(results_2000, orient='index', columns=['precision','recall','fscore','preds'])

df_1850 = df_1850.rename(index=renamed_columns)
df_1920 = df_1920.rename(index=renamed_columns)
df_2000 = df_2000.rename(index=renamed_columns)
df_fscores = pd.concat([df_1850[['precision', 'recall', 'fscore']], df_1920[['precision', 'recall', 'fscore']], df_2000[['precision', 'recall', 'fscore']]],axis=1) # ,df_2000['fscore']
cols_baselines = [c for c in df_fscores.index if not c.startswith("bert_")]
cols_bert = [c for c in df_fscores.index if c.startswith("bert_") and not 'ts' in c and not 'contrast' in c]
df_fscores = df_fscores.loc[cols_baselines + cols_bert]
print(df_fscores.to_latex())
rows_1850 = [r for r in df_fscores.index if '1850' in r]
rows_1920 = [r for r in df_fscores.index if '1900' in r]
rows_2000 = [r for r in df_fscores.index if 'base' in r]
diff_1850_2000 = df_1850.loc[df_1850.index.isin(rows_1850)].fscore.values - df_1850.loc[df_1850.index.isin(rows_2000)].fscore.values
diff_1920_2000 = df_1920.loc[df_1920.index.isin(rows_1920)].fscore.values - df_1920.loc[df_1920.index.isin(rows_2000)].fscore.values
df_diff = pd.DataFrame([diff_1850_2000,diff_1920_2000],columns=rows_2000).T
#df_diff['sum'] = df_diff.sum(axis=1)
print(df_diff.to_latex())<jupyter_output>\begin{tabular}{lrr}
\toprule
{} &      0 &      1 \\
\midrule
BERT\_base binary centroid & -0.010 & -0.011 \\
BERT\_base sense centroid  &  0.011 &  0.025 \\
BERT\_base perceptron      & -0.009 &  0.001 \\
\bottomrule
\end{tabular}<jupyter_text>Table 3: Time-sensitive methods<jupyter_code>results_ts_1850 = evaluate_results(Path("results_ts_1850"),avg=avg)
results_ts_1920 = evaluate_results(Path("results_ts_1920"),avg=avg)
df_ts_1850 = pd.DataFrame.from_dict(results_ts_1850, orient='index', columns=['precision','recall','fscore','preds'])
df_ts_1920 = pd.DataFrame.from_dict(results_ts_1920, orient='index', columns=['precision','recall','fscore','preds'])
df_fscores = pd.concat([df_ts_1850['fscore'], df_ts_1920['fscore']],axis=1)
df_fscores = df_fscores.rename(index=renamed_columns)
df_fscores.index
print(df_fscores.to_latex())<jupyter_output>\begin{tabular}{lrr}
\toprule
{} &  fscore &  fscore \\
\midrule
BERT\_base sense centroid          &   0.575 &   0.552 \\
BERT\_base nearest sense centroid  &   0.458 &   0.433 \\
BERT\_base weighted sense centroid &   0.593 &   0.556 \\
BERT\_1900 sense centroid          &   0.604 &   0.592 \\
BERT\_1900 nearest sense centroid  &   0.505 &   0.464 \\
BERT\_1900 weighted sense centroid &   0.627 &   0.584 \\
BERT\_1850 sense centroid          &   0.602 &   0.579 \\
BERT\_1850 nearest sense centroid  &   0.489 &   0.441 \\
BERT\_1850 weighted sense centroid &   0.609 &   0.562 \\
\bottomrule
\end{tabular}<jupyter_text>Curated examples<jupyter_code>results_curated_seed = evaluate_results(Path(f'results_curated_1920_seed'),avg=avg)
results_curated_synonym = evaluate_results(Path(f'results_curated_1920_syn'),avg=avg)
df_curated_seed = pd.DataFrame.from_dict(results_curated_seed, orient='index', columns=['precision','recall','fscore','preds'])
df_curated_synonym = pd.DataFrame.from_dict(results_curated_synonym, orient='index', columns=['precision','recall','fscore','preds'])
df_fscore = pd.concat([df_curated_seed['fscore'],df_curated_synonym['fscore']], axis=1)
df_fscore = df_fscore.rename(index=renamed_columns)
df_fscore.columns=['vertical','horizontal']
print(df_fscore.to_latex())<jupyter_output>\begin{tabular}{lrr}
\toprule
{} &  vertical &  horizontal \\
\midrule
BERT\_base sense centroid          &     0.691 &       0.536 \\
BERT\_base weighted sense centroid &     0.582 &       0.521 \\
BERT\_base perceptron              &     0.710 &       0.493 \\
BERT\_1900 sense centroid          &     0.700 &       0.554 \\
BERT\_1900 weighted sense centroid &     0.613 &       0.566 \\
BERT\_1900 perceptron              &     0.612 &       0.526 \\
BERT\_1850 sense centroid          &     0.658 &       0.563 \\
BERT\_1850 weighted sense centroid &     0.564 &       0.540 \\
BERT\_1850 perceptron              &     0.621 &       0.482 \\
\bottomrule
\end{tabular}<jupyter_text>Assess Statistical Significance<jupyter_code>res = evaluate_results(Path("results_1920/"),avg=avg)

selected = "bert_centroid_sense_vector_bert_base_-1,-2,-3,-4_mean"

selected_pred = res[selected][3]
print (selected, res[selected][:3], "\n\nIs the difference significant?\n")

for method,values in res.items():
    if method != selected:
        pred = values[3]
        p_value = scipy.stats.ttest_rel(selected_pred[0],pred[0])[1]
        if p_value<0.05:
            print (method, values[:3], "YES")
        else:
            print (method, values[:3], "NO p_value:",round(p_value,5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15198782575613468, 'fraction_numerical': 0.09206771923150085, 'mean_word_length': 4.569915254237288, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '519882', 'license': 'MIT', 'n_tokens_mistral': 2257, 'n_tokens_neox': 1946, 'n_words': 315, 'path': 'create_results_tables.ipynb', 'repo_name': 'Living-with-machines/TargetedSenseDisambiguation', 'size': '5257'}