<jupyter_start><jupyter_text>Read in results<jupyter_code>from collections import OrderedDict
paths = OrderedDict([
    ('baselines', '../output/baselines-results.csv'),
    ('sieve', '../output/stanford-sieve-results.csv'),
#     ('e2e', '../output/e2e/results.csv'),
    ('cort', '../output/cort/results.csv'),
    ('stduents', '../output/mturk/students-results.csv')
])
import pandas as pd
name2ret = OrderedDict((name, pd.read_csv(path)) for name, path in paths.items())<jupyter_output><empty_output><jupyter_text>Some fixes to match the new format, to be removed<jupyter_code>all_results = pd.concat(name2ret.values())
orig = all_results[all_results['manipulation'] == 'orig']
men_0, nonmen_0 = orig.copy(), orig.copy()
men_0['manipulation'] = 'men_0'
nonmen_0['manipulation'] = 'nonmen_0'
results = pd.concat([all_results, men_0, nonmen_0])
results.head()
results[results.system == 'human'].head()
results.manipulation.unique()<jupyter_output><empty_output><jupyter_text>Check the validity of results The precision mention identification should always be 100%. Recall might differ because some systems filter out singletons before returning results.<jupyter_code>results[results.mention_p < 100]
results[results.mention_r < 100].system.unique()<jupyter_output><empty_output><jupyter_text>Summary<jupyter_code>results[(results.manipulation == 'orig') & (results.auto_or_gold == 'gold')]<jupyter_output><empty_output><jupyter_text>Comparing local names and global names<jupyter_code>dev_gold_names_results = results[(results['dataset'] == 'dev') & 
                                 ((results['auto_or_gold'] == 'gold') | results.auto_or_gold.isna()) &
                                 results['manipulation'].isin(['orig', 'no-external', 'no-internal', 'no-name'])]
dev_gold_names_results.sort_values(['system', 'manipulation'])
sns.set_palette('cubehelix', n_colors=3)
_ = sns.barplot(x='system', y='f1_conll', hue='manipulation', data=dev_gold_names_results)
dev_auto_names_results = results[(results['dataset'] == 'dev') & 
                                 ((results['auto_or_gold'] == 'gold') | results.auto_or_gold.isna()) &
                                 results['manipulation'].isin(['orig', 'no-external', 'no-internal', 'no-name'])]
dev_auto_names_results.sort_values(['system', 'manipulation'])
sns.set_palette('cubehelix', n_colors=3)
_ = sns.barplot(x='system', y='f1_conll', hue='manipulation', data=dev_auto_names_results)<jupyter_output><empty_output><jupyter_text>Results on gold datasets (with retraining)<jupyter_code>dev_gold_results = results[((results['dataset'] == 'dev') & 
                            ((results['auto_or_gold'] == 'gold') | results.auto_or_gold.isna())) |
                           results['system'].isin(['student3', 'student2'])]
men_results = dev_gold_results[dev_gold_results['manipulation'].str.contains('^men_')].copy()
men_results['percentage'] = men_results['manipulation'].apply(lambda s: int(s[4:]))
# don't use sns.pointplot because its horizonal axis takes categorical values 
# instead of numerical values
# def line_plot(x, y, hue, data, title=None):
#     ax = sns.pointplot(x=x, y=y, hue=hue, data=data, palette='cubehelix')
#     ax.set_title(title)
    
def line_plot(x, y, hue, data, title=None, legend=False):
    hue_vals = data[hue].drop_duplicates()
    sns.set_palette('cubehelix', n_colors=len(hue_vals))
    for hue_val in hue_vals:
        data_hue = data[data[hue] == hue_val].sort_values(x)
        plt.plot(data_hue[x], data_hue[y])
    plt.ylabel('F1 (CoNLL)')
    plt.xlabel('Percentage')
    if title: plt.title(title)
    if legend: plt.legend(labels=hue_vals, loc='center left', bbox_to_anchor=(1, 0.5))
men_results[men_results.percentage == 100]
line_plot(x='percentage', y='f1_conll', hue='system', data=men_results, 
          title="Performance vs percent of mentions masked", legend=True)
import scipy

def plot_auc(df, hue):
    df = df.sort_values('percentage')
    hue_vals = df[hue].drop_duplicates()
    auc_df = []
    for hue_val in hue_vals:
        my_df = df[df[hue] == hue_val]
        x = my_df['percentage'].values
        f = my_df['f1_conll'].values
        max_f = [100]*len(x)
        auc = scipy.integrate.simps(f, x) / scipy.integrate.simps(max_f, x)
        auc_df.append({'system': hue_val, 'men_auc': auc})
    sns.set_palette('cubehelix', n_colors=len(hue_vals))
    ax = sns.barplot(x='system', y='men_auc', data=pd.DataFrame(auc_df))
    ax.set_xticklabels(ax.get_xticklabels(), rotation=30)
    
plot_auc(men_results, 'system')
nonmen_results = dev_gold_results[dev_gold_results['manipulation'].str.contains('^nonmen_')].copy()
nonmen_results['percentage'] = nonmen_results['manipulation'].apply(lambda s: int(s[7:]))
nonmen_results[nonmen_results.percentage == 100]
line_plot(x='percentage', y='f1_conll', hue='system', data=nonmen_results,
          title="Performance vs percent of non-mentions masked", legend=True)
plot_auc(nonmen_results, 'system')
def plot_context_plots(df):
    results = df[df['manipulation'].str.contains('^(?:non)?men_')].copy()
    results['percentage'] = results['manipulation'].apply(lambda s: int(re.sub('(?:non)?men_', '', s)))
    results['manipulation'] = results['manipulation'].apply(lambda s: re.sub('_\d+', '', s))
    results = results.sort_values(['manipulation', 'percentage'])
    hues = list(results['system'].drop_duplicates())
    g = sns.FacetGrid(results, col="manipulation", hue="system", hue_order=hues, palette='cubehelix')
    g.map(plt.plot, 'percentage', 'f1_conll').add_legend()
plot_context_plots(dev_gold_results)<jupyter_output><empty_output><jupyter_text>Results on auto datasets (without retraining)<jupyter_code>dev_auto_results = results[(results['dataset'] == 'dev') & 
                           ((results['auto_or_gold'] == 'auto') | results.auto_or_gold.isna())]
plot_context_plots(dev_auto_results)
men_auto_results = dev_auto_results[dev_auto_results['manipulation'].str.contains('^men_')].copy()
men_auto_results['percentage'] = men_auto_results['manipulation'].apply(lambda s: int(s[4:]))
plot_auc(men_auto_results, 'system')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16699250570218313, 'fraction_numerical': 0.007983056370153145, 'mean_word_length': 5.328865979381443, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '300862', 'license': 'MIT', 'n_tokens_mistral': 2192, 'n_tokens_neox': 2122, 'n_words': 331, 'path': 'notebooks/results.ipynb', 'repo_name': 'natemalek/molen-pater-nathan-rma-thesis-coreference-with-singletons', 'size': '6138'}