<jupyter_start><jupyter_text>Compute various properties<jupyter_code>anns_paths = ['../data/annotations/Coref-annotation.csv',
              '../data/annotations/Coref-annotation-too-long.csv']
anns_gform = pd.concat([pd.read_csv(adhoc_fix(p)) for p in anns_paths], sort=False)
# sometimes people submit the same answer twice, we'll keep only the last submission 
anns_gform = anns_gform.drop_duplicates(['Document', 'Username'], keep='last')
anns_chains = anns_gform.Annotations.apply(unpack_json)
anns = pd.concat([anns_gform, anns_chains], axis=1)
anns['events'] = anns['events'].apply(lambda s: s.replace(' ', '') if isinstance(s, str) else None)
anns['Timestamp_dt'] = pd.to_datetime(anns.Timestamp)
anns['transformation'] = anns.conll_file.str.extract(r'/([\w-]+)/\s*(?:dev|test)')
assert anns.transformation.isna().sum() == 0
periods = pd.date_range(freq='7D', start='2019-03-06', periods=52)

def compute_period_name(d):
    insert_position = periods.searchsorted(d)
    period = periods[insert_position-1]
    return str(period)[:10]

anns['period'] = anns.Timestamp_dt.apply(compute_period_name)
import traceback
def chains_str_from_events_pd(row):
    try:
        return chains_str_from_events(row['events'])
    except:
        print('Error at document %s, ignored converting events to chains' % row['Document'], file=sys.stderr)
        traceback.print_exc()
anns['events'] = anns['events'].apply(lambda s: s.replace(' ', '') if isinstance(s, str) else None)
anns['chains'] = anns['chains'].apply(lambda s: s.replace(' ', '') if isinstance(s, str) else None)
anns['chains_old'] = anns['chains']
anns['chains_from_events'] = anns.apply(chains_str_from_events_pd, axis=1)
anns['chains'] = anns.apply(lambda row: (row['chains_from_events'] 
                             if (str(row['Timestamp_dt']) < '2019-04-18') and row['events'] 
                             else row['chains_old']), axis=1)
anns['num_mentions'] = anns.chains.apply(lambda x: len(set(mention for chain in (x or '').split(',') for mention in chain.split('='))))
anns['num_chains'] = anns.chains.apply(lambda x: sum(1 for chain in (x or '').split(',')))
anns['num_mentions_per_chain'] = anns.num_mentions / anns.num_chains
anns['genre'] = anns.conll_file.str.extract(r'(?:train|dev|test)/(\w\w)_')
# there's no difference between using auto or gold because humans don't see our syntactic and semantic annotations
# some *.auto_conll files are missing so I replace them with the *.gold_conll equivalence
anns['conll_file'] = anns.conll_file.str.replace('auto_conll', 'gold_conll')
anns['conll_file_found'] = anns.conll_file.apply(lambda p: os.path.exists(os.path.join('..', p)))
assert anns['conll_file_found'].sum() / len(anns) == 1<jupyter_output><empty_output><jupyter_text>Filter irrelevant annotations<jupyter_code># remove erroneous documents that weren't reannotated (11 of them)
anns = anns[anns.chains.notna() & (anns.chains != '')]
# ignore authors' annotations
anns = anns[~anns.Username.str.startswith('author')]
len(anns)<jupyter_output><empty_output><jupyter_text>Remove annotations that coincide with bad control annotations<jupyter_code>anns_orig = anns[anns.transformation == 'orig'].copy()
results_orig = (anns_orig.groupby(['Username', 'period'])
                .apply(evaluate_submissions, proj_dir='..').reset_index())
sns.barplot(x='Username', y='f1_conll', hue='period', data=results_orig.sort_values('Username'))
plt.title('Performance on original documents (control)')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
None
bad_user_period = results_orig[results_orig.f1_conll < 80].copy()
anns_checked = pd.merge(anns, bad_user_period, on=['Username', 'period'], how='outer', indicator=True)
anns = anns_checked[anns_checked._merge == 'left_only'].drop(columns='_merge')
len(anns)<jupyter_output><empty_output><jupyter_text>General statistics  Distribution by annotator<jupyter_code>_ = anns.groupby('Username').agg({'Annotations': 'count'})\
    .reset_index().plot.bar('Username', 'Annotations')<jupyter_output><empty_output><jupyter_text>Distribution by genre<jupyter_code>anns[anns.transformation == 'men_100'].groupby(['Username', 'genre']).agg({'conll_file': len})
anns[anns.transformation == 'nonmen_100'].groupby(['Username', 'genre']).agg({'conll_file': len})
anns[anns.transformation == 'nonmen_100'].groupby(['Username', 'genre']).agg({'conll_file': len})<jupyter_output><empty_output><jupyter_text>Distribution by single- vs double-annotation<jupyter_code>anns_count_by_conll_file = (anns.groupby('conll_file').agg({'Annotations': 'count', 'Username': list})
                            .rename(columns={'Annotations': 'num_anns'}).reset_index())
num_docs_by_num_annotation = (anns_count_by_conll_file.groupby('num_anns').agg({'conll_file': 'count'}))
num_docs_by_num_annotation<jupyter_output><empty_output><jupyter_text>Distribution by transformation<jupyter_code>num_docs_by_transformation = anns.groupby('transformation') \
    .agg({'Annotations': 'count'}).rename(columns={'Annotations': 'num_anns'}) \
    .reset_index()
_ = num_docs_by_transformation.plot.bar('transformation', 'num_anns')
num_docs_by_transformation.describe()<jupyter_output><empty_output><jupyter_text>Joint distribution of genre and annotation/transformation<jupyter_code># shared_labels = sorted(anns['transformation'].unique())
shared_labels = sorted(anns['genre'].unique())
from collections import OrderedDict

def custom_bar(labels, values, **kwargs):
    label2val = OrderedDict((l, 0) for l in shared_labels)
    for l, v in zip(labels, values): label2val[l] = v
    
    ax = plt.gca()
    g = plt.bar(label2val.keys(), label2val.values(), **kwargs)
    ax.set_xticklabels(shared_labels, rotation=90)
    return g
sns.set_palette("Paired", n_colors=14)
df = (anns.groupby(['Username', 'genre']).agg({'Annotations': 'count'})
      .reset_index().rename(columns={'Annotations': 'annotation count'}))
g = sns.FacetGrid(df, col="Username", sharex=False)
g = g.map(custom_bar, "genre", "annotation count")
df = (anns.groupby(['genre', 'transformation']).agg({'Annotations': 'count'})
      .reset_index().rename(columns={'Annotations': 'annotation count'}))
g = sns.FacetGrid(df, col="transformation", col_wrap=4, sharex=False)
g = g.map(custom_bar, "genre", "annotation count")<jupyter_output><empty_output><jupyter_text>As we can see from above, the distribution w.r.t to genre differs substantially between transformations/users. So plotting results of different users next to each other or comparing results between transformations would be unfair. The answer is to resample to normalize the distribution of genre.<jupyter_code>def examine_corpus(df, transformations=['orig', 'men_100', 'nonmen_100']):
    examined_df = df.copy()
    examined_df['UserTransf'] = examined_df.Username + '/' + examined_df.transformation
    examined_df = examined_df[examined_df.transformation.isin(transformations)]
    df = (examined_df.groupby(['genre', 'UserTransf']).agg({'Annotations': 'count'})
          .reset_index().rename(columns={'Annotations': 'annotation count'}))
    g = sns.FacetGrid(df, col="UserTransf", col_wrap=4, sharex=False)
    g = g.map(custom_bar, "genre", "annotation count")
examine_corpus(anns)
examine_corpus(anns, ['orig', 'no-external', 'no-internal', 'no-name'])<jupyter_output><empty_output><jupyter_text>Resample to correct the genre distributionWe will group the data by annotator-transformation and then resample each group to match a reference distribution of genres.<jupyter_code># obtaining reference distribution
paths = glob('../data/conll-2012-flat/train/*_gold_conll')
ref_genre_counts = Counter(os.path.basename(p)[:2] for p in paths)
ref_genre_counts
mean_ann_per_user_transf, = anns.groupby(['Username', 'transformation']).agg({'Annotations': len}).mean().values
mean_ann_per_user_transf
def resample(group_df):
    # compute the share of each strata
    available_genres = group_df.genre.unique().tolist()
    raw_shares = {g: ref_genre_counts[g] for g in available_genres}
    total_shares = sum(raw_shares.values())
    sample_size = min(mean_ann_per_user_transf, len(group_df)*0.8)
    shares = {g: int(v/total_shares*sample_size) for g, v in raw_shares.items()}
    # sample according to the number of shares
    dfs = []
    for genre, target_count in shares.items():
        genre_df = group_df[group_df.genre == genre].sample(target_count, replace=True)
        genre_df = genre_df.drop(columns=['Username', 'transformation'])
        dfs.append(genre_df)
    return pd.concat(dfs)
anns_all = anns.copy()
anns_all['Username'] = 'all'
anns_with_all = pd.concat([anns, anns_all], sort=False)
samples = [
    anns_with_all.groupby(['Username', 'transformation']).apply(resample).reset_index()
    for _ in tqdm(range(25))
]
examine_corpus(samples[0])
examine_corpus(samples[1])
examine_corpus(samples[2], ['orig', 'no-external', 'no-internal', 'no-name'])
examine_corpus(samples[3], ['orig', 'no-external', 'no-internal', 'no-name'])<jupyter_output><empty_output><jupyter_text>Unfortunately, some genres are just absent in certain groups, I can't do anything about it :-(  Evaluate per annotator-transformation<jupyter_code>%%time
results = []
for sample_no, sample in enumerate(samples):
    tqdm.pandas(desc='Measuring performance')
    results_sample = (sample.groupby(['Username', 'transformation'])
                                      .progress_apply(evaluate_submissions, proj_dir='..')
                                      .reset_index())
    results_sample['sample'] = sample_no
    results.append(results_sample)
results = pd.concat(results)
results.columns
results['sample'].unique()
# write to CSV to present in another notebook
df = results.copy().rename(columns={'Username': 'system', 'transformation': 'manipulation'})
df['auto_or_gold'] = 'auto'
df['dataset'] = 'dev+test'
df.to_csv('../output/mturk/students-results.csv', index=False)
filtered_results = (results[results.transformation.isin(['orig', 'men_100', 'nonmen_100'])]
                    [['Username', 'transformation', 'f1_conll']])
_ = sns.barplot(x='Username', y='f1_conll', data=filtered_results, hue='transformation')
filtered_results = (results[results.transformation.isin(['orig', 'no-external', 'no-internal', 'no-name'])]
                    [['Username', 'transformation', 'f1_conll']])
_ = sns.barplot(x='Username', y='f1_conll', data=filtered_results, hue='transformation')
confidence_data = []
for _, row in anns.iterrows():
    confidence_scores = re.findall(r':confidence:(\d+);', row['events'] or '')
    for conf_score in confidence_scores:
        conf_score = int(conf_score)
        if conf_score > 5:
            print('Weird confidence score found in file:', row['Document'])
        else:
            confidence_data.append({
                'Username': row['Username'],
                'transformation': row['transformation'],
                'confidence': conf_score
            })
confidence_df = pd.DataFrame(confidence_data)
(confidence_df[confidence_df.transformation.isin(['orig', 'no-external', 'no-internal', 'no-name', 'men_100', 'nonmen_100'])]
 .groupby(['Username', 'transformation']).agg(['mean', 'std']))
# student3 apparently isn't serious about confidence score
confidence_df = confidence_df[confidence_df.Username != 'student3']
g = sns.barplot(y='confidence', x='transformation', data=confidence_df.sort_values('transformation'))
_ = g.set_xticklabels(g.get_xticklabels(), rotation=90)<jupyter_output><empty_output><jupyter_text>Compute precision-recall errors<jupyter_code># ann_conll_dir = '../output/mturk/student-official-annotations'

# %%bash
# cd ..
# INPUT0=data/annotations/Coref-annotation.csv
# INPUT1=data/annotations/Coref-annotation-too-long.csv
# OUTPUT0=output/mturk/student-official-annotations
# rm -rf $OUTPUT0
# python3 -m mturk.score_submissions --out=$OUTPUT0 $INPUT0 $INPUT1

# results = pd.DataFrame([{'path': p, **extract_results_from_log_file(p)}
#                         for p in glob(os.path.join(ann_conll_dir, '*.log'))])

# results['transformation'] = results.path.str.extract(r'/([\w_-]+)\.log$')
# results.sort_values('transformation')

# errors = []
# ann_paths = glob(os.path.join(ann_conll_dir, '*.ann_conll'))
# ref_paths = [p.replace('ann_conll', 'gold_conll') for p in ann_paths]
# for ref_path, ann_path in tqdm(list(zip(ref_paths, ann_paths))):
#     errors_in_one_file = extract_errors(ref_path, ann_path)
#     errors.append({
#         'ann_path': ann_path, 'ref_path': ref_path,
#         'precision_errors': errors_in_one_file['ann']['precision_errors'],
#         'recall_errors': errors_in_one_file['ann']['recall_errors']
#     })

# errors_df = pd.DataFrame(errors)
# errors_df['transformation'] = errors_df.ann_path.str.extract(r'/([\w_-]+)\.ann_conll$')
# errors_df['errorcount_precision'] = errors_df.precision_errors.apply(lambda x: len(x))
# errors_df['errorcount_recall'] = errors_df.recall_errors.apply(lambda x: len(x))
# results_with_errors = pd.merge(results, errors_df, on='transformation')
# results_with_errors.sort_values('transformation')[['f1_conll', 'errorcount_precision', 'errorcount_recall', 'transformation']]<jupyter_output><empty_output><jupyter_text>Plot the shape of performance and look for anomaly<jupyter_code># g = sns.distplot(anns_with_results.f1_conll, kde=False)
# _ = g.set_title('Histogram of F1 CoNLL score per document')

# formatted_results[formatted_results.f1_conll <= 50]

# formatted_results[(formatted_results.f1_conll >= 90) & (formatted_results.transformation == 'men_60')]

# formatted_results[(formatted_results.f1_conll >= 90) & (formatted_results.transformation == 'nonmen_60')]

# ref_path = f'{ann_conll_dir}/student2-2019-03-29-question-051.html_by_student2.gold_conll'
# ann_path = f'{ann_conll_dir}/student2-2019-03-29-question-051.html_by_student2.ann_conll'
# visualize(ref_path, ann_path)

# ref_path = f'{ann_conll_dir}/student2-2019-03-29-question-178.html_by_student2.gold_conll'
# ann_path = f'{ann_conll_dir}/student2-2019-03-29-question-178.html_by_student2.ann_conll'
# visualize(ref_path, ann_path)

# ref_path = f'{ann_conll_dir}/student2-2019-03-29-question-107.html_by_student2.gold_conll'
# ann_path = f'{ann_conll_dir}/student2-2019-03-29-question-107.html_by_student2.ann_conll'
# visualize(ref_path, ann_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15823623805204773, 'fraction_numerical': 0.014093351008162981, 'mean_word_length': 5.891346153846154, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '257674', 'license': 'MIT', 'n_tokens_mistral': 5077, 'n_tokens_neox': 4869, 'n_words': 836, 'path': 'notebooks/processing-annotation.ipynb', 'repo_name': 'natemalek/molen-pater-nathan-rma-thesis-coreference-with-singletons', 'size': '14333'}