<jupyter_start><jupyter_text>Submission statisticShowing some particular statistic over user submission. The main aspects are an evolution in time and success rate. * number of submmission over time * succesful submission over time * leaderboard evolving - scores and rankingThe ANHIR challenge is hosted on https://anhir.grand-challenge.org.These particular results are for ANHIR workshop hosted at ISBI 2019 in Venice, Italy.In case you want to get some further evaluation related to new submission, you may contact one of the ANHIR administrators because the full submission is required, see https://anhir.grand-challenge.org/Organizers.<jupyter_code>%matplotlib inline
%load_ext autoreload
%autoreload 2

import os, sys
import glob, json
import shutil

import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
from birl.utilities.data_io import update_path
from birl.utilities.drawing import draw_matrix_user_ranking
from birl.utilities.evaluate import grouping_cumulative, aggregate_user_score_timeline<jupyter_output><empty_output><jupyter_text>Loading submission tableThe required input table is exported from https://anhir.grand-challenge.org/evaluation/jobs/ which is accessible for ANHIR administrators.<jupyter_code># temporary folder for figures submissions
PATH_TEMP = os.path.abspath(os.path.expanduser('~/Desktop/ANHIR_submissions'))
PATH_SUBMISSIONS = os.path.join(update_path('bm_ANHIR'), 'submissions.csv')
df = pd.read_csv(PATH_SUBMISSIONS)
# conver the data-time columns
for col in ['Created', 'Updated']:
    df[col] = pd.to_datetime(df[col])
df['Team'] = df['User'].apply(lambda txt: txt[txt.index(' (') + 2: txt.index(')')] 
                                          if all(c in txt for c in '()') else None)
df['User'] = df['User'].apply(lambda txt: txt[:txt.index(' (')] if all(c in txt for c in '()') else txt)
# compute time remaining to deadline
df['Till_Deadline'] = - df['Created'].sub(pd.to_datetime('April 1, 2019, 9:00 a.m.'), axis=0)
df['Deadline_days'] = (df['Till_Deadline'] / np.timedelta64(1, 'D')).astype(int)
print ('loaded records: %i' % len(df))
df.head(3)<jupyter_output>loaded records: 279<jupyter_text>Visualise participants infoShowing charts related to individual participants<jupyter_code>dfx = grouping_cumulative(df, col_index='User', col_column='Status')
ax = dfx.plot(kind='bar', stacked=True, figsize=(len(dfx) / 2, 4), grid=True, cmap='RdYlGn')
_= plt.ylabel('# submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_team-submit-succes.pdf'))

# normalize over the sum of all cases
dfx = (dfx.T / dfx.T.sum().astype(float)).T
ax = dfx.plot(kind='bar', stacked=True, figsize=(len(dfx) / 2, 4), grid=True, cmap='RdYlGn')
ax.legend(loc='upper center', bbox_to_anchor=(1.15, 0.7), ncol=1)
_= plt.ylabel('% submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'bar_team-submit-succes.pdf'))
dfx = grouping_cumulative(df, col_index='Deadline_days', col_column='Status')
_= dfx.plot(kind='area', cmap='RdYlGn', figsize=(len(dfx) / 3, 4), grid=True)
_= plt.xlim([len(dfx), 0]), plt.ylabel('# submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_time-submit-succes.pdf'))

# normalize over the sum of all cases
dfx = (dfx.T / dfx.T.sum().astype(float)).T
ax = dfx.plot(kind='bar', stacked=True, cmap='RdYlGn', figsize=(len(dfx) / 3, 3), grid=True)
ax.legend(loc='upper center', bbox_to_anchor=(1.15, 0.7), ncol=1)
_= plt.xlim([len(dfx), 0]), plt.ylabel('% submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'bar_time-submit-succes.pdf'))<jupyter_output><empty_output><jupyter_text>Visualise submissions infoShowing charts related to particular submissions<jupyter_code>dfx = grouping_cumulative(df, col_index='Deadline_days', col_column='User')
ax = dfx.plot(kind='area', cmap='nipy_spectral', figsize=(len(dfx) / 2, 4), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05), ncol=int(np.ceil(len(dfx.columns) / 3)), fancybox=True, shadow=True)
_= plt.xlim([len(dfx), 0]), plt.ylabel('# submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_time-submit-team.pdf'))

ax = dfx.plot(kind='bar', stacked=True, cmap='nipy_spectral', figsize=(len(dfx) / 2, 3), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05), ncol=int(np.ceil(len(dfx.columns) / 3)), fancybox=True, shadow=True)
_= plt.xlim([len(dfx), 0]), plt.ylabel('# submissions')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'bar_time-submit-team.pdf'))
dfx = grouping_cumulative(df, col_index='Deadline_days', col_column='Team')
_= dfx.plot(kind='area', cmap='nipy_spectral', figsize=(len(dfx) / 2, 4), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05), ncol=int(np.ceil(len(dfx.columns) / 2)), fancybox=True, shadow=True)
_= plt.xlim([len(dfx), 0]), plt.ylabel('# submissions')

_= dfx.plot(kind='bar', stacked=True, cmap='nipy_spectral', figsize=(len(dfx) / 2, 3), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05), ncol=int(np.ceil(len(dfx.columns) / 2)), fancybox=True, shadow=True)
_= plt.xlim([len(dfx), 0]), plt.ylabel('# submissions')<jupyter_output><empty_output><jupyter_text>Leader-board evolvingEvolving of the automatic leader board over time with respect to a particular score. Exporting on rob results of a particular evalation method is required, see for example https://anhir.grand-challenge.org/evaluation/methods/80cebd88-9133-4802-98cf-53d878e1bb30/ (to do so you have to be challnge admin)<jupyter_code>import yaml # https://stackoverflow.com/questions/988228

PATH_SUBMISSIONS = os.path.join(update_path('bm_ANHIR'), 'submission-results.csv')
df_res = pd.read_csv(PATH_SUBMISSIONS)
print ('loaded records: %i' % len(df_res))
df_res.head(3)<jupyter_output>loaded records: 183<jupyter_text>Merge results with user submissions<jupyter_code>df2 = df.merge(df_res[['Job', 'Submission', 'Result']], how='right', left_on='ID', right_on='Job')
print ('loaded records: %i' % len(df2))
# get aggregated results
result_str = df2.iloc[0]['Result']
# parse keys from aggregated results
agg_cols = list(yaml.load(result_str)['aggregates'].keys())
# convert aggregated resulst to table columns
for col in agg_cols:
    df2[col] = df2['Result'].apply(lambda ar: yaml.load(ar)['aggregates'][col]
                                   if isinstance(ar, str) else None)
df2 = df2[['Created', 'User', 'Team', 'Till_Deadline', 'Deadline_days'] + agg_cols]
df2.head(3)<jupyter_output>loaded records: 183<jupyter_text>VisualisationsShowing the evolving over time from perspective of the score and mean-time ranking **Timeline: Average Median rTRE**<jupyter_code>dfx = aggregate_user_score_timeline(df2, 'Deadline_days', 'User', 'Average-Median-rTRE',
                                    lower_better=True, top_down=False, interp=True)
ax = dfx.plot(style='-', cmap='nipy_spectral', figsize=(len(dfx) / 2, 3), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(1.2, 1.1), ncol=1)
_= plt.xlim([len(dfx), 0]), plt.ylim([0, 0.1]), 
_= plt.ylabel('Average-Median-rTRE'), plt.xlabel('Deadline_days')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_time-score_avg-median-rTRE.pdf'))

dfx.index = ['D-%i' % c for c in dfx.index]
fig = draw_matrix_user_ranking(dfx.T[dfx.index[::-1]])
fig.savefig(os.path.join(PATH_TEMP, 'chess_time-ranking_avg-median-rTRE.pdf'))<jupyter_output>/home/jb/Dropbox/Workspace/BIRL/birl/utilities/evaluate.py:301: RuntimeWarning: All-NaN axis encountered
  mtx[aggrs.index(agg), users.index(usr)] = fn_best(dfgg[col_score])
/home/jb/Dropbox/Workspace/BIRL/birl/utilities/evaluate.py:309: RuntimeWarning: All-NaN slice encountered
  mtx[i, j] = fn_best(vals)
/home/jb/Dropbox/Workspace/BIRL/birl/utilities/evaluate.py:223: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  ranking = np.zeros(df_stat.as_matrix().shape)
/home/jb/Dropbox/Workspace/BIRL/birl/utilities/visualisation.py:454: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  fig, _ = plt.subplots(figsize=np.array(df_stat.as_matrix().shape[::-1]) * 0.35)<jupyter_text>**Timeline: Execution time**<jupyter_code>dfx = aggregate_user_score_timeline(df2, 'Deadline_days', 'User', 'Average-Norm-Time',
                                    lower_better=True, top_down=False, interp=True)
ax = dfx.plot(style='-', cmap='nipy_spectral', figsize=(len(dfx) / 2, 3), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(1.2, 1.1), ncol=1)
_= plt.xlim([len(dfx), 0]), plt.yscale('log')
_= plt.ylabel('Average-Norm-Time'), plt.xlabel('Deadline_days')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_time-score_avg-time.pdf'))

dfx.index = ['D-%i' % c for c in dfx.index]
fig = draw_matrix_user_ranking(dfx.T[dfx.index[::-1]])
fig.savefig(os.path.join(PATH_TEMP, 'chess_time-ranking_avg-time.pdf'))<jupyter_output><empty_output><jupyter_text>**Timeline: Robustness**<jupyter_code>dfx = aggregate_user_score_timeline(df2, 'Deadline_days', 'User', 'Average-Robustness',
                                    lower_better=False, top_down=False, interp=True)
ax = dfx.plot(style='-', cmap='nipy_spectral', figsize=(len(dfx) / 2, 3), grid=True)
_= plt.legend(loc='upper center', bbox_to_anchor=(1.2, 1.1), ncol=1)
_= plt.xlim([len(dfx), 0]), plt.ylim([0.8, 1.0]), 
_= plt.ylabel('Average-Robustness'), plt.xlabel('Deadline_days')
ax.get_figure().tight_layout()
ax.get_figure().savefig(os.path.join(PATH_TEMP, 'plot_time-score_avg-robust.pdf'))

dfx.index = ['D-%i' % c for c in dfx.index]
fig = draw_matrix_user_ranking(dfx.T[dfx.index[::-1]], higher_better=True)
fig.savefig(os.path.join(PATH_TEMP, 'chess_time-ranking_avg-robust.pdf'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1666003976143141, 'fraction_numerical': 0.017693836978131212, 'mean_word_length': 6.0405878236529045, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '290921', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3918, 'n_tokens_neox': 3739, 'n_words': 708, 'path': 'notebooks/ANHIR_submissions-stat.ipynb', 'repo_name': 'Borda/pyImageRegistrationBenchmark', 'size': '10060'}