<jupyter_start><jupyter_text>Annotation DashboardHere is an example of a full-fledged annotation dashboard to see detailed information on the cases annotated and the breakdown<jupyter_code>%matplotlib inline
import os
import jupyanno as ja
from jupyanno.sheets import get_task_sheet
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
import seaborn as sns
ja.setup_appmode()
USERNAME = ja.get_app_user_id()
task_data = ja.read_task_file('../task.json')
annot_df = ja.read_annotation(get_task_sheet(task_data))
print('Showing most recent 3 annotations')
annot_df.tail(3)
annot_df.groupby(['annotator_class', 'annotator_name']).\
    agg({'viewing_time': ['sum', 'mean'], 'label': len}).\
    reset_index().\
    rename({'label': 'count'}, axis=1).\
    round(2).\
    sort_values(('count', 'len'), ascending=False).\
    style.\
    bar(color='#d65f5f')
results_list = []
# we need to combine the results from binary class and multiclass problems distinctly

for c_mode, group_annot_df in annot_df.groupby('annotation_mode'):
    group_annot_df = group_annot_df.copy()
    group_annot_df['answer'] = group_annot_df['label']
    if c_mode == 'BinaryClass':
        # make the binary result look like a more standard question
        group_annot_df['label'] = group_annot_df.apply(
            lambda c_row: c_row['task'] if c_row['label'] == 'Yes' else None, 1)
    elif c_mode == 'MultiClass':
        pass
    else:
        print('Dashboard does not support {} problems yet!'.format(c_mode))

    c_results_df = pd.merge(group_annot_df,
                            task_data.data_df,
                            how='left',
                            left_on='item_id',
                            right_on=task_data.image_key_col)

    if c_mode == 'BinaryClass':
        c_results_df['correct'] = c_results_df.apply(lambda x: ja.task.binary_correct(x, task_data.label_col), 1)
    elif c_mode == 'MultiClass':
        c_results_df['correct'] = c_results_df.apply(
            lambda c_row: c_row['label'] == c_row[task_data.label_col], 1)

    results_list += [c_results_df]
results_df = pd.concat(results_list)<jupyter_output><empty_output><jupyter_text>My Personal ResultsHere we show the questions you directly answered<jupyter_code>my_answers = results_df[results_df['annotator'] == USERNAME]['correct'].values
ja.task.show_my_result([USERNAME], [my_answers.sum()], my_answers.shape[0])
my_answers_df = results_df[results_df['annotator'] == USERNAME][[
    'Timestamp', 'correct', task_data.image_key_col, 'task', 'answer', task_data.label_col]].copy()

my_answers_df = my_answers_df.sort_values(
    ['Timestamp'], ascending=True).drop(['Timestamp'], 1)
my_answers_df[task_data.image_key_col] = my_answers_df[task_data.image_key_col].map(
    lambda x: ja.utils.path_to_img(os.path.join(task_data.base_img_dir, x)))
out_html = ja.utils.raw_html_render(my_answers_df.rename({task_data.image_key_col: 'Image',
                                           'task': 'Question',
                                           'answer': 'Your Answer',
                                           task_data.label_col: 'Real Answer'}, axis=1))
from IPython.display import HTML
HTML(out_html)<jupyter_output><empty_output><jupyter_text>Overall ResultsWe can show the overall results by person / type of user<jupyter_code>results_df.groupby(['annotator_class', 'annotator_name']).\
    agg({'viewing_time': 'mean',
         'label': len,
         'correct': lambda x: 100*np.mean(x),
         'answer_negativity': lambda x: 100*np.mean(x)}).\
    reset_index().\
    rename({'label': 'Count',
            'correct': 'Accuracy (%)',
            'viewing_time': 'Average Viewing Time (s)',
            'annotator_class': 'Type of User',
            'annotator_name': 'Name',
            'answer_negativity': 'Negative Responses (%)'}, axis=1).\
    round(1).\
    sort_values('Accuracy (%)', ascending=False).\
    style.\
    background_gradient(cmap='hot', low=.5, high=0).\
    set_properties(**{'font-size': '12pt'})<jupyter_output><empty_output><jupyter_text>Disease to be identified Here we show the breakdown based on which condition the patient actually had<jupyter_code>results_df.groupby(task_data.label_col).\
    agg({'viewing_time': 'mean', 'label': len, 'correct': lambda x: 100*np.mean(x)}).\
    reset_index().\
    rename({'label': 'count',
            'correct': 'Accuracy (%)',
            'viewing_time': 'Average Viewing Time (s)'
            }, axis=1).\
    round(1).\
    sort_values('Accuracy (%)', ascending=False).\
    style.\
    background_gradient(cmap='hot', low=.5, high=0).\
    set_properties(**{'font-size': '12pt'})<jupyter_output><empty_output><jupyter_text>Question AskedFinally we show the breakdown based on the question asked<jupyter_code>results_df.groupby('task').\
    agg({'viewing_time': 'mean', 'label': len, 'correct': lambda x: 100*np.mean(x)}).\
    reset_index().\
    rename({'label': 'count',
            'correct': 'Accuracy (%)',
            'viewing_time': 'Average Viewing Time (s)',
            'task': 'Question Asked'
            }, axis=1).\
    round(1).\
    sort_values('Accuracy (%)', ascending=False).\
    style.\
    background_gradient(cmap='hot', low=.5, high=0).\
    set_properties(**{'font-size': '12pt'})<jupyter_output><empty_output><jupyter_text>Results vs Random GuessesShow the results vs Random Guesses for each user<jupyter_code>res_df = results_df.groupby(['annotator', 'annotator_name']).\
    agg({'Timestamp': 'count', 'correct': 'sum'}).\
    reset_index().\
    rename({'Timestamp': 'count'}, axis=1)
for count, c_df in res_df.groupby('count'):
    ax1 = ja.task.show_my_result(c_df['annotator_name'].values.tolist(),
                         c_df['correct'].values.astype(int).tolist(),
                         count)
    ax1.set_title('{} questions answered'.format(count))<jupyter_output><empty_output><jupyter_text>Export ResultsThe full results can be exported and viewed in Excel or Google Sheets and analyzed even further<jupyter_code>results_df.to_csv('results.csv', index=False)
from IPython.display import FileLink
print('Download Results')
FileLink('results.csv')<jupyter_output><empty_output><jupyter_text>View by ImagesHere we show the summarized physicians answers to each question by image<jupyter_code>from IPython.display import display, Markdown
def magnify(th_size='4pt'):
    return [dict(selector="th",
                 props=[("font-size", th_size)]),
            dict(selector="td",
                 props=[('padding', "0em 0em")]),
            dict(selector="th:hover",
                 props=[("font-size", "12pt")]),
            dict(selector="tr:hover td:hover",
                 props=[('max-width', '200px'),
                        ('font-size', '12pt')])
]

with pd.option_context('display.precision', 2):
    display(results_df.pivot_table(index=[task_data.label_col, 'item_id'], 
                           columns =['task'], 
                           values='answer_negativity', 
                           aggfunc=lambda x: 1-np.mean(x)).\
            style.\
            set_properties(**{'max-width': '80px', 'font-size': '1pt'}).\
            background_gradient(cmap='viridis', low=1, high=0).\
            highlight_null('black').\
            set_table_styles(magnify())
           )
from collections import Counter
get_most_common = lambda items: max(Counter(items).items(), key = lambda x: x[1])[0]
results_df['_idx'] = pd.Categorical(results_df['item_id']).codes
pivot_df = results_df[~results_df['label'].isin(['None'])].\
    groupby([task_data.label_col,'annotator', 'annotator_name', '_idx']).\
    agg({'label': get_most_common}).\
    reset_index().\
    pivot_table(columns=[task_data.label_col, '_idx'], 
                       index =['annotator_name'], 
                       values='label', 
                       aggfunc=get_most_common)
lab_enc = {k: i for i, k in enumerate(set(pivot_df.melt()['value'].dropna()))}
c_palette = sns.color_palette("Set2", len(lab_enc)+1)
c_css_p = (np.array(c_palette)*255).clip(0, 255).astype(np.uint8)
def color_by_label(val):
    c_idx = lab_enc.get(val, -1)+1
    color = c_css_p[c_idx]
    return 'background-color: rgb({}, {}, {})'.format(*color)
pd.DataFrame({'Finding': list(lab_enc.keys())}).style.applymap(color_by_label)
pivot_df.\
    style.\
    applymap(color_by_label).\
    set_properties(**{'max-width': '120px', 'font-size': '1pt'}).\
    highlight_null('black').\
    set_table_styles(magnify('5pt'))
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]
for c_label, raw_rows in results_df.groupby(task_data.label_col):
    display(Markdown('## {}'.format(c_label)))
    c_rows = raw_rows.copy()
    c_rows['answer_value'] = 1.0-c_rows['answer_negativity']
    display(c_rows.pivot_table(index=['item_id'], 
                       columns =['task', 'annotator_name'], 
                       values='answer_value', 
                       aggfunc='mean').\
        style.\
        background_gradient(cmap='viridis', low=1, high=0).\
        highlight_null('black').\
        set_properties(**{'font-size': '12pt'}).\
            set_table_styles(styles))<jupyter_output><empty_output><jupyter_text>Image InteractionHere we look at the interaction that took place with the images<jupyter_code>from itertools import chain
# flatten out all the zoom results
has_zoom = results_df['viewing_info_dict'].map(lambda x: 'zoom' in x)

zoom_df = results_df[has_zoom].copy()

print(zoom_df.shape[0], 'results with zoom information')
if zoom_df.shape[0]>0:
    zoom_df['zoom'] = zoom_df.apply(
        lambda c_row: [dict(y, img_idx=c_row.name)
                       for y in c_row['viewing_info_dict']['zoom']
                       if ('x' in y) and ('y' in y)],
        axis=1
    )
    zooms_df = pd.DataFrame(list(chain(*zoom_df['zoom'].values.tolist())))
    zooms_df['img_cat'] = pd.factorize(zooms_df['img_idx'])[0]
else:
    zooms_df = pd.DataFrame(dict(x=[], y=[], img_idx=[], img_cat=[]))

from cornerstone_widget import get_bbox_handles
    
results_df['bbox_info'] = results_df['viewing_info_dict'].map(get_bbox_handles)

has_bbox = results_df['bbox_info'].map(lambda x: len(x)>0)
bbox_df = results_df[has_bbox].copy()
if sum(has_bbox)>0:
    bbox_df['zoom'] = bbox_df.apply(
        lambda c_row: [dict(y, img_idx=c_row.name)
                       for y in c_row['bbox_info']
                       if ('x' in y) and ('y' in y)],
        axis=1
    )
    bbox_df = pd.DataFrame(list(chain(*bbox_df['zoom'].values.tolist())))
    bbox_df['img_cat'] = pd.factorize(bbox_df['img_idx'])[0]
    
    if bbox_df.shape[0]>0:
        zooms_df = pd.concat([zooms_df, bbox_df], 0, sort=False)   
from matplotlib.patches import Rectangle
for col_name in 'xy':
    for i in range(2):
        zooms_df['{}_{}'.format(col_name, i)] = zooms_df[col_name].map(
            lambda x: x[i])
    zooms_df['d_{}'.format(col_name, i)] = zooms_df[col_name].map(
        lambda x: max(x)-min(x))
fig, ax1 = plt.subplots(1, 1, figsize=(10, 10))
if zooms_df.shape[0]>1:
    ax1.set_xlim([zooms_df['x_0'].min(), zooms_df['x_1'].max()])
    ax1.set_ylim([zooms_df['y_0'].min(), zooms_df['y_1'].max()])
max_cat = zooms_df['img_cat'].max()


for c_idx, c_rows in zooms_df.groupby('img_idx'):
    bonus_args = {'label': c_idx}
    for _, c_row in c_rows.iterrows():
        ax1.add_patch(Rectangle(xy=(c_row['x_0'], c_row['y_0']),
                                width=c_row['d_x'], height=c_row['d_y'],
                                color=plt.cm.hot(c_row['img_cat']/max_cat),
                                **bonus_args,
                                alpha=0.1))
        bonus_args = {}
ax1.set_title('Map of Zoom Regions')
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
ax1.hist((zooms_df['d_x']*zooms_df['d_y'])/1e3, bins=20)
ax1.set_title('kilopixels examined')
for c_idx, c_rows in zooms_df.groupby('img_idx'):
    ax2.semilogy((c_rows['d_x'].values*c_rows['d_y'].values),
                 label='Img: {}'.format(c_idx))
ax2.set_xlabel('Interaction Step')
ax2.set_ylabel('Pixels Visible')
ax2.set_title('Pixels visible vs Interaction Count')
fig, m_axs = plt.subplots(3, 3, figsize=(15, 15), dpi=300)
[c_ax.axis('off') for c_ax in m_axs.flatten()]
for c_ax, (img_idx, zoom_rows) in zip(m_axs.flatten(),
                                      zooms_df.groupby('img_idx')):
    img_row = results_df.loc[img_idx]
    c_img_path = os.path.join(task_data.base_img_dir, img_row[task_data.image_key_col])
    img_data = ja.utils.load_image_multiformat(c_img_path)
    c_ax.imshow(img_data)
    for idx, (_, c_row) in enumerate(zoom_rows.iterrows()):
        c_ax.add_patch(Rectangle(xy=(c_row['x_0'], 
                                     img_data.shape[1]-c_row['y_1']),
                                 width=c_row['d_x'], height=c_row['d_y'],
                                 color=plt.cm.viridis(idx/zoom_rows.shape[0]),
                                 label=str(idx),
                                 linewidth=2,
                                 fill=False,
                                 alpha=0.5))
    c_ax.legend()
    legend = c_ax.legend(loc='upper right', shadow=True, frameon=True)
    # black on black is hard to read
    legend.get_frame().set_facecolor('#FFFFFF')
    c_ax.set_title(
        '{annotator}, Task:{task}\nTime: {viewing_time:2.1f}s, Correct: {correct}'.format(**img_row))<jupyter_output><empty_output><jupyter_text>Other Fun GraphsJust a few other graphs to look at<jupyter_code>sns.lmplot(x='time',
           y='viewing_time',
           col='annotator_name',
           sharex=False,
           sharey=False,
           col_wrap=4,
           ci=None,
           data=results_df)
fig, ax1 = plt.subplots(1, 1, figsize=(20, 5))
sns.swarmplot(x='annotator_name',
              y='correct',
              hue='answer',
              size=5,
              ax=ax1,
              data=results_df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15660456646589158, 'fraction_numerical': 0.012396694214876033, 'mean_word_length': 3.0925766695328174, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '561447', 'license': 'Apache-2.0', 'n_tokens_mistral': 5061, 'n_tokens_neox': 4847, 'n_words': 880, 'path': 'notebooks/dashboard.ipynb', 'repo_name': 'betatim/jupyanno', 'size': '14278'}