<jupyter_start><jupyter_text>IntroductionNotebook to analyse the PyBossa taskruns from the Expert App Version 2.  Load Libraries and Data<jupyter_code>from mod_finder_util import mod_finder_util
mod_finder_util.add_modules_origin_search_path()
import pandas as pd
import seaborn as sns
import numpy as np

import modules.utils.firefox_dataset_p2 as fd

from sklearn.metrics import cohen_kappa_score

taskruns = fd.TaskRuns.read_expert_taskruns_df()<jupyter_output>TaskRuns shape: (93, 11)<jupyter_text>Grouping Contributions By Time<jupyter_code>taskruns['created'] = pd.to_datetime(taskruns['created'], yearfirst=True)
taskruns['created_month'] = taskruns.apply(lambda row: row['created'].month, axis=1)
taskruns['created_day'] = taskruns.apply(lambda row: row['created'].day, axis=1)
taskruns['created_hour'] = taskruns.apply(lambda row: row['created'].hour, axis=1)

grouped_trs = taskruns.groupby(by=['created_month','created_day','created_hour']).count()
grouped_trs<jupyter_output><empty_output><jupyter_text>Count TaskRuns by Task<jupyter_code>cols = ['task_id','id']

df = taskruns[cols].groupby(by='task_id').count()
df.rename(columns={"id" :'count_trs'}, inplace=True)
display(df.tail(10))
print(df.shape)<jupyter_output><empty_output><jupyter_text>Check All Tasks Have At Least One Answer<jupyter_code>print(set(df.index) - set(range(1928,2020+1,1)))<jupyter_output>{1721, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813}<jupyter_text>Analysis of Taskruns Infos<jupyter_code>taskruns[['bug_id','answers']].head(10)
taskruns.dtypes<jupyter_output><empty_output><jupyter_text>Compare Order of TaskrunsThe order of the taskruns must be the same, ordered by the finish time or by the bug_id fields.<jupyter_code>taskruns_ordered_by_finish_time = taskruns.sort_values(by='finish_time')
taskruns_ordered_by_bug_id = taskruns.copy()  ## already ordered by bug_id

diffs = taskruns_ordered_by_finish_time.bug_id - taskruns_ordered_by_bug_id.bug_id
d = np.sum(diff)

assert d == 0<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13041749502982108, 'fraction_numerical': 0.15745526838966203, 'mean_word_length': 8.115942028985508, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '510054', 'license': 'MIT', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1001, 'n_words': 219, 'path': 'preprocessing/trs_analysis/exp_trs_analysis.ipynb', 'repo_name': 'guilhermemg/trace-links-tc-br', 'size': '2515'}