<jupyter_start><jupyter_text>Get summaries of a lot of projects<jupyter_code>summaries_file = os.path.join(get_data_dir(), 'summaries.json')
summaries = {} 
with open(summaries_file) as f:
    summaries = json.load(f)
for project_id in tqdm(range(5636, 5650)):
    if project_id not in summaries or summaries[project_id]['status'] != 'ARCHIVED':
        summaries[project_id] = download_summary_data(project_id)
        time.sleep(0.5 + random.random())
for project_id in tqdm(range(5650, 6100)):
    if project_id not in summaries or summaries[project_id]['status'] != 'ARCHIVED':
        summaries[project_id] = download_summary_data(project_id)
        time.sleep(0.5 + random.random())
with open(summaries_file, 'w') as outfile:
    json.dump(summaries, outfile)<jupyter_output><empty_output><jupyter_text>Data extraction and cleaning<jupyter_code>summary_df = pd.DataFrame()
for project_id in summaries:
    if 'Error' in summaries[project_id]:
        print(str(project_id) + ' : ' + summaries[project_id]['Error'])
        continue
    summary_df = pd.concat([summary_df, pd.DataFrame(data=[(project_id, summaries[project_id]['created'],
                                                            summaries[project_id]['lastUpdated'],
                                                            summaries[project_id]['status'],
                                                            summaries[project_id]['percentValidated'],
                                                            summaries[project_id]['organisationTag'],
                                                            summaries[project_id]['projectArea(in sq.km)'])],
                                                     columns=['project_id', 'created', 'lastUpdated', 'status',
                                                              'percentValidated', 'organisation', 'area'])],
                           axis=0, ignore_index=True)
summary_df = summary_df.set_index('project_id')
summary_df['created'] = pd.to_datetime(summary_df['created'])
summary_df['lastUpdated'] = pd.to_datetime(summary_df['lastUpdated'])
summary_df.info()
summary_df.head()<jupyter_output><empty_output><jupyter_text>Qualitative analysis<jupyter_code>summary_df.hist('percentValidated')
summary_df['status'].unique()
summary_df['organisation'].unique()
summary_df['organisation'] = summary_df['organisation'].apply(lambda s: 'MSF' if s=='Médecins Sans Frontières' else s)
summary_df['organisation'].unique()<jupyter_output><empty_output><jupyter_text>Restrict to S1 2019 Definitions of the restriction- lastUpdated after 1st January 2019- lastUpdated before 1st July 2019- validated > 90%<jupyter_code>summary_df[summary_df['organisation'] == 'CartONG']
summary_df[summary_df['organisation'] == 'AIT']
df = summary_df[(summary_df['lastUpdated'] > '2019-01-01') &
                (summary_df['lastUpdated'] < '2019-07-19') &
                (summary_df['percentValidated'] >= 90)]
df.head()
df['organisation'].unique()
pd.DataFrame(df.groupby('organisation').count().area.sort_values(ascending=False))
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), dpi=100, sharex=True)
g = df.groupby('organisation').count().area.sort_values()
ax.bar(np.arange(len(g)), g.values, color=['black' if index == 'CartONG' else 'lightgray' for index in g.index])
plt.xticks(np.arange(len(g)), g.index, rotation=90)
ax.set_xlabel('Organisations')
ax.set_ylabel('Number of finished projects')
ax.set_title('Number of finished projects in S1 2019')
plt.savefig('finished_project_nb.png', dpi=100)
df.groupby('organisation').sum().sort_values('area', ascending=False)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), dpi=100, sharex=True)
g = df.groupby('organisation').sum().sort_values('area')
ax.bar(np.arange(len(g)), g['area'], color=['black' if index == 'CartONG' else 'lightgray' for index in g.index])
plt.xticks(np.arange(len(g)), g.index, rotation=90)
ax.set_xlabel('Organisations')
ax.set_ylabel('Area in km²')
ax.set_title('Total area of finished projects in S1 2019')
plt.savefig('finished_project_total_area.png', dpi=100)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), dpi=100, sharex=True)
g = df.groupby('organisation').sum().sort_values('area')[:6]
ax.bar(np.arange(len(g)), g['area'], color=['black' if index == 'CartONG' else 'lightgray' for index in g.index])
plt.xticks(np.arange(len(g)), g.index, rotation=90)
ax.set_xlabel('Organisations')
ax.set_ylabel('Area in km²')
ax.set_title('[ZOOM IN] Total area of finished projects in S1 2019')
plt.savefig('finished_project_total_area_bis.png', dpi=100)
df.groupby('organisation').mean().sort_values('area', ascending=False)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), dpi=100, sharex=True)
g = df.groupby('organisation').mean().sort_values('area')
ax.bar(np.arange(len(g)), g['area'], color=['black' if index == 'CartONG' else 'lightgray' for index in g.index])
plt.xticks(np.arange(len(g)), g.index, rotation=90)
ax.set_xlabel('Organisations')
ax.set_ylabel('Area in km²')
ax.set_title('Mean area of finished projects in S1 2019')
plt.savefig('finished_project_mean_area.png', dpi=100)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), dpi=100, sharex=True)
g = df.groupby('organisation').mean().sort_values('area')[:4]
ax.bar(np.arange(len(g)), g['area'], color=['black' if index == 'CartONG' else 'lightgray' for index in g.index])
plt.xticks(np.arange(len(g)), g.index, rotation=90)
ax.set_xlabel('Organisations')
ax.set_ylabel('Area in km²')
ax.set_title('[ZOOM IN] Mean area of finished projects in S1 2019')
plt.savefig('finished_project_mean_area_bis.png', dpi=100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15885049365303244, 'fraction_numerical': 0.026974612129760225, 'mean_word_length': 4.34180790960452, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '111095', 'license': 'MIT', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1868, 'n_words': 348, 'path': 'notebooks/HOT_global_stats.ipynb', 'repo_name': 'NicolasGrosjean/tasking_manager_stats', 'size': '5672'}