<jupyter_start><jupyter_text>CORDIS Data IO Manual dataset inspection for writing functions to load and parse data.<jupyter_code>import pandas as pd
import json
pd.set_option('max_columns', 100)
fps = ['h2020', 'fp7', 'fp6', 'fp5', 'fp4', 'fp3', 'fp2', 'fp1']
project_read_opts = {'sep': ';', 'decimal': ',', 'parse_dates': ['startDate', 'endDate']}
project_parse_opts = {'list_sep': ';', 'list_cols': ['participants', 'participantCountries', 'programme'], 'drop_cols': ['subjects']}


h2020_org_read_opts = {'sep': ';', 'decimal': ',', 'parse_dates': ['lastUpdateDate']}
h2020_org_parse_opts = {'list_sep': ';', 'list_cols': ['programme'], 'drop_cols': []}

fp7_org_read_opts = {'sep': ';', 'decimal': ',', 'parse_dates': []}
fp7_org_parse_opts = {'list_sep': ';', 'list_cols': ['programme'], 'drop_cols': []}
read_opts = {'projects': {f: project_read_opts for f in fps},
             'organizations': {
                 'h2020': h2020_org_read_opts,
                 'fp7'
                              },
            }
parse_opts = {'projects': project_parse_opts,
             'organizations': org_parse_opts,
             }
import json
with open('../../data/aux/cordis_parse_opts.json', 'w') as f:
    json.dump(parse_opts, f)

with open('../../data/aux/cordis_read_opts.json', 'w') as f:
    json.dump(read_opts, f)
cordis_raw_params<jupyter_output><empty_output><jupyter_text>H2020  H2020 Projects<jupyter_code>h2020_projects = pd.read_csv('../../data/raw/cordis/h2020/raw/h2020_projects.csv', sep=';', decimal=',', 
                             parse_dates=['startDate', 'endDate'])
print(h2020_projects.shape)
h2020_projects.head(2)<jupyter_output><empty_output><jupyter_text>H2020 Orgs<jupyter_code>h2020_orgs = pd.read_csv('../../data/raw/cordis/h2020/h2020_organizations.csv', sep=';', decimal=',')
print(h2020_orgs.shape)
h2020_orgs.head(2)<jupyter_output><empty_output><jupyter_text>FP7  FP7 Projects<jupyter_code>fp7_projects = pd.read_csv('../../data/raw/cordis/fp7/raw/fp7_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp7_projects.shape)
fp7_projects.head(2)<jupyter_output><empty_output><jupyter_text>FP7 Orgs<jupyter_code>fp7_orgs = pd.read_csv('../../data/raw/cordis/fp7/fp7_organizations.csv', sep=';', decimal=',')
print(fp7_orgs.shape)
fp7_orgs.head(5)<jupyter_output><empty_output><jupyter_text>FP6<jupyter_code>fp6_projects = pd.read_csv('../../data/raw/cordis/fp6/raw/fp6_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp6_projects.shape)
fp6_projects.head(2)
fp6_orgs = pd.read_csv(
    '../../data/raw/cordis/fp6/fp6_organizations.csv', sep='\t', decimal=',')
print(fp6_orgs.shape)
fp6_orgs.head(5)<jupyter_output><empty_output><jupyter_text>FP5<jupyter_code>fp5_projects = pd.read_csv('../../data/raw/cordis/fp5/raw/fp5_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp5_projects.shape)
fp5_projects.head(2)<jupyter_output><empty_output><jupyter_text>FP4<jupyter_code>fp4_projects = pd.read_csv('../../data/raw/cordis/fp4/raw/fp4_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp4_projects.shape)
fp4_projects.head(2)<jupyter_output><empty_output><jupyter_text>FP3<jupyter_code>fp3_projects = pd.read_csv('../../data/raw/cordis/fp3/raw/fp3_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp3_projects.shape)
fp3_projects.head(2)<jupyter_output><empty_output><jupyter_text>FP2<jupyter_code>fp2_projects = pd.read_csv('../../data/raw/cordis/fp2/raw/fp2_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp2_projects.shape)
fp2_projects.head(2)<jupyter_output><empty_output><jupyter_text>FP1<jupyter_code>fp1_projects = pd.read_csv('../../data/raw/cordis/fp1/raw/fp1_projects.csv', sep=';', decimal=',',
                             parse_dates=['startDate', 'endDate'])
print(fp1_projects.shape)
fp1_projects.head(2)
rcn = []
for fp in [h2020_projects, fp7_projects, fp6_projects, fp5_projects, fp4_projects, fp3_projects, fp2_projects, fp1_projects]:
    rcn.extend(fp['rcn'].values)
import numpy as np
len(np.unique(rcn))
len(rcn)
set(fp5_projects['rcn']).intersection(set(fp6_projects['rcn']))
fp5_projects[~pd.isnull(fp5_projects.status)]
print(h2020_sdg.shape)
h2020_sdg.head()
pred_cols = [f'{str(i)}_pred' for i in range(1, 17)]
sdg_cols = [f'sdg_{str(i)}' for i in range(1, 17)]
h2020_projects = h2020_projects.set_index('rcn').merge(h2020_sdg.set_index('rcn')[pred_cols], 
                                      left_index=True, right_index=True, how='right')
h2020_projects = h2020_projects.rename(columns={old: new for old, new in zip(pred_cols, sdg_cols)})
h2020_projects[sdg_cols].sum().plot.barh()
h2020_projects.groupby(h2020_projects['startDate'].dt.year)[sdg_cols].sum().sum(axis=1)
h2020_projects.groupby(h2020_projects['startDate'].dt.year)[sdg_cols].sum().divide(
    h2020_projects.groupby(h2020_projects['startDate'].dt.year)['id'].count(), axis=0).plot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1866001151852563, 'fraction_numerical': 0.043962372816279516, 'mean_word_length': 4.776053215077606, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '291991', 'license': 'MIT', 'n_tokens_mistral': 2092, 'n_tokens_neox': 1964, 'n_words': 232, 'path': 'notebooks/dev/gr_01_cordis_data_io.ipynb', 'repo_name': 'nestauk/sdg-mapping', 'size': '5209'}