<jupyter_start><jupyter_text>Dataset preprocessing<jupyter_code>df = pd.read_csv('merged_mob_owid.csv', parse_dates=['date'], infer_datetime_format=True).set_index(['date','country_region']).swaplevel()
df.info()
# fill mobility data with 0
df.iloc[:,:6].fillna(0.0, inplace=True)
df.dropna(axis='columns', thresh = int(df.shape[0]*0.5), inplace=True)
df.drop(labels=['stringency_index', 'extreme_poverty', 'total_cases_per_million', 'new_cases_per_million', 'total_deaths_per_million','new_deaths_per_million'],axis='columns', inplace=True)


for c in df.index.levels[0]:
    if np.sum(np.isnan(df.loc[c].to_numpy())) > 0:
        df.drop(c, level=0, inplace=True)
#     if c in ['France', 'Germany', 'Greece', 'Estonia','Hong Kong', 'Hungary','Ireland','Japan', 'Lebanon', 'Mexico','Netherlands', 'Puerto Rico','Singapore','South Africa','Switzerland','United Arab Emirates']:
#         print(c)
#         print(df.loc[c].info())
#     print(c, np.sum(np.isnan(df.loc[c].to_numpy())))
    
    
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19922254616132168, 'fraction_numerical': 0.009718172983479106, 'mean_word_length': 6.62962962962963, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '378673', 'license': 'MIT', 'n_tokens_mistral': 412, 'n_tokens_neox': 391, 'n_words': 54, 'path': 'Untitled.ipynb', 'repo_name': 'hannanabdul55/neural-nets-research-playground', 'size': '1029'}