<jupyter_start><jupyter_text>Base data setup from databases<jupyter_code># new module data_config imported by data.py as well as Cluster.py
import data_config
data_config.report_correct = True
from data import *
continents
# continents + Middle_East
for cc in countries_owid:
    print(cc,continent[cc])
set([continent[cc] for cc in countries_owid])
countries_in_continent<jupyter_output><empty_output><jupyter_text>Data saveExecute this section once to produce file `data_all_base.pk` in directory pks.<jupyter_code># miscnms = ['clusdata_all','cases','datasets','contact_dic','age_group_dic']
miscnms = ['acute_dict','age_group_dic','base','contact_dic','final_date','gdp_per_capita_owid',
 'icu_dict','owid_file','stringency_owid','tests_owid','translate_age','translate_contact']
deathnms = [x for x in dir() if 'deaths' in x]
casenms = [x for x in dir() if 'cases' in x if not callable(eval(x))]
covidnms = [x for x in dir() if 'covid' in x]
popnms = [x for x in dir() if 'population' in x]
testnms = [x for x in dir() if 'testing' in x if not callable(eval(x))]
recovnms = [x for x in dir() if 'recovered' in x]
confnms = [x for x in dir() if 'confirmed' in x]
countrynms = [x for x in dir() if 'countr' in x and not callable(eval(x))]
countrynms = [x for x in dir() if 'countr' in x and (isinstance(eval(x),dict) or isinstance(eval(x),list) or isinstance(eval(x),tuple))]
continentnms = [x for x in dir() if 'continent' in x]

allnmsdb = countrynms + covidnms + miscnms + deathnms + confnms + recovnms + casenms + popnms + testnms + ['allnmsdb']
data_all = {nm:eval(nm) for nm in allnmsdb}

start = time()
pk.dump(data_all,open('./pks/data_all_base.pk','wb'))
print('elapsed: ',time()-start)
print(allnmsdb)
# the remaining list from dir() below should only contain temporary variables and system modules
[x for x in dir() if '__' not in x and not callable(eval(x)) and not x in allnmsdb and not x.startswith('_') and not x.endswith('nms')]<jupyter_output><empty_output><jupyter_text>Data LoadUse this code to read in the data, e.g. at the top of another notebook, as an alternative to loading data.py<jupyter_code># read in data
start=time()
print('reading in data...')
with open('./pks/data_all_base.pk','rb') as fp:
    foo = pk.load(fp)
print('elapsed: ',time()-start)

# make each element of the dictionary a global variable named with key:
for x in foo:
    stmp = x+"= foo['"+x+"']"
    exec(stmp)<jupyter_output>reading in data...
elapsed:  8.589014768600464
{'fraction_non_alphanumeric': 0.13708046906591184, 'fraction_numerical': 0.00646987464617873, 'mean_word_length': 5.686486486486486, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '509918', 'license': 'MIT', 'n_tokens_mistral': 928, 'n_tokens_neox': 896, 'n_words': 287, 'path': 'Notebooks/covid-19-caution/Data_base.ipynb', 'repo_name': 'ProtoLife/covid-recovery', 'size': '2473'}