<jupyter_start><jupyter_text>|  name  |  grouping | period    |granularity| n_X  |countries|   contents                          |meta cols| comments                                        || -------|-----------|-----------|---- -----|-------|---------|-------------------------------------|---------|---------------------------------------------------------------------|| ECON  | country   |1960-2016   |yearly    | 128   | 218     | Economic data                       | 4 (62)  | Also contains country grouping                           | HNP   | country   |1960-2016   |yearly    | 39    | 218     | Health and population statistics    | 4 (62)  | Also contains country grouping                                || POP   | country   |1960-2016   |yearly    | 39    | 218     | Population statistics               | 4 (62)  | Also contains country grouping<jupyter_code>pd.options.display.float_format = '{:20,.2f}'.format
raw_df = pd.read_csv('world_bank/ECON.csv', sep=',')
raw_df.head(50)
raw_df.size
econ_indicators = set(raw_df['Series Name'].values.tolist())
len(econ_indicators)
print('\033[1mEconomic indicators\033[0m\n\n%s' % '\n'.join(list(map(lambda x: str(x), econ_indicators))))
hnp_indicators = set(raw_df['Series Name'].values.tolist())
len(hnp_indicators)
print('Health and population indicators\n\n%s' % hnp_indicators)
pop_indicators = set(raw_df['Series Name'].values.tolist())
len(pop_indicators)
print('Population indicators\n\n%s' % pop_indicators)
wb_indicators = econ_indicators | hnp_indicators | pop_indicators
len(wb_indicators)
print('World bank annual indicators\n\n%s' % wb_indicators)
country_codes = set(raw_df['Country Name'].values.tolist())
len(country_codes)
country_groups = ('nan', 'Lower middle income', 'Post-demographic dividend', 'High income', 'Pre-demographic dividend',
                  'East Asia & Pacific (IDA & IBRD countries)', 'Europe & Central Asia (excluding high income)',
                  'Heavily indebted poor countries (HIPC)', 'Caribbean small states', 'Pacific island small states',
                  'Middle income', 'Late-demographic dividend', 'OECD members', 'IDA & IBRD total', 'Not classified',
                  'East Asia & Pacific (excluding high income)', 'Latin America & the Caribbean (IDA & IBRD countries)',
                  'Low income', 'Low & middle income', 'IDA blend', 'IBRD only', 'Sub-Saharan Africa (excluding high income)', 
                  'Fragile and conflict affected situations', 'Europe & Central Asia (IDA & IBRD countries)', 'Euro area',
                  'Other small states', 'Europe & Central Asia', 'Arab World', 'Latin America & Caribbean (excluding high income)',
                  'Sub-Saharan Africa (IDA & IBRD countries)', 'Early-demographic dividend', 'IDA only', 'Small states',
                  'Middle East & North Africa (excluding high income)', 'East Asia & Pacific', 'South Asia', 'European Union',
                  'Least developed countries: UN classification', 'Middle East & North Africa (IDA & IBRD countries)',
                  'Upper middle income', 'South Asia (IDA & IBRD)', 'Central Europe and the Baltics', 'Sub-Saharan Africa',
                  'Latin America & Caribbean', 'Middle East & North Africa', 'IDA total', 'North America',
                  'Last Updated: 11/14/2018', 'Data from database: World Development Indicators')
len(country_groups)
countries = list(filter(lambda c: str(c) not in country_groups, country_codes))
len(countries)
print(countries)<jupyter_output>['Cayman Islands', 'Guam', 'Trinidad and Tobago', 'Portugal', 'Macedonia, FYR', 'Macao SAR, China', 'Oman', 'Romania', 'Namibia', 'Marshall Islands', 'Dominica', 'Dominican Republic', 'Rwanda', 'Turkey', 'Djibouti', 'Lithuania', 'Northern Mariana Islands', 'Lesotho', 'Niger', 'United Arab Emirates', 'Gabon', 'Morocco', 'Cambodia', 'Croatia', 'Venezuela, RB', "Cote d'Ivoire", 'Bolivia', 'El Salvador', 'Equatorial Guinea', 'St. Lucia', 'Aruba', 'Iceland', 'Mongolia', 'Pakistan', 'Slovenia', 'Ghana', 'Vietnam', 'Puerto Rico', 'Tonga', 'Argentina', 'Belgium', 'Bosnia and Herzegovina', 'Kosovo', 'Albania', 'St. Kitts and Nevis', 'Korea, Rep.', 'Nicaragua', 'Bulgaria', 'Spain', 'South Sudan', 'United States', 'Honduras', 'Congo, Dem. Rep.', 'Jamaica', 'Philippines', 'Afghanistan', 'New Caledonia', 'Sierra Leone', 'Suriname', 'Greece', 'Kiribati', 'France', 'Uzbekistan', 'Estonia', 'Gibraltar', 'United Kingdom', 'Jordan', 'San Marino', 'Singapore', 'Tunisia', 'Austria', 'Guyana', 'Finland', '[...]
{'fraction_non_alphanumeric': 0.18308611910560108, 'fraction_numerical': 0.015497011290679655, 'mean_word_length': 3.0339285714285715, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '341722', 'license': 'MIT', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1325, 'n_words': 387, 'path': 'dataprep/world_bank_datadictionary.ipynb', 'repo_name': 'braadbaart/macroeconomics', 'size': '4517'}