<jupyter_start><jupyter_text>Change to your local directory where data files are copied<jupyter_code>import pandas as pd
import os

os.chdir("./data")
os.getcwd()<jupyter_output><empty_output><jupyter_text>Read and filter HDI (Human Development Index) file<jupyter_code>filename = "hdi/Human Development Index (HDI).csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='latin-1') 

df = df[df.columns.drop(list(df.filter(regex='Unnamed')))]

df_hdi = pd.melt(df, id_vars=['Country', "HDI Rank"], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019'],\
                                            var_name='Year', value_name='HDI')
df_hdi.head()
# Many countries have ".." or null values for HDI. Those countries are filtered out.

missing_hdi_countries = df_hdi[(df_hdi["HDI"]== "..") | (df_hdi["HDI"].isnull())].Country.unique()
missing_hdi_countries
# Remove countries with missing HDI values

df_hdi = df_hdi[~df_hdi["Country"].isin(missing_hdi_countries)]

# Remove entries which HDI Rank is not null

df_hdi = df_hdi[df_hdi["HDI Rank"].notnull()]
df_hdi = df_hdi[["Country", "Year", "HDI"]]
df_hdi["Country"] = df_hdi["Country"].str.strip()
df_hdi[df_hdi["Country"] == "United States"].head()<jupyter_output><empty_output><jupyter_text>Read and filter Inflation file<jupyter_code>filename = "inflation/API_FP.CPI.TOTL.ZG_DS2_en_csv_v2_2917215.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8') 
df = df.drop(columns = ["Country Code", "Indicator Name", "Indicator Code"])
df.rename(columns = {'Country Name': 'Country'}, inplace = True)


df_inflation = pd.melt(df, id_vars=['Country'], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019', '2020'],\
                                            var_name='Year', value_name='Inflation')
df_inflation.head()<jupyter_output><empty_output><jupyter_text>Read and filter Infant Mortality file<jupyter_code>filename = "infant_mortality/fusion_CME_UNICEF_1.0_all.csv"
df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df = df[(df['INDICATOR:Indicator'] =='CME_MRY0: Infant mortality rate') \
        & (df['SEX:Sex'] == '_T: Total')\
        & (df['TIME_PERIOD:Time period'] > 1989)]

df = df[['REF_AREA:Country', 'TIME_PERIOD:Time period', 'OBS_VALUE:Observation Value']]
df = df.rename(columns = {"REF_AREA:Country" : "Country", \
                          "TIME_PERIOD:Time period" : "Year", \
                          "OBS_VALUE:Observation Value" : "Infant_Mortality"})

df['Country'] = df['Country'].str[5:]

df_infant_mortality = df.copy()
df_infant_mortality.head()<jupyter_output><empty_output><jupyter_text>Read and filter GDP (Gross Domestic Product) file<jupyter_code>filename = "gdp/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_2916517.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df = df.drop(columns = ["Country Code", "Indicator Name", "Indicator Code"])
df = df.rename(columns = {"Country Name": "Country"})

df_gdp = pd.melt(df, id_vars=['Country'], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019', '2020'],\
                                            var_name='Year', value_name='GDP')

df_gdp.head()<jupyter_output><empty_output><jupyter_text>Read and filter GNI (Gross National Income) file<jupyter_code>filename = "gni/API_NY.GNP.PCAP.CD_DS2_en_csv_v2_2924224.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df = df.drop(columns = ["Country Code", "Indicator Name", "Indicator Code"])
df = df.rename(columns = {"Country Name": "Country"})

df_gni = pd.melt(df, id_vars=['Country'], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019', '2020'],\
                                            var_name='Year', value_name='GNI')

df_gni.head()<jupyter_output><empty_output><jupyter_text>Read and filter Forest Area file<jupyter_code>filename = "forest_area/API_AG.LND.FRST.ZS_DS2_en_csv_v2_2917381.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df = df.drop(columns = ["Country Code", "Indicator Name", "Indicator Code"])
df = df.rename(columns = {"Country Name": "Country"})

df_forest = pd.melt(df, id_vars=['Country'], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019', '2020'],\
                                            var_name='Year', value_name='Forest_Area')

df_forest.head()
# Filter out countries that have Forest_Area as 0 or blank in 1990. Eg Nauru, Qatar, Luxembourg

missing_forest_countries = df_forest[(df_forest["Forest_Area"] == 0) | (df_forest["Forest_Area"].isnull())].Country.unique()
missing_forest_countries

df_forest = df_forest[~df_forest["Country"].isin(missing_forest_countries)]
df_forest.head()<jupyter_output><empty_output><jupyter_text>Read and filter Population file<jupyter_code>filename = "population/API_EN.POP.DNST_DS2_en_csv_v2_2917301.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df = df.drop(columns = ["Country Code", "Indicator Name", "Indicator Code"])
df = df.rename(columns = {"Country Name": "Country"})

df_pop = pd.melt(df, id_vars=['Country'], value_vars=['1990', '1991', '1992', '1993', '1994', '1995', '1996', \
                                             '1997', '1998', '1999', '2000', '2001', '2002', '2003', \
                                             '2004', '2005', '2006', '2007', '2008', '2009', '2010', \
                                             '2011', '2012', '2013', '2014', '2015', '2016', '2017', \
                                             '2018', '2019', '2020'],\
                                            var_name='Year', value_name='Population')

df_pop.head()<jupyter_output><empty_output><jupyter_text>Read and filter Unemployment file<jupyter_code>filename = "unemployment/unemployment-rate.csv"

df = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8', low_memory= False) 
df_unemp = df.drop('Code',axis=1)\
    .rename(columns={'Entity':'Country', 
    'Unemployment, total (% of total labor force) (modeled ILO estimate)':'Unemployment'})

df_unemp.head()
# Strip whitespaces from country

df_hdi['Country'] = df_hdi['Country'].str.strip()
df_inflation['Country'] = df_inflation['Country'].str.strip()
df_infant_mortality['Country'] = df_infant_mortality['Country'].str.strip()
df_gdp['Country'] = df_gdp['Country'].str.strip()
df_gni['Country'] = df_gni['Country'].str.strip()
df_forest['Country'] = df_forest['Country'].str.strip()
df_pop['Country'] = df_pop['Country'].str.strip()
df_unemp['Country'] = df_unemp['Country'].str.strip()



# Convert year column to string in all factor dataframes

df_hdi["Year"] = df_hdi["Year"].astype('str')
df_inflation["Year"] = df_inflation["Year"].astype('str')
df_infant_mortality["Year"] = df_infant_mortality["Year"].astype('str')
df_gdp["Year"] = df_gdp["Year"].astype('str')
df_gni["Year"] = df_gni["Year"].astype('str')
df_forest["Year"] = df_forest["Year"].astype('str')
df_pop["Year"] = df_pop["Year"].astype('str')
df_unemp["Year"] = df_unemp["Year"].astype('str')

df_hdi['Country'] = df_hdi['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_inflation['Country'] = df_inflation['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_infant_mortality['Country'] = df_infant_mortality['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_gdp['Country'] = df_gdp['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_gni['Country'] = df_gni['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_forest['Country'] = df_forest['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_pop['Country'] = df_pop['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_unemp['Country'] = df_unemp['Country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

# Merge all the factor dataframes

temp_df = pd.merge(df_hdi, df_inflation, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_infant_mortality, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_gdp, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_gni, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_forest, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_pop, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])
temp_df = pd.merge(temp_df, df_unemp, how = "outer", left_on = ["Country", "Year"], right_on = ["Country", "Year"])

# temp_df.count
# len(temp_df["Country"].unique())
# temp_df.loc[temp_df["Country"] == "India"].head()
# Country list downloaded from https://unstats.un.org/unsd/methodology/m49/overview/

filename = "./country/UNSD - Country_List.csv"

df_country = pd.read_csv(filename, header = 0, skiprows=0, encoding='utf-8') 
df_country = df_country[["Country or Area", "Region Name", "Developed / Developing Countries"]]
df_country = df_country.rename(columns = {"Country or Area" : "Country", \
                          "Region Name" : "Continent", \
                          "Developed / Developing Countries" : "Development_Status"})

df_country = df_country.reset_index()
df_country = df_country.rename(columns={"index":"Country_ID"})
df_country.head()
df_final = pd.merge(temp_df, df_country, how = "outer", left_on = ["Country"], right_on = ["Country"])
df_final.head()
# Data imputation for missing values

df_final = df_final.ffill().bfill()
df_final.columns=[str(i).lower() for i in df_final.columns]

df_final.head()
# Convert all columns in df_country to lower case
df_country.columns=[str(i).lower() for i in df_country.columns]

# Convert all str columns in df_country to upper case and write to file
df_country.apply(lambda x: x.astype(str).str.upper()).to_csv('data_countries.csv', index=False)

# Convert all columns to lower case in df_final
df_final.columns=[str(i).lower() for i in df_final.columns]
df_final["hdi"] = df_final["hdi"].astype("float")
df_final.dtypes<jupyter_output><empty_output><jupyter_text>Calculate %age change in factors on a YoY basis<jupyter_code>exclude_list = ['inflation', 'forest_area', 'unemployment']

for col in df_final.columns:
    if (df_final.dtypes[col] == "float64") and (col not in exclude_list):
        temp = ''
        temp = col + "_pct_change"
        df_final['shifted']=df_final.sort_values(by='year',ascending=True).groupby(['country'])[col].shift(1)
        df_final[temp]=(df_final[col] - df_final['shifted']) / df_final[col]
        df_final.drop('shifted',axis=1,inplace=True)
df_final = df_final[['year', 'hdi_pct_change', 'inflation', 'infant_mortality_pct_change', 'gdp_pct_change', 'gni_pct_change', 'population_pct_change', 'unemployment', 'country_id', 'forest_area']]        

# Inflation, Unemployment and forest area are already in %age. So no need to calculate %age YoY change

df_final["forest_area"] = df_final["forest_area"] / 100
df_final["inflation"] = df_final["inflation"] / 100
df_final["unemployment"] = df_final["unemployment"] / 100

# Rename percentage change columns
df_final.rename(columns = {'hdi_pct_change' : 'hdi',
                           'infant_mortality_pct_change' : 'infant_mortality',
                           'gdp_pct_change' : 'gdp', 
                           'gni_pct_change' : 'gni', 
                           'forest_area' : 'forest', 
                           'population_pct_change' : 'population'}, inplace= True)

# Add a column called "info" which will store actual / predicted value
df_final['info'] = 'actual'

df_final = df_final[["year", "hdi", "inflation", "infant_mortality", "gdp","gni", 
                     "forest", "population", "unemployment","country_id", "info" ]]

# filter out 1990 as it is the first year and has not percentage change associated with it

df_final = df_final[df_final['year'] != "1990"]

# df_final[df_final['country_id'] == 142].head()
# import matplotlib.pyplot as plt
# import seaborn as sns

# df_plot = df_final[["hdi", "gdp", "gni", "inflation", "infant_mortality", "unemployment", "population"]]

# fig = plt.figure(figsize = (12,8))
# sns.boxplot(data=df_plot)
# plt.show()
# filter for outliers - countries that have unusually high values for KPIs.

df_final = df_final[(df_final["inflation"] > -10) & (df_final["inflation"] < 10) & \
        (df_final["gdp"] > -100) & (df_final["gdp"] < 100) & \
        (df_final["gni"] > -100) & (df_final["gni"] < 100) &\
        (df_final["population"] > -100) & (df_final["population"] < 100)]

df_final.head()
# Sample time series forecast for country_id = 116

import matplotlib.pyplot as plt 
from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt

x = df_final.copy()
x = x[x["country_id"] == 116]
x = x[["year", "forest"]]
# len(x)

train=x[1:22] 
test= x[21:]

# train

train.Timestamp = pd.to_datetime(train.year,format='%Y') 
train.index = train.Timestamp 
train = train.resample('Y').mean() 

test.Timestamp = pd.to_datetime(test.year,format='%Y') 
test.index = test.Timestamp
test = test.resample('Y').mean()

train.forest.plot(figsize=(15,8), title= 'Train', fontsize=14)
test.forest.plot(figsize=(15,8), title= 'Test', fontsize=14)
plt.show()
# Fitting the model for country_id = 116
import numpy as np
y_hat_avg = test.copy()
fit1 = Holt(np.asarray(train['forest'])).fit(smoothing_level = 0.9)

# y_hat_avg['SES'] = fit2.forecast(len(test))
y_hat_avg['Holt_linear'] = fit1.forecast(len(test))

# y_hat_avg
plt.figure(figsize=(16,8))
plt.plot(train['forest'], label='Train')
plt.plot(test['forest'], label='Test')
plt.plot(y_hat_avg['Holt_linear'], label='Holt_linear')
plt.legend(loc='best')
plt.show()
# Calculate accuracy of fit

from sklearn.metrics import mean_squared_error
from math import sqrt
rms = sqrt(mean_squared_error(test.forest, y_hat_avg.Holt_linear))
print(rms)
# Predict for year 2021 to 2025
data = {'years': [2021,2022,2023,2024,2025], 'forecast': [0,0,0,0,0]}
predict = pd.DataFrame(data=data)
predict['forecast'] = fit1.forecast(len(predict))
predict<jupyter_output><empty_output><jupyter_text>Forest cover Prediction for next 5 years for all countries based on Holt Winter model<jupyter_code># Predict next 5 years score for all countries based on Holt Winter model 
country_list = []
country_list =  list(df_final.country_id.unique())

for country_id in country_list:
    
    data = {'year': [2021,2022,2023,2024,2025], 'forest': [0,0,0,0,0]}
    predict = pd.DataFrame(data=data)
    print(country_id)
    
    x = df_final.copy()
    x = x[x["country_id"] == country_id]
    x = x[["year", "forest"]]

    train=x[1:25] 
    test= x[24:]

    train.Timestamp = pd.to_datetime(train.year,format='%Y') 
    train.index = train.Timestamp 
    train = train.resample('Y').mean() 

    test.Timestamp = pd.to_datetime(test.year,format='%Y') 
    test.index = test.Timestamp
    test = test.resample('Y').mean()
    
    y_hat_avg = test.copy()
#     fit2 = SimpleExpSmoothing(np.asarray(train['forest_area'])).fit(smoothing_level=0.75,optimized=False)
#     y_hat_avg['SES'] = fit2.forecast(len(test))
    fit1 = Holt(np.asarray(train['forest'])).fit(smoothing_level = 0.8,smoothing_slope = 0.8)
    y_hat_avg['Holt_linear'] = fit1.forecast(len(test))
    

    predict['country_id'] = country_id
    predict['info'] = 'predicted'
#     predict['forest_area'] = fit2.forecast(len(predict))
    predict['forest'] = fit1.forecast(len(predict))
    
    
    df_final = pd.concat([df_final, predict], join = "outer")
    
    
    

# Round to forest cover % and KPI %age to 4  decimal places

for col in df_final.columns:
    if df_final.dtypes[col] == "float64" :
        df_final[col] = df_final[col].round(decimals = 4)

df_final["country_id"] = df_final["country_id"].astype(int)

df_final.tail()
# Write to measurements file

df_final.to_csv('data_measurements.csv', index=False)<jupyter_output><empty_output><jupyter_text>Web Scraping to update income classification for all countries (df_country)<jupyter_code>from urllib.request import urlopen
url = "https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups"
page = urlopen(url)
page
html_bytes = page.read()
html = html_bytes.decode("utf-8")
# print(html)
# LOW-INCOME ECONOMIES ($1,045 OR LESS)
low_income_start_index = html.find("LOW-INCOME ECONOMIES\n($1,045 OR LESS)")
low_income_start_index

low_income_end_index = html.find("LOWER-MIDDLE INCOME")
low_income_end_index

# html[low_income_start_index: low_income_end_index]
# Using regular expression to parse countries
import regex as re

pattern = re.compile(r"""<td[^>]*>(?:<[^>]*>|&\w*;|\\u[a-f\d]*)*((?:[^<&\\]+|\\n)+)""", re.VERBOSE)

matches = re.finditer(pattern, html[low_income_start_index: low_income_end_index])
low_income_countries = []
for match in matches:
    low_income_countries.append(match[1].upper())

low_income_countries = sorted(low_income_countries)
low_income_countries = low_income_countries[2:-1]
print(low_income_countries)
# LOWER-MIDDLE INCOME ECONOMIES ($1,046 TO $4,095)
low_middle_income_start_index = html.find("LOWER-MIDDLE INCOME")
low_middle_income_start_index
low_middle_income_end_index = html.find("UPPER-MIDDLE-INCOME")
low_middle_income_end_index
# html[low_middle_income_start_index: low_middle_income_end_index]
matches = re.finditer(pattern, html[low_middle_income_start_index: low_middle_income_end_index])
low_middle_income_countries = []
for match in matches:
    low_middle_income_countries.append(match[1].upper())

low_middle_income_countries = sorted(low_middle_income_countries)
low_middle_income_countries = low_middle_income_countries[1:] 
print(low_middle_income_countries)
# UPPER-MIDDLE-INCOME ECONOMIES ($4,096 TO $12,695)
upper_middle_income_start_index = html.find("UPPER-MIDDLE-INCOME")
upper_middle_income_start_index
upper_middle_income_end_index = html.find("HIGH-INCOME ECONOMIES")
upper_middle_income_end_index
# html[upper_middle_income_start_index: upper_middle_income_end_index]
matches = re.finditer(pattern, html[upper_middle_income_start_index: upper_middle_income_end_index])
upper_middle_income_countries = []
for match in matches:
    upper_middle_income_countries.append(match[1].upper())

upper_middle_income_countries = sorted(upper_middle_income_countries)
upper_middle_income_countries = upper_middle_income_countries[ 1:-1]
print(upper_middle_income_countries)
# HIGH-INCOME ECONOMIES ($12,696 OR MORE)
high_income_start_index = upper_middle_income_end_index
high_income_start_index
high_income_end_index = html.find("<br>IDA</b>")
high_income_end_index
# html[high_income_start_index:high_income_end_index]
matches = re.finditer(pattern, html[high_income_start_index: high_income_end_index])
high_income_countries = []
for match in matches:
    high_income_countries.append(match[1].upper())

high_income_countries = sorted(high_income_countries)
high_income_countries = high_income_countries[2:]
print(high_income_countries)
# Concatentate all income class countries into one dictionary and convert to a dataframe

dict_income_class = {}
for c in low_income_countries:
        dict_income_class[c] = "Low Income"
for c in low_middle_income_countries:
        dict_income_class[c] = "Low Middle Income"
for c in upper_middle_income_countries:
        dict_income_class[c] = "Upper Middle Income"
for c in high_income_countries:
        dict_income_class[c] = "High Income"

df_income_class = pd.DataFrame(dict_income_class.items(), columns=['country', 'income'])
df_income_class
df_country["country"] = df_country["country"].str.upper()
df_country
# Standardize df_income_class and df_country for country names

df_income_class['country'] = df_income_class['country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())

df_country['country'] = df_country['country'].map(lambda x: x.replace("\n", "").replace("  ", " ")\
                                                            .split("(", 1)[0].split(", T", 1)[0]\
                                                            .split(", A", 1)[0]\
                                                            .split(", I", 1)[0].strip())


# Standardize country names where the names dont match between the country file and scrapped country name

change_country_list = {"HONG KONG SAR, CHINA" : "HONG KONG, CHINA",
                        "KOREA, DEM. PEOPLE'S REP" : "DEMOCRATIC PEOPLE'S REPUBLIC OF KOREA",
                        "KOREA, REP." : "REPUBLIC OF KOREA",
                        "YEMEN, REP." : "YEMEN",
                        "CONGO, DEM. REP" : "DEMOCRATIC REPUBLIC OF THE CONGO",
                        "CONGO, REP." : "CONGO",
                        "VIETNAM" :"VIET NAM",
                        "TANZANIA" : "UNITED REPUBLIC OF TANZANIA",
                        "KYRGYZ REPUBLIC" : "KYRGYZSTAN",
                        "MOLDOVA" : "REPUBLIC OF MOLDOVA",
                        "CZECH REPUBLIC" :"CZECHIA",
                        "SLOVAK REPUBLIC" : "SLOVAKIA",
                        "LAO PDR":"LAO PEOPLE'S DEMOCRATIC REPUBLIC"
}

for country, correct_country in change_country_list.items():
    df_income_class.loc[df_income_class["country"] == country, "country"] = change_country_list[country]
# Add country names to country file which are missing in the website
extra_countries = list({37, 75, 80, 81, 83, 87, 161, 162, 238, 240})
extra_countries

for c in list(df_country[df_country['country_id'].isin(extra_countries)]["country"]):
    df_income_class = df_income_class.append({'country': c, 'income' : 'Not Available' }, ignore_index=True)
    
df_income_class[df_income_class["income"].str.contains("Not")]
# Delete ids for which there are no entries in measurement file

remove_from_df_country = [69, 84, 97, 123, 143]
df_country = df_country[~df_country['country_id'].isin(remove_from_df_country)]
df_country
df_country = df_country[["country_id", "country", "continent", "development_status"]]
df_country = pd.merge(df_country, df_income_class, on = "country", how = "inner")

# Remove extra countries for which data is not available for analysis

df_country = df_country[~df_country["country_id"].isnull()]

df_country[df_country["country"].str.contains("RUS")]
df_country.to_csv('data_countries.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1628210872162485, 'fraction_numerical': 0.04316009557945042, 'mean_word_length': 2.908507223113965, 'pattern_counts': {'":': 6, '<': 58, '<?xml version=': 0, '>': 60, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 129, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '535114', 'license': 'MIT', 'n_tokens_mistral': 9630, 'n_tokens_neox': 8576, 'n_words': 1727, 'path': 'CODE/Project_Wrangling.ipynb', 'repo_name': 'sunsmasher16/green-analytics', 'size': '26784'}