<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
%matplotlib inline
from google.colab import files
uploads = files.upload()
data = pd.read_csv('Health Systems Dataset.csv')
# data.describe()
data.info()
data.head()<jupyter_output><empty_output><jupyter_text>**Data Cleaning**<jupyter_code># Drop useless columns
df = pd.DataFrame(data)
df = data.drop(['Country_Region', 'Province_State'], axis = 1)
df.head()
# Dealing with NAN values
# We can see that most of the columns have null values (True)
df.isnull().any()<jupyter_output><empty_output><jupyter_text>**To deal with null values, we need to fill them with a measure of central tendency. We need to find out which measure is the best to use.**<jupyter_code>from scipy.stats import norm
fig, axes = plt.subplots(5, 3, figsize=(15, 25))
fig.suptitle('Data Distribution Visualization', fontsize=20)

for i, col in enumerate(df.columns[1:]):
  ax=fig.add_subplot(5, 3, i+1)
  sns.distplot(df[col],fit=norm, kde=False,ax=ax)
# plt.plot()<jupyter_output><empty_output><jupyter_text>**We can observe that our data is either skewed to the right or to the left & the median would be the most suitable measure of central tendency to replace with**<jupyter_code># Filling NAN values with their median
df = df.fillna(df.median())
df.isnull().any()<jupyter_output><empty_output><jupyter_text>**We now don't have null values as shown by bolean value, False**<jupyter_code>df.head()<jupyter_output><empty_output><jupyter_text>**Data Visualization** **We first sort the countries based on their total expenditures and compare the high spenders vs the low spenders**<jupyter_code>total_exp = df.sort_values('Health_exp_pct_GDP_2016', ascending = False)
total_exp.head()
top_ten_exp = total_exp.head(10)
total_exp = total_exp.sort_values('Health_exp_pct_GDP_2016')
bot_ten_exp = total_exp.head(10)
from plotly.subplots import make_subplots
import plotly.graph_objects as go
fig = make_subplots(rows=1, cols=2, shared_yaxes=True)

fig.add_trace(
    go.Bar(x=top_ten_exp['World_Bank_Name'], y=top_ten_exp['Health_exp_pct_GDP_2016']),
    row=1, col=1
)

fig.add_trace(
    go.Bar(x=bot_ten_exp['World_Bank_Name'], y=bot_ten_exp['Health_exp_pct_GDP_2016']),
    row=1, col=2
)

fig.update_layout(
    title={
        'text': "Top Ten & Bottom ten spenders",
        'y':0.9,
        'x':0.5,
        'xanchor': 'center',
        'yanchor': 'top'},
    plot_bgcolor= 'white',
    paper_bgcolor= 'white',
    yaxis_title="Health_exp_pct_GDP_2016 (%)",
    showlegend=False,
    font=dict(
        size=14,
        color="black"
    )
)
fig.show()<jupyter_output><empty_output><jupyter_text>**Observations:**The top 10 spenders are:1.   Marshall Islands2. United States3. Sierra Leone4. Tuvalu5. Micronesia. Fed. Sts6. Switzerland7. Cuba8. Kiribati9. Brazil10. PalauThe bottom 10 spenders are:1. Monaco2. Papua New Guinea3. Brunei Darussalam4. Bangladesh5. Lao PDR6. Pakistan7. Angola8. Eritrea9. Gabon10. Indonesia  **Using KNN to map the spenders** **First check the data distribution for the data column we are trying to predict (Health_exp_pct_GDP_2016)**<jupyter_code>print('Mean: ', df['Health_exp_pct_GDP_2016'].mean())
print('Median: ', df['Health_exp_pct_GDP_2016'].median())
print('Mode: ', df['Health_exp_pct_GDP_2016'].mode()[0])<jupyter_output>Mean:  6.656190476190477
Median:  6.2
Mode:  6.2<jupyter_text>The median is smaller than mean and we may use it to categorize byTransforming the column *Health_exp_pct_GDP_2016* to categorical data<jupyter_code>def cat_Health_exp_pct_GDP_2016(row):
    if row > 6.2:
        row = 1
    else:
        row = 0
    return row
# create a new column for spending category
df['Spending_category'] = df['Health_exp_pct_GDP_2016'].apply(cat_Health_exp_pct_GDP_2016)
# We now no longer need the Health_exp_pct_GDP_2016 column
df.drop('Health_exp_pct_GDP_2016', axis=1, inplace=True)
df.head()
# Also remove the World_Bank_Name column since it's not categorical and may interfere with accuracy
df.drop('World_Bank_Name', axis=1, inplace=True)
df.head()
# Get dependent and independent variables which will be used in the training and testing of our model
y = df['Spending_category']
print(y[:5])
X = df.drop('Spending_category', axis=1)
X.head()<jupyter_output>0    1
1    1
2    1
3    1
4    0
Name: Spending_category, dtype: int64<jupyter_text>**Spliting dataset into training set and test set**<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 99)
# from sklearn import preprocessing
# from sklearn import utils

# lab_enc = preprocessing.LabelEncoder()
# encoded = lab_enc.fit_transform(y_train)<jupyter_output><empty_output><jupyter_text>** Fitting clasifier to the Training set**<jupyter_code>from sklearn.preprocessing import PowerTransformer

pow_trans = PowerTransformer() # Used in data normalization during preprocessing

pow_trans.fit(X_train)# Fit the X_training data to PowerTransformer

X_train_normalized = pow_trans.transform(X_train)
X_test_normalized = pow_trans.transform(X_test)

# This is now our final normalized data
X_train_normalized = pd.DataFrame(X_train_normalized, columns=X_train.columns, index=X_train.index)
X_test_normalized = pd.DataFrame(X_test_normalized, columns=X_test.columns, index=X_test.index)<jupyter_output><empty_output><jupyter_text>**We need to evaluate our model's predictions using a confusion matrix since it gives a clear insight of the performance.**<jupyter_code># from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
# from sklearn.model_selection import cross_val_score

# Instantiate learning model (k = 3)
classifier = KNeighborsClassifier(n_neighbors=3)

# Fitting the model
classifier.fit(X_train, y_train)

# # Predicting the Test set results
y_pred = classifier.predict(X_test)

# The confusion matrix:
print(confusion_matrix(y_test, y_pred))

# The classification report...
print('\n', classification_report(y_test, y_pred))<jupyter_output>[[19  7]
 [ 8  8]]

               precision    recall  f1-score   support

           0       0.70      0.73      0.72        26
           1       0.53      0.50      0.52        16

    accuracy                           0.64        42
   macro avg       0.62      0.62      0.62        42
weighted avg       0.64      0.64      0.64        42<jupyter_text>**And now to judge the performance of our model....**<jupyter_code>accuracy = accuracy_score(y_test, y_pred)*100
print('Accuracy of our model is ' + str(round(accuracy, 2)) + ' %.')<jupyter_output>Accuracy of our model is 64.29 %.<jupyter_text>**Using cross-validation for parameter tuning:** **The model did not perform so well, so we need a better value of k (number of neighbours)**<jupyter_code># creating list of K for KNN
k_list = list(range(1,50,2))

# creating list of cv scores
cv_scores = []

# perform 10-fold cross validation
for k in k_list:
    knn = KNeighborsClassifier(n_neighbors=k)
    scores = cross_val_score(knn, X_train, y_train, cv=10, scoring='accuracy')
    cv_scores.append(scores.mean())
# changing to misclassification error
MSE = [1 - x for x in cv_scores]

plt.figure()
plt.figure(figsize=(15,10))
plt.title('The optimal number of neighbors', fontsize=20, fontweight='bold')
plt.xlabel('Number of Neighbors K', fontsize=15)
plt.ylabel('Misclassification Error', fontsize=15)
sns.set_style("whitegrid")
plt.plot(k_list, MSE)

plt.show()
# finding best k
best_k = k_list[MSE.index(min(MSE))]
print("The optimal number of neighbors is %d." % best_k)<jupyter_output>The optimal number of neighbors is 21.<jupyter_text>**Now we train again our model using a k value of 21**<jupyter_code># Instantiate learning model (k = 21)
classifier = KNeighborsClassifier(n_neighbors=21)

# Fitting the model
classifier.fit(X_train, y_train)

# # Predicting the Test set results
y_pred = classifier.predict(X_test)

# The confusion matrix:
print(confusion_matrix(y_test, y_pred))

# The classification report...
print('\n', classification_report(y_test, y_pred))
accuracy = accuracy_score(y_test, y_pred)*100
print('Accuracy of our model is ' + str(round(accuracy, 2)) + ' %.')<jupyter_output>Accuracy of our model is 69.05 %.<jupyter_text>**The model accuracy has greatly improved with more neighbours****Our precision and recall values are 70% and 88% respectively, which means the model performed fairly well in the prediction of the countries with high and low spending**<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1077207582061951, 'fraction_numerical': 0.030744336569579287, 'mean_word_length': 5.128186968838527, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '116200', 'license': 'MIT', 'n_tokens_mistral': 3152, 'n_tokens_neox': 2954, 'n_words': 801, 'path': 'Comparing Health Systems Using World Bank COVID-19 Data.ipynb', 'repo_name': 'manuel103/Machine-Learning', 'size': '8652'}