<jupyter_start><jupyter_text>Covid-19 CDC Dataset Analysis and Predictions <NAME>  1. Preparing Data Quality Report for the provided CSV file.- Read CSV- Check shape- Check first rows- Check last rows- Check the data types- Check info for each feature- Check column names- Check cardinality- Print description tables- Check for duplicates and drop them if needed- Save the cleaned data<jupyter_code># Import pandas, numpy, matplotlib, seaborn libraries to be used in the project
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import seaborn as sns
from matplotlib.backends.backend_pdf import PdfPages
from numpy import mean
from numpy import std
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import StackingClassifier
from matplotlib import pyplot

import pandas as pd

# Import package numpy for numeric computing
import numpy as np

import matplotlib
# Import package matplotlib for visualisation/plotting
import matplotlib.pyplot as plt

#For showing plots directly in the notebook run the command below
%matplotlib inline

# # For saving multiple plots into a single pdf file
from matplotlib.backends.backend_pdf import PdfPages
import seaborn as sns
import matplotlib.pyplot as plt
sns.set_theme(style="ticks", color_codes=True)

import matplotlib.patches as mpatches
from patsy import dmatrices
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import cross_validate
from sklearn.model_selection import cross_val_score
from sklearn.tree import export_graphviz

import graphviz
from graphviz import Source
from graphviz import Digraph

# ignore warnings
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Cleaning the dataset<jupyter_code># read in data from the provided csv file to pandas dataframe.  
df = pd.read_csv('covid19-cdc-20206769.csv',  keep_default_na=True, sep=',\s+', delimiter=',', skipinitialspace=True)
# check the shape of the data frame
df.shape
# print the first 5 rows of the data
df.head(5)
# print the last 5 rows of the data
df.tail(5)
# check the type of each column in the data frame
df.dtypes
df.info()
df.describe().T
df.columns
# Display the cardinality of each continuous feature

features_cardinality = list(df.columns.values)

print('{0:35}  {1}'.format("Feature", "Cardinality"))
print('{0:35}  {1}'.format("-------", "--------------- \n"))

for c in features_cardinality:
    print('{0:35}  {1}'.format(c, str(len(df[c].unique()))))
df.describe().T
# check the number of null values for each row
df.isnull().sum()
# check missing data

medcond_columns_missing  = 100 * (df['medcond_yn'].value_counts()/df.shape[0])
print(medcond_columns_missing)

icu_columns_missing  = 100 * (df['icu_yn'].value_counts()/df.shape[0])
print(icu_columns_missing)

hosp_columns_missing  = 100 * (df['hosp_yn'].value_counts()/df.shape[0])
print(hosp_columns_missing)

age_columns_missing  = 100 * (df['age_group'].value_counts()/df.shape[0])
print(age_columns_missing)

sex_missing  = 100 * (df['sex'].value_counts()/df.shape[0])
print(sex_missing)

race_ethnicity_combined  = 100 * (df['race_ethnicity_combined'].value_counts()/df.shape[0])
print(race_ethnicity_combined)
# Check for irregular cardinality & permitted values in categorical features. 

print("Unique values for:\ncurrent status:", pd.unique(df["current_status"].ravel()))
print('')
print("sex:", pd.unique(df["sex"].ravel()))
print('')
print("age_group:", pd.unique(df["age_group"].ravel()))
print('')
print("race_ethnicity_combined:", pd.unique(df["race_ethnicity_combined"].ravel()))
print('')
print("hosp_yn:", pd.unique(df["hosp_yn"].ravel()))
print('')
print("icu_yn:", pd.unique(df["icu_yn"].ravel()))
print('')
print("medcond_yn:", pd.unique(df["medcond_yn"].ravel()))
# printing the number of 'Unknown' or 'Missing' values for the columns that might have them

print('{0:35}  {1}'.format("race_ethnicity_combined", np.isin(df['race_ethnicity_combined'], ['Missing', 'Unknown']).sum()))
print('{0:35}  {1}'.format("sex", np.isin(df['sex'], ['Missing', 'Unknown']).sum()))
print('{0:35}  {1}'.format("hosp_yn", np.isin(df['hosp_yn'], ['Missing', 'Unknown']).sum()))
print('{0:35}  {1}'.format("icu_yn", np.isin(df['icu_yn'], ['Missing', 'Unknown']).sum()))
print('{0:35}  {1}'.format("medcond_yn", np.isin(df['medcond_yn'], ['Missing', 'Unknown']).sum()))
print('{0:35}  {1}'.format("age_group", np.isin(df['age_group'], ['Missing', 'Unknown']).sum()))

# printing the number of 'Unknown' or 'Missing' values for a column that does not have them to make sure the code above works as expected
print('{0:35}  {1}'.format("death_yn", np.isin(df['death_yn'], ['Missing', 'Unknown']).sum()))
# check the number of unique values for each row
df.nunique()
df_duplicates = df[df.duplicated()]
# Print the number of duplicates, without the original rows that were duplicated

print('Number of duplicate (excluding first) rows in the table is: ', df_duplicates.duplicated().sum())
print('Number of duplicate rows (including first) in the table is:', df_duplicates[df_duplicates.duplicated(keep=False)].shape[0])
# Check for duplicate columns

dfT = df_duplicates.T
print("Number of duplicate (excluding first) columns in the table is: ", dfT.duplicated().sum())
print("Number of duplicate (including first) columns in the table is: ",  dfT[dfT.duplicated(keep=False)].shape[0])
df_duplicates.shape
df_duplicates.head()
df_duplicates.tail()<jupyter_output><empty_output><jupyter_text>Duplicates:- There are no duplicate columns- Duplicates do exist but looking at some of them it looks like the `age_group` is different so they might not be true duplicates.- Due to the large number of `Missing` and `Unknown` values it might be possible that data was not actually collected when the forms were filled out for various reasons such as a large number of cases overwhelming the staff.<jupyter_code># Save a copy of the data frame in a new df
df_cleaned = df
df_cleaned.shape
df_cleaned.drop('cdc_report_dt', axis=1, inplace=True)
df_cleaned.shape
# print each feature and all unique values

for col in df_cleaned[['current_status','sex','age_group','race_ethnicity_combined','hosp_yn','icu_yn','death_yn','medcond_yn']].columns:
    print(f"Column {col}:\n{df_cleaned[col].unique().tolist()}\n")<jupyter_output>Column current_status:
['Laboratory-confirmed case', 'Probable Case']

Column sex:
['Male', 'Unknown', 'Female', 'Missing']

Column age_group:
['50 - 59 Years', '30 - 39 Years', '0 - 9 Years', '20 - 29 Years', '10 - 19 Years', '70 - 79 Years', '40 - 49 Years', '80+ Years', '60 - 69 Years', 'Missing']

Column race_ethnicity_combined:
['Unknown', 'White, Non-Hispanic', 'Hispanic/Latino', 'American Indian/Alaska Native, Non-Hispanic', 'Black, Non-Hispanic', 'Multiple/Other, Non-Hispanic', 'Asian, Non-Hispanic', 'Missing', 'Native Hawaiian/Other Pacific Islander, Non-Hispanic']

Column hosp_yn:
['Missing', 'No', 'Unknown', 'Yes', 'OTH']

Column icu_yn:
['Missing', 'No', 'Unknown', 'Yes']

Column death_yn:
['No', 'Yes']

Column medcond_yn:
['Missing', 'No', 'Yes', 'Unknown']<jupyter_text>More cleaning:- All the `Unknown` features could be dropped from `sex` and `age_group` as they represent a small portion of the data and will not affect the predictions later- For each feature merge `Missing` into `Unknown`. In the case of `1hosp_yn` also merge `OTH` into `Unknown`<jupyter_code># Merge Missing and OTH into Unknown

df_cleaned.replace({'hosp_yn' : {'Missing' : 'Unknown', 'OTH' : 'Unknown'}}, inplace=True)
df_cleaned.replace({'icu_yn' : {'Missing' : 'Unknown'}}, inplace=True)
df_cleaned.replace({'medcond_yn' : {'Missing' : 'Unknown'}}, inplace=True)
df_cleaned.replace({'race_ethnicity_combined' : {'Missing' : 'Unknown'}}, inplace=True)
df_cleaned.replace({'sex' : {'Missing' : 'Unknown'}}, inplace=True)
# Drop all Unknowns for sex and age_group

df_cleaned = df_cleaned[df_cleaned.sex != 'Unknown']
df_cleaned = df_cleaned[df_cleaned.age_group != 'Unknown']
# Check after merging and dropping Unknowns

for col in df_cleaned[['current_status','sex','age_group','race_ethnicity_combined','hosp_yn','icu_yn','death_yn','medcond_yn']].columns:
    print(f"Column {col}:\n{df_cleaned[col].unique().tolist()}\n")<jupyter_output>Column current_status:
['Laboratory-confirmed case', 'Probable Case']

Column sex:
['Male', 'Female']

Column age_group:
['50 - 59 Years', '30 - 39 Years', '0 - 9 Years', '20 - 29 Years', '10 - 19 Years', '70 - 79 Years', '40 - 49 Years', '80+ Years', '60 - 69 Years', 'Missing']

Column race_ethnicity_combined:
['Unknown', 'White, Non-Hispanic', 'Hispanic/Latino', 'American Indian/Alaska Native, Non-Hispanic', 'Black, Non-Hispanic', 'Multiple/Other, Non-Hispanic', 'Asian, Non-Hispanic', 'Native Hawaiian/Other Pacific Islander, Non-Hispanic']

Column hosp_yn:
['Unknown', 'No', 'Yes']

Column icu_yn:
['Unknown', 'No', 'Yes']

Column death_yn:
['No', 'Yes']

Column medcond_yn:
['Unknown', 'No', 'Yes']<jupyter_text>Checks:Trying to check if I can use `icu_yn` in order to infer data for `hosp_yn`. This has proven not to be the case as `hosp_yn` could be infered only in the case of `icu_yn` being Yes. A person would be hospitalized if they were in ICU.<jupyter_code>df_cleaned.loc[df_cleaned['hosp_yn'] != df_cleaned['icu_yn'], ['hosp_yn', 'icu_yn']]
df_cleaned.loc[(df_cleaned['hosp_yn'] != df_cleaned['icu_yn']) & (df_cleaned['icu_yn'] == 'Yes'), ['hosp_yn', 'icu_yn']]
df_cleaned.head()
# Separate categorical data and convert it to 'category' from 'object'
categorical_columns = df_cleaned[['current_status','sex','age_group','race_ethnicity_combined','hosp_yn','icu_yn','death_yn','medcond_yn']].columns

for column in categorical_columns:
    df_cleaned[column] = df_cleaned[column].astype('category')

# Separate datetime data and convert it to 'datetime' from 'object'
datetime_columns = df_cleaned[['onset_dt','cdc_case_earliest_dt','pos_spec_dt']].columns

for column in datetime_columns:
    df_cleaned[column] = pd.to_datetime(df_cleaned[column])
     
df_cleaned.dtypes
# Descriptive table of the categorical data
df_cleaned.select_dtypes(['category']).describe().T
# Write the dataframe to a csv file

df_cleaned.to_csv('cleaned-covid19-cdc-20206769.csv', index=False)
df_cleaned.head()<jupyter_output><empty_output><jupyter_text>Categorical feature plotting- Separate features into `conitnuous` and `categorical` (datetime for categorical)<jupyter_code>continuous_columns = df_cleaned[["cdc_case_earliest_dt", "pos_spec_dt", "onset_dt"]]
continuous_columns.dtypes
continuous_columns.info()
categorical_columns = df_cleaned[["current_status", "sex", "age_group", "race_ethnicity_combined","hosp_yn", 'icu_yn','medcond_yn']]
categorical_columns.dtypes
categorical_columns.info()
# Plot bar plots for all the catagorical features 
# All plots are saved to a PDF file
with PdfPages('categorical_barplots_1-1.pdf') as pp:
    for col in categorical_columns:
        f = df_cleaned[col].value_counts().plot(kind='bar', figsize=(12,10))
        plt.title(col)
        plt.ylabel('number of people')
        pp.savefig(f.get_figure())
        plt.show()<jupyter_output><empty_output><jupyter_text>1 Preparation Encode `death_yn`<jupyter_code># Encode death_tn and change it to int64

df_cleaned.replace({'death_yn' : {'No' : 0, 'Yes' : 1}}, inplace=True)
df_cleaned['death_yn'] = df_cleaned['death_yn'].astype('int64')<jupyter_output><empty_output><jupyter_text>Randomly shuffle the rows of your dataset and split the dataset into two datasets: 70% training and 30% test.<jupyter_code>df_prepared = df_cleaned.sample(frac = 1,  random_state= 420)
# Split

y = pd.DataFrame(df_prepared["death_yn"])
X = df_prepared.drop(["death_yn"],1)
# Split the dataset into two datasets: 70% training and 30% test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state=420)

print("original range is: ",df_prepared.shape[0])
print("training range (70%):\t rows 0 to", round(X_train.shape[0]))
print("test range (30%): \t rows", round(X_train.shape[0]), "to", round(X_train.shape[0]) + X_test.shape[0])
X_train<jupyter_output><empty_output><jupyter_text>Plot the correlations between all the categorical features.<jupyter_code>y_train = y_train.astype("category")
# Create temp dataframe for these plots, mergeing X_train and y_train
df_temp= pd.concat([X_train, y_train], axis=1)
for categorical_feature in categorical_columns:
    maxDelqEver = pd.unique(df_temp[categorical_feature].ravel())
    df_temp['percent'] = 0

    # Header
    print("\n", categorical_feature)
    print("Index \t Count")
    
    for i in maxDelqEver:

        count = df_temp[df_temp[categorical_feature] == i].count()['death_yn']
        count_percentage = (1 / count) * 100
        print(i, "\t", count)
        index_list = df_temp[df_temp[categorical_feature] == i].index.tolist()
        for ind in index_list:
            df_temp.loc[ind, 'percent'] = count_percentage

    group = df_temp[['percent',categorical_feature,'death_yn']].groupby([categorical_feature,'death_yn']).sum()
    my_plot = group.unstack().plot(kind='bar', stacked=True, title=f"Death_yn vs {categorical_feature}", figsize=(15,7), grid=True)

    # Legend
    red_patch = mpatches.Patch(color='orange', label='Yes')
    blue_patch = mpatches.Patch(color='blue', label='No')
    my_plot.legend(handles=[red_patch, blue_patch], frameon = True)

    # Gridlines
    plt.grid(b=True, which='major', color='#666666', linestyle='-')
    plt.minorticks_on()
    plt.grid(b=True, which='minor', color='#999999', linestyle='-', alpha=0.2)

    # Labels
    my_plot.set_xlabel("Feature values")
    my_plot.set_ylabel("% of rows")
    my_plot.set_ylim([0,100])
    
# Drop 'percent' that was used only for stacked bar plot
df_temp = df_temp.drop("percent", 1)<jupyter_output>current_status
Index 	 Count
Laboratory-confirmed case 	 6525
Probable Case 	 412

 sex
Index 	 Count
Female 	 3650
Male 	 3287

 age_group
Index 	 Count
20 - 29 Years 	 1270
40 - 49 Years 	 1013
30 - 39 Years 	 1143
0 - 9 Years 	 331
10 - 19 Years 	 682
60 - 69 Years 	 735
70 - 79 Years 	 406
50 - 59 Years 	 992
80+ Years 	 359
Missing 	 6

 race_ethnicity_combined
Index 	 Count
Unknown 	 2874
Multiple/Other, Non-Hispanic 	 341
White, Non-Hispanic 	 2324
Black, Non-Hispanic 	 507
Hispanic/Latino 	 657
Asian, Non-Hispanic 	 168
American Indian/Alaska Native, Non-Hispanic 	 49
Native Hawaiian/Other Pacific Islander, Non-Hispanic 	 17

 hosp_yn
Index 	 Count
Unknown 	 2877
No 	 3573
Yes 	 487

 icu_yn
Index 	 Count
Unknown 	 6183
Yes 	 60
No 	 694

 medcond_yn
Index 	 Count
Unknown 	 5655
No 	 680
Yes 	 602<jupyter_text>Interpreting features paired with `death_yn`:- current_status: there is a correlation between the two but not a strong one. The low number of `death_yn_Yes` might cause that.- sex: there is a correlation between the two but not a strong one. The low number of `death_yn_Yes` might cause that.- age_group: there seems to be a strong correlation- race_ethnicity_combined: there is not a strong correlation.- hosp_yn: there is a strong enough correlation- medcond_yn: there is a strong enough correlation- icu_yn: this feature pair has the strongest correlation  Drop features that are not neededAfter visually analizing the data I have decided to drop:- cdc_case_earliest_dt- sex- current_status- race_ethnicity_combined- onset_dt- pos_spec_dtThe features listed above will be dropped as it seems that they don't have a big impact and there is not a strong correlation.<jupyter_code>df_prepared = df_prepared.drop(['cdc_case_earliest_dt'], 1)
df_prepared = df_prepared.drop(['sex'], 1)
df_prepared = df_prepared.drop(['current_status'], 1)
df_prepared = df_prepared.drop(['race_ethnicity_combined'], 1)
df_prepared = df_prepared.drop(['onset_dt'], 1)
df_prepared = df_prepared.drop(['pos_spec_dt'], 1)
df_prepared<jupyter_output><empty_output><jupyter_text>Data Quality Plan: Observations:- The data sample contains 12 columns and 10 000 rows- The type for all the features is 'object' and needs to be changed to datetime and category- There are no numeric values- All the features are categorical, there are no continuous features in the data set- The data sample contains duplicates that have to be dropped- The data sample contains a high number of Null, Unknown or Missing values for multiple features Summary of data quality plan:| Variable Names                     | Data Quality Issue            | Handling Strategy              ||------------------------------------|-------------------------------|--------------------------------|| cdc_case_earliest_dt               | OK                            | Do Nothing                     || cdc_report_dt                      | Deprecated                    | Remove                         || pos_spec_dt                        | Null values                   | Do Nothing                     || onset_dt                           | Missing or Unknown Values     | Merge Missing and Unknown      || current_status                     | OK                            | Do Nothing                     || sex                                | Missing or Unknown Values     | Drop Missing and Unknown       || age_group                          | Missing or Unknown Values     | Drop Missing and Unknown       || race_ethnicity_combined            | Missing or Unknown Values     | Merge Missing and Unknown      || hosp_yn                            | Missing or Unknown Values     | Merge Missing and Unknown      || icu_yn                             | Missing or Unknown Values     | Merge Missing and Unknown      || death_yn                           | OK                            | Do Nothing                     || medcond_yn                         | Missing or Unknown Values     | Merge Missing and Unknown      |  Prepare dataset for modeling- Convert the categorical variables into dummies variable to allow modeling- Remove the redundant dummies which contain no additional information- Set up the train test split again based on the dataset with the dummies included<jupyter_code># Use a copy of the prepared data set to work on
df_rev1 = df_prepared.copy()
df_rev1 = pd.get_dummies(df_rev1, columns=['age_group','hosp_yn','icu_yn','medcond_yn'], drop_first=True)
df_rev1
y = df_rev1["death_yn"]
X = df_rev1.drop(["death_yn"],1)

# Split the dataset into two datasets: 70% training and 30% test
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1,  test_size=0.3)

print("original range is: ",df_rev1.shape[0])
print("training range (70%):\t rows 0 to", round(X_train.shape[0]))
print("test range (30%): \t rows", round(X_train.shape[0]), "to", round(X_train.shape[0]) + X_test.shape[0])
print("\nDescriptive features in X:\n", X_train.head(5))
print("\nTarget feature in y:\n", y_train.head(5))<jupyter_output>Descriptive features in X:
       age_group_10 - 19 Years  age_group_20 - 29 Years  \
6959                        0                        0   
7789                        0                        1   
2911                        0                        0   
8108                        0                        0   
1060                        0                        0   

      age_group_30 - 39 Years  age_group_40 - 49 Years  \
6959                        0                        1   
7789                        0                        0   
2911                        0                        0   
8108                        0                        0   
1060                        0                        0   

      age_group_50 - 59 Years  age_group_60 - 69 Years  \
6959                        0                        0   
7789                        0                        0   
2911                        0                        0   
8108                        1            [...]<jupyter_text>Reset index<jupyter_code>X_train.head(5)
# Resetting the index to allow contatenation with predicted values
X_train.reset_index(drop=True, inplace=True)
y_train.reset_index(drop=True, inplace=True)
X_test.reset_index(drop=True, inplace=True)
y_test.reset_index(drop=True, inplace=True)
X_train.head(5)<jupyter_output><empty_output><jupyter_text>2 Linear regression modelTrain a linear regression model using only the descriptive features selected from part 1 above<jupyter_code># Train a model using all continuous and categorical features.
multiple_linreg = LinearRegression().fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Print out the coefficients learned by the model<jupyter_code># Print the weights learned for each feature.

print("\nFeatures are: \n", X_train.columns)
print("\nCoeficients are: \n", multiple_linreg.coef_)
print("\nIntercept is: \n", multiple_linreg.intercept_)
print("\nFeatures and coeficients: \n", list(zip(X_train.columns, multiple_linreg.coef_)))<jupyter_output>Features are: 
 Index(['age_group_10 - 19 Years', 'age_group_20 - 29 Years',
       'age_group_30 - 39 Years', 'age_group_40 - 49 Years',
       'age_group_50 - 59 Years', 'age_group_60 - 69 Years',
       'age_group_70 - 79 Years', 'age_group_80+ Years', 'age_group_Missing',
       'hosp_yn_Unknown', 'hosp_yn_Yes', 'icu_yn_Unknown', 'icu_yn_Yes',
       'medcond_yn_Unknown', 'medcond_yn_Yes'],
      dtype='object')

Coeficients are: 
 [ 5.02263383e-04  3.43928064e-06 -1.31707688e-03 -3.73216382e-03
  5.26148352e-03  2.62830431e-02  1.00840152e-01  2.45390268e-01
  2.12753785e-03  9.97612566e-03  1.72175864e-01  3.04410756e-02
  3.96976170e-01 -5.21003079e-03  2.71512107e-02]

Intercept is: 
 -0.030253904741784604

Features and coeficients: 
 [('age_group_10 - 19 Years', 0.0005022633825153252), ('age_group_20 - 29 Years', 3.4392806420263293e-06), ('age_group_30 - 39 Years', -0.0013170768811897973), ('age_group_40 - 49 Years', -0.0037321638210315853), ('age_group_50 - 59 Years', 0.0052[...]<jupyter_text>Interpreting the linear regression modelThe Linear Regression model is based on the formula:$target\_feature = w_0 + w_1 * feature_1 + w_2*feature_2 + ...+ w_n*feature_n $The intercept represents the point where the line meets the Y axis wneh X is 0.Positive coeficiemts:- A positive coefficient indicates that as the value of the independent variable increases, the mean of the dependent variable also tends to increase.- A person with `ycu_yn_Yes` has a higher chance of dying than a person in the `age_group10 - 19`Negative coefficients:- A negative coefficient indicates that as the value of the independent variable increases, the mean of the dependent variable tends to decrease.- A person with `age_group_30 - 39 Years` has a higher chance of dying than a person in the `medcond_yn_Unknown`Linear Regression is not recommended as we are trying to solve a classification problem.  Printing 10 predicted target features and evaluate the prediction<jupyter_code>multiple_linreg_predictions_train = (multiple_linreg.predict(X_train) >= 0.5) * 1.0

print("\nPredictions with multiple linear regression: \n")
actual_vs_predicted_multiplelinreg = pd.concat([y_train, pd.DataFrame(multiple_linreg_predictions_train, columns=['Predicted'])], axis=1)
print(actual_vs_predicted_multiplelinreg.head(10))<jupyter_output>Predictions with multiple linear regression: 

   death_yn  Predicted
0         0        0.0
1         0        0.0
2         0        0.0
3         0        0.0
4         0        0.0
5         0        0.0
6         0        0.0
7         0        0.0
8         0        0.0
9         0        0.0<jupyter_text>Evaluation metrics based on training data<jupyter_code># Some more evaluation metrics.

print("==================== Train Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_train, multiple_linreg_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, multiple_linreg_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, multiple_linreg_predictions_train))
print("======================================================")<jupyter_output>==================== Train Data =======================
Accuracy:  0.9664119936572005
Confusion matrix: 
 [[6672   19]
 [ 214   32]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      1.00      0.98      6691
           1       0.63      0.13      0.22       246

    accuracy                           0.97      6937
   macro avg       0.80      0.56      0.60      6937
weighted avg       0.96      0.97      0.96      6937

======================================================<jupyter_text>Interpretation of results- Accuracy    - This is simply stating how often the model is correct. We have an accuracy of 73%- Confusion Matrix    - [0][0] TRUE NEGATIVE - The number that is predicted 0 that was actually 0    - [0][1] FALSE POSITIVE - The number that is predicted 1 that is actually 0    - [1][0] FALSE NEGATIVE - The number that is predicted 0 that is actually 1    - [1][1] TRUE POSITIVE - The number that is predicted 1 that is actually 1- Precision - How good the model is at predicting the positive class    - What % of the predicted positive are actually positive    - It is the number values correctly predicted positive over the total number of  positive values    - Precision Positive is 0.97    - Precision Negative is 0.63- Recall    - What % of the positive values did we predict    - Is the number correctly predicted positive over the total number actual positive    - Recall Positive is 1.00    - Recall Negative is 0.13- F1 Score    - Is an a weighted average of Precision and recall    - F1 Score Positive is 0.98    - F1 Score Negative is 0.22Conclusion:- The mode is better at predictiong a negative `death_yn`- This makes sense as most of the training data had `death_yn_No`  Evaluate the model using the hold-out (30% examples) test set<jupyter_code># calculate the prediction and threshold the value. If >= 0.5 its true
multiple_linreg_predictions_test = (multiple_linreg.predict(X_test) >= 0.5) * 1.0

print("\nPredictions with multiple linear regression: \n")
actual_vs_predicted_multiplelinreg = pd.concat([y_test, pd.DataFrame(multiple_linreg_predictions_test, columns=['Predicted'])], axis=1)
print(actual_vs_predicted_multiplelinreg.head(100))
# Some more evaluation metrics for train/test
print("==================== Test Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_test, multiple_linreg_predictions_test))
print("Confusion matrix: \n", metrics.confusion_matrix(y_test, multiple_linreg_predictions_test))
print("Classification report - Test data:\n ", metrics.classification_report(y_test, multiple_linreg_predictions_test))
print("\n==================== Train Data ======================")
print("Accuracy: ", metrics.accuracy_score(y_train, multiple_linreg_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, multiple_linreg_predictions_train))
print("\nClassification report: - Training data\n ", metrics.classification_report(y_train, multiple_linreg_predictions_train))<jupyter_output>==================== Test Data =======================
Accuracy:  0.9630127774041695
Confusion matrix: 
 [[2850    9]
 [ 101   14]]
Classification report - Test data:
                precision    recall  f1-score   support

           0       0.97      1.00      0.98      2859
           1       0.61      0.12      0.20       115

    accuracy                           0.96      2974
   macro avg       0.79      0.56      0.59      2974
weighted avg       0.95      0.96      0.95      2974


==================== Train Data ======================
Accuracy:  0.9664119936572005
Confusion matrix: 
 [[6672   19]
 [ 214   32]]

Classification report: - Training data
                precision    recall  f1-score   support

           0       0.97      1.00      0.98      6691
           1       0.63      0.13      0.22       246

    accuracy                           0.97      6937
   macro avg       0.80      0.56      0.60      6937
weighted avg       0.96      0.97      0.96      6937<jupyter_text>**Interpretation of test results and comparison with training results**- Accuracy    - The accuracy of the test data is lower. - Precision    - The precision score for predicting the positive case hasn't changed.    - The precision score for predicting the negative case has lowered.- Recall    - The recall score for predicting the positive case hasn't changed.    - The recall score for predicting the negative case has lowered.- F1    - The f1 score for predicting the positive case has increased.    - The f1 score for predicting the negative case has lowered.   Cross validation- Perform cross validation on the linear regression model. - Perform the same evaluation as above but multiple times- Store the results in a dictionary for later use<jupyter_code>def cross_val_LinReg(X, y, cv=3, scoring='accuracy'):
    results = []
    for i in range(cv):
        X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=i , test_size=0.3)
        multiple_linreg = LinearRegression().fit(X_train, y_train)
        multiple_linreg_predictions = (multiple_linreg.predict(X_test) >= 0.5) * 1.0
        if scoring=='accuracy':
            score = metrics.accuracy_score(y_test, multiple_linreg_predictions)
        elif scoring=='precision':
            score = metrics.precision_score(y_test, multiple_linreg_predictions)
        elif scoring=='f1':
            score = metrics.f1_score(y_test, multiple_linreg_predictions)
        elif scoring=='recall':
            score = metrics.recall_score(y_test, multiple_linreg_predictions)
        results.append(score)
        
    return results<jupyter_output><empty_output><jupyter_text>Additional function to perform 10 fold cross validation and store results into a dataframe<jupyter_code>def cross_val_LinReg_DF(X,y):
    linRegResults = {}
    test_metrics = ['accuracy','precision','recall', 'f1']
    
    for metric in test_metrics:
        result = cross_val_LinReg(X, y, cv=10, scoring=metric)
        length = len(result)
        linRegResults[metric] = sum(result)/length

    LinRegDF = pd.DataFrame.from_dict(linRegResults, orient='index', columns=['Linear_Regression'])
    
    return LinRegDF<jupyter_output><empty_output><jupyter_text>Preform 10 fold cross validation using cross_val_LinReg_DF function<jupyter_code>linRegDF = cross_val_LinReg_DF(X,y)
print(f"Mean results from 10 fold cross validation are:")
linRegDF<jupyter_output>Mean results from 10 fold cross validation are:<jupyter_text>Conclusion:The results are lower than the previous ones as we took the mean of 10 sets of results.  3 Logistic Regression   Train a logistic regression model using only the descriptive features selected from part 1 above<jupyter_code># Train a model using all continuous and categorical features.
multiple_logisticreg = LogisticRegression().fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Print out the coefficients learned by the model<jupyter_code># Print the weights learned for each feature.

print("\nFeatures are: \n", X_train.columns)
print("\nCoeficients are: \n", multiple_logisticreg.coef_[0])
print("\nIntercept is: \n", multiple_logisticreg.intercept_)
print("\nFeatures and coeficients: \n", list(zip(X_train.columns, multiple_logisticreg.coef_[0])))<jupyter_output>Features are: 
 Index(['age_group_10 - 19 Years', 'age_group_20 - 29 Years',
       'age_group_30 - 39 Years', 'age_group_40 - 49 Years',
       'age_group_50 - 59 Years', 'age_group_60 - 69 Years',
       'age_group_70 - 79 Years', 'age_group_80+ Years', 'age_group_Missing',
       'hosp_yn_Unknown', 'hosp_yn_Yes', 'icu_yn_Unknown', 'icu_yn_Yes',
       'medcond_yn_Unknown', 'medcond_yn_Yes'],
      dtype='object')

Coeficients are: 
 [-1.37088025 -1.21538489 -1.18429001 -0.63803911  0.44416535  1.29949637
  2.21657284  3.10845775 -0.07251322  0.76028996  2.39317276  0.9279907
  2.33513412  0.64114062  1.37097645]

Intercept is: 
 [-6.78565913]

Features and coeficients: 
 [('age_group_10 - 19 Years', -1.3708802531714452), ('age_group_20 - 29 Years', -1.215384890218537), ('age_group_30 - 39 Years', -1.1842900118838937), ('age_group_40 - 49 Years', -0.6380391133912737), ('age_group_50 - 59 Years', 0.4441653531576263), ('age_group_60 - 69 Years', 1.2994963695227575), ('age_group_70 - 7[...]<jupyter_text>Interpreting the results The model estimated in logistic regression is given by the logistic function:$probability(target=1|descriptive\_features)=logistic(w_0 + w_1 * feature_1 + w_2*feature_2 + ...+ w_n*feature_n)$ where $logistic(x)$ is defined as: $logistic(x) = \frac{e ^ x}{1 + e ^ x} = \frac{1}{1+e^{-x}}$Logistic regression, as oppose to Linear Regression can be used for solving a Classification Problem.The Intercept is at **-6.785**, this represents the starting point. If all the other coeficients were 0 X would be -6.785.The coeficients have the same effect on X as in Linear Regression.*The Logistic Regression outputs a probability as a value between 0 and 1, representing the likelyhood of dying.*  Printing 10 predicted target features and evaluate the prediction- Print the predicted target feature value for the first 100 training examples based on training data<jupyter_code>multiple_logisticreg_predictions_train = multiple_logisticreg.predict(X_train)

print("\nPredictions with multiple linear regression: \n")
actual_vs_predicted_multiplelogisticreg = pd.concat([y_train, pd.DataFrame(multiple_logisticreg_predictions_train, columns=['Predicted'])], axis=1)
print(actual_vs_predicted_multiplelogisticreg.head(10))<jupyter_output>Predictions with multiple linear regression: 

   death_yn  Predicted
0         0          0
1         0          0
2         0          0
3         0          0
4         0          0
5         0          0
6         0          0
7         0          0
8         0          0
9         0          1<jupyter_text>Evaluation metrics based on training data- Print the classification evaluation measures computed on the training set (e.g. Accuracy, Confusion matrix, Precision, Recall, F1)<jupyter_code># Some more evaluation metrics.

print("==================== Train Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_train, multiple_logisticreg_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, multiple_logisticreg_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, multiple_logisticreg_predictions_train))
print("======================================================")<jupyter_output>==================== Train Data =======================
Accuracy:  0.9658353755225602
Confusion matrix: 
 [[6633   58]
 [ 179   67]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      6691
           1       0.54      0.27      0.36       246

    accuracy                           0.97      6937
   macro avg       0.75      0.63      0.67      6937
weighted avg       0.96      0.97      0.96      6937

======================================================<jupyter_text>Interpretation of results- Accuracy    - We have an accuracy of 97%- Precision - How good model is a prediction the positive class    - Precision Positive is 0.97    - Precision Negative is 0.54- Recall - What % of the positive values did we predict    - Recall Positive is 0.99    - Recall Negative is 0.27- F1 Score - Is an a weighted average of Precision and recall    - F1 Score Positive is 0.98    - F1 Score Negative is 0.36Conclusion:- The mode is better at predictiong a negative `death_yn`- This makes sense as most of the training data had `death_yn_No`  Evaluate the model using the hold-out (30% examples) test set<jupyter_code>multiple_logisticreg_predictions_test = multiple_logisticreg.predict(X_test)

print("\nPredictions with multiple linear regression: \n")
actual_vs_predicted_multiplelogisticreg = pd.concat([y_test, pd.DataFrame(multiple_logisticreg_predictions_test, columns=['Predicted'])], axis=1)
print(actual_vs_predicted_multiplelogisticreg.head(30))
# Some more evaluation metrics.

print("==================== Test Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_test, multiple_logisticreg_predictions_test))
print("Confusion matrix: \n", metrics.confusion_matrix(y_test, multiple_logisticreg_predictions_test))
print("Classification report:\n ", metrics.classification_report(y_test, multiple_logisticreg_predictions_test))
print("==================== Train Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_train, multiple_logisticreg_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, multiple_logisticreg_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, multiple_logisticreg_predictions_train))
print("======================================================")<jupyter_output>==================== Test Data =======================
Accuracy:  0.9626765299260256
Confusion matrix: 
 [[2834   25]
 [  86   29]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      2859
           1       0.54      0.25      0.34       115

    accuracy                           0.96      2974
   macro avg       0.75      0.62      0.66      2974
weighted avg       0.95      0.96      0.96      2974

==================== Train Data =======================
Accuracy:  0.9658353755225602
Confusion matrix: 
 [[6633   58]
 [ 179   67]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      6691
           1       0.54      0.27      0.36       246

    accuracy                           0.97      6937
   macro avg       0.75      0.63      0.67      6937
weighted avg       0.96      0.97      0.96      6937

===============================[...]<jupyter_text>Interpretation of test results and comparison with training results - Accuracy    - The accuracy of the test data is lower. - Precision    - The precision score for predicting the positive case hasn't change.    - The precision score for predicting the negative case hasn't change.- Recall    - The recall score for predicting the positive case hasn't change.    - The recall score for predicting the negative case has lowered.- F1    - The f1 score for predicting the positive case hasn't change.    - The f1 score for predicting the negative case has lowered.**There is not a big change in between the Train and Test**  Cross validation<jupyter_code>def cross_val_LogReg_DF(X, y, depth=None, estimators=100):
    logRegResults = {}
    test_metrics = ['accuracy','precision','recall', 'f1']

    for metric in test_metrics:
        result = cross_val_score(LogisticRegression(), X, y, scoring=metric, cv=10)
        logRegResults[metric] = result.mean()
        
    LogRegDF = pd.DataFrame.from_dict(logRegResults, orient='index', columns=['Logistic_Regression'])
    
    return LogRegDF
logRegDF = cross_val_LogReg_DF(X,y)
print(f"Mean results from 10 fold cross validation are:")
logRegDF<jupyter_output>Mean results from 10 fold cross validation are:<jupyter_text>The results are in line with the previous ones.   Random Forest Model Train a random forest model using only the descriptive features selected from part 1 above - Train a random forest model - Use the RandomForestClassifier() function - The random state will be set to 1 to allow the results to be repeated<jupyter_code># Train RF with 100 trees

rfc = RandomForestClassifier(n_estimators=100, max_features='auto', oob_score=True, random_state=1)
rfc.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Understanding the Random Forest model- Decision tree    - A decision tree is a set of if-then-else rules based on splitting the data based on specific features    - The decision on which feature to split on is based on the information gain (Gini index).     - The tree will always split on the feature that gives the highest information gain    - The feature with the highest information gain will sit at the root of each decision tree    - Each decision tree will be a single prediction- Random Forest     - The random forest is made up of a number of decision trees    - The output of the random forest is base on the majority output of all decision trees combined    - This gives reliable results but can be hard to interpret    - The random forest will keep track of the importance of features in terms of information gain, which we will see      Train Decision tree- One with max_depth=4 and the other max_depth=10- Max depth determines how deep a decision tree can go before it must give an answer<jupyter_code>dtc4 = DecisionTreeClassifier(max_depth=4, random_state=1)
dtc10 = DecisionTreeClassifier(max_depth=10, random_state=1)
dtc4.fit(X_train, y_train)
dtc10.fit(X_train, y_train)
print("Max depth 4: \n",dtc4)
print("Max depth 10: \n",dtc10)
# Graphviz png
with open("DecisionTree4.dot", 'w') as f1:
    f1 = export_graphviz(dtc4, out_file=f1, feature_names=X_train.columns)
with open("DecisionTree10.dot", 'w') as f2:
    f2 = export_graphviz(dtc10, out_file=f2, feature_names=X_train.columns)
!dot -Tpng DecisionTree4.dot -o DecisionTree4.png
!dot -Tpng DecisionTree10.dot -o DecisionTree10.png<jupyter_output><empty_output><jupyter_text>Interpretation of the random forest model<jupyter_code>importance = pd.DataFrame({'feature': X_train.columns, 'importance':rfc.feature_importances_})
importance.sort_values('importance', ascending=False)<jupyter_output><empty_output><jupyter_text>*Decision Tree with maxdepth=4* *Decision tree with maxdepth=10*   Printing predicted target features and evaluate the prediction - Print the predicted target feature value for the first 100 training examples.<jupyter_code>rfc_predictions_train = rfc.predict(X_train)
df_true_vs_rfc_predicted = pd.DataFrame({'ActualClass': y_train, 'PredictedClass': rfc_predictions_train})
df_true_vs_rfc_predicted.head(20)<jupyter_output><empty_output><jupyter_text>Evaluation metrics based on training data- Print the classification evaluation measures computed on the training set (e.g. Accuracy, Confusion matrix, Precision, Recall, F1)<jupyter_code>print("==================== Train Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_train, rfc_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, rfc_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, rfc_predictions_train))<jupyter_output>==================== Train Data =======================
Accuracy:  0.968862620729422
Confusion matrix: 
 [[6674   17]
 [ 199   47]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      1.00      0.98      6691
           1       0.73      0.19      0.30       246

    accuracy                           0.97      6937
   macro avg       0.85      0.59      0.64      6937
weighted avg       0.96      0.97      0.96      6937<jupyter_text>Evaluate the model using the hold-out (30% examples) test set<jupyter_code>rfc_predictions_test = rfc.predict(X_test)
df_true_vs_rfc_predicted_test = pd.DataFrame({'ActualClass': y_test, 'PredictedClass': rfc_predictions_test})
df_true_vs_rfc_predicted_test.head(20)
print("==================== Test Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_test, rfc_predictions_test))
print("Confusion matrix: \n", metrics.confusion_matrix(y_test, rfc_predictions_test))
print("Classification report:\n ", metrics.classification_report(y_test, rfc_predictions_test))
print()
print("==================== Train Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_train, rfc_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, rfc_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, rfc_predictions_train))
print("======================================================")<jupyter_output>==================== Test Data =======================
Accuracy:  0.9616677874915938
Confusion matrix: 
 [[2842   17]
 [  97   18]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      2859
           1       0.51      0.16      0.24       115

    accuracy                           0.96      2974
   macro avg       0.74      0.58      0.61      2974
weighted avg       0.95      0.96      0.95      2974


==================== Train Data =======================
Accuracy:  0.968862620729422
Confusion matrix: 
 [[6674   17]
 [ 199   47]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      1.00      0.98      6691
           1       0.73      0.19      0.30       246

    accuracy                           0.97      6937
   macro avg       0.85      0.59      0.64      6937
weighted avg       0.96      0.97      0.96      6937

===============================[...]<jupyter_text>Interpretation of Test resulst compared to Train results- Accuracy    - The accuracy of the test data is lower. - Precision    - The precision score for predicting the positive case has lowered.    - The precision score for predicting the negative case hasn't change.- Recall    - The recall score for predicting the positive case has lowered.    - The recall score for predicting the negative case has lowered.- F1    - The f1 score for predicting the positive case has lowered.    - The f1 score for predicting the negative case hasn't change.   Cross validation- Preform cross validation and store the results in a dictionary for later use<jupyter_code>def cross_val_RandomForest_DF(X,y, depth=None, estimators=100):
    RandomForestResults = {}
    test_metrics = ['accuracy','precision','recall', 'f1']

    for metric in test_metrics:
        result = cross_val_score(RandomForestClassifier(n_estimators=estimators, max_features='auto', oob_score=True, random_state=1, max_depth=depth), X, y, scoring=metric, cv=10)
        RandomForestResults[metric] = result.mean()
    
    RandomForestDF = pd.DataFrame.from_dict(RandomForestResults, orient='index', columns=['Random_Forests'])

    return RandomForestDF
RandomForestDF = cross_val_RandomForest_DF(X,y)
print(f"Mean results from 10 fold cross validation are:")
RandomForestDF
rfc.oob_score_<jupyter_output><empty_output><jupyter_text>Improving Predictive Models<jupyter_code>ResultsDF = pd.concat([linRegDF, logRegDF, RandomForestDF], axis=1)
ResultsDF<jupyter_output><empty_output><jupyter_text>Conclusion:From the table shown above we can conclude that the best model, in my case, is the Logistic Regression Model as it has the best `recall` and `f1-score`.The next best performing model is Random Forest. We can try to optimize Random Forests and maybe combine it with Logistic Regression in order to get a better prediction.Further down I will try to improve both Random Forest and Logistic Regression and stack them.   Check if the model is more accurate than a simple model that always predicts the majority class<jupyter_code>good_count = y_test[y_test == 1].count()
bad_count = y_test[y_test == 0].count()
total = good_count+ bad_count

print(f'From original dataset: \t\tCount {total}')
print(f'Total number death yes:\t {good_count}\t{round(good_count/len(y_test)*100,2)}%')
print(f'Total number death no:\t {bad_count}\t{round(bad_count/len(y_test)*100,2)}%')
# generate array of 2974 zeros (length of test dataset)
majority = np.zeros(2974)
df_majorityClass = pd.DataFrame(majority, columns=['prediction'])

print("Accuracy: ", metrics.accuracy_score(y_test, df_majorityClass))
print("Confusion matrix: \n", metrics.confusion_matrix(y_test, df_majorityClass))
print("Classification report:\n ", metrics.classification_report(y_test, df_majorityClass))<jupyter_output>Accuracy:  0.9613315400134499
Confusion matrix: 
 [[2859    0]
 [ 115    0]]
Classification report:
                precision    recall  f1-score   support

           0       0.96      1.00      0.98      2859
           1       0.00      0.00      0.00       115

    accuracy                           0.96      2974
   macro avg       0.48      0.50      0.49      2974
weighted avg       0.92      0.96      0.94      2974<jupyter_text>Compare Model performance using different feature subsets  All features- Repeat the tests using all features - Set up the dataframe using the original features<jupyter_code>df_all = pd.get_dummies(df_prepared)
y = df_all["death_yn"]
X = df_all.drop(["death_yn"],1)

print("number of features incl dummies: ", len(X.columns))
# calculate Linear regression, logistic regression, random forest results again using all features
linRegDF = cross_val_LinReg_DF(X,y)
logRegDF = cross_val_LogReg_DF(X,y)
RandomForestDF = cross_val_RandomForest_DF(X,y)
# merge all 3 models into dataframe
ResultsDF_allFeatures = pd.concat([linRegDF, logRegDF, RandomForestDF], axis=1)
ResultsDF_allFeatures
print("\nScores using all features: 28 features (0 features dropped)")
print("===================================================================")
print(ResultsDF_allFeatures)
print("\nScores using subset of features: 18 features (10 features dropped)")
print("===================================================================")
print(ResultsDF)<jupyter_output>Scores using all features: 28 features (0 features dropped)
===================================================================
           Linear_Regression  Logistic_Regression  Random_Forests
accuracy            0.963618             0.964787        0.964585
precision           0.638890             0.541888        0.570565
recall              0.132446             0.274249        0.155255
f1                  0.218606             0.360121        0.239474

Scores using subset of features: 18 features (10 features dropped)
===================================================================
           Linear_Regression  Logistic_Regression  Random_Forests
accuracy            0.963618             0.964887        0.964585
precision           0.638890             0.546201        0.566248
recall              0.132446             0.265916        0.155255
f1                  0.218606             0.352780        0.239298<jupyter_text>The results above show that there is not a big difference in accuracy between the two tables aas the features we removed have a low value.  Further optimisations- Optimizing Random Forests by changing the Depth and Number of Estimators- Combine Logistic Regression with Random Forests to see if there is an improvement  Optimise Max Depth<jupyter_code># setup dataframe
df_all = pd.get_dummies(df_prepared)
y = df_all["death_yn"]
X = df_all.drop(["death_yn"],1)

RandomForest_Depth = pd.DataFrame()

for i in range(1,18):
    df1 = cross_val_RandomForest_DF(X,y,i)
    df1.rename(columns={'Random_Forests':f'depth={i}'}, inplace=True)
    RandomForest_Depth = pd.concat([RandomForest_Depth, df1], axis=1)

no_max_depth = cross_val_RandomForest_DF(X,y)
no_max_depth.rename(columns={'Random_Forests':f'no_max_depth'}, inplace=True)
RandomForest_Depth = pd.concat([RandomForest_Depth, no_max_depth], axis=1)
RandomForest_Depth<jupyter_output><empty_output><jupyter_text>It stabilizes after a depth of 14  Optimise Number of Estimators<jupyter_code># initialised empty daraframe
RandomForest_Estimators = pd.DataFrame()

n_estimators = [1, 2, 4, 8, 16, 32, 64, 100, 200]

for item in n_estimators:
    df1 = cross_val_RandomForest_DF(X,y,estimators=item)
    df1.rename(columns={'Random_Forests':f'estimators={item}'}, inplace=True)
    RandomForest_Estimators = pd.concat([RandomForest_Estimators, df1], axis=1)

RandomForest_Estimators<jupyter_output><empty_output><jupyter_text>Best performance with 1 Estimator<jupyter_code># setup dataframe
df_all = pd.get_dummies(df_prepared)
y = df_all["death_yn"]
X = df_all.drop(["death_yn"],1)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1 , test_size=0.3)

rfc = LogisticRegression()
rfc.fit(X_train, y_train)
importance_lr = pd.DataFrame({'feature': X_train.columns, 'importance':rfc.coef_[0]})
importance_lr.sort_values('importance', ascending=False, inplace=True)
importance_lr.set_index('feature',1, inplace=True)
importance_lr
# Train RF with 1 tree
rfc = RandomForestClassifier(n_estimators=1, max_features='auto', oob_score=True, random_state=1)
rfc.fit(X_train, y_train)
importance_rf = pd.DataFrame({'feature': X_train.columns, 'importance':rfc.feature_importances_})
importance_rf.sort_values('importance', ascending=False, inplace=True)
importance_rf.set_index('feature',1, inplace=True)
importance_rf<jupyter_output><empty_output><jupyter_text>Check number of features and their impact Random forest<jupyter_code># set up empty list to store features
list_features = []

# set up empty dataframe to store results
RandomForest_Features = pd.DataFrame()
count=0

for index, row in importance_rf.iterrows():
    if count < 20:
        list_features.append(index)
        X = df_all[list_features]
        df1 = cross_val_RandomForest_DF(X,y, depth=14)
        df1.rename(columns={'Random_Forests':f'features={len(list_features)}'}, inplace=True)
        RandomForest_Features = pd.concat([RandomForest_Features, df1], axis=1)
        count+=1
    else:
        break


RandomForest_Features<jupyter_output><empty_output><jupyter_text>Logistic regression<jupyter_code># set up empty list to store features
list_features = []

# set up empty dataframe to store results
LogReg_Features = pd.DataFrame()
count=0

for index, row in importance_lr.iterrows():
    if count < 20:
        list_features.append(index)
        X = df_all[list_features]
        df1 = cross_val_LogReg_DF(X,y, depth=14)
        df1.rename(columns={'Logistic_Regression':f'features={len(list_features)}'}, inplace=True)
        LogReg_Features = pd.concat([LogReg_Features, df1], axis=1)
        count+=1
    else:
        break


LogReg_Features<jupyter_output><empty_output><jupyter_text>Conclusion: - Random Forest: It oscilates but starts to stabilize a bit after 5 features- Logistic Regression: It oscilates but starts to stabilize a bit after 4 featuresWe can now define our list of optimum features<jupyter_code># Set list of optimum features
optimum_features = list_features[0:11]<jupyter_output><empty_output><jupyter_text>Review performance of all models using what we have learned so far- For all models use optimum_features based on random forests feature importance evaluation- For random forests model use max_depth=16 and keep default number estimators=16<jupyter_code>df_optimum_features = pd.get_dummies(df_prepared)
y = df_optimum_features["death_yn"]
X = df_optimum_features[optimum_features]

print("Number of features: ", len(X.columns))

# calculate Linear regression, logistic regression, random forest results again
linRegDF = cross_val_LinReg_DF(X,y)
logRegDF = cross_val_LogReg_DF(X,y)
RandomForestDF = cross_val_RandomForest_DF(X,y,depth=14,estimators=1)
# merge all 3 models into dataframe
ResultsDF_optimum_features = pd.concat([linRegDF, logRegDF, RandomForestDF], axis=1)
ResultsDF_optimum_features
print("\nScores using all features: 28 features (0 features dropped)")
print("===================================================================")
print(ResultsDF_allFeatures)
print("\nScores using subset of features: 18 features (10 features dropped)")
print("===================================================================")
print(ResultsDF)
print("\nScores using optimum features: 10 features (18 features dropped)")
print("===================================================================")
print(ResultsDF_optimum_features)<jupyter_output>Scores using all features: 28 features (0 features dropped)
===================================================================
           Linear_Regression  Logistic_Regression  Random_Forests
accuracy            0.963618             0.964787        0.964585
precision           0.638890             0.541888        0.570565
recall              0.132446             0.274249        0.155255
f1                  0.218606             0.360121        0.239474

Scores using subset of features: 18 features (10 features dropped)
===================================================================
           Linear_Regression  Logistic_Regression  Random_Forests
accuracy            0.963618             0.964887        0.964585
precision           0.638890             0.546201        0.566248
recall              0.132446             0.265916        0.155255
f1                  0.218606             0.352780        0.239298

Scores using optimum features: 10 features (18 features dropped)
=========[...]<jupyter_text>Combining Random Forests with Logistic Regression  Functions used for stacking the models<jupyter_code># get a list of models to evaluate
def get_models():
    models = dict()
    models['lr'] = multiple_logisticreg
    models['rf'] = rfc
    models['stacking'] = get_stacking()
    return models

# get a stacking ensemble of models
def get_stacking():
    # define the base models
    level0 = list()
    level0.append(('lr', multiple_logisticreg))
    level0.append(('rf', rfc))
    # define meta learner model
    level1 = LogisticRegression()
    # define the stacking ensemble
    model_combined = StackingClassifier(estimators=level0, final_estimator=level1, cv=5)
    return model_combined

# evaluate a give model using cross-validation
def evaluate_model(model, X, y):
    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
    scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
    return scores<jupyter_output><empty_output><jupyter_text>Stacking the models and comparing the results<jupyter_code># get the models to evaluate
models = get_models()
# evaluate the models and store results
results, names = list(), list()
for name, model in models.items():
    scores = evaluate_model(model, X, y)
    results.append(scores)
    names.append(name)
    print('>%s %.3f (%.3f)' % (name, mean(scores), std(scores)))
# plot model performance for comparison
pyplot.boxplot(results, labels=names, showmeans=True)
pyplot.show()<jupyter_output>>lr 0.965 (0.003)
>rf 0.964 (0.003)
>stacking 0.965 (0.004)<jupyter_text>Conclusion:- Combining Logistic Regression and Random Forests does not improve the prediction- The best model in my case is Logistic Regression as it has the best recall and f1-score- There is not a big difference between dropping various numbers of features- Logistic regression is only one that performs better after dropping 18 features (compared to not dropping any feature)  Running the model on the provided data setI will predict using Logistic Regression as that is the best performing one in my case.Steps:- Read in data set- Merge `Missing` and `OTH` into `Unknown`- Drop featuires to match the train set- Encode `death_yn`- Split data- Predict- Compare to `y_test` resultsAs the *provided data set* is not used for training a model and only for predicting I am not cleaning the data set, just preparing it as in a real life situation I would like the model to be able to predict on a data set that contains NaN values or even values that were not used for training and they made their way into the data set.<jupyter_code>df_provided = pd.read_csv('24032021-covid19-cdc-deathyn-recent-10k.csv', keep_default_na=True, sep=',\s+', delimiter=',', skipinitialspace=True)
df_provided.replace({'hosp_yn' : {'Missing' : 'Unknown', 'OTH' : 'Unknown'}}, inplace=True)
df_provided.replace({'icu_yn' : {'Missing' : 'Unknown'}}, inplace=True)
df_provided.replace({'medcond_yn' : {'Missing' : 'Unknown'}}, inplace=True)
df_provided.replace({'race_ethnicity_combined' : {'Missing' : 'Unknown'}}, inplace=True)
df_provided.replace({'sex' : {'Missing' : 'Unknown'}}, inplace=True)
df_provided.replace({'age_group' : {'Missing' : 'Unknown'}}, inplace=True)
#df_provided.isnull().sum()
print("death_yn:", pd.unique(df["death_yn"]))
df_provided = df_provided.drop(["cdc_case_earliest_dt", "cdc_report_dt", "pos_spec_dt", "current_status", "sex", "onset_dt","race_ethnicity_combined"], 1)
df_provided
df_rev2 = df_provided.copy()
df_rev2.replace({'death_yn' : {'No' : 0, 'Yes' : 1}}, inplace=True)
df_rev2['death_yn'] = df_rev2['death_yn'].astype('int64')

df_rev2_X = df_rev2.drop(["death_yn"], 1)
df_rev2_Y = df_rev2['death_yn']


df_rev2_dummy = pd.get_dummies(df_rev2_X, columns=['age_group','hosp_yn','icu_yn','medcond_yn'], drop_first=True)
df_rev2_X.shape
df_rev2_dummy.shape
multiple_logreg_predictions_provided = multiple_logisticreg.predict(df_rev2_dummy)

print("\nPredictions with multiple linear regression: \n")
actual_vs_predicted_multiplelogreg_provided = pd.concat([df_rev2_Y, pd.DataFrame(multiple_logreg_predictions_provided, columns=['Predicted'])], axis=1)
print(actual_vs_predicted_multiplelinreg.head(10))
# Some more evaluation metrics.

print("==================== Test Data =======================")
print("Accuracy: ", metrics.accuracy_score(y_test, multiple_logisticreg_predictions_test))
print("Confusion matrix: \n", metrics.confusion_matrix(y_test, multiple_logisticreg_predictions_test))
print("Classification report:\n ", metrics.classification_report(y_test, multiple_logisticreg_predictions_test))
print("==================== Train Data =======================")
print()
print("Accuracy: ", metrics.accuracy_score(y_train, multiple_logisticreg_predictions_train))
print("Confusion matrix: \n", metrics.confusion_matrix(y_train, multiple_logisticreg_predictions_train))
print("Classification report:\n ", metrics.classification_report(y_train, multiple_logisticreg_predictions_train))
print()
print("==================== Provided Data ====================")
print("Accuracy: ", metrics.accuracy_score(df_rev2_Y, multiple_logreg_predictions_provided))
print("Confusion matrix: \n", metrics.confusion_matrix(df_rev2_Y, multiple_logreg_predictions_provided))
print("Classification report:\n ", metrics.classification_report(df_rev2_Y, multiple_logreg_predictions_provided))
print("======================================================")<jupyter_output>==================== Test Data =======================
Accuracy:  0.9626765299260256
Confusion matrix: 
 [[2834   25]
 [  86   29]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      2859
           1       0.54      0.25      0.34       115

    accuracy                           0.96      2974
   macro avg       0.75      0.62      0.66      2974
weighted avg       0.95      0.96      0.96      2974

==================== Train Data =======================

Accuracy:  0.9658353755225602
Confusion matrix: 
 [[6633   58]
 [ 179   67]]
Classification report:
                precision    recall  f1-score   support

           0       0.97      0.99      0.98      6691
           1       0.54      0.27      0.36       246

    accuracy                           0.97      6937
   macro avg       0.75      0.63      0.67      6937
weighted avg       0.96      0.97      0.96      6937


==================== Provided[...]
{'fraction_non_alphanumeric': 0.12883416361186828, 'fraction_numerical': 0.04792117053607795, 'mean_word_length': 3.8613966948442617, 'pattern_counts': {'":': 0, '<': 167, '<?xml version=': 0, '>': 173, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 85, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '46', 'dir': 'jupyter-structured-clean-dedup', 'id': '439211', 'license': 'Unlicense', 'n_tokens_mistral': 21500, 'n_tokens_neox': 19214, 'n_words': 5612, 'path': 'Stefan_Bordei_20206769_COMP47350_Homework2.ipynb', 'repo_name': 'stefan-bordei/python-CDC-covid19-dataAnalysis', 'size': '63841'}