<jupyter_start><jupyter_text>Assignment 2<jupyter_code>import pandas as pd
Records='heart_failure_clinical_records_dataset.csv'
Rd = pd.read_csv(Records, encoding='ISO-8859-1',sep=',', decimal='.')

Rd.shape<jupyter_output><empty_output><jupyter_text>Task 1. Retrieving and Preparing the Data<jupyter_code>Rd.dtypes
Rd.head(300)
print(Rd['DEATH_EVENT'].value_counts())
print(Rd['time'].value_counts())
print(Rd['smoking'].value_counts())
print(Rd['sex'].value_counts())
print(Rd['serum_sodium'].value_counts())
print(Rd['serum_creatinine'].value_counts())
print(Rd['platelets'].value_counts())
print(Rd['platelets'].max())
print(Rd['platelets'].min())
print(Rd['serum_creatinine'].max())
print(Rd['serum_creatinine'].min())
print(Rd['serum_sodium'].min())
print(Rd['serum_sodium'].max())
print(Rd['time'].max())
print(Rd['time'].min())
Rd.dtypes
Rd['platelets']=Rd['platelets'].div(1000).round(2)
print(Rd['platelets'].value_counts())

print(Rd['platelets'].max())
print(Rd['platelets'].min())
Rd['gender']=Rd['sex']
Rd['gender']=Rd['gender'].apply(str)
Rd.gender.replace({'1':'male','0':'female'},inplace=True)   
print(Rd['gender'].value_counts())
Rd['Smoking']=Rd['smoking']
Rd['Smoking']=Rd['Smoking'].apply(str)
Rd.Smoking.replace({'1':'Yes','0':'No'},inplace=True)   
print(Rd['Smoking'].value_counts())
Rd['Status']=Rd['DEATH_EVENT']
Rd['Status']=Rd['Status'].apply(str)
Rd.Status.replace({'1':'Dead','0':'Alive'},inplace=True)   
print(Rd['Status'].value_counts())
Rd['Serum_Sodium']=Rd['serum_sodium']
Rd.Serum_Sodium.replace({113:114},inplace=True)   
print(Rd['Serum_Sodium'].value_counts())
Rd.head(5)
print(Rd['Serum_Sodium'].min())
# ===== Now cleaning the first half columns of the dataset =====
# Clean the age column
# Check for min, max and null values
Rd.sort_values(by='age',ascending=False)
# Clean anaemia column
# Check if there is anything other than 0 and 1
Rd['anaemia'].value_counts()
# Check creatinine_phosphokinase
# Check for min, max and null values
Rd.sort_values(by='creatinine_phosphokinase',ascending=False)
# Clean diabetes column
# Check if there is anything other than 0 and 1
Rd['diabetes'].value_counts()
# Check ejection_fraction
# Check for min, max and null values
Rd.sort_values(by='ejection_fraction',ascending=False)
# Clean diabetes column
# Check if there is anything other than 0 and 1
Rd['high_blood_pressure'].value_counts()<jupyter_output><empty_output><jupyter_text>Task 2. Data Exploration<jupyter_code>from matplotlib import pyplot as plt
import numpy as np
from matplotlib.pyplot import pie, axis, show
def label_function(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'
Rd.groupby('gender').size().plot(kind='pie', autopct=label_function, textprops={'fontsize': 20},
                                 colors=['violet', 'lime'])
plt.ylabel('Gender', size=22)
plt.show()
def label_function_deadAlive(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'
Rd.groupby('Status').size().plot(kind='pie', autopct=label_function_deadAlive, textprops={'fontsize': 20},
                                 colors=['gold', 'skyblue'])
plt.ylabel('STATUS', size=22)
plt.show()
def label_function_SMOKING(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'
Rd.groupby('Smoking').size().plot(kind='pie', autopct=label_function_SMOKING, textprops={'fontsize': 20},
                                 colors=['skyblue', 'red'])
plt.ylabel('smoking', size=22)
plt.show()
import numpy as np
import scipy as sp
from scipy import stats
mu = round(Rd['Serum_Sodium'].mean(), 2)
print(mu)
sigma = round(Rd['Serum_Sodium'].std(), 2)
print(sigma)
Rd.dtypes
Rd.Serum_Sodium.plot.density(color='green')
plt.title('Density plot for serum_sodium')
plt.show()
Rd['Serum_Sodium'].describe()
Rd.platelets.plot.density(color='green')
plt.title('Density plot for platelets')
plt.show()
Rd['platelets'].describe()
mu = round(Rd['platelets'].mean(), 2)
print(mu)
#Check the number of female/male patients
patient_seen=Rd['gender'].value_counts()

#creating masks for male/female
mask_f=Rd['gender']=='female'
mask_m=Rd['gender']=='male'

f_seen=Rd.loc[mask_f,'Smoking'].value_counts()
m_seen=Rd.loc[mask_m,'Smoking'].value_counts()
print(m_seen)
print('female smoker:', f_seen[1])
print('male smoker :', m_seen[1])


rate=[f_seen[1],m_seen[1]]


plt.bar(list(range(2)),rate,color='purple',align='center')
plt.xticks(list(range(2)),['FEMALE','MALE'])
plt.ylabel('number of people')
plt.xlabel('Gender')
plt.title('smoking ')
lol = Rd.groupby('gender').sum()
print(lol)
graph = Rd.groupby('Smoking').sum()
print(graph)
rate=graph['DEATH_EVENT']
#ploting the rate as bar graph
plt.bar(list(range(2)),rate,color='purple',align='center')
plt.xticks(list(range(2)),['Non Smokers','Smokers'])
plt.ylabel('number of people')
plt.xlabel('Smoker')
plt.title('smoking ')
# ===== Explore each column in the first 6 columns =====
# Explore Age column
# See the distribution of the age (Draw mean: https://stackoverflow.com/questions/16180946/drawing-average-line-in-histogram-matplotlib)
Rd['age'].plot(kind='hist',bins=50, figsize=(10,5))
plt.title('Age Info')
plt.xlabel('Age')
plt.axvline(Rd['age'].mean(), color='k', linestyle='dashed', linewidth=3)
min_ylim, max_ylim = plt.ylim()
plt.text(Rd['age'].mean()*1.01, max_ylim*0.95, 'Average age: {:.2f}'.format(Rd['age'].mean()))
plt.show()
# Explore anaemia column
# Group to see the total and percentage
group_anaemia = Rd.groupby('anaemia').size()

def label_function_anaemia(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'

group_anaemia.plot(kind='pie', autopct=label_function_anaemia, textprops={'fontsize': 20}, figsize=(5,5), colors=['lightgreen', 'indianred'])
plt.title('Anaemia Info', size=22)
plt.ylabel(None)
plt.legend(["Negative", "Positive"]);
plt.show()
# Explore creatinine_phosphokinase column
# See the distribution of creatinine_phosphokinase 
Rd.boxplot(column=['creatinine_phosphokinase'], figsize=(5,5))
plt.ylabel('mcg/L')
plt.title('Creatinine Phosphokinase Info')
plt.show()
# Explore diabetes column
# Group to see the total and percentage
group_diabetes = Rd.groupby('diabetes').size()

def label_function_diabetes(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'

group_diabetes.plot(kind='pie', autopct=label_function_diabetes, textprops={'fontsize': 20}, figsize=(5,5), colors=['lightgreen', 'indianred'])
plt.title('Diabetes Info', size=22)
plt.ylabel(None)
plt.legend(["Negative", "Positive"]);
plt.show()
# Explore ejection_fraction column
# See the distribution of ejection_fraction 
Rd.boxplot(column=['ejection_fraction'], figsize=(5,5))
plt.ylabel('Percentage')
plt.title('Ejection Fraction Info')
plt.show()
# Explore high_blood_pressure column
# Group to see the total and percentage
group_high_blood_pressure = Rd.groupby('high_blood_pressure').size()

def label_function_HBP(val):
    return f'{val / 100 * len(Rd):.0f}\n{val:.0f}%'

group_high_blood_pressure.plot(kind='pie', autopct=label_function_HBP, textprops={'fontsize': 20}, figsize=(5,5), colors=['lightgreen', 'indianred'])
plt.title('High Blood Pressure Info', size=22)
plt.ylabel(None)
plt.legend(["Negative", "Positive"]);
plt.show()
# ===== End explore each column in the first 6 columns =====
male = Rd.loc[Rd['gender']=='male']
print(male)
female = Rd.loc[Rd['gender']=='female']
print(female)
x = male['platelets']
#print(x)
y = female['platelets']
#print(y)
#import plotly.express as px
import seaborn as sns
sns.scatterplot(x = "age", y = "platelets", data = Rd)
sns.scatterplot(x = "age", y = "serum_creatinine", data = Rd)
sns.scatterplot(x = "age", y = "Serum_Sodium", data = Rd)
# Relationship between age and creatinine_phosphokinase
plt.figure(figsize=(10, 10))
plt.scatter('age', 'creatinine_phosphokinase', data=Rd, color='royalblue')
plt.xlabel('Age')
plt.ylabel('mcg/L')
plt.title('Distribution of creatinine phosphokinase level by age')
plt.show()
# Relationship between creatinine_phosphokinase and DEATH_EVENT
group_cpk_death = Rd[['creatinine_phosphokinase', 'DEATH_EVENT']].groupby('DEATH_EVENT').mean()

group_cpk_death.plot(kind='bar', xlabel='Death Event (0 - Alive, 1 - Death)', ylabel='mcg/L', rot=0, color='royalblue', figsize=(7,7))
plt.title('Average creatinine phosphokinase level by death event')
plt.legend()
plt.show()
# Relationship between age and ejection_fraction
plt.figure(figsize=(10, 10))
plt.scatter('age', 'ejection_fraction', data=Rd, color='indianred')
plt.xlabel('Age')
plt.ylabel('Percentage')
plt.title('Distribution of ejection fraction level by age')
plt.show()
# Relationship between ejection_fraction and DEATH_EVENT
group_eject_death = Rd[['ejection_fraction', 'DEATH_EVENT']].groupby('DEATH_EVENT').mean()

group_eject_death.plot(kind='bar', xlabel='Death Event (0 - Alive, 1 - Death)', ylabel='Percentage', rot=0, color='indianred', figsize=(7,7))
plt.title('Average ejection fraction level by death event')
plt.legend()
plt.show()
# Relationship between age and time
group_age_time = Rd[['age', 'time']].groupby('age').mean()

plt.figure(figsize=(10, 5))
plt.plot('time', data=group_age_time, color='blue')
plt.xlabel('Age')
plt.ylabel('Days')
plt.title('Average follow up period by age')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Task 3. Data Modelling  3.1 Decision Tree<jupyter_code>#copied Data frame data into the new frame df for decision tree 
Rd.dtypes
copy_column = Rd[['age','anaemia','creatinine_phosphokinase','diabetes','ejection_fraction','high_blood_pressure','platelets','serum_creatinine','Serum_Sodium','sex','smoking','time','DEATH_EVENT']]
df = copy_column.copy()

df.head(5)

X = df.drop('DEATH_EVENT',axis=1)
y=df.DEATH_EVENT #dependand variable 
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.3,random_state=4)

from sklearn.metrics import accuracy_score ,confusion_matrix,precision_score,recall_score,f1_score
def print_score(clf,X_train,y_train,X_test,y_test,train=True):
    if train:       
        pred = clf.predict(X_train)
        print("Train Result:\n#####################")
        print(f"accuracy score:{accuracy_score(y_train,pred):4f}\n")
        print(f"accuracy percentage:{(accuracy_score(y_train,pred)*100).round(2)}\n")
        print(f"Classification Report:\n \tPrecision:{precision_score(y_train,pred)}\n\tRecall Score:{recall_score(y_train,pred)}\n")
        
        print(f"Confusion Matrix:\n{confusion_matrix(y_train,clf.predict(X_train))}\n")
    elif train == False:
        pred = clf.predict(X_test)
        print("Test Result:\n#####################")
        print(f"accuracy score:{accuracy_score(y_test,pred):4f}\n")
        print(f"accuracy percentage:{(accuracy_score(y_test,pred)*100).round(2)}\n")
        print(f"Classification Report:\n \tPrecision:{precision_score(y_test,pred)}\n\tRecall Score:{recall_score(y_test,pred)}\n")
        
        print(f"Confusion Matrix:\n{confusion_matrix(y_test,pred)}\n") 



from sklearn.tree import DecisionTreeClassifier 
tree = DecisionTreeClassifier(random_state=4)
tree.fit(X_train,y_train)
print_score(tree,X_train,y_train,X_test,y_test,train=True)
print_score(tree,X_train,y_train,X_test,y_test,train=False)<jupyter_output>Train Result:
#####################
accuracy score:1.000000

accuracy percentage:100.0

Classification Report:
 	Precision:1.0
	Recall Score:1.0

Confusion Matrix:
[[137   0]
 [  0  72]]

Test Result:
#####################
accuracy score:0.755556

accuracy percentage:75.56

Classification Report:
 	Precision:0.5384615384615384
	Recall Score:0.5833333333333334

Confusion Matrix:
[[54 12]
 [10 14]]<jupyter_text>Decision Tree Hyperparameter Tuning<jupyter_code># Find the best k for Decision Tree for max depth
for k in np.arange(1, 16):
    clf = DecisionTreeClassifier(max_depth=k,random_state=4)
    clf.fit(X_train, y_train)
    predicted = clf.predict(X_test)
    # Now check the prediction accuracy
    print("[k = {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# Find the best k for Decision Tree for criterion
for k in np.arange(1,3):
    
    if k ==1:
        clf = DecisionTreeClassifier(criterion="gini",random_state=4)
        clf.fit(X_train, y_train)
        predicted = clf.predict(X_test)
        print("[k = {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
    elif k==2:
        clf = DecisionTreeClassifier(criterion="entropy")
        clf.fit(X_train, y_train)
        predicted = clf.predict(X_test)
        print("[k = {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
    

   
# Find the best k for Decision Tree for min_samples_leaf
for k in np.arange(1, 16):
    clf = DecisionTreeClassifier(min_samples_leaf=k,random_state=4)
    clf.fit(X_train, y_train)
    predicted = clf.predict(X_test)
    # Now check the prediction accuracy
    print("[k = {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# Looks like k = 7 gives the most accurate result, so set k = 7 for max depth
# Looks like k = 2 gives the most accurate result, so set k = 2 for criterion means set to entropy 
# Looks like k = 5 gives the most accurate result, so set k = 12 for min_samples_leaf
clf = DecisionTreeClassifier(criterion="entropy",max_depth=7,min_samples_leaf=12,random_state=4)

# Find the set of features the gives the best results using hill climbing technique
from sklearn.utils import shuffle
new_Ind = []
cur_MaxScore = 0.0
col_num = 12
col_Ind_Random = shuffle(range(0,col_num), random_state=4)
for cur_f in range(0, col_num):
    new_Ind.append(col_Ind_Random[cur_f])
    newData = X.iloc[:, new_Ind]
    X_train, X_test, y_train, y_test = train_test_split(newData, y, test_size=0.30, random_state=4)
    clf.fit(X_train, y_train)
    cur_Score = clf.score(X_test, y_test)
    if cur_Score < cur_MaxScore:
        new_Ind.remove(col_Ind_Random[cur_f])
    else:
        cur_MaxScore = cur_Score
        print("Score with " + str(len(new_Ind)) + " selected features: " + str(cur_Score))
print("There are " + str(len(new_Ind)) + " features selected:")
print(new_Ind)
# Set the features to the 4
ds_selected_data = X.iloc[:, [3, 9, 11, 0]]
ds_selected_data
# Test the accuracy again with the finding about k and selected columns
X_train,X_test,y_train,y_test=train_test_split(ds_selected_data,y,test_size=0.3,random_state=42)
clf.fit(X_train,y_train)
print_score(clf,X_train,y_train,X_test,y_test,train=True)
print_score(clf,X_train,y_train,X_test,y_test,train=False)
# Now use K-Folds Cross Validation to validate the model - with selected k and selected columns
from sklearn.model_selection import KFold
kf = KFold(n_splits=5)
clf_entropy = DecisionTreeClassifier(ccp_alpha=0.0,class_weight=None,criterion="entropy",
                             max_depth=7,max_features= None,max_leaf_nodes= None,min_impurity_decrease = 0.0,
                             min_impurity_split=None, min_samples_leaf = 12,min_samples_split=2,
                             min_weight_fraction_leaf=0.0, random_state=42,splitter='best')
for k, (train_index, test_index) in enumerate(kf.split(ds_selected_data)):
    X_train, X_test = ds_selected_data.iloc[train_index], ds_selected_data.iloc[test_index]
    y_train, y_test = y.iloc[train_index], y.iloc[test_index]
    clf.fit(X_train, y_train)
    print("[fold {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# Now use K-Folds Cross Validation to validate the model from lab 4 - with all columns and default k
for k, (train_index, test_index) in enumerate(kf.split(X)):

    X_train, X_test = X.iloc[train_index], X.iloc[test_index]
    y_train, y_test = y.iloc[train_index], y.iloc[test_index]

    clf_entropy.fit(X_train, y_train)

    print("[fold {0}] score: {1:.5f}".format(k, clf_entropy.score(X_test, y_test)))
print_score(clf_entropy,X_train,y_train,X_test,y_test,train=True)
print_score(clf_entropy,X_train,y_train,X_test,y_test,train=False)    
# The result clearly shows that with decision tree the accuracy is around 89.83 %
# =================== End Decision Tree Classifier ===================<jupyter_output><empty_output><jupyter_text>Visualization of Tree<jupyter_code>from sklearn.tree import export_graphviz
import graphviz
#displaying the decision tree without hypertuning 
export_graphviz(tree, out_file="mytree.dot",feature_names = X.columns)
with open("mytree.dot") as f:
    dot_graph = f.read()
graphviz.Source(dot_graph)
# #displaying the decision tree with hyperparameter tuning criterion gini
# export_graphviz(clf_gini, out_file="mytree.dot",feature_names = X.columns)
# with open("mytree.dot") as f:
#     dot_graph = f.read()
# graphviz.Source(dot_graph)
#displaying the decision tree with hyperparameter tuning criterion entropy
export_graphviz(clf_entropy, out_file="mytree.dot",feature_names = X.columns)
with open("mytree.dot") as f:
    dot_graph = f.read()
graphviz.Source(dot_graph)<jupyter_output><empty_output><jupyter_text>3.2 KNeighborsClassifier<jupyter_code># Set data and target for the dataset using all columns, also set the random state to make the result consistent
ds_data = Rd[['age','anaemia','creatinine_phosphokinase','diabetes','ejection_fraction','high_blood_pressure','platelets','serum_creatinine', 'serum_sodium', 'sex', 'smoking', 'time']]
ds_target = Rd['DEATH_EVENT']
random_state_value = 4
X_train, X_test, y_train, y_test = train_test_split(ds_data, ds_target, test_size=0.30, random_state=random_state_value)
# Print the train and test set to check
X_train.shape
X_test.shape
y_train.shape
y_test.shape
# Use KNN to train the dataset using 70% train and 30% test and k default = 5
from sklearn.neighbors import KNeighborsClassifier
clf = KNeighborsClassifier(5)
clf.fit(X_train, y_train)
predicted = clf.predict(X_test)
# Now check the prediction accuracy
print("[Train/test split] score: {:.5f}".format(clf.score(X_test, y_test)))
# Find the best k for KNN
for k in np.arange(1, 16):
    clf = KNeighborsClassifier(k)
    clf.fit(X_train, y_train)
    predicted = clf.predict(X_test)
    # Now check the prediction accuracy
    print("[k = {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# Looks like k = 7 gives the most accurate result, so set k = 7
clf = KNeighborsClassifier(7)
# Find the set of features the gives the best results using hill climbing technique from lab 6
from sklearn.utils import shuffle
new_Ind = []
cur_MaxScore = 0.0
col_num = 12
col_Ind_Random = shuffle(range(0,col_num), random_state=random_state_value)
for cur_f in range(0, col_num):
    new_Ind.append(col_Ind_Random[cur_f])
    newData = ds_data.iloc[:, new_Ind]
    X_train, X_test, y_train, y_test = train_test_split(newData, ds_target, test_size=0.30, random_state=random_state_value)
    clf.fit(X_train, y_train)
    cur_Score = clf.score(X_test, y_test)
    if cur_Score < cur_MaxScore:
        new_Ind.remove(col_Ind_Random[cur_f])
    else:
        cur_MaxScore = cur_Score
        print("Score with " + str(len(new_Ind)) + " selected features: " + str(cur_Score))
print("There are " + str(len(new_Ind)) + " features selected:")
print(new_Ind)
# Set the features to the 5 selected columns since after 5 features, the accuracy remains the same
ds_selected_data = ds_data.iloc[:, [3, 4, 9, 8, 11]]
ds_selected_data
# Test the accuracy again with the finding about k and selected columns
X_train, X_test, y_train, y_test = train_test_split(ds_selected_data, ds_target, test_size=0.30, random_state=random_state_value)
clf = KNeighborsClassifier(7)
clf.fit(X_train, y_train)
predicted = clf.predict(X_test)
# Now check the prediction accuracy
print("[Train/test split] score: {:.5f}".format(clf.score(X_test, y_test)))
# Now use K-Folds Cross Validation to validate the model from lab 4 - with selected k and selected columns
from sklearn.model_selection import KFold
kf = KFold(n_splits=5)
for k, (train_index, test_index) in enumerate(kf.split(ds_selected_data)):
    X_train, X_test = ds_selected_data.iloc[train_index], ds_selected_data.iloc[test_index]
    y_train, y_test = ds_target.iloc[train_index], ds_target.iloc[test_index]
    clf.fit(X_train, y_train)
    print("[fold {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# Now use K-Folds Cross Validation to validate the model from lab 4 - with all columns and default k
clf = KNeighborsClassifier(5)
kf = KFold(n_splits=5)
for k, (train_index, test_index) in enumerate(kf.split(ds_data)):
    X_train, X_test = ds_data.iloc[train_index], ds_data.iloc[test_index]
    y_train, y_test = ds_target.iloc[train_index], ds_target.iloc[test_index]
    clf.fit(X_train, y_train)
    print("[fold {0}] score: {1:.5f}".format(k, clf.score(X_test, y_test)))
# The result clearly shows that with selected k and columns, the accuracy is much higher
# =================== End KNeighborsClassifier ===================<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15607828015039044, 'fraction_numerical': 0.01860599633664321, 'mean_word_length': 6.1075710859883525, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '398394', 'license': 'BSL-1.0', 'n_tokens_mistral': 8030, 'n_tokens_neox': 7471, 'n_words': 1540, 'path': 'assignment2.ipynb', 'repo_name': 'shantanu779/data_science', 'size': '20746'}