<jupyter_start><jupyter_text>***Heart Attack Prediction***Data Dictionary* age - Age of the patient* sex - Sex of the patient* cp - Chest pain type ~ 0 = Typical Angina, 1 = Atypical Angina, 2 = Non-anginal Pain, 3 = Asymptomatic* trtbps - Resting blood pressure (in mm Hg)* chol - Cholestoral in mg/dl fetched via BMI sensor* fbs - (fasting blood sugar > 120 mg/dl) ~ 1 = True, 0 = False* restecg - Resting electrocardiographic results ~ 0 = Normal, 1 = ST-T wave normality, 2 = Left ventricular hypertrophy* thalachh - Maximum heart rate achieved* oldpeak - Previous peak* slp - Slope* caa - Number of major vessels* thall - Thalium Stress Test result ~ (0,3)* exng - Exercise induced angina ~ 1 = Yes, 0 = No* output - Target variable 1. reading the data<jupyter_code># Load libraries
import pandas as pd
import numpy as np
from pandas.plotting import scatter_matrix
from matplotlib import pyplot
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix
import warnings
warnings.filterwarnings('ignore')

%matplotlib inline
...

filepath = 'https://raw.githubusercontent.com/yoel2000/Computational-learning/main/Project/heart.csv'
data = pd.read_csv(filepath, sep=',')<jupyter_output><empty_output><jupyter_text>2. explore the data:<jupyter_code># shape
data.shape
# head
data.head(10)
# descriptions
data.describe()
# continuous fetures distrebutaiom by histograms
data[['age', 'trtbps', 'chol', 'thalachh', 'oldpeak']].hist(figsize=(35, 5), layout=(1,6))
pyplot.show()

# As we can see most of the fetures have a normal distribution 
# Count plot of categorical features
fig, ax = plt.subplots(2,4,figsize=(35,15))
sns.countplot(data['sex'], ax=ax[0][0])
sns.countplot(data['cp'], ax=ax[0][1])
sns.countplot(data['fbs'], ax=ax[0][2])
sns.countplot(data['restecg'], ax=ax[0][3])
sns.countplot(data['exng'], ax=ax[1][0])
sns.countplot(data['slp'], ax=ax[1][1])
sns.countplot(data['caa'], ax=ax[1][2])
sns.countplot(data['thall'], ax=ax[1][3])

for ax_r in ax:
  for ax_c in ax_r:
    for i, bar in enumerate(ax_c.patches):
        h = bar.get_height()
        ax_c.text(
            i, # bar index (x coordinate of text)
            h/2, # y coordinate of text
            '{}'.format(int(h)),  # y label
            ha='center', va='center', fontweight='bold', size=14, color='white')
    
fig.show()
# Count plot to label column
fig, ax = plt.subplots(1)
sns.countplot(data['output'])

for i, bar in enumerate(ax.patches):
    h = bar.get_height()
    ax.text(
        i, # bar index (x coordinate of text)
        h/2, # y coordinate of text
        '{}'.format(int(h)),  # y label
        ha='center', va='center', fontweight='bold', size=14, color='white')<jupyter_output><empty_output><jupyter_text>**Inspect correlation of continuous fetures**<jupyter_code>df_corr = data[['age', 'trtbps', 'chol', 'thalachh', 'oldpeak']].corr().transpose()
df_corr
fig = plt.figure(figsize=(10,10))
gs = fig.add_gridspec(1,1)
gs.update(wspace=0.3, hspace=0.15)
ax0 = fig.add_subplot(gs[0,0])

color_palette = ["#5833ff","#da8829"]
mask = np.triu(np.ones_like(df_corr))
ax0.text(1.5,-0.1,"Correlation Matrix",fontsize=22, fontweight='bold', fontfamily='serif', color="#000000")
df_corr = data[con_cols].corr().transpose()
sns.heatmap(df_corr,mask=mask,fmt=".1f",annot=True,cmap='YlGnBu')
plt.show()<jupyter_output><empty_output><jupyter_text>As we can above see there is no high correlation between the *features* **Check if there is missing data**<jupyter_code>data.isna().sum()
data.dtypes<jupyter_output><empty_output><jupyter_text>***Scaling the data***<jupyter_code>from sklearn.preprocessing import RobustScaler

# Scale features using statistics that are robust to outliers.

# This Scaler removes the median and scales the data according to the quantile range (defaults to IQR: Interquartile Range).

df = data
# define the columns to be encoded and scaled
cat_cols = ['sex','exng','caa','cp','fbs','restecg','slp','thall']
con_cols = ["age","trtbps","chol","thalachh","oldpeak"]


# defining the features and target
X = df.drop(['output'],axis=1) # features
y = df[['output']] # target

# encoding the categorical columns
df = pd.get_dummies(df, columns = cat_cols, drop_first = True)

# instantiating the scaler
scaler = RobustScaler()

# scaling the continuous featuree
X[con_cols] = scaler.fit_transform(X[con_cols])
print("The first 5 rows of X are")
X.head()
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.2, random_state = 42)
print("The shape of X_train is ", X_train.shape)
print("The shape of X_test is ",X_test.shape)
print("The shape of y_train is ",y_train.shape)
print("The shape of y_test is ",y_test.shape)
# Count plot to label column
fig, ax =plt.subplots(1,2,figsize=(15,6))
sns.countplot(y_test['output'], ax=ax[0]).set_title("y_test")
sns.countplot(y_train['output'], ax=ax[1]).set_title("y_train")

for i, bar in enumerate(ax[0].patches):
    h = bar.get_height()
    ax[0].text(
        i, # bar index (x coordinate of text)
        h/2, # y coordinate of text
        '{}'.format(int(h)),  # y label
        ha='center', va='center', fontweight='bold', size=14, color='white')

for i, bar in enumerate(ax[1].patches):
    h = bar.get_height()
    ax[1].text(
        i, # bar index (x coordinate of text)
        h/2, # y coordinate of text
        '{}'.format(int(h)),  # y label
        ha='center', va='center', fontweight='bold', size=14, color='white')
feature_importances = np.mean([tree.feature_importances_ for tree in  BC.estimators_], axis=0)
feature_importances
def measure_error(y_true, y_pred, label):
    return pd.Series({'accuracy':accuracy_score(y_true, y_pred),
                      'precision': precision_score(y_true, y_pred),
                      'recall': recall_score(y_true, y_pred),
                      'f1': f1_score(y_true, y_pred)},
                      name=label)

dt = DecisionTreeClassifier(random_state=42)
dt = dt.fit(X_train, y_train)
dt.tree_.node_count, dt.tree_.max_depth
# The error on the training and test data sets
y_train_pred = dt.predict(X_train)
y_test_pred = dt.predict(X_test)

train_test_full_error = pd.concat([measure_error(y_train, y_train_pred, 'train'),
                              measure_error(y_test, y_test_pred, 'test')],
                              axis=1)

train_test_full_error<jupyter_output><empty_output><jupyter_text>> We can see that as we learned in class, in decision tree, there is a problem of overfitting. We cas see here because the pourcentage of success in the test set is really smaller that the train set.<jupyter_code>from sklearn.model_selection import GridSearchCV

param_grid = {'max_depth':range(1, dt.tree_.max_depth+1, 2),
              'max_features': range(1, len(dt.feature_importances_)+1)}

GR = GridSearchCV(DecisionTreeClassifier(random_state=42),
                  param_grid=param_grid,
                  scoring='accuracy',
                  n_jobs=-1)

GR = GR.fit(X_train, y_train)
GR.best_estimator_.tree_.node_count, GR.best_estimator_.tree_.max_depth

y_train_pred_gr = GR.predict(X_train)
y_test_pred_gr = GR.predict(X_test)

train_test_gr_error = pd.concat([measure_error(y_train, y_train_pred_gr, 'train'),
                                 measure_error(y_test, y_test_pred_gr, 'test')],
                                axis=1)

train_test_gr_error<jupyter_output><empty_output><jupyter_text>> When we use Grid Search and we find the best paramaters, we increase the success. But the results are not what we hoped for.As we just see, decision trees tend to overfit. So we can use bagging to improve results.<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=10)
knn.fit(X_train, y_train)

knn_predicted = knn.predict(X_test)
knn_train_predicted = knn.predict(X_train)


train_test_full_error = pd.concat([measure_error(y_train, knn_train_predicted, 'train'),
                              measure_error(y_test, knn_predicted, 'test')],
                              axis=1)

train_test_full_error
from sklearn.ensemble import BaggingClassifier
from sklearn.svm import SVC

BC = BaggingClassifier(n_estimators=40, base_estimator=SVC())
BC = BC.fit(X_train, y_train)

y_train_pred = BC.predict(X_train)
y_test_pred = BC.predict(X_test)
train_test_full_error = pd.concat([measure_error(y_train, y_train_pred, 'train'),
                              measure_error(y_test, y_test_pred, 'test')],
                              axis=1)

train_test_full_error
from sklearn.ensemble import RandomForestClassifier

# Initialize the random forest estimator
# Note that the number of trees is not setup here
RF = RandomForestClassifier(oob_score=True, 
                            max_depth=2,
                            warm_start=True,
                            random_state=42,
                            criterion="entropy")

oob_list = list()

# Iterate through all of the possibilities for 
# number of trees
for n_trees in [15, 20, 30, 40, 50, 100, 150, 200, 300, 400]:
    
    # Use this to set the number of trees
    RF.set_params(n_estimators=n_trees)

    # Fit the model
    RF.fit(X_train, y_train)

    # Get the oob error
    oob_error = 1 - RF.oob_score_
    
    # Store it
    oob_list.append(pd.Series({'n_trees': n_trees, 'oob': oob_error}))

rf_oob_df = pd.concat(oob_list, axis=1).T.set_index('n_trees')

rf_oob_df
y_train_pred_rf = RF.predict(X_train)
y_test_pred_rf = RF.predict(X_test)

train_test_rf_error = pd.concat([measure_error(y_train, y_train_pred_rf, 'train'),
                                 measure_error(y_test, y_test_pred_rf, 'test')],
                                axis=1)

train_test_rf_error
# Initialize the random forest estimator
# Note that the number of trees is not setup here
EF = ExtraTreesClassifier(oob_score=True, 
                          random_state=42, 
                          warm_start=True,
                          bootstrap=True,
                          n_jobs=-1)

oob_list = list()

# Iterate through all of the possibilities for 
# number of trees
for n_trees in [15, 20, 30, 40, 50, 100, 150, 200, 300, 400]:
    
    # Use this to set the number of trees
    EF.set_params(n_estimators=n_trees)
    EF.fit(X_train, y_train)

    # oob error
    oob_error = 1 - EF.oob_score_
    oob_list.append(pd.Series({'n_trees': n_trees, 'oob': oob_error}))

et_oob_df = pd.concat(oob_list, axis=1).T.set_index('n_trees')

et_oob_df
y_train_pred_ef = EF.predict(X_train)
y_test_pred_ef = EF.predict(X_test)

train_test_ef_error = pd.concat([measure_error(y_train, y_train_pred_ef, 'train'),
                                 measure_error(y_test, y_test_pred_ef, 'test')],
                                axis=1)

train_test_ef_error
sns.set_context('talk')
sns.set_palette('dark')
sns.set_style('white')

ax = et_oob_df.plot(marker='o')
ax.set(ylabel='out-of-bag error');
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import cross_val_score

GNB = GaussianNB()
cv_N = 4

scores_gaussian = cross_val_score(GNB, X, y, n_jobs=cv_N, cv=cv_N)
print(scores_gaussian)
np.mean(scores_gaussian)
from sklearn.metrics import classification_report, confusion_matrix

def evaluating_svm(kernel_type, X_train, y_train, X_test, y_test):
  kernel_type.fit(X_train, y_train)
  y_pred = kernel_type.predict(X_test)

  print("confusion matrix: ")
  print(confusion_matrix(y_test,y_pred))
  print()
  print("classification_report: ")
  print(classification_report(y_test,y_pred))
from sklearn.svm import LinearSVC

LSVC = LinearSVC()
evaluating_svm(LSVC, X_train, y_train, X_test, y_test)<jupyter_output>confusion matrix: 
[[25  4]
 [ 5 27]]

classification_report: 
              precision    recall  f1-score   support

           0       0.83      0.86      0.85        29
           1       0.87      0.84      0.86        32

    accuracy                           0.85        61
   macro avg       0.85      0.85      0.85        61
weighted avg       0.85      0.85      0.85        61<jupyter_text>> We will also test different types of kernel.<jupyter_code>from sklearn.svm import SVC

svclassifier = SVC(kernel='poly', degree=8)
evaluating_svm(svclassifier, X_train, y_train, X_test, y_test)
# Gaussian
gammas = [.5, 1, 2, 10]
for gamma in gammas:
  svm_gaussian = SVC(kernel='rbf', gamma=gamma)
  print("gamma: " + str(gamma))
  evaluating_svm(svm_gaussian, X_train, y_train, X_test, y_test)
# Sigmoid
svm_sigmoid = SVC(kernel='sigmoid')
evaluating_svm(svm_sigmoid, X_train, y_train, X_test, y_test)
from sklearn.linear_model import LogisticRegression

# Standard logistic regression
lr = LogisticRegression().fit(X_train, y_train)
from sklearn.linear_model import LogisticRegressionCV

# L1 regularized logistic regression
lr_l1 = LogisticRegressionCV(Cs=10, cv=4, penalty='l1', solver='liblinear').fit(X_train, y_train)
#Try with different solvers like ‘newton-cg’, ‘lbfgs’, ‘sag’, ‘saga’ and give your observations
lr_newton = LogisticRegression(solver="newton-cg").fit(X_train, y_train)
lr_sag = LogisticRegression(solver="sag").fit(X_train, y_train)
lr_saga = LogisticRegression(solver="saga").fit(X_train, y_train)
# L2 regularized logistic regression
lr_l2 = LogisticRegressionCV(Cs=10, cv=4, penalty='l2').fit(X_train, y_train)
# Predict the class and the probability for each

y_pred = list()
y_prob = list()

coeff_labels = ['lr', 'l1', 'l2', 'newton', 'sag', 'saga']
coeff_models = [lr, lr_l1, lr_l2, lr_newton, lr_sag, lr_saga]

for lab,mod in zip(coeff_labels, coeff_models):
    y_pred.append(pd.Series(mod.predict(X_test), name=lab))
    y_prob.append(pd.Series(mod.predict_proba(X_test).max(axis=1), name=lab))
    
y_pred = pd.concat(y_pred, axis=1)
y_prob = pd.concat(y_prob, axis=1)

y_pred.head()
y_prob.head()
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import confusion_matrix, accuracy_score

metrics = list()

for lab in coeff_labels:

    # Preciision, recall, f-score from the multi-class support function
    precision, recall, fscore, _ = score(y_test, y_pred[lab], average='weighted')

    # The usual way to calculate accuracy
    accuracy = accuracy_score(y_test, y_pred[lab])
    
    metrics.append(pd.Series({'precision':precision, 'recall':recall, 
                              'fscore':fscore, 'accuracy':accuracy,
                              }, 
                             name=lab))

metrics = pd.concat(metrics, axis=1)
metrics<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11679738562091503, 'fraction_numerical': 0.01869281045751634, 'mean_word_length': 3.899455651617035, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '26536', 'license': 'BSL-1.0', 'n_tokens_mistral': 5533, 'n_tokens_neox': 5254, 'n_words': 1258, 'path': 'Project/heart_attack.ipynb', 'repo_name': 'yoel2000/Computational-learning', 'size': '15300'}