<jupyter_start><jupyter_text>Medical Appointment No Shows Capstone  5. MODELING> "All models are wrong, but some are useful." – George Box<jupyter_code>#Import necessary libraries
import pandas as pd
import numpy as np

# Plotting modules
import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('ggplot')


# Analysing datetime
from datetime import datetime as dt

# File system manangement
import os,sys

# Suppress warnings 
import warnings
warnings.filterwarnings('ignore')

from IPython.core.interactiveshell import InteractiveShell  
InteractiveShell.ast_node_interactivity = "all"
%matplotlib inline
#SKLEARN 

#MODEL_SELECTION
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV

from sklearn.tree import DecisionTreeClassifier

#LINEAR_MODEL
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso, Ridge


#ENSEMBLE 
from sklearn.ensemble import VotingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import BaggingClassifier

#SVM
from sklearn.svm import SVC

from sklearn.neighbors import KNeighborsClassifier

from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB

#METRICS
from sklearn.metrics import r2_score 
from sklearn.metrics import accuracy_score
from sklearn.metrics import make_scorer
from sklearn.metrics import balanced_accuracy_score, precision_score
from sklearn.metrics import roc_curve, roc_auc_score, auc
from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.metrics import classification_report, confusion_matrix



#PIPELINE 
from sklearn.pipeline import Pipeline
from sklearn.pipeline import make_pipeline as mpl

import pandas.util.testing as tm
from xgboost import XGBClassifier
from sklearn.preprocessing import StandardScaler
#SCIPY
from scipy.stats import randint<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>path = 'data/df_ohe.csv'
df_ohe = pd.read_csv(path, index_col=None) 
df_ohe.sample(5)<jupyter_output><empty_output><jupyter_text>Train/Test Data<jupyter_code>X = df_ohe.drop(['NoShow_1'], axis = 1)
y = df_ohe['NoShow_1']

SEED = 42
TS = 0.25

# Create training and test sets
X_train, X_test, y_train, y_test = \
    train_test_split(X, y, test_size = TS, random_state=SEED, stratify=y)

#Feature Scaling to prevent information leakage
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

print (X_train.shape)
print (y_train.shape)
print (X_test.shape)
print (y_test.shape)<jupyter_output>(82887, 112)
(82887,)
(27630, 112)
(27630,)<jupyter_text>Select a Performance MeasureROC_AUC Will be used since it is a typical performance measure for classification problems in Supervised Learning ___  1. K-NEAREST NEIGHBORS- Model That classifies data based on its distance to training set data. A new data point is assigned a label based on the class that the majority of surrounding data points belong to. - “The k-nearest neighbors algorithm (KNN) is a non-parametric method used for classification and regression. In both cases, the input consists of the k closest training examples in the feature space. The output depends on whether k-NN is used for classification or regression”-Wikipedia<jupyter_code>MODEL_PARAMS = {
    "n_neighbors": 5,
    "weights":'uniform',
    "algorithm": 'auto',
    "leaf_size":30,
    "p":2,
    "metric":'minkowski'
}

steps = [('knn', KNeighborsClassifier(**MODEL_PARAMS) )]

pipeline = Pipeline(steps)

# Fit/Train the classifier to the Training data.
pipeline.fit(X_train,y_train)  

# Predict the labels for the training data X
y_pred = pipeline.predict(X_test)  
y_pred_proba = pipeline.predict_proba(X_test)[:,1]

# Generate the confusion matrix and classification report
print("Confusion Matrix \n", confusion_matrix(y_test,y_pred))
print("Classification Report \n",classification_report(y_test,y_pred))


knn_score = round(pipeline.score(X_train, y_train) * 100, 2)
print("KNN Score:", knn_score)

knn_test_score = round(pipeline.score(X_test, y_test) * 100, 2)
print("KNN Test Score:", knn_score)


#Checking performance on our model with ROC Score.
knn_roc_score = round(roc_auc_score(y_test,y_pred_proba)*100,3)
print("ROC Score:", knn_roc_score)<jupyter_output><empty_output><jupyter_text>Overfitting and underfitting<jupyter_code># Setup arrays to store train and test accuracies
neighbors = np.arange(1, 9)
train_accuracy = np.empty(len(neighbors))
test_accuracy = np.empty(len(neighbors))

# Loop over different values of k
for i, k in enumerate(neighbors):
    # Setup a k-NN Classifier with k neighbors: knn
    knn = KNeighborsClassifier(n_neighbors=k)

    # Fit the classifier to the training data
    knn.fit(X_train, y_train)
    
    #Compute accuracy on the training set
    train_accuracy[i] = knn.score(X_train,y_train)

    #Compute accuracy on the testing set
    test_accuracy[i] = knn.score(X_test, y_test)

# Generate plot
plt.title('k-NN: Varying Number of Neighbors')
plt.plot(neighbors, test_accuracy, label = 'Testing Accuracy')
plt.plot(neighbors, train_accuracy, label = 'Training Accuracy')
plt.legend()
plt.xlabel('Number of Neighbors')
plt.ylabel('Accuracy')
plt.show()

param_grid = {'n_neighbors': np.arange(1,200)}
knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn, param_grid,cv=5)
knn_cv.fit(X_train,y_train)
knn_cv.best_params_<jupyter_output><empty_output><jupyter_text>___  2. Logistic Regression ModelLogistic regression is a supervised learning classification algorithm used to predict the probability of a target variable. The nature of target or dependent variable is dichotomous, which means there would be only two possible classes.<jupyter_code>MODEL_PARAMS = {
    "solver":'lbfgs',
    "penalty":'l2',
    "n_jobs":None,
    "verbose":0,
    #"max_iter":200  
}

# Create logistic regression model
logreg = LogisticRegression(**MODEL_PARAMS)

# Train the model using the training sets
logreg.fit(X_train, y_train)

# Make predictions using the testing set
y_pred = logreg.predict(X_test)
y_pred_proba = logreg.predict_proba(X_test)[:,1]

# Compute and print the confusion matrix and classification report
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))

logreg_score = round(logreg.score(X_train, y_train) * 100, 2)
logreg_test_score = round(logreg.score(X_test, y_test) * 100, 2)

print('Logistic Regression Training Score: \n', logreg_score)
print('Logistic Regression Test Score: \n', logreg_test_score)

print('Accuracy: \n', accuracy_score(y_test,y_pred))

#Checking performance on our model with ROC Score.
logreg_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", logreg_roc_score)
# 10-fold cross-validation scores:
cv_scores = cross_val_score(logreg, X_train, y_train, cv=10,scoring='roc_auc').mean()
print(cv_scores*100)
print("Average 5-Fold CV Score: {}".format(np.mean(cv_scores)))<jupyter_output>71.39965783205375
Average 5-Fold CV Score: 0.7139965783205375<jupyter_text>Plot ROC curve<jupyter_code># Generate ROC curve values: fpr, tpr, thresholds
fpr,tpr, thresholds = roc_curve(y_test,y_pred_prob)

# Plot ROC curve
plt.plot([0, 1], [0, 1], 'k--', label='naive classifier')
plt.plot(fpr,tpr, label='model')



plt.legend(prop={'size':10})
plt.xlabel('False Positive Rate', size=12)
plt.ylabel('True Positive Rate', size=12)
plt.title('ROC Curve', size=15)
plt.show()<jupyter_output><empty_output><jupyter_text>___  3. Gaussian Naive Bayes<jupyter_code>gaussian = GaussianNB()
gaussian.fit(X_train, y_train)

#Predict Output
y_pred = gaussian.predict(X_test)
y_pred_prob = gaussian.predict_proba(X_test)[:,1]

gauss_score = round(gaussian.score(X_train, y_train) * 100, 2)
gauss_test_score = round(gaussian.score(X_test, y_test) * 100, 2)

print(confusion_matrix(y_test,y_pred))
print(classification_report(y_test,y_pred))


print('Gaussian Score: \n', gauss_score)
print('Gaussian Test Score: \n', gauss_test_score)
print('Accuracy: \n', accuracy_score(y_test,y_pred))

#Checking performance on our model with ROC Score.
gauss_roc_score = round(roc_auc_score(y_test, y_pred_prob)*100,3)
print("ROC Score:", gauss_roc_score)<jupyter_output><empty_output><jupyter_text>___  Boosting- Boosting: Ensemble method combining several weak learners to form a strong learner.- Weak learner: Model doing slightly better than random guessing.* Example of weak learner: Decision stump (CART whose maximum depth is 1).  4. AdaBoost Model<jupyter_code>MODEL_PARAMS = {
    "max_depth": 1,
    "random_state": SEED
}

dt = DecisionTreeClassifier(**MODEL_PARAMS)

adb = AdaBoostClassifier(base_estimator=dt,n_estimators=100)

adb.fit(X_train,y_train)

y_pred = adb.predict(X_test)
y_pred_proba = adb.predict_proba(X_test)[:,1]

# Generate the confusion matrix and classification report
print("Confusion Matrix \n", confusion_matrix(y_test,y_pred))
print("Classification Report \n",classification_report(y_test,y_pred))

adb_score = round(adb.score(X_train, y_train) * 100, 2)
adb_test_score = round(adb.score(X_test, y_test) * 100, 2)

print('AdaBoost Training Score: ', adb_score)
print('AdaBoost Test Score: ', adb_test_score)

#Checking performance on our model with ROC Score.
adb_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", adb_roc_score)
# Generate ROC curve values: fpr, tpr, thresholds
fpr,tpr, thresholds = roc_curve(y_test,y_pred_prob)

# Plot ROC curve
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr,tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.show()

auc(fpr, tpr)<jupyter_output><empty_output><jupyter_text>5. Gradient Boosting<jupyter_code>MODEL_PARAMS = {
    "max_depth":4,
    "n_estimators":300,
    "random_state":SEED
}

gb = GradientBoostingClassifier(**MODEL_PARAMS)

# Fit gb to the training set
gb.fit(X_train,y_train)

# Predict test set labels
y_pred = gb.predict(X_test)
y_pred_proba = gb.predict_proba(X_test)[:,1]

# Generate the confusion matrix and classification report
print("Confusion Matrix \n", confusion_matrix(y_test,y_pred))
print("Classification Report \n",classification_report(y_test,y_pred))

#Training And Testing Scores
gb_score = round(gb.score(X_train, y_train) * 100, 2)
gb_test_score = round(gb.score(X_test, y_test) * 100, 2)

print('Gradient Boosting Training Score: ', gb_score)
print('Gradient Boosting Test Score: ', gb_test_score)


#Checking performance on our model with ROC Score.
gb_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", gb_roc_score)<jupyter_output><empty_output><jupyter_text>Stochastic Gradient Boosting- Each Tree is trained on a random subset of rows of the training data.<jupyter_code>sgbt = GradientBoostingClassifier(subsample=0.9,
                                  max_depth=4, 
                                  n_estimators=300,
                                  random_state=SEED)
sgbt.fit(X_train,y_train)
y_pred = sgbt.predict(X_test)
y_pred_proba = sgbt.predict_proba(X_test)[:,1]

sgbt_score = round(sgbt.score(X_train, y_train) * 100, 2)
sgbt_test_score = round(sgbt.score(X_test, y_test) * 100, 2)

print('XGBoost Training Score: ', sgbt_score)
print('XGBoost Test Score: ', sgbt_test_score)

#Checking performance on our model with ROC Score.
sgbt_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", sgbt_roc_score)<jupyter_output><empty_output><jupyter_text>6. XGBOOST<jupyter_code>MODEL_PARAMS = {
    "base_score":0.5,
    "booster":'gbtree',
    "colsample_bytree":1,
    "gamma": 0,
    "learning_rate": 0.1,
    "n_estimators": 100,
    "max_depth": 6,
    "objective": 'binary:logistic',
    "min_child_weight":1,
    "subsample":0.8,
    "random_state":SEED,
    "scale_pos_weight":1,
    "nthread":4,
    "n_jobs":0,
    "seed": SEED,
    "validate_parameters":1,
    "verbosity":None,
}
xgb = XGBClassifier(**MODEL_PARAMS)

xgb.fit(X_train,y_train)
y_pred = xgb.predict(X_test)
y_pred_proba = xgb.predict_proba(X_test)[:,1]

accuracy = float(np.sum(y_pred==y_test))/y_test.shape[0]
print("Accuracy: %.2f" % (accuracy*100),"%")

xgb_score = round(xgb.score(X_train, y_train) * 100, 2)
xgb_test_score = round(xgb.score(X_test, y_test) * 100, 2)

print('XGBoost Training Score: ', xgb_score)
print('XGBoost Test Score: ', xgb_test_score)

#Checking performance on our model with ROC Score.
xgb_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", xgb_roc_score)<jupyter_output><empty_output><jupyter_text>7. Decision Tree Model  Gini Criterion<jupyter_code>MODEL_PARAMS = {
    "criterion": "gini",
    "random_state":SEED,
    #"max_depth": 7,
}
dtc = DecisionTreeClassifier(**MODEL_PARAMS)

# Call fit() on entr_model, and pass in X_train and y_train, in that order 
dtc.fit(X_train, y_train)

# Call predict() on entr_model with X_test passed to it, and assign the result to a variable y_pred 
y_pred = dtc.predict(X_test)
y_pred_proba = dtc.predict_proba(X_test)[:,1]

# Assign y_pred the following: pd.Series(y_pred)
y_pred = pd.Series(y_pred)

print(dtc)

dtc_score = round(gb.score(X_train, y_train) * 100, 2)
dtc_test_score = round(gb.score(X_test, y_test) * 100, 2)

print('Decision Tree Training Score: ', dtc_score)
print('Decision Tree  Test Score: ', dtc_test_score)

#Checking performance on our model with ROC Score.
dtc_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", dtc_roc_score)
#Hyperparameter tuning with RandomizedSearchCV
# Setup the parameters and distributions to sample from: param_dist
param_dist = {"max_depth": [1,2,3,4,5,6,7,8,9, None],
              "max_features": randint(1, 9),
              "min_samples_leaf": randint(1, 9),
              "criterion": ["gini", "entropy"]}

# Instantiate a Decision Tree classifier: tree
tree = DecisionTreeClassifier()

# Instantiate the RandomizedSearchCV object: tree_cv
tree_cv = RandomizedSearchCV(tree, param_dist, cv=5)

# Fit it to the data
tree_cv.fit(X_train,y_train)

# Print the tuned parameters and score
print("Tuned Decision Tree Parameters: {}".format(tree_cv.best_params_))
print("Best score is {}".format(tree_cv.best_score_))
MODEL_PARAMS = {
    "criterion":"gini",
    "random_state":SEED,
    "max_depth": 5,
    "max_features":1,
    "min_samples_leaf":4,
}
dtce = DecisionTreeClassifier(**MODEL_PARAMS)

# Call fit() on entr_model, and pass in X_train and y_train, in that order 
dtce.fit(X_train, y_train)

# Call predict() on entr_model with X_test passed to it, and assign the result to a variable y_pred 
y_pred = dtc.predict(X_test)
y_pred_proba = dtce.predict_proba(X_test)[:,1]

print(dtc)

dtce_score = round(dtce.score(X_train, y_train) * 100, 2)
dtce_test_score = round(dtce.score(X_test, y_test) * 100, 2)

print('Decision Tree Training Score: ', dtce_score)
print('Decision Tree  Test Score: ', dtce_test_score)

#Checking performance on our model with ROC Score.
dtc_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", dtc_roc_score)<jupyter_output><empty_output><jupyter_text>8. Bagging Classifier<jupyter_code>dt = DecisionTreeClassifier(random_state=SEED)


bgc = BaggingClassifier(base_estimator=dt, n_estimators=50, random_state=SEED)

bgc.fit(X_train, y_train)

y_pred = bgc.predict(X_test)
y_pred_proba = bgc.predict_proba(X_test)[:,1]

# Evaluate acc_test
acc_test = accuracy_score(y_test, y_pred)
print('Test set accuracy of bc: {:.2f}'.format(acc_test)) 

bgc_score = round(bgc.score(X_train, y_train) * 100, 2)
bgc_test_score = round(bgc.score(X_test, y_test) * 100, 2)

print('BaggingClassifier Training Score:', bgc_score)
print('BaggingClassifier Test Score:', bgc_test_score)

#Checking performance on our model with ROC Score.
bgc_roc_score = round(roc_auc_score(y_test, y_pred_proba)*100,3)
print("ROC Score:", bgc_roc_score)<jupyter_output><empty_output><jupyter_text>9. Random Forest Model  Default Gini criterion<jupyter_code>MODEL_PARAMS = {
    "n_estimators": 100,
    "criterion": 'gini',
    "max_features":'sqrt',
    "random_state":SEED
}

rf = RandomForestClassifier(**MODEL_PARAMS)
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)
y_pred_proba = rf.predict_proba(X_test)[:,1]



rf_score = round(rf.score(X_train, y_train) * 100, 2)
rf_test_score = round(rf.score(X_test, y_test) * 100, 2)

print('Random Forest Training Score:', rf_score)
print('Random Forest Test Score:', rf_test_score)


#Checking performance on our model with ROC Score.
rf_roc_score = round(roc_auc_score(y_test,y_pred_proba)*100,3)
print("ROC Score:", rf_roc_score)<jupyter_output><empty_output><jupyter_text>Entropy criterion<jupyter_code>MODEL_PARAMS = {
    "n_estimators": 300,
    "criterion": 'entropy',
    "max_depth": None,
    "min_samples_split":2,
    "min_samples_leaf":1,
    "min_weight_fraction_leaf":0.0,
    "max_features":'auto',
    "max_leaf_nodes":None,
    "min_impurity_decrease":0.0,
    "min_impurity_split":None,
    "bootstrap":True,
    "oob_score":False,
    "n_jobs":None,
    "random_state": SEED,
    "verbose":0,
    "warm_start":False,
    "class_weight":None,
    "ccp_alpha":0.0,
    "max_samples":None,
}

rf = RandomForestClassifier(**MODEL_PARAMS)
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)
y_pred_proba = rf.predict_proba(X_test)[:,1]



rf_score = round(rf.score(X_train, y_train) * 100, 2)
rf_test_score = round(rf.score(X_test, y_test) * 100, 2)

print('Random Forest Training Score:', rf_score)
print('Random Forest Test Score:', rf_test_score)


#Checking performance on our model with ROC Score.
rf_roc_score = round(roc_auc_score(y_test,y_pred_proba)*100,3)
print("ROC Score:", rf_roc_score)<jupyter_output><empty_output><jupyter_text>___  Model Evaluation<jupyter_code>models = pd.DataFrame({
    
    'Model': [ 
        'k-Nearest Neighbors',
        'Logistic Regression',
        'Gaussian Naive Bayes',
        'AdaBoost',
        'Gradient Boosting',
        'XGBoost',
        'Decision Tree',
        'Bagging Classifier',
        'Random Forest',
        
    ],
    
    
    'ROC-AUC Score': [ 
        knn_roc_score, 
        logreg_roc_score,
        gauss_roc_score,
        adb_roc_score,
        gb_roc_score,
        xgb_roc_score,
        dtc_roc_score,
        bgc_roc_score,
        rf_roc_score
    ],             
    
    'Score': [ 
        knn_score, 
        logreg_score,
        gauss_score,
        adb_score,
        gb_score,
        xgb_score,
        dtc_score,
        bgc_score,
        rf_score
    ],
    
    'Test Score': [ 
        knn_test_score,
        logreg_test_score,
        gauss_test_score,
        adb_test_score,
        gb_test_score,
        xgb_test_score,
        dtc_test_score,
        bgc_test_score,
        rf_test_score
    ]})


models.sort_values(by='ROC-AUC Score', ascending=False)
knn = KNeighborsClassifier()
lr = LogisticRegression(random_state=SEED)
dt = DecisionTreeClassifier()
rf = RandomForestClassifier()

classifiers = [
    ('KNN', knn),
    ('Logistic Regression', lr),
    ('Decision Tree', dt),
    ('Random forest', rf)
]

print('ROC Score:\n')
for clf_name, clf in classifiers:
    clf.fit(X_train, y_train)
    y_pred_proba = clf.predict_proba(X_test)[:,1]
    
    # Calculate accuracy
    accuracy = roc_auc_score(y_test,y_pred_proba) 
    print('[{:s}] : {:.3f}%'.format(clf_name, accuracy * 100))<jupyter_output>Accuracy Score:
{'fraction_non_alphanumeric': 0.11185580096471186, 'fraction_numerical': 0.017974105102817974, 'mean_word_length': 5.40520325203252, 'pattern_counts': {'":': 68, '<': 66, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '388749', 'license': 'MIT', 'n_tokens_mistral': 7623, 'n_tokens_neox': 7166, 'n_words': 1588, 'path': '5_Modeling.ipynb', 'repo_name': 'asaadkhattab/MedicalAppointmentNoShowsCapstone', 'size': '19695'}