<jupyter_start><jupyter_text>Basic Models ExperimentUse the new processed dataset called `hotel_bookings_processed.csv`<jupyter_code>import time

import numpy as np
import pandas as pd
import datetime as dt
import plotly.express as px
import seaborn as sns
import matplotlib.pyplot as plt
# from sklearn.metrics import plot_roc_curve
# import seaborn as sns

from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix, auc
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler

# Basic models
# from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC, LinearSVC
# from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
path = "/content/drive/Shareddrives/CIS 520/final project/hotel_bookings_processed.csv"
data = pd.read_csv(path)
data
data.corr()['is_canceled'].abs().sort_values(ascending=False)[1:11]
## Initialize X, y and split dataset
df = data.copy()
y = df['is_canceled']
X = df.drop(['is_canceled'], axis=1)
X_train, X_test, y_train, y_test = \
  train_test_split(X, y, test_size=0.20, random_state=42)
clf_rf = DecisionTreeClassifier(max_depth=12).fit(X_train, y_train)
pd.DataFrame(data = clf_rf.feature_importances_ * 100, 
             columns = ['Importance'], 
             index = X_train.columns)\
                .sort_values('Importance', ascending=True)[-20:]\
                .plot(kind='barh', color='r')

plt.xlabel("Top 20 Feature Importance (%)")<jupyter_output><empty_output><jupyter_text>Imbalance<jupyter_code>from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler
from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
def sampler_compare(samplers):

    accus, accu_balanceds = [], []

    for sampler in samplers.values():
        X_train_balanced, y_train_balanced = sampler.fit_sample(X_train, y_train)
        ## Test by logistic regression accuracy
        logit = LogisticRegression(solver='liblinear')
        clf = logit.fit(X_train, y_train)
        accu = roc_auc_score(y_test, clf.predict(X_test))
        clf_balanced = logit.fit(X_train_balanced, y_train_balanced)
        accu_balanced = roc_auc_score(y_test, clf.predict(X_test))

        accus.append(accu)
        accu_balanceds.append(accu_balanced)

    return accus, accu_balanceds
samplers = {'Undersample': RandomUnderSampler(random_state=0),
           'Oversample': RandomOverSampler(random_state=0),
           'SMOTE': SMOTE()}
accus, accu_balanceds = np.round(sampler_compare(samplers), 4)
## Output comparsion table
table = pd.DataFrame({"Balancing Method": samplers.keys(), 
                      "Accuracy (imbalanced)": accus, 
                      "Accuracy (balanced)": accu_balanceds})

pd.pivot_table(table, index=['Balancing Method'])\
    .sort_values(by="Accuracy (balanced)", ascending=False)
print(pd.pivot_table(table, index=['Balancing Method'])\
    .sort_values(by="Accuracy (balanced)", ascending=False).to_latex())<jupyter_output><empty_output><jupyter_text>It's obvious that the balancing method can imporve the accuracy and among them, SMOTE preforms the best, thus we deploy the SMOTE on the train set.<jupyter_code>X_train_balanced, y_train_balanced = SMOTE().fit_sample(X_train, y_train)
print(X_train_balanced.shape)
print(y_train_balanced.shape)<jupyter_output>(46280, 194)
(46280,)<jupyter_text>Basic Models Implementation  Tuning   Random Forest<jupyter_code># rf_para = {'n_estimators': [int(x) for x in np.linspace(200, 2000, num=10)],
#            'max_features': ['auto', 'sqrt'],
#            'max_depth': [int(x) for x in np.linspace(10, 110, num=11)],
#            'min_samples_split': [2, 5, 10],
#            'bootstrap': [True, False]}
rf_para = {'n_estimators': [100, 200, 500],
           'max_features': ['auto', 'sqrt', 'log2'],
           'min_samples_split': [2, 5, 10]}

rf = RandomForestClassifier()
rf_cv = GridSearchCV(rf, rf_para, cv=10, n_jobs=-1, verbose=2)
rf_cv.fit(X_test[:1000], y_test[:1000])
# rf_cv.fit(X_test, y_test)
print("Best paramters (RF): {}".format(rf_cv.best_params_))<jupyter_output>Fitting 10 folds for each of 27 candidates, totalling 270 fits<jupyter_text>Best paramters (RF): {'max_features': 'sqrt', 'min_samples_split': 2, 'n_estimators': 500}  Support Vector Machine<jupyter_code>svm_para = {'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
            'C': [50, 10, 1.0, 0.1, 0.01],
            'gamma': ['scale']}

svm = SVC()
nn_cv = GridSearchCV(svm, svm_para, cv=10, n_jobs=-1, verbose=2)
nn_cv.fit(X_test[:1000], y_test[:1000])
# nn_cv.fit(X_test, y_test)
print("Best paramters (SVM): {}".format(nn_cv.best_params_))<jupyter_output>Fitting 10 folds for each of 20 candidates, totalling 200 fits<jupyter_text>Best paramters (SVM): {'C': 0.1, 'gamma': 'scale', 'kernel': 'linear'}<jupyter_code>## Visualize by heatmap
pvt = pd.pivot_table(pd.DataFrame(nn_cv.cv_results_),
                     values='mean_test_score',
                     index='param_C',
                     columns='param_kernel')
ax = sns.heatmap(pvt, annot=True, linewidths=.5)<jupyter_output><empty_output><jupyter_text>Neural Netowrk<jupyter_code>nn_para = {'alpha': [1, 0.1, 0.01, 0.001],
           'hidden_layer_sizes': [(50,50,50), (100,100)],
           'solver': ["adam", "sgd"],
           'activation': ["logistic", "relu"]}

nn = MLPClassifier()
nn_cv = GridSearchCV(nn, nn_para, cv=10, n_jobs=-1, verbose=2)
nn_cv.fit(X_test, y_test)
print("Best paramters (NN): {}".format(nn_cv.best_params_))<jupyter_output>Fitting 10 folds for each of 32 candidates, totalling 320 fits<jupyter_text>Best paramters (NN): {'activation': 'logistic', 'alpha': 0.001, 'hidden_layer_sizes': (100, 100), 'solver': 'adam'}  All Models ComparisonNotes: SVC runs very slowly due to the fact of its complexity $\mathcal{O}(n^2 p)$ ([source](https://stackoverflow.com/questions/40077432/why-is-scikit-learn-svm-svc-extremely-slow))<jupyter_code>def compute(name, model, X_train, X_test, y_train, y_test):

    fpr, tpr = [], []
    accu = auc = exectime = 0

    start = time.process_time()
    clf = model.fit(X_train, y_train)
    y_prob = clf.predict_proba(X_test)[:, 1]
    y_pred = clf.predict(X_test)

    exectime = round(time.process_time() - start, 4)
    
    fpr, tpr, _ = roc_curve(y_test, y_prob)
    accu = round(accuracy_score(y_test,y_pred), 4)
    auc = round(roc_auc_score(y_test, y_prob), 4)
    cm = confusion_matrix(y_test, y_pred)

    ## Plot accuracy, AUC and confusion matrix 
    print("# {}:\nAccuracy Score: {}\nAUC Score: {}\nConfusion Matrix:\n{}"
            .format(name, accu, auc, cm))
    print("-------------------- \n")

    return fpr, tpr, accu, auc, exectime

def implement(models, X_train, X_test, y_train, y_test, balanced=False):

    accus, aucs, times, fprs, tprs = [], [], [], [], []

    for name, model in models.items():
    
        ## XGBoost requires the same column number of training and test set
        if name == "XGBoost" and balanced == True:
            X_test_balanced, y_test_balanced = SMOTE().fit_sample(X_test, y_test)
            fpr, tpr, accu, auc, exectime = \
                compute(name, model, X_train, X_test_balanced, y_train, y_test_balanced)
        ## Neural Network prefers standardization
        elif name == "Neural Network (vanilla)" or "Neural Network (tuned)":
            scaler = StandardScaler()
            scaler.fit(X_train)
            X_train_scaled = scaler.transform(X_train)
            X_test_scaled = scaler.transform(X_test)
            fpr, tpr, accu, auc, exectime = \
                compute(name, model, X_train_scaled, X_test_scaled, y_train, y_test)
        else:
            fpr, tpr, accu, auc, exectime = \
                compute(name, model, X_train, X_test, y_train, y_test)
      
        fprs.append(fpr)
        tprs.append(tpr)
        times.append(exectime)
        accus.append(accu)
        aucs.append(auc)
        
    return accus, aucs, times, fprs, tprs

def plot_ROC(fprs, tprs, aucs, names):

    plt.figure(0).clf()

    for fpr, tpr, name, auc in zip(fprs, tprs, names, aucs):
        plt.plot(fpr, tpr, label=name+" (AUC="+str(auc)+")")

    ## Plot ROC comparison
    plt.legend(loc=0)
    plt.title("Comparison of Models by ROC")
    plt.ylabel("True Positive Rate")
    plt.xlabel("False Positive Rate")
## Dictionary: {model name -- classifier}
models = \
{"Logistic Regression (baseline)": LogisticRegression(solver='liblinear'),
 "Decisioion Tree": DecisionTreeClassifier(max_depth=12),
 "Random Forest (vanilla)": RandomForestClassifier(),
 "Random Forest (Tuned)": RandomForestClassifier(max_features='sqrt',
                                                 min_samples_split=2,
                                                 n_estimators=500),
 "Extra Trees": ExtraTreesClassifier(min_samples_leaf=7,
                                     min_samples_split=2,
                                     n_estimators=500),
 "Neural Network (vanilla)": MLPClassifier(),
 "Neural Network (Tuned)": MLPClassifier(alpha=0.001, 
                                         hidden_layer_sizes=(100,100),
                                         solver='adam',
                                         activation='logistic'), 
#  "Gaussian Naive Bayes": GaussianNB(),
 "XGBoost": XGBClassifier(),
 "AdaBoost": AdaBoostClassifier(n_estimators=500),
 "SVM (Tuned)": SVC(kernel='linear', C=0.1, probability=True)}<jupyter_output><empty_output><jupyter_text>Imbalanced Training and Plot<jupyter_code>accus, aucs, times, fprs, tprs = implement(models, X_train, X_test, y_train, y_test, False)
plot_ROC(fprs, tprs, aucs, models.keys())<jupyter_output><empty_output><jupyter_text>Balanced Training and Plot<jupyter_code>accus_b, aucs_b, times_b, fprs_b, tprs_b = \
    implement(models, X_train_balanced, X_test, y_train_balanced, y_test, True)
plot_ROC(fprs_b, tprs_b, aucs_b, models.keys())
## Plot runtime comparison
pd.DataFrame(data=times, columns=['Runtime'], index=models.keys())\
                .sort_values('Runtime', ascending=False)\
                .plot(kind='barh', color='g')

plt.xlabel("Runtime Comparison (s)")
## Output comparsion table
table = pd.DataFrame({"Model": models.keys(), 
                      "Accuracy": accus, 
                      "AUC": aucs,
                      "Runtime(s)": times,
                      "Accuracy_B": accus_b, 
                      "AUC_B": aucs_b,
                      "Runtime_B(s)": times_b})

pd.pivot_table(table, index=['Model'])\
    .sort_values(by="Accuracy", ascending=False)<jupyter_output><empty_output><jupyter_text>Compare the accuracy difference among models and balance/imbalance.<jupyter_code>print(pd.pivot_table(table, index=['Model'])\
    .sort_values(by="Accuracy", ascending=False).to_latex())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13043478260869565, 'fraction_numerical': 0.018546058962470476, 'mean_word_length': 3.722015695993391, 'pattern_counts': {'":': 22, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '119147', 'license': 'MIT', 'n_tokens_mistral': 4179, 'n_tokens_neox': 3873, 'n_words': 828, 'path': 'Basic_Models_Experiment.ipynb', 'repo_name': 'realliyifei/ML-Project-Hotel-Cancellation-Prediction', 'size': '11431'}