<jupyter_start><jupyter_text>Impute Data<jupyter_code>import math

hotel_bookings_transform = hotel_bookings
hotel_bookings_transform= hotel_bookings_transform.drop(columns=['agent','company', 'country'])

hotel_bookings_transform['meal']= hotel_bookings_transform['meal'].apply(lambda x: 'SC' if  x == 'Undefined'  else x)
hotel_bookings_transform['children']= hotel_bookings_transform['children'].apply(lambda x: 0 if  math.isnan(x)  else x)
hotel_bookings_transform['market_segment']= hotel_bookings_transform['market_segment'].apply(lambda x: 'Online TA' if  x == 'Undefined'  else x)
hotel_bookings_transform['distribution_channel']= hotel_bookings_transform['distribution_channel'].apply(lambda x: 'TA/TO' if  x == 'Undefined'  else x)


hotel_bookings_transform['hotel']=hotel_bookings_transform['hotel'].astype('category').cat.codes
hotel_bookings_transform['market_segment']=hotel_bookings_transform['market_segment'].astype('category').cat.codes
hotel_bookings_transform['meal']=hotel_bookings_transform['meal'].astype('category').cat.codes
hotel_bookings_transform['reservation_status']=hotel_bookings_transform['reservation_status'].astype('category').cat.codes
hotel_bookings_transform['distribution_channel']=hotel_bookings_transform['distribution_channel'].astype('category').cat.codes
hotel_bookings_transform['reserved_room_type']=hotel_bookings_transform['reserved_room_type'].astype('category').cat.codes
hotel_bookings_transform['assigned_room_type']=hotel_bookings_transform['assigned_room_type'].astype('category').cat.codes
hotel_bookings_transform['deposit_type']=hotel_bookings_transform['deposit_type'].astype('category').cat.codes
hotel_bookings_transform['customer_type']=hotel_bookings_transform['customer_type'].astype('category').cat.codes

assign_month = {"arrival_date_month": {"January": 1, "February": 2, "March": 3, "April": 4,"May": 5,"June": 6,"July": 7,"August": 8,"September": 9,"October": 10,"November": 11,"December": 12}}
hotel_bookings_transform = hotel_bookings_transform.replace(assign_month)

# Convert Date to Ordinal
import datetime as dt
hotel_bookings_transform['reservation_status_date'] = pd.to_datetime(hotel_bookings_transform['reservation_status_date'])
hotel_bookings_transform['reservation_status_date'] = hotel_bookings_transform['reservation_status_date'].map(dt.datetime.toordinal)
# To convert back to Date
# hotel_bookings_transform['reservation_status_date'] = hotel_bookings_transform['reservation_status_date'].map(dt.datetime.fromordinal)
# hotel_bookings_transform['reservation_status_date']
hotel_bookings_transform.info()
hotel_bookings_transform.describe()
hotel_bookings_transform.head()<jupyter_output><empty_output><jupyter_text>Training and Testing Split<jupyter_code>from sklearn.model_selection import ShuffleSplit

y = hotel_bookings_transform['is_canceled']
X = hotel_bookings_transform.drop(columns=['is_canceled'])<jupyter_output><empty_output><jupyter_text>Standarization<jupyter_code>from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
scaler.fit(X)

X_scl = scaler.transform(X)<jupyter_output><empty_output><jupyter_text>Split Data for Train and Test<jupyter_code># to use the cross validation object in scikit learn, we need to grab an instance
#    of the object and set it up. This object will be able to split our data into 
#    training and testing splits

cv_object = ShuffleSplit(n_splits=10,
                         test_size  = 0.2, random_state=10)

print(cv_object)<jupyter_output>ShuffleSplit(n_splits=10, random_state=10, test_size=0.2, train_size=None)<jupyter_text>Logistic Regression<jupyter_code>from sklearn.model_selection import ShuffleSplit
cv = ShuffleSplit(n_splits=10, test_size=0.10, random_state=0)
#Logisitic regression 10-fold cross-validation 
from sklearn.linear_model import LogisticRegression
regEstimator = LogisticRegression()


parameters = { 'penalty':['l2']
              ,'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]
              ,'class_weight': ['balanced', 'none']
              ,'random_state': [10]
              ,'solver': ['lbfgs']
              ,'max_iter':[100,500]
             }

#Create a grid search object using the  
from sklearn.model_selection import GridSearchCV
regGridSearch = GridSearchCV(estimator=regEstimator
                   , n_jobs=8 # jobs to run in parallel
                   , verbose=1 # low verbosity
                   , param_grid=parameters
                   , cv=cv # KFolds = 10
                   , scoring='accuracy')

#Perform hyperparameter search to find the best combination of parameters for our data
regGridSearch.fit(X, y)
#Diplay the top model parameters
regGridSearch.best_estimator_
from sklearn.model_selection import cross_validate

def EvaluateClassifierEstimator(classifierEstimator, X, y, cv):
   
    #Perform cross validation 
    scores = cross_validate(classifierEstimator, X, y, scoring=['accuracy','precision','recall']
                            , cv=cv, return_train_score=True)

    Accavg = scores['test_accuracy'].mean()
    Preavg = scores['test_precision'].mean()
    Recavg = scores['test_recall'].mean()

    print_str = "The average accuracy for all cv folds is: \t\t\t {Accavg:.5}"
    print_str2 = "The average precision for all cv folds is: \t\t\t {Preavg:.5}"
    print_str3 = "The average recall for all cv folds is: \t\t\t {Recavg:.5}"

    print(print_str.format(Accavg=Accavg))
    print(print_str2.format(Preavg=Preavg))
    print(print_str3.format(Recavg=Recavg))
    print('*********************************************************')

    print('Cross Validation Fold Mean Error Scores')
    scoresResults = pd.DataFrame()
    scoresResults['Accuracy'] = scores['test_accuracy']
    scoresResults['Precision'] = scores['test_precision']
    scoresResults['Recall'] = scores['test_recall']

    return scoresResults

def EvaluateClassifierEstimator2(classifierEstimator, X, y, cv):
    
    #Perform cross validation 
    from sklearn.model_selection import cross_val_predict
    predictions = cross_val_predict(classifierEstimator, X, y, cv=cv)
    
    #model evaluation 
    from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
    
    #pass true test set values and predictions to classification_report
    classReport = classification_report(y,predictions)
    confMat = confusion_matrix(y,predictions)
    acc = accuracy_score(y,predictions)
    
    print(classReport)
    print(confMat)
    print(acc)
#Use the best parameters for our Linear Regression object
classifierEst = regGridSearch.best_estimator_

#Evaluate the regression estimator above using our pre-defined cross validation and scoring metrics. 
EvaluateClassifierEstimator(classifierEst, X, y, cv)
EvaluateClassifierEstimator2(classifierEst, X, y, 10)<jupyter_output>precision    recall  f1-score   support

           0       0.67      0.87      0.76     75166
           1       0.55      0.28      0.37     44224

    accuracy                           0.65    119390
   macro avg       0.61      0.57      0.56    119390
weighted avg       0.63      0.65      0.61    119390

[[65307  9859]
 [32044 12180]]
0.6490242063824441<jupyter_text>After Standarization<jupyter_code>#Logisitic regression 10-fold cross-validation 
from sklearn.linear_model import LogisticRegression
regEstimator = LogisticRegression()

cv = ShuffleSplit(n_splits=10, test_size=0.20, random_state=0)

parameters = { 'penalty':['l2']
              ,'C': [1, 10, 100, 1000]
              ,'class_weight': ['balanced', 'none']
              ,'random_state': [10]
              ,'solver': ['lbfgs']
              ,'max_iter':[100,500]
             }


#Create a grid search object using the  
from sklearn.model_selection import GridSearchCV
regGridSearch_std = GridSearchCV(estimator=regEstimator
                   , n_jobs=1 # jobs to run in parallel
                   , verbose=1 # low verbosity
                   , param_grid=parameters
                   , cv=cv # KFolds = 10
                   , scoring='accuracy')

#Perform hyperparameter search to find the best combination of parameters for our data
regGridSearch_std.fit(X_scl, y)
#Diplay the top model parameters
regGridSearch_std.best_estimator_
#Use the best parameters for our Linear Regression object
classifierEst = regGridSearch_std.best_estimator_

#Evaluate the regression estimator above using our pre-defined cross validation and scoring metrics. 
EvaluateClassifierEstimator(classifierEst, X_scl, y, cv)
EvaluateClassifierEstimator2(classifierEst, X_scl, y, 10)<jupyter_output>C:\Users\Chris\anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
C:\Users\Chris\anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_[...]<jupyter_text>Feature Importance for the Top Performing Model<jupyter_code>#Create a classification estimator with best parameters for cross validation
classifierEst = regGridSearch.best_estimator_

#Fit the model using all of the scaled training data
classifierEst.fit(X, y)

from sklearn.metrics import confusion_matrix
#Load the model's coefficient weights and feature names into a dataframe sorted by weights
weights = classifierEst.coef_.ravel()
feature_names = X.columns.values
linreg_ft_imp_df = pd.DataFrame({'feature_names':feature_names, 'weights':weights, 'absolute_weights': np.abs(weights)})
linreg_ft_imp_df.sort_values(by='absolute_weights', inplace=True, ascending=False )
# Examine categorical variables of interest  
import matplotlib.pyplot as plt
%matplotlib inline

#Plot the model's feature importances
# REFERENCE:  <NAME>, https://github.com/eclarson/DataMiningNotebooks
plt.style.use('ggplot')

wt_plt_df = linreg_ft_imp_df.head(75)

weights = pd.Series(wt_plt_df['weights'].values,index=wt_plt_df['feature_names'])
ax = weights.plot(kind='bar', figsize=(20,8))

ax.set_title("Top Feature Correlations")
ax.set_ylabel("Coefficient Magnitude\n(z-score)")
ax.set_xlabel("Feature Names")
plt.show()
from sklearn.feature_selection import RFECV

#Create a regression estimator with best parameters for cross validation
classifierEst = regGridSearch.best_estimator_

rfecv = RFECV(estimator=classifierEst, step=1, cv=cv, scoring='accuracy')
rfecv.fit(X, y)
# Examine categorical variables of interest  
import matplotlib.pyplot as plt
%matplotlib inline

print("Optimal number of features : %d" % rfecv.n_features_)

# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()
#Plot the RFE Rankings
plt.style.use('ggplot')

rfe_ft_imp_df = pd.DataFrame({'feature_names':X.columns, 'weights':rfecv.grid_scores_})
rfe_ft_imp_df.sort_values(by='weights', inplace=True, ascending=False )

top50features = rfe_ft_imp_df.head(50)

top50features<jupyter_output><empty_output><jupyter_text>After Feature Selection  With Feature and Without Standarization<jupyter_code>X_sel = hotel_bookings_transform[['hotel'
,'lead_time'
,'arrival_date_year'
,'arrival_date_month'
,'arrival_date_week_number'
,'arrival_date_day_of_month'
,'stays_in_weekend_nights'
,'stays_in_week_nights'
,'adults'
,'children'
,'babies'
,'previous_cancellations'
,'previous_bookings_not_canceled'
,'meal'
,'is_repeated_guest'
,'distribution_channel'
,'market_segment'
,'reserved_room_type'
,'assigned_room_type'
,'booking_changes'
,'days_in_waiting_list'
,'customer_type'
,'adr']]
#Logisitic regression 10-fold cross-validation 
from sklearn.linear_model import LogisticRegression
regEstimator = LogisticRegression()


parameters = { 'penalty':['l2']
              ,'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]
              ,'class_weight': ['balanced', 'none']
              ,'random_state': [10]
              ,'solver': ['lbfgs']
              ,'max_iter':[100,500]
             }


#Create a grid search object using the  
from sklearn.model_selection import GridSearchCV
regGridSearch = GridSearchCV(estimator=regEstimator
                   , n_jobs=8 # jobs to run in parallel
                   , verbose=1 # low verbosity
                   , param_grid=parameters
                   , cv=cv # KFolds = 10
                   , scoring='accuracy')

#Perform hyperparameter search to find the best combination of parameters for our data
regGridSearch.fit(X_sel, y)
#Diplay the top model parameters
regGridSearch.best_estimator_
#Use the best parameters for our Linear Regression object
classifierEst = regGridSearch.best_estimator_

#Evaluate the regression estimator above using our pre-defined cross validation and scoring metrics. 
EvaluateClassifierEstimator(classifierEst, X_sel, y, cv)
EvaluateClassifierEstimator2(classifierEst, X_sel, y, 10)<jupyter_output>C:\Users\Chris\anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
C:\Users\Chris\anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_[...]<jupyter_text>With Feature Selection and With Standarization<jupyter_code>X_sel = hotel_bookings_transform[['hotel'
,'lead_time'
,'arrival_date_year'
,'arrival_date_month'
,'arrival_date_week_number'
,'arrival_date_day_of_month'
,'stays_in_weekend_nights'
,'stays_in_week_nights'
,'adults'
,'children'
,'babies'
,'previous_cancellations'
,'previous_bookings_not_canceled'
,'meal'
,'is_repeated_guest'
,'distribution_channel'
,'market_segment'
,'reserved_room_type'
,'assigned_room_type'
,'booking_changes'
,'days_in_waiting_list'
,'customer_type'
,'adr']]
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
scaler.fit(X_sel)

X_sel_scl = scaler.transform(X_sel)
#Logisitic regression 10-fold cross-validation 
from sklearn.linear_model import LogisticRegression
regEstimator = LogisticRegression()


parameters = { 'penalty':['l2']
              ,'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]
              ,'class_weight': ['balanced', 'none']
              ,'random_state': [10]
              ,'solver': ['lbfgs']
              ,'max_iter':[100,500]
             }


#Create a grid search object using the  
from sklearn.model_selection import GridSearchCV
regGridSearch = GridSearchCV(estimator=regEstimator
                   , n_jobs=8 # jobs to run in parallel
                   , verbose=1 # low verbosity
                   , param_grid=parameters
                   , cv=cv # KFolds = 10
                   , scoring='accuracy')

#Perform hyperparameter search to find the best combination of parameters for our data
regGridSearch.fit(X_sel_scl, y)
#Diplay the top model parameters
regGridSearch.best_estimator_
#Use the best parameters for our Linear Regression object
classifierEst = regGridSearch.best_estimator_

#Evaluate the regression estimator above using our pre-defined cross validation and scoring metrics. 
EvaluateClassifierEstimator(classifierEst, X_sel_scl, y, cv)
EvaluateClassifierEstimator2(classifierEst, X_sel_scl, y, 10)<jupyter_output>precision    recall  f1-score   support

           0       0.66      0.76      0.70     75166
           1       0.45      0.33      0.38     44224

    accuracy                           0.60    119390
   macro avg       0.55      0.55      0.54    119390
weighted avg       0.58      0.60      0.59    119390

[[56972 18194]
 [29499 14725]]
0.6005276823854594<jupyter_text>With Feature Selection, Standarization, and Clean up Outliers<jupyter_code>hotel_clr_outliers = hotel_bookings_transform[['hotel'
,'lead_time'
,'arrival_date_year'
,'arrival_date_month'
,'arrival_date_week_number'
,'arrival_date_day_of_month'
,'stays_in_weekend_nights'
,'stays_in_week_nights'
,'adults'
,'children'
,'babies'
,'previous_cancellations'
,'previous_bookings_not_canceled'
,'meal'
,'is_repeated_guest'
,'distribution_channel'
,'market_segment'
,'reserved_room_type'
,'assigned_room_type'
,'booking_changes'
,'days_in_waiting_list'
,'customer_type'
,'adr'
,'is_canceled']]

X_clr_out = hotel_clr_outliers[
    (np.abs(stats.zscore(hotel_clr_outliers[['hotel']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['lead_time']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['arrival_date_year']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['arrival_date_month']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['arrival_date_week_number']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['arrival_date_day_of_month']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['stays_in_weekend_nights']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['stays_in_week_nights']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['adults']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['adr']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['children']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['babies']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['previous_cancellations']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['previous_bookings_not_canceled']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['is_repeated_guest']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['distribution_channel']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['market_segment']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['reserved_room_type']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['assigned_room_type']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['booking_changes']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['days_in_waiting_list']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['customer_type']])) <= 3).all(axis=1) &
    (np.abs(stats.zscore(hotel_clr_outliers[['adr']])) <= 3).all(axis=1)]
X_clr_out.shape
X_sel.shape
y_clr_out = X_clr_out['is_canceled']
y_clr_out.shape
X_clr_out.drop(['is_canceled'], axis=1, inplace=True)
X_clr_out.shape
X_clr_out.info()
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
scaler.fit(X_clr_out)

X_clr_out_scl = scaler.transform(X_clr_out)
#Logisitic regression 10-fold cross-validation 
from sklearn.linear_model import LogisticRegression

regEstimator = LogisticRegression()


parameters = { 'penalty':['l2']
              ,'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]
              ,'class_weight': ['balanced', 'none']
              ,'random_state': [10]
              ,'solver': ['lbfgs']
              ,'max_iter':[100,500]
             }


#Create a grid search object using the  
from sklearn.model_selection import GridSearchCV
regGridSearch = GridSearchCV(estimator=regEstimator
                   , n_jobs=8 # jobs to run in parallel
                   , verbose=1 # low verbosity
                   , param_grid=parameters
                   , cv=cv # KFolds = 10
                   , scoring='accuracy')

#Perform hyperparameter search to find the best combination of parameters for our data
regGridSearch.fit(X_clr_out_scl, y_clr_out)

#Use the best parameters for our Linear Regression object
classifierEst = regGridSearch.best_estimator_

#Evaluate the regression estimator above using our pre-defined cross validation and scoring metrics. 
EvaluateClassifierEstimator(classifierEst, X_clr_out_scl, y_clr_out, cv)
regGridSearch.best_index_
regGridSearch.best_params_
regGridSearch.best_score_
regGridSearch.best_estimator_.intercept_
regGridSearch.best_estimator_.coef_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11665257819103973, 'fraction_numerical': 0.0241852164929088, 'mean_word_length': 4.861546930911093, 'pattern_counts': {'":': 13, '<': 60, '<?xml version=': 0, '>': 37, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '382979', 'license': 'MIT', 'n_tokens_mistral': 7595, 'n_tokens_neox': 6974, 'n_words': 1392, 'path': 'Mini-lab/minilab2_logistic.ipynb', 'repo_name': 'olmosjorge28/DS7331_ML1_ASSINGMENTS', 'size': '21299'}