<jupyter_start><jupyter_text>Commercial Bank Customer Retention Prediction APSTA-GE.2401: Statistical Consulting Scripts Created on: 12/07/2020Modified on: 12/28/2020  Supervised Learning Models ----  DescriptionThis script contains the machine learning models. Research DesignThe strategy of supervised learning is to train models using the `X_train` data and validate model performance using the `y_train` data. After training, we fit the model to the `X_test` data. The model will then generate predictions, `y_test`, based on `X_test`. To increase model performance, we splited the train set into two sets: 80% of the train data goes to the `X_train` set and 20% of the data goes to the `X_test` set. Then, we conducted a 5-fold cross validation and selected the best performed model output. We also find tuned hyperparameters using randomized search.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from itertools import cycle
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegressionCV, ElasticNet
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import roc_auc_score, roc_curve, auc
from sklearn.preprocessing import StandardScaler, label_binarize
from sklearn.decomposition import TruncatedSVD
from sklearn.multioutput import ClassifierChain
from sklearn.multiclass import OneVsRestClassifier
from scipy import interp

print('SUCCESS! All modules are imported.')<jupyter_output>SUCCESS! All modules are imported.<jupyter_text>----<jupyter_code>X = pd.read_csv('../data/X_train.csv')
y = pd.read_csv('../data/y_train.csv')
X_hold = pd.read_csv('../data/X_test.csv')
print('The model-ready training set has {} rows and {} columns.'.format(X.shape[0], X.shape[1]))
print('The model-ready validation set has {} rows and {} columns.'.format(y.shape[0], y.shape[1]))
print('The model-ready testing set has {} rows and {} columns.'.format(X_hold.shape[0], X_hold.shape[1]))
ID_y = y.drop('cust_no', axis=1)
X['label'] = ID_y
X['label'].value_counts()
X_1 = X[X['label'] == 1]
X = X[X['label'] != 1]
X_1_down = X_1.sample(frac=len(X_1)/145296, replace=True, random_state=1061)
X = X.append(X_1_down)
X['label'].value_counts()
y = X['label']
X = X.drop('label', axis=1)
y = label_binarize(y, classes=[-1, 0, 1])
n_classes = y.shape[1]<jupyter_output><empty_output><jupyter_text>Train Test Split<jupyter_code>X = X.drop('cust_no', axis=1)
ID_hold = X_hold['cust_no']
X_hold = X_hold.drop('cust_no', axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1061)
print('After train test split, the training set has {} rows and {} columns.'.format(X_train.shape[0], X_train.shape[1]))
print('After train test split, the train has {} labels.'.format(y_train.shape[0]))
print('After train test split, the test set has {} rows and {} columns.'.format(X_test.shape[0], X_test.shape[1]))
print('After train test split, the test has {} labels.'.format(y_test.shape[0]))<jupyter_output>After train test split, the training set has 89417 rows and 77 columns.
After train test split, the train has 89417 labels.
After train test split, the test set has 22355 rows and 77 columns.
After train test split, the test has 22355 labels.<jupyter_text>----  Feature Importance Analysis SVD<jupyter_code>scaler = StandardScaler()
scaler.fit(X_train)
scaled_X_train = pd.DataFrame(scaler.transform(X_train), columns = X_train.columns)
scaled_X_test = pd.DataFrame(scaler.transform(X_test), columns = X_test.columns)
scaled_X_hold = pd.DataFrame(scaler.transform(X_hold), columns = X_hold.columns)
s = np.linalg.svd(scaled_X_train, compute_uv=False)
np.cumsum(s**2)/np.sum(s**2)<jupyter_output><empty_output><jupyter_text>SVD ResultsSVD results indicate that the number of components to retain 80% of information is 37.<jupyter_code>plt.style.use('seaborn')
fig = plt.figure(num=None, figsize=(5, 5), dpi=100, tight_layout=True)
plt.plot(np.cumsum(s**2)/np.sum(s**2))
plt.hlines(y=0.8, xmin=-5, xmax=85, linestyles='dashed', colors='red')
plt.vlines(x=37, ymin=0, ymax=1.0, linestyles='dashed', colors='red')
plt.xlabel('Number of Features')
plt.ylabel('Cumulative Information Rate')
plt.title('Singular Value Decomposition Results')
plt.show()
fig.savefig('../results/svd.png', dpi=fig.dpi)
svd = TruncatedSVD(n_components=37)
svd.fit(scaled_X_train)
best_features = [scaled_X_train.columns[col] for col in svd.components_[0].argsort()[: : -1][:15]]
best_features
best_feature_names = ['Gold', '# of Products', 'Age', 'Platinum', '# of mobile Bank Login',
                 'is_Loan', '# of transfer-in', 'Diamond', 'Assets Amount', 'Structural Savings',
                 '# of Seasonal Transfers', '# of transfer-out', 'End of Month CD Amount,', 'Date of Biggest Transfer-in', 'Gender']
best_features_explained_variance = svd.explained_variance_
plt.style.use('seaborn')
fig = plt.figure(num=None, figsize=(10, 8), dpi=150, tight_layout=True)
plt.bar(best_feature_names, height=best_features_explained_variance[0:15])
plt.xticks(rotation=90)
plt.xlabel('Feature Names')
plt.ylabel('Feature Importance')
plt.title('15 Most Important Features')
plt.show()
fig.savefig('../results/feature_importance.png', dpi=fig.dpi)
X_train_svd = X_train[best_features]
X_test_svd = X_test[best_features]
scaled_X_train_svd = scaled_X_train[best_features]
scaled_X_test_svd = scaled_X_test[best_features]
X_hold_svd = X_hold[best_features]
scaled_X_hold_svd = scaled_X_hold[best_features]
y_train<jupyter_output><empty_output><jupyter_text>----  Logistic Regression (SVD)<jupyter_code>logreg = LogisticRegressionCV(max_iter=5000, random_state=1061, cv=5, verbose=0.1)
ovr = OneVsRestClassifier(logreg)
ovr.fit(scaled_X_train_svd, y_train)
y_score = ovr.decision_function(scaled_X_test_svd)
y_score
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
fpr['micro'], tpr['micro'], _ = roc_curve(y_test.ravel(), y_score.ravel())
roc_auc['micro'] = auc(fpr['micro'], tpr['micro'])
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])
mean_tpr /= n_classes
fpr['macro'] = all_fpr
tpr['macro'] = mean_tpr
roc_auc['macro'] = auc(fpr['macro'], tpr['macro'])
# Plot Logreg
plt.figure()
plt.style.use('seaborn')
plt.plot(fpr['micro'], tpr['micro'],
         label='micro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['micro']),
         color='red', linestyle=':', linewidth=4)

plt.plot(fpr['macro'], tpr['macro'],
         label='macro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['macro']),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['blue', 'darkorange', 'green'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=2,
            label='ROC curve of class {0} (area = {1:0.2f})'
            ''.format(i, roc_auc[i]))
    
plt.plot([0, 1], [0, 1], 'k--', lw=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True positive Rate')
plt.title('ROC Curves of Multi-class Logistic Regression, Class = 3')
plt.legend(loc='lower right')
plt.savefig('../results/logreg.png', dpi=150)
plt.show()<jupyter_output><empty_output><jupyter_text>----  Random Forest<jupyter_code>rf = RandomForestClassifier(criterion='entropy', random_state=1061, verbose=0.1)
ovr = OneVsRestClassifier(rf)
ovr.fit(scaled_X_train_svd, y_train)
y_score = ovr.predict_proba(scaled_X_test_svd)

fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
    
fpr['micro'], tpr['micro'], _ = roc_curve(y_test.ravel(), y_score.ravel())
roc_auc['micro'] = auc(fpr['micro'], tpr['micro'])

all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])
    
mean_tpr /= n_classes

fpr['macro'] = all_fpr
tpr['macro'] = mean_tpr
roc_auc['macro'] = auc(fpr['macro'], tpr['macro'])
# Plot RF
plt.figure()
plt.style.use('seaborn')
plt.plot(fpr['micro'], tpr['micro'],
         label='micro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['micro']),
         color='red', linestyle=':', linewidth=4)

plt.plot(fpr['macro'], tpr['macro'],
         label='macro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['macro']),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['blue', 'darkorange', 'green'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=2,
            label='ROC curve of class {0} (area = {1:0.2f})'
            ''.format(i, roc_auc[i]))
    
plt.plot([0, 1], [0, 1], 'k--', lw=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True positive Rate')
plt.title('ROC Curves of Multi-class Random Forest, Class = 3')
plt.legend(loc='lower right')
plt.savefig('../results/rf.png', dpi=150)
plt.show()<jupyter_output><empty_output><jupyter_text>----  Gradient Boosting Machine<jupyter_code>gb = GradientBoostingClassifier()
ovr = OneVsRestClassifier(gb)
ovr.fit(scaled_X_train_svd, y_train)
y_score = ovr.predict_proba(scaled_X_test_svd)

fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
    
fpr['micro'], tpr['micro'], _ = roc_curve(y_test.ravel(), y_score.ravel())
roc_auc['micro'] = auc(fpr['micro'], tpr['micro'])

all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])
    
mean_tpr /= n_classes

fpr['macro'] = all_fpr
tpr['macro'] = mean_tpr
roc_auc['macro'] = auc(fpr['macro'], tpr['macro'])
# Plot GBM
plt.figure()
plt.style.use('seaborn')
plt.plot(fpr['micro'], tpr['micro'],
         label='micro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['micro']),
         color='red', linestyle=':', linewidth=4)

plt.plot(fpr['macro'], tpr['macro'],
         label='macro-average ROC curve (area = {0:0.2f})'
                ''.format(roc_auc['macro']),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['blue', 'darkorange', 'green'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=2,
            label='ROC curve of class {0} (area = {1:0.2f})'
            ''.format(i, roc_auc[i]))
    
plt.plot([0, 1], [0, 1], 'k--', lw=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True positive Rate')
plt.title('ROC Curves of Multi-class Gradient Boosting Machine, Class = 3')
plt.legend(loc='lower right')
plt.savefig('../results/gbm.png', dpi=150)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16956599713055953, 'fraction_numerical': 0.02187948350071736, 'mean_word_length': 5.290468133107727, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '332104', 'license': 'MIT', 'n_tokens_mistral': 4492, 'n_tokens_neox': 4214, 'n_words': 919, 'path': 'dev/models.ipynb', 'repo_name': 'tong-jin-nyu/predicting-bank-customer-churn', 'size': '11152'}