<jupyter_start><jupyter_text>Churn Predict   Import the librarys<jupyter_code>! pip install shap
# library and Data Manipulation
import pandas as pd
import numpy as np 

# Visualization libraries 
import matplotlib as mpl
import matplotlib.pyplot as plt 
import seaborn as sns
import matplotlib.gridspec as gridspec
from matplotlib.legend_handler import HandlerLine2D

# Data preparation
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.pipeline import Pipeline

# Machine Learning Models
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier 
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBClassifier, train
from sklearn.neural_network import MLPClassifier

# Evaluating the models
from sklearn import metrics
from sklearn.metrics import accuracy_score, precision_score, f1_score, recall_score, classification_report 
from sklearn.metrics import plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay, roc_curve, auc, precision_recall_curve
from sklearn .metrics import roc_auc_score
# Finding the best parameter for the Machine Learning Model
from sklearn.model_selection import GridSearchCV

# SHAP Values
import shap

# Serializing the model 
import joblib

# Avoiding unnecessary messages
from pprint import pprint
import warnings
warnings.filterwarnings("ignore")
import pandas as pd<jupyter_output><empty_output><jupyter_text>Reading the Data<jupyter_code>df = pd.read_csv('churn.csv')<jupyter_output><empty_output><jupyter_text>Preparing the dataset<jupyter_code>def treate_data(df):
  df['AgeGroup'] = df['Age'].apply(lambda x: 1 if x >=55  else 0)  
  df["AgeCategory"] = 'Adult'
  df.loc[df['Age'] >= 55, 'AgeCategory'] ='Senior'
  df.loc[df['Age'] <= 25, 'AgeCategory'] = 'Young'
  df['Gender']= df['Gender'].map({'Male':0, 'Female':1})
  return df
def get_dummies(df):
  Products = pd.get_dummies(df['NumOfProducts'])
  Geo = pd.get_dummies(df['Geography'])
  df = pd.concat([df, Products, Geo], axis=1)
  return df
def delete(df):
  df = df.drop(['RowNumber', 'CustomerId', 'Surname', 'Geography', 'AgeCategory'], axis=1)
  return df
df = (df.
    pipe(treate_data).
    pipe(get_dummies).
    pipe(delete))
    
df1 = df[["CreditScore", "Gender","Age", "Tenure", "Balance", "NumOfProducts", "HasCrCard", "IsActiveMember", 
          "EstimatedSalary", 1, 2	,3,	4,'France',	'Germany'	,'Spain', "Exited"]] #"AgeCategory"

df1.columns
df.columns<jupyter_output><empty_output><jupyter_text>Spiltting the features<jupyter_code>y = df1['Exited']
X = df1.drop(['Exited'], axis = 1)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)<jupyter_output><empty_output><jupyter_text>Balancing the classes<jupyter_code>from imblearn.over_sampling import RandomOverSampler
# Only apply the over/under sampling in the train data
sampling = RandomOverSampler()
X_train_sampled, y_train_sampled = sampling.fit_resample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Evaluating the model<jupyter_code># Choosing the models
models = {'KNN':               KNeighborsClassifier(),  
          'Extra Tree':        ExtraTreesClassifier(),  
          'Random Forest':     RandomForestClassifier(),   
          'XGBoost':           XGBClassifier(), 
          'AdaBoost':          AdaBoostClassifier(),  
          'Gradient Boosting': GradientBoostingClassifier(),  
          'Redes Neurais':     MLPClassifier()}  
          
# evaluating the models
for reg, modelo in zip(models.values(),models.keys()):
  regressor = reg
  regressor.fit(X_train, y_train)
  y_pred = regressor.predict(X_test)
  print(modelo)
  print(f"Acccuracy: {accuracy_score(y_test, y_pred)*100}%")
  print(f"Precision: {precision_score(y_test, y_pred)*100}%")
  print(f"Recall:    {recall_score(y_test, y_pred)*100}%")
  print(f"F1-Score:  {f1_score(y_test, y_pred)*100}%")
  print('---------------------------------')<jupyter_output>KNN
Acccuracy: 76.9090909090909%
Precision: 25.514403292181072%
Recall:    9.642301710730948%
F1-Score:  13.99548532731377%
---------------------------------
Extra Tree
Acccuracy: 86.57575757575758%
Precision: 73.69668246445498%
Recall:    48.367029548989116%
F1-Score:  58.40375586854459%
---------------------------------
Random Forest
Acccuracy: 86.87878787878788%
Precision: 75.60975609756098%
Recall:    48.21150855365474%
F1-Score:  58.87939221272555%
---------------------------------
XGBoost
Acccuracy: 87.03030303030303%
Precision: 76.28361858190709%
Recall:    48.52255054432349%
F1-Score:  59.31558935361216%
---------------------------------
AdaBoost
Acccuracy: 85.75757575757575%
Precision: 70.06960556844548%
Recall:    46.96734059097978%
F1-Score:  56.238361266294234%
---------------------------------
Gradient Boosting
Acccuracy: 86.9090909090909%
Precision: 75.669099756691%
Recall:    48.367029548989116%
F1-Score:  59.01328273244781%
---------------------------------
<NAME>[...]<jupyter_text>Training the best model<jupyter_code>EXC1 = ExtraTreesClassifier()
EXC1.fit(X_train_sampled, y_train_sampled)
# Check the best parameters
EXC1.fit(X_train,y_train) 
y_pred_EXC1 = EXC1.predict(X_test) 
print(f"Acccuracy: {accuracy_score(y_test, y_pred_EXC1)*100}%")
print(f"Precision: {precision_score(y_test, y_pred_EXC1)*100}%")
print(f"Recall:    {recall_score(y_test, y_pred_EXC1)*100}%")
print(f"F1-Score:  {f1_score(y_test, y_pred_EXC1)*100}%")<jupyter_output>Acccuracy: 86.48484848484848%
Precision: 73.96593673965937%
Recall:    47.27838258164852%
F1-Score:  57.685009487666036%<jupyter_text>Founding the [best parameters](https://medium.com/all-things-ai/in-depth-parameter-tuning-for-gradient-boosting-3363992e9bae)<jupyter_code>print('Hyperparameters currently in use:\n')
pprint(EXC1.get_params())<jupyter_output>Hyperparameters currently in use:

{'bootstrap': False,
 'ccp_alpha': 0.0,
 'class_weight': None,
 'criterion': 'gini',
 'max_depth': None,
 'max_features': 'auto',
 'max_leaf_nodes': None,
 'max_samples': None,
 'min_impurity_decrease': 0.0,
 'min_samples_leaf': 1,
 'min_samples_split': 2,
 'min_weight_fraction_leaf': 0.0,
 'n_estimators': 100,
 'n_jobs': None,
 'oob_score': False,
 'random_state': None,
 'verbose': 0,
 'warm_start': False}<jupyter_text>Checking the mertrics of the current model<jupyter_code># Check the best parameters
#model = RandomForestClassifier(bootstrap='True',  criterion='gini',  max_features= 5, min_samples_split= 3,  n_estimators= 100)
EXC1.fit(X_train_sampled,y_train_sampled)
y_pred_EXC1 = EXC1.predict(X_test) 
print(f"Acccuracy: {accuracy_score(y_test, y_pred)*100}%")
print(f"Precision: {precision_score(y_test, y_pred)*100}%")
print(f"Recall:    {recall_score(y_test, y_pred)*100}%")
print(f"F1:        {f1_score(y_test, y_pred)*100}%")
confusion_matrix(y_pred= y_pred, y_true = y_test)
print(classification_report(y_pred= y_pred, y_true = y_test))
from sklearn.metrics import plot_confusion_matrix
cf_matrix = confusion_matrix(y_test, y_pred)
ax = sns.heatmap(cf_matrix, annot=True, cmap='Blues')

ax.set_title('Seaborn Confusion Matrix with labels\n\n');
ax.set_xlabel('\nPredicted Values')
ax.set_ylabel('Actual Values ');

## Ticket labels - List must be in alphabetical order
ax.xaxis.set_ticklabels(['False','True'])
ax.yaxis.set_ticklabels(['False','True'])

## Display the visualization of the Confusion Matrix.
plt.show()
ax = sns.heatmap(cf_matrix/np.sum(cf_matrix), annot=True, 
            fmt='.2%', cmap='Blues')

ax.set_title('Seaborn Confusion Matrix with labels\n\n');
ax.set_xlabel('\nPredicted Values')
ax.set_ylabel('Actual Values ');

## Ticket labels - List must be in alphabetical order
ax.xaxis.set_ticklabels(['False','True'])
ax.yaxis.set_ticklabels(['False','True'])

## Display the visualization of the Confusion Matrix.
plt.show()<jupyter_output><empty_output><jupyter_text>Founding the best parameters Here we start to find the best parameters. Let's improve:* n_estimators * max_depth * min_samples_split * min_samples_leaf * max_features  Learning rate<jupyter_code>learning_rates = [1, 0.5, 0.25, 0.1, 0.05, 0.01]
train_results = []
test_results = []
for eta in learning_rates:
   model = ExtraTreesClassifier(learning_rates=eta)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)
   
#plotting the best parameters
line1, = plt.plot(learning_rates, train_results, 'b', label="Train AUC")
line2, = plt.plot(learning_rates, test_results, 'r', label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("learning_rates")
plt.show()<jupyter_output><empty_output><jupyter_text>N_Estimators<jupyter_code>n_estimators = [1, 2, 4, 8, 16, 32, 64, 100, 200]
train_results = []
test_results = []
for eta in n_estimators:
   model = ExtraTreesClassifier(n_estimators=eta)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)
   
#plotting the best parameters
line1, = plt.plot(n_estimators, train_results, 'b', label="Train AUC")
line2, = plt.plot(n_estimators, test_results, 'r', label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("n_estimators")
plt.show()<jupyter_output><empty_output><jupyter_text>Max_depths<jupyter_code># 
max_depths = np.linspace(1, 32, 32, endpoint=True)
train_results = []
test_results = []
#
for max_depth in max_depths:
   model = ExtraTreesClassifier(max_depth=max_depth)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)
   
#plotting the best parameters
line1, = plt.plot(max_depths, train_results, 'b', label="Train AUC")
line2, = plt.plot(max_depths, test_results, 'r', label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("Tree depth")
plt.title("GBC max_depth")
plt.show()<jupyter_output><empty_output><jupyter_text>min_samples_split<jupyter_code>min_samples_splits = np.linspace(0.1, 1.0, 10, endpoint=True)
train_results = []
test_results = []

for min_samples_split in min_samples_splits:
   model = ExtraTreesClassifier(min_samples_split=min_samples_split)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)


line1, = plt.plot(min_samples_splits, train_results, 'b', label="Train AUC")
line2, = plt.plot(min_samples_splits, test_results, 'r', label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("min samples split")
plt.show()<jupyter_output><empty_output><jupyter_text>min_samples_leaf<jupyter_code>min_samples_leafs = np.linspace(0.1, 0.5, 5, endpoint=True)
train_results = []
test_results = []
for min_samples_leaf in min_samples_leafs:
   model = ExtraTreesClassifier(min_samples_leaf=min_samples_leaf)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)


line1, = plt.plot(min_samples_leafs, train_results, "b", label="Train AUC")
line2, = plt.plot(min_samples_leafs, test_results, "r", label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("min samples leaf")
plt.show()<jupyter_output><empty_output><jupyter_text>max_features<jupyter_code>max_features = list(range(1,X_train.shape[1]))

train_results = []
test_results = []

for max_feature in max_features:
   model = ExtraTreesClassifier(max_features=max_feature)
   model.fit(X_train, y_train)
   train_pred = model.predict(X_train)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   train_results.append(roc_auc)
   y_pred = model.predict(X_test)
   false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
   roc_auc = auc(false_positive_rate, true_positive_rate)
   test_results.append(roc_auc)

line1, = plt.plot(max_features, train_results, 'b', label="Train AUC")
line2, = plt.plot(max_features, test_results, 'r', label="Test AUC")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel("AUC score")
plt.xlabel("max features")
plt.show()<jupyter_output><empty_output><jupyter_text>Update of best parameters<jupyter_code>GBC = GradientBoostingClassifier()
GBC.fit(X_train_sampled, y_train_sampled)
EXC1 = ExtraTreesClassifier()
EXC1.fit(X_train_sampled, y_train_sampled)
GBC_final = GradientBoostingClassifier(learning_rate= 0.2, n_estimators=100, max_depth=15, max_features=12)
GBC_final.fit(X_train_sampled, y_train_sampled)
EXC_final = ExtraTreesClassifier(criterion="gini", n_estimators=100, max_depth=15, max_features=10)
EXC_final.fit(X_train_sampled, y_train_sampled)<jupyter_output><empty_output><jupyter_text>Checking the metrics of each model<jupyter_code># Check the best parameters
EXC_final.fit(X_train_sampled, y_train_sampled) 
y_pred_EXC_final = EXC_final.predict(X_test)
print(f"Acccuracy: {accuracy_score(y_test, y_pred_EXC_final)*100}%")
print(f"Precision: {precision_score(y_test, y_pred_EXC_final)*100}%")
print(f"Recall:    {recall_score(y_test, y_pred_EXC_final)*100}%")
print(f"F1-Score:  {f1_score(y_test, y_pred)*100}%")
GBC_final.fit(X_train_sampled,y_train_sampled)
y_pred_GBC_final= GBC_final.predict(X_test)
print(f"Acccuracy: {accuracy_score(y_test, y_pred_GBC_final)*100}%")
print(f"Precision: {precision_score(y_test, y_pred_GBC_final)*100}%")
print(f"Recall:    {recall_score(y_test, y_pred_GBC_final)*100}%")
print(f"F1-Score:  {f1_score(y_test, y_pred_GBC_final)*100}%")
# https://towardsdatascience.com/deep-learning-with-python-neural-networks-complete-tutorial-6b53c0b06af0
y_pred1 = EXC_final.predict_proba(X_test)[:, 1]
auc = metrics.roc_auc_score(y_test, y_pred1)
false_positive_rate, true_positive_rate, thresolds = metrics.roc_curve(y_test, y_pred1)
plt.axis('scaled')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.title("AUC & ROC Curve")
plt.plot(false_positive_rate, true_positive_rate, 'g')
plt.fill_between(false_positive_rate, true_positive_rate, facecolor='lightgreen', alpha=0.7)
plt.text(0.95, 0.05, 'AUC = %0.4f' % auc, ha='right', fontsize=12, weight='bold', color='blue')
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.show()<jupyter_output><empty_output><jupyter_text>Explorying the model prediction with SHAP<jupyter_code>feature_dict = dict(enumerate(df.drop("Exited", 1).columns))
# Let's try to explain the model with shap values
explainer = shap.Explainer(model_final)
shap_values = explainer.shap_values(X_test)
shap.summary_plot(shap_values, X_test, feature_names = list(feature_dict.values()), plot_type = "bar", color = "lightblue")
# Get expected value and shap values array
expected_value = explainer.expected_value
shap_array = explainer.shap_values(X)

#Descion plot for first 10 observations
shap.decision_plot(expected_value, shap_array[0:10],feature_names=list(X.columns))<jupyter_output><empty_output><jupyter_text>Exporting the model<jupyter_code>import joblib
model_FV = {
'model': model_final, 
'features_modelo': list(X_train.columns)
}
joblib.dump(modelo, 'modelo.pickle')
modelo = joblib.load('modelo.pickle')<jupyter_output><empty_output><jupyter_text>Verifying the version of each packpage Don't forget to use the code  `pip install -r requirements.txt`<jupyter_code>! pip freeze<jupyter_output>absl-py==1.0.0
alabaster==0.7.12
albumentations==0.1.12
altair==4.2.0
appdirs==1.4.4
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
arviz==0.12.0
astor==0.8.1
astropy==4.3.1
astunparse==1.6.3
atari-py==0.2.9
atomicwrites==1.4.0
attrs==21.4.0
audioread==2.1.9
autograd==1.4
Babel==2.10.1
backcall==0.2.0
beautifulsoup4==4.6.3
bleach==5.0.0
blis==0.4.1
bokeh==2.3.3
Bottleneck==1.3.4
branca==0.5.0
bs4==0.0.1
CacheControl==0.12.11
cached-property==1.5.2
cachetools==4.2.4
catalogue==1.0.0
certifi==2021.10.8
cffi==1.15.0
cftime==1.6.0
chardet==3.0.4
charset-normalizer==2.0.12
click==7.1.2
cloudpickle==1.3.0
cmake==3.22.4
cmdstanpy==0.9.5
colorcet==3.0.0
colorlover==0.3.0
community==1.0.0b1
contextlib2==0.5.5
convertdate==2.4.0
coverage==3.7.1
coveralls==0.5
crcmod==1.7
cufflinks==0.17.3
cvxopt==1.2.7
cvxpy==1.0.31
cycler==0.11.0
cymem==2.0.6
Cython==0.29.28
daft==0.0.4
dask==2.12.0
datascience==0.10.6
debugpy==1.0.0
decorator==4.4.2
defusedxml==0.7.1
descartes==1.1.0
dill==0.3.4
distributed=[...]
{'fraction_non_alphanumeric': 0.1413713405238829, 'fraction_numerical': 0.051067576491305305, 'mean_word_length': 6.28376753507014, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 80, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '434096', 'license': 'MIT', 'n_tokens_mistral': 7690, 'n_tokens_neox': 6941, 'n_words': 1169, 'path': 'scripts/MLChurn.ipynb', 'repo_name': 'MEziliano/Bank', 'size': '18178'}