<jupyter_start><jupyter_text>Predicting Bank's Term Deposit Subscription - XGBoost Classification Model Author: Guansu(Frances) Niu Data Resource: https://archive.ics.uci.edu/ml/datasets/Bank+Marketing<jupyter_code># Imports:

import numpy as np
import pandas as pd
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import fbeta_score
# Read data:

raw_data = pd.read_csv("data/raw data.csv",sep=';')
preprocessed_data = pd.read_csv("data/preprocessed data.csv")

pp_data = preprocessed_data.drop(preprocessed_data.columns[0], axis=1)

X = pp_data.drop(['y'], inplace = False, axis = 1)
y = pp_data['y']
# Calculate baseline of F1 score:

# Method reference: 
# https://stats.stackexchange.com/questions/390200/what-is-the-baseline-of-the-f1-score-for-a-binary-classifier

pp_data['y'].value_counts()
count_0 = 36531
prob_0 = count_0/len(pp_data['y'])
prob_1 = (len(pp_data['y']) - count_0)/len(pp_data['y'])
baseline = (2*prob_1)/(prob_1+1)
print(baseline)
# Split data:

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=random_state,stratify=y)
# Apply preprocessing:

cat_ftrs = ['job', 'marital', 'default', 'housing','loan', 'contact','poutcome']

ordinal_ftrs = ['education','month','day_of_week']

ordinal_cats = [['basic.4y', 'basic.6y', 'basic.9y', 'high.school', 'illiterate', 'professional.course', 
                'university.degree','missing'],['mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep','oct',
                'nov','dec'],['mon', 'tue', 'wed', 'thu', 'fri']]

num_ftrs = ['age','campaign','previous','emp.var.rate','cons.price.idx','cons.conf.idx', 
                 'euribor3m','nr.employed']

# one-hot encoder
categorical_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='constant',fill_value='missing')),
    ('onehot', OneHotEncoder(sparse=False,handle_unknown='ignore'))])

# ordinal encoder
ordinal_transformer = Pipeline(steps=[
    ('imputer2', SimpleImputer(strategy='constant',fill_value='NA')),
    ('ordinal', OrdinalEncoder(categories = ordinal_cats))])

# standard scaler
numeric_transformer = Pipeline(steps=[
    ('scaler', StandardScaler())])

# collect the transformers
preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, num_ftrs),
        ('cat', categorical_transformer, cat_ftrs),
        ('ord', ordinal_transformer, ordinal_ftrs)])

# fit_transform the training set
X_prep = preprocessor.fit_transform(X_train)

# Collect feature names

feature_names = preprocessor.transformers_[0][-1] + \
                list(preprocessor.named_transformers_['cat'][1].get_feature_names(cat_ftrs)) + \
                preprocessor.transformers_[2][-1]

X_train = pd.DataFrame(data=X_prep,columns=feature_names)

# transform the test
df_test = preprocessor.transform(X_test)
X_test = pd.DataFrame(data=df_test,columns = feature_names)
# Tune parameters and get best F1 score:

max_depths = range(7,14) 
f1_xgb=[]

for max_depth in max_depths:
    xgb = XGBClassifier(max_depth=max_depth,learning_rate=0.1,random_state=random_state)
    xgb.fit(X_train, y_train)
    y_pred_xgb = xgb.predict(X_test)
    f1_xgb.append(fbeta_score(y_test, y_pred_xgb,1))
    
best_max_depth = max_depths[np.argmax(f1_xgb)]

print(f"xgboost: max_depth={best_max_depth},f1_score={max(f1_xgb)}")
# Cross validation using parameters tuned from the best f1 score:

xgb = XGBClassifier(max_depth=11, learning_rate=0.1,random_state=random_state)
xgb.fit(X_train, y_train)
y_pred_xgb = xgb.predict(X_test)

kf = StratifiedKFold(n_splits=10,shuffle=True,random_state=1)
CV = (cross_val_score(xgb, X_train, y_train, cv=kf, n_jobs=1, scoring = 'f1').mean())
print(CV)
# Uncertainty due to splitting:

uncertainty_split_xgb = []

for i in range(1, 12, 2): 
    
    # Split data
    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=i,stratify=y)

    # Run the model using the tuned parameters:
    xgb = XGBClassifier(max_depth=11,learning_rate=0.001,random_state=123)
    xgb.fit(X_train, y_train)
    y_pred_xgb = xgb.predict(X_test)

    # Cross validation
    CV = (cross_val_score(xgb, X_train, y_train, cv=kf, n_jobs=1, scoring = 'f1').mean())
    uncertainty_split_xgb.append(CV)
print(np.std(uncertainty_split_xgb))
# Uncertainty due to non-deterministic ML methods:

uncertainty_ndeter_xgb = []

for i in range(123, 500, 80): 
    
    # Split data
    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=58,stratify=y)

    # Run the model using the tuned parameters:
    xgb = XGBClassifier(max_depth=11,learning_rate=0.001,random_state=i)
    xgb.fit(X_train, y_train)
    y_pred_xgb = xgb.predict(X_test)

    # Cross validation
    CV = (cross_val_score(xgb, X_train, y_train, cv=kf, n_jobs=1, scoring = 'f1').mean())
    uncertainty_ndeter_xgb.append(CV)
print(np.std(uncertainty_ndeter_xgb))
# Confusion matrix:

confusion_matrix(y_test,y_pred_xgb)
# Accuracy score:

accuracy_score(y_test, y_pred_xgb)
# Baseline of accuracy score:

baseline_accuracy = np.array(pp_data["y"].value_counts()/pp_data.shape[0])[0]
print(baseline_accuracy)<jupyter_output>0.8873855272426944
{'fraction_non_alphanumeric': 0.13938832252085265, 'fraction_numerical': 0.021316033364226137, 'mean_word_length': 5.703105590062112, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '188907', 'license': 'Apache-2.0', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2008, 'n_words': 345, 'path': 'src/xgboost model.ipynb', 'repo_name': 'francesniu/Predicting-Bank-Term-Deposit-Subscription', 'size': '5395'}