<jupyter_start><jupyter_text>Loading Data, change encoding and Defining score metrics<jupyter_code># loading data from UCI ML Repository

url = "https://archive.ics.uci.edu/ml/machine-learning-databases/00350/default%20of%20credit%20card%20clients.xls"
df = pd.read_excel(url, skiprows=1)

# renaming the target variable

df.rename(columns = {"default payment next month":"target"}, inplace=True)

# dropping ID

df.drop(columns='ID', inplace=True)

# Changing numeric encoding to labels

df['SEX'].replace([1, 2], ['Male', 'Female'],
                  inplace=True)

df['MARRIAGE'].replace([1, 2, 3, 0],
                       ['Married', 'Single', 'Other', 'Other'], inplace=True)

df['EDUCATION'].replace([1, 2, 3, 4, 0, 5, 6],
                        ['GraduateSchool', 'University', 'HighSchool', 'Other',
                         'Other', 'Other', 'Other'], inplace=True)
# split the data 80/20

train_df, test_df = train_test_split(df, test_size=0.2, random_state=2)
train_df.shape
print(
    "Distribution of the target variable :\n",
    train_df["target"].value_counts(normalize=True), sep=''
)<jupyter_output>Distribution of the target variable :
0    0.779125
1    0.220875
Name: target, dtype: float64<jupyter_text>Class imbalance in the target variable.<jupyter_code># Separate X and Y for training and testing 
X_train, y_train = train_df.drop(columns=["target"]), train_df["target"]
X_test, y_test = test_df.drop(columns=["target"]), test_df["target"]
# evalution metric for scoring
f1_micro = make_scorer(f1_score, average="micro")

scoring = {
    "f1": f1_micro,
    "recall": "recall",
    "precision": "precision",
    'roc_auc': 'roc_auc'
}<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>f, ax = plt.subplots(figsize=(12, 10))
corr = train_df.drop(columns=["target"]).corr().round(decimals=2)
sns.set(font_scale=0.8)
sns.heatmap(corr, cmap='Blues',
            square=True, ax=ax, annot=True)
plt.title('Correlation Matrix', fontsize=15)
plt.show();
f, ax = plt.subplots(figsize=(8, 8))
violin= sns.violinplot(x="target",
                       y="LIMIT_BAL",
                       hue='target',
               data=train_df)
plt.title('Distribution Of Amount of the Given Credit', fontsize=15)
plt.legend().remove()
plt.xlabel('Default', fontsize=13)
plt.ylabel('Amount of Given Credit', fontsize=13)

labels = [item.get_text() for item in ax.get_xticklabels()]
labels[0] = 'No Default'
labels[1] = 'Default'
ax.set_xticklabels(labels)
plt.show();
plt.bar(train_df['SEX'].unique().tolist(),
       train_df['SEX'].value_counts().tolist())
plt.title('Genre', fontsize=15);
sns.barplot(x = 'SEX',y='LIMIT_BAL',data = df)
plt.xlabel('Genre', fontsize=13)
plt.ylabel('Amount of Given Credit', fontsize=13)
plt.title('Amount of the Given Credit By Genre', fontsize=15)
plt.show();
fig, ax = plt.subplots()
ax.hist(train_df['AGE'], bins=20, linewidth=0.5, edgecolor="white")
plt.xlabel('Age')
plt.ylabel('Count')
plt.title('Distribution of Age')
plt.show();
fig, ax = plt.subplots()
ax = sns.countplot(y="EDUCATION", hue="target", data=train_df)
plt.xlabel('Count')
plt.ylabel('Education')
plt.title('Education')
plt.legend(title='Default', labels = ['No Default','Default'])
plt.show();
fig, ax = plt.subplots()
ax = sns.countplot(y="MARRIAGE", hue="target", data=train_df)
plt.xlabel('Count')
plt.ylabel('MARRIAGE')
plt.title('MARRIAGE')
plt.legend(title='Default', labels = ['No Default','Default'])
plt.show();
ax = sns.displot(x="LIMIT_BAL", hue="target", data=train_df, kind="kde")
ax.set(xlabel=' Amount of Given Credit', title='Distribution of Given Credit')
ax._legend.set_title('Default')
new_labels = ['No Default', 'Default']
for t, l in zip(ax._legend.texts, new_labels):
    t.set_text(l)
sns.move_legend(ax, "center right")
# Preprocessing and transformations

numeric_features = ["LIMIT_BAL", "AGE",
                    "PAY_0", "PAY_2",
                    "PAY_3", "PAY_4",
                    "PAY_5", "PAY_6",
                    'BILL_AMT1', 'BILL_AMT2',
                    'BILL_AMT3', 'BILL_AMT4',
                    'BILL_AMT5', 'BILL_AMT6',
                    'PAY_AMT1', 'PAY_AMT2',
                    'PAY_AMT3', 'PAY_AMT4',
                    'PAY_AMT5', 'PAY_AMT6']

categorical_features = ["MARRIAGE"]

binary_features = ["SEX"]

ordinal_features = ["EDUCATION"]

education_levels = ['Other',  'HighSchool', 'University', 'GraduateSchool']


# preprocessor

preprocessor = make_column_transformer(

    (StandardScaler(), numeric_features),

    (
        OrdinalEncoder(categories=[education_levels], dtype=int),
        ordinal_features
    ),
    (OneHotEncoder(drop="if_binary"), binary_features),
    (
        OneHotEncoder(handle_unknown="ignore", sparse=False),
        categorical_features
    )

)
## Code attribution: UBC-MDS DSCI-571 <NAME> 

def mean_std_cross_val_scores(model, X_train, y_train, **kwargs):
    """
    Returns mean and std of cross validation

    Parameters
    ----------
    model :
        scikit-learn model
    X_train : numpy array or pandas DataFrame
        X in the training data
    y_train :
        y in the training data

    Returns
    ----------
        pandas Series with mean scores from cross_validation
    """

    scores = cross_validate(model, X_train, y_train, **kwargs)

    mean_scores = pd.DataFrame(scores).mean()
    std_scores = pd.DataFrame(scores).std()
    out_col = []

    for i in range(len(mean_scores)):
        out_col.append((f"%0.3f (+/- %0.3f)" % (mean_scores[i], std_scores[i])))

    return pd.Series(data=out_col, index=mean_scores.index)<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code># Dictionary of Results

results = {}

models = {
    "Logistic Regression": LogisticRegression(max_iter=1000),
    "SVC": SVC(random_state=123),
    "Random Forest": RandomForestClassifier(random_state=123),
    "XGBoost": XGBClassifier(random_state=123, eval_metric="logloss",
                             verbosity=0, use_label_encoder=False,
                             class_weight="balanced"),
    "LightGBM": LGBMClassifier(random_state=123),
    "CatBoost": CatBoostClassifier(verbose=0,
                                   random_state=123)
}

for name, model in models.items():
    pipe = make_pipeline(preprocessor, model)
    results[name] = mean_std_cross_val_scores(
        pipe, X_train, y_train, return_train_score=True, scoring=scoring
    )

pd.DataFrame(results).T
# Building Confusion Matrix

fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2,
                                             figsize=(12, 12))

fig.subplots_adjust(wspace = 0.5, hspace = 0.5 )

fig.suptitle("Confusion Matrices", horizontalalignment='center',
             fontsize=16)

axes = [ax1, None, None, ax2, ax3, ax4 ]
labels = ["Non Fraud", "Fraud"]

for (k, m), ax in zip(models.items(), axes):

    if (k == "Random Forest" or k == "SVC"):
        continue

    pipe = make_pipeline(preprocessor, m)
    pipe.fit(X_train, y_train);

    ConfusionMatrixDisplay.from_estimator(

        pipe, X_test, y_test, ax=ax, display_labels=labels,
        normalize='true'
    )
    ax.set_title(f"{k}")

fig;
f, ax = plt.subplots(figsize=(8, 8))

for k, m in models.items():

    if (k == "Random Forest" or k == "SVC"):
        continue

    pipe = make_pipeline(preprocessor, m)
    pipe.fit(X_train, y_train);

    precision, recall, thresholds = precision_recall_curve(
        y_train, pipe.predict_proba(X_train)[:, 1])

    plt.plot(precision, recall, label=k)

plt.title("Precision Recall Curves")
plt.xlabel("Precision")
plt.ylabel("Recall")
plt.legend(loc="best");
f, ax = plt.subplots(figsize=(8, 8))

for k, m in models.items():

    if (k == "Random Forest" or k == "SVC"):
        continue

    pipe = make_pipeline(preprocessor, m)
    pipe.fit(X_train, y_train);

    precision, recall, thresholds = roc_curve(
        y_train, pipe.predict_proba(X_train)[:, 1])

    plt.plot(precision, recall, label=k)

plt.title("ROC curves Curves")
plt.xlabel("False Positive Rate")
plt.ylabel("True Negative Rate")
plt.legend(loc="best");<jupyter_output><empty_output><jupyter_text>Class Balancing - SMOTE<jupyter_code>results_smote = {}
for name, model in models.items():
    pipe = make_imb_pipeline(preprocessor,
                             SMOTE(),
                             model)
    results_smote[name] = mean_std_cross_val_scores(
        pipe, X_train, y_train, return_train_score=True, scoring=scoring
    )

pd.DataFrame(results_smote).T<jupyter_output><empty_output><jupyter_text>Hyperparameter Tunning LGBM<jupyter_code># Light GBM
pipe_lgbm = make_imb_pipeline(preprocessor, LGBMClassifier(random_state=123))

params = {
    "lgbmclassifier__boosting_type": ['gbdt', 'dart'],
    "lgbmclassifier__num_leaves ": randint(low=10, high=2000),
    "lgbmclassifier__n_estimators":randint(low=10, high=2000),
    "lgbmclassifier__class_weight": [None, "balanced"]
}

search = RandomizedSearchCV(
    pipe_lgbm,
    params,
    verbose=0,
    n_jobs=-1,
    n_iter=50,
    return_train_score=True,
    scoring= f1_micro,
    random_state=123,
)

search.fit(X_train, y_train);
print("Best score: %0.3f" % (search.best_score_))

print("Best hyperparameter values: ", search.best_params_)
b_type = search.best_params_['lgbmclassifier__boosting_type']
n_estimators= search.best_params_['lgbmclassifier__n_estimators']
num_leaves= search.best_params_['lgbmclassifier__num_leaves ']
pipe_lgbm_best = make_imb_pipeline(preprocessor,
                                   LGBMClassifier(random_state=123,
                                                  boosting_type=b_type,
                                                  n_estimators= n_estimators,
                                                  num_leaves= num_leaves))
pipe_lgbm_best.fit(X_train, y_train);<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>print(
    f"Grid Search best model validation score: {round(search.best_score_, 3)}"
)


print(f"Test f1 score:  {round(f1_score(y_test, pipe_lgbm_best.predict(X_test), average= 'micro'), 3)}")
print(f"Test f1 score - micro-average:  {round(f1_score(y_test, pipe_lgbm_best.predict(X_test), average= 'micro'), 3)}")
print(f"Test Accuracy score: {round(accuracy_score(y_test, pipe_lgbm_best.predict(X_test)), 3)}")
print(f"Test Recall score: {round(recall_score(y_test, pipe_lgbm_best.predict(X_test)), 3)}")
print(f"Test ROC_AUC score: {round(roc_auc_score(y_test, pipe_lgbm_best.predict(X_test)), 3)}")
print(f"Test precision score: {round(precision_score(y_test, pipe_lgbm_best.predict(X_test)), 3)}")
binary_names = pipe_lgbm_best.named_steps["columntransformer"].named_transformers_['onehotencoder-1'].get_feature_names_out().tolist()
categorical_names = pipe_lgbm_best.named_steps["columntransformer"].named_transformers_['onehotencoder-2'].get_feature_names_out().tolist()
feature_names = numeric_features + ordinal_features + binary_names + categorical_names
X_train_enc = pd.DataFrame(
    data=preprocessor.transform(X_train),
    columns=feature_names,
    index=X_train.index,
)


X_test_enc = pd.DataFrame(
    data=preprocessor.transform(X_test),
    columns=feature_names,
    index=X_test.index,
)

lgbm_explainer = shap.TreeExplainer(pipe_lgbm_best.named_steps["lgbmclassifier"])
train_lgbm_shap_values = lgbm_explainer.shap_values(X_train_enc)
test_lgbm_shap_values = lgbm_explainer.shap_values(X_test_enc)
shap.summary_plot(train_lgbm_shap_values[1], X_train_enc, plot_type="bar")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14322916666666666, 'fraction_numerical': 0.017447916666666667, 'mean_word_length': 3.963808703145196, 'pattern_counts': {'":': 14, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '658221', 'license': 'MIT', 'n_tokens_mistral': 4266, 'n_tokens_neox': 4001, 'n_words': 734, 'path': 'notebook/Credit_card_default.ipynb', 'repo_name': 'RamiroMejia/Default_Credit_Cards', 'size': '11530'}