<jupyter_start><jupyter_text>ObjectiveCreate a model using advanced techniques for a case study of a prediction model that can be used to evaluate loan repayment behaviour of currently excluded customers of formal financial serviceshttps://www.kaggle.com/c/home-credit-default-risk  Potentential improvements - Use unbalanced dataset vs artificially balanced- Use a different method of feature selection (*e.g. correlation, Recursive feature selection*) - Include more or different algorithms- Include more parameters as part of the Hyper Parameter Tuning process  Preparation  Load libraries<jupyter_code>import numpy as np
import pandas as pd
import statsmodels.api as sm
import pylab as pl
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets
from sklearn import preprocessing
from sklearn import metrics as met
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectFromModel
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc
from sklearn.ensemble import GradientBoostingClassifier, AdaBoostClassifier, RandomForestClassifier
from sklearn.neighbors import RadiusNeighborsClassifier, KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB, BernoulliNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold, train_test_split
from sklearn.utils import resample
from sklearn.metrics import roc_curve, roc_auc_score, f1_score
from scipy.stats import randint
import os
import errno
%load_ext pycodestyle_magic
%flake8_on
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Create folder structure<jupyter_code>os.chdir('..')
output_folder = './reports/figures/advanced/'
cleaned_folder = './data/processed/advanced/'
external_data = './data/external/advanced/'<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>x_train = pd.read_csv(cleaned_folder + 'x_train.csv')
x_test = pd.read_csv(cleaned_folder + 'x_test.csv')
y_train = pd.read_csv(cleaned_folder + 'y_train.csv', dtype='int64')
y_test = pd.read_csv(cleaned_folder + 'y_test.csv', dtype='int64')
test_df = pd.read_csv(cleaned_folder + 'test_df.csv')
test_ids_df = pd.read_csv(cleaned_folder + 'test_ids_df.csv')
y_test = y_test['TARGET']
y_train = y_train['TARGET']<jupyter_output><empty_output><jupyter_text>Balance dataset<jupyter_code># Verify that the dataset is balanced
y_train.value_counts(normalize=False)<jupyter_output><empty_output><jupyter_text>**Result**: dataset unbalanced with only 9% customers with payment difficulties, which can be expected.*While some algorithms can work with unbalanced datasets, we will be balancing the dataset to allow for algorithms that require a balanced sample.*<jupyter_code># Merge outcome variable & features
train_df = pd.concat([x_train, y_train], axis=1)
# Separate dataset based on outcome variable
no_pay_prob = train_df[train_df['TARGET'] == 0]
pay_prob = train_df[train_df['TARGET'] == 1]
# upsample - artificially add customers with payment difficulties
# Reason for upsampling is that our dataset is relatively small
pay_prob2 = resample(pay_prob,
                     replace=True,  # sample with replacement
                     n_samples=len(no_pay_prob),  # dataset to match customers without payment problems
                     random_state=18)
# Count of customers with payment difficulties
pay_prob.shape
# New count of customers without payment difficulties
pay_prob2.shape
# Combine dataset with added cases
train_df = pd.concat([pay_prob2, no_pay_prob])
# Separate dataset in preparation of modelling
y_train = train_df['TARGET']
x_train = train_df.drop('TARGET', axis=1)<jupyter_output><empty_output><jupyter_text>Feature selection<jupyter_code>rf_feature_select = SelectFromModel(RandomForestClassifier(n_estimators=100))
rf_feature_select.fit(x_train, y_train)

rf_sel_feature_count = rf_feature_select.get_support()
rf_selected_features = x_train.loc[:, rf_sel_feature_count].columns.tolist()
print(str(len(rf_selected_features)), 'selected features')
print('Selected features:', rf_selected_features)
# Select strongest features
x_train = x_train[rf_selected_features]
x_test = x_test[rf_selected_features]
test_df = test_df[rf_selected_features]<jupyter_output><empty_output><jupyter_text>Algorithm selection<jupyter_code>classifiers = {'KNN': KNeighborsClassifier(),
               'Random Forest Classifier': RandomForestClassifier(),
               'Decision Tree Classifier': DecisionTreeClassifier(),
               'Logistic Regression': LogisticRegression()}
# Evaluate strongest predicting algorithm in default setting
base_score = 0
model_outcomes = []
for Name, classify in classifiers.items():
    classify.fit(x_train, y_train)
    predicting_y = classify.predict(x_test)
    model_outcomes.append({'Algorithm': str(Name),
                           'f1_score': str(met.f1_score(y_test, predicting_y))})
    if met.f1_score(y_test, predicting_y) > base_score:
        # prediction = classify.predict(test_df)
        base_score = met.f1_score(y_test, predicting_y)

    else:
        continue<jupyter_output><empty_output><jupyter_text>Best performing algorithms according to the F1 score<jupyter_code>model_scores = pd.DataFrame(model_outcomes, columns=['Algorithm', 'f1_score'])
model_scores.sort_values(by=['f1_score'], ascending=False)<jupyter_output><empty_output><jupyter_text>Hyper parameter tuningOptimise the top performing algorithm parameters to create the best possible prediction.<jupyter_code>classifiers = {'Random_forest': {'model': RandomForestClassifier(),
                                 'params': {'n_estimators': [33, 34, 35, 36, 37]}},
               'Logistic_regression': {'model': LogisticRegression(solver='liblinear', multi_class='auto'),
                                       'params': {'C': [5, 10, 30, 50],
                                                  'penalty': ['l1', 'l2'], }},
               'KNearestNeighbors': {'model': KNeighborsClassifier(),
                                     'params': {'n_neighbors': [2, 3, 5],
                                                'metric': ['euclidean', 'minkowski']}},
               'DecisionTreeClassifier': {'model': DecisionTreeClassifier(),
                                          'params': {'criterion': ["gini", "entropy"],
                                                     'splitter': ['best', 'random'],
                                                     'max_depth': [3, None],
                                                     'max_features': [1, 2, 3, 4],
                                                     'min_samples_leaf': [1, 2, 3]}}}
# Select classifier algorithms to optimise
scores = []
for model_name, mp in classifiers.items():
    grid = GridSearchCV(mp['model'],
                        mp['params'],
                        cv=10,
                        scoring='f1',
                        return_train_score=False,
                        n_jobs=-1)
    grid.fit(x_train, y_train)
    scores.append({
        'model': model_name,
        'best_score': grid.best_score_,
        'best_params': grid.best_params_
    })
# Create table with best parameters per algorithm
model_parameters = pd.DataFrame(scores, columns=['model',
                                                 'best_score',
                                                 'best_params'])
model_parameters.sort_values(by=['best_score'], ascending=False)<jupyter_output><empty_output><jupyter_text>**Result:** models perform slightly better than before parameter optimisation  Modelling Use the earlier identified top 3 algorithms with best performing parameters<jupyter_code>model1 = RandomForestClassifier(n_estimators=33)
model1.fit(x_train, y_train)
model2 = DecisionTreeClassifier(criterion='gini',
                                max_depth=None,
                                max_features=1,
                                min_samples_leaf=1,
                                splitter='best')
model2.fit(x_train, y_train)
model3 = KNeighborsClassifier(metric='euclidean',
                              n_neighbors=2)
model3.fit(x_train, y_train)
model4 = LogisticRegression(C=5,
                            penalty='l2')
model4.fit(x_train, y_train)<jupyter_output><empty_output><jupyter_text>Model evaluation<jupyter_code># Create predictions from the training data on the test data
y_pred_mod_1 = model1.predict(x_test)
y_pred_mod_2 = model2.predict(x_test)
y_pred_mod_3 = model3.predict(x_test)
y_pred_mod_4 = model4.predict(x_test)
# Evaluate model based on training data
pred_prob_1 = model1.predict_proba(x_test)
pred_prob_2 = model2.predict_proba(x_test)
pred_prob_3 = model3.predict_proba(x_test)
pred_prob_4 = model4.predict_proba(x_test)
# roc curve for models
fpr1, tpr1, thresh1 = roc_curve(y_test, pred_prob_1[:, 1], pos_label=1)
fpr2, tpr2, thresh2 = roc_curve(y_test, pred_prob_2[:, 1], pos_label=1)
fpr3, tpr3, thresh3 = roc_curve(y_test, pred_prob_3[:, 1], pos_label=1)
fpr4, tpr4, thresh4 = roc_curve(y_test, pred_prob_4[:, 1], pos_label=1)
# Create benchmark predictions based on random chance for ROC comparison
random_probs = [0 for i in range(len(y_test))]
p_fpr, p_tpr, _ = roc_curve(y_test, random_probs, pos_label=1)
# Calculate auc scores
auc_score1 = roc_auc_score(y_test, pred_prob_1[:, 1])
auc_score2 = roc_auc_score(y_test, pred_prob_2[:, 1])
auc_score3 = roc_auc_score(y_test, pred_prob_3[:, 1])
auc_score4 = roc_auc_score(y_test, pred_prob_4[:, 1])

print('Model1 AUC:', auc_score1, 'Model2 AUC:', auc_score2, 'Model3 AUC:', auc_score3, 'Model4 AUC:', auc_score4)
# Model names
model_1 = 'Random Forest'
model_2 = 'DecisionTreeClassifier'
model_3 = 'KNeighborsClassifier'
model_4 = 'LogisticRegression'
# matplotlib
import matplotlib.pyplot as plt
plt.style.use('seaborn')

# plot roc curves
plt.plot(fpr1, tpr1, linestyle='--', color='orange', label=model_1)
plt.plot(fpr2, tpr2, linestyle='--', color='green', label=model_2)
plt.plot(fpr3, tpr3, linestyle='--', color='purple', label=model_3)
plt.plot(fpr4, tpr4, linestyle='--', color='blue', label=model_4)
plt.plot(p_fpr, p_tpr, linestyle='--', color='red')
plt.title('ROC curve')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')

plt.legend(loc='best')
plt.savefig(output_folder + 'ROC_graph_advanced', dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Based on the above AUC, the Logistic Regression Classifier is the best performing algorithm.<jupyter_code># Create confusion matrix
print("Confusion matrix model 1")
y_actual = pd.Series(y_test, name='Actual')
y_predicted = pd.Series(y_pred_mod_1, name='Predicted')
pd.crosstab(y_actual, y_predicted)
print(classification_report(y_predicted, y_actual))
# Create confusion matrix
print("Confusion matrix model 2")
y_actual = pd.Series(y_test, name='Actual')
y_predicted = pd.Series(y_pred_mod_2, name='Predicted')
pd.crosstab(y_actual, y_predicted)
print(classification_report(y_predicted, y_actual))
# Create confusion matrix
print("Confusion matrix model 3")
y_actual = pd.Series(y_test, name='Actual')
y_predicted = pd.Series(y_pred_mod_3, name='Predicted')
pd.crosstab(y_actual, y_predicted)
print(classification_report(y_predicted, y_actual))
# Create confusion matrix
print("Confusion matrix model 4")
y_actual = pd.Series(y_test, name='Actual')
y_predicted = pd.Series(y_pred_mod_4, name='Predicted')
pd.crosstab(y_actual, y_predicted)
print(classification_report(y_predicted, y_actual))<jupyter_output>precision    recall  f1-score   support

           0       0.49      0.94      0.65     40348
           1       0.64      0.10      0.18     43150

    accuracy                           0.51     83498
   macro avg       0.57      0.52      0.41     83498
weighted avg       0.57      0.51      0.41     83498<jupyter_text>Create final model<jupyter_code># Combine train and test datasets
x_df = pd.concat([x_train, x_test])
y_df = pd.concat([y_train, y_test])
# Apply the best performing model with best parameters to the train and test dataset
model4.fit(x_df, y_df)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code># Create predictions based on generated model
prediction = model4.predict(test_df)
# Generate Submission File
SK_ID_CURR = list(test_ids_df['SK_ID_CURR'])
predicted_test_values = pd.DataFrame({'SK_ID_CURR': SK_ID_CURR, 'TARGET': prediction})
predicted_test_values.to_csv(external_data + 'Submission_file.csv', index=False)
ID_fin_included_df = pd.read_csv(cleaned_folder + 'ID_fin_included_df.csv')
prediction.shape<jupyter_output><empty_output><jupyter_text>Evaluate financially excluded<jupyter_code># Add column 'fin_included'
IDs_fin_included = list(ID_fin_included_df['SK_ID_CURR'])
predicted_test_values['fin_included'] = predicted_test_values['SK_ID_CURR'].apply(lambda x: x in IDs_fin_included).astype(int)
# create separate dataset with financially excluded
fin_excluded_df = predicted_test_values[predicted_test_values['fin_included'] == 0]
fin_excluded_df['TARGET'].value_counts()
fin_excluded_df['TARGET'].nunique()
fin_excluded_df['TARGET'].value_counts(normalize=True)
ax = fin_excluded_df['TARGET'].value_counts(normalize=False).plot(kind='bar',
                                                                  title='Predicted financially excluded customers with payment difficulties',
                                                                  ylabel='Financially excluded')
fig = ax.get_figure()mod
fig.savefig(output_folder + 'figure.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10189175370681469, 'fraction_numerical': 0.01723760134394858, 'mean_word_length': 4.110862262038074, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '532133', 'license': 'MIT', 'n_tokens_mistral': 4455, 'n_tokens_neox': 4243, 'n_words': 954, 'path': 'notebooks/Homecred - ML modelling - Advanced.ipynb', 'repo_name': 'Dutchmick/loan_repayment_prediction', 'size': '13691'}