<jupyter_start><jupyter_text>Business Understanding - Use customers' demographic and balances/payments history to predict credit card default 💳   Data Understanding & Processing<jupyter_code># customer - credit lines
cust_line = train.groupby(['customer_id'])['credit_line'].nunique().reset_index().rename(columns={'credit_line':'num_lines'})
print('The max number of credit lines a customer holds is', cust_line['num_lines'].max())
# gender, education, marital status 
# they all ordinal columns, to convert them into categorical features
def ordinal2categorical(df):
    """Convert ordinal columns into categorical columns."""
    df['gender'] = df['gender'].replace({1:'male', 2:'female'})
    df['education'] = df['education'].replace({0:'others', 1:'master', 2:'bachelor', 3:'high school', 4:'others', 5:'others', 6:'others'})
    df['marital_status'] = df['marital_status'].replace({0:'others', 1:'married', 2:'single', 3:'others'})
    
    return df

train = ordinal2categorical(train)
fig, (ax1, ax2, ax3) = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))
sns.barplot(data=train['gender'].value_counts().reset_index().rename(columns={'index':'gender', 'gender':'freq'}), x='gender', y='freq', ax=ax1)
sns.barplot(data=train['education'].value_counts().reset_index().rename(columns={'index':'education', 'education':'freq'}), x='education', y='freq', ax=ax2)
sns.barplot(data=train['marital_status'].value_counts().reset_index().rename(columns={'index':'marital_status', 'marital_status':'freq'}), x='marital_status', y='freq', ax=ax3)

for ax in (ax1, ax2, ax3):
    ax.set_xticklabels(labels=ax.get_xticklabels(), rotation=15)
fig.tight_layout()
# age
assert train['age'].dtype == int

plt.figure(figsize=(10, 5))
sns.histplot(data=train, x='age', bins=train['age'].nunique())
# pay, bill statement, pay amount
# note: payment follows the bill statement, usually payment on the following month means pay 2 month ahead
on_time_ratio = pd.DataFrame(np.zeros((6, 2)), columns=['y_m', 'pay_on_time'])
on_time_ratio['y_m'] = ['pay_04_2005', 'pay_05_2005', 'pay_06_2005', 'pay_07_2005', 'pay_08_2005', 'pay_09_2005']
on_time_ratio.iloc[0, 1] = train['pay_04_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]
on_time_ratio.iloc[1, 1] = train['pay_05_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]
on_time_ratio.iloc[2, 1] = train['pay_06_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]
on_time_ratio.iloc[3, 1] = train['pay_07_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]
on_time_ratio.iloc[4, 1] = train['pay_08_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]
on_time_ratio.iloc[5, 1] = train['pay_09_2005'].apply(lambda x: 'yes' if x <= 0 else 'no').value_counts(normalize=True)[0]

plt.figure(figsize=(10, 5))
sns.barplot(data=on_time_ratio, x='y_m', y='pay_on_time', alpha=0.9)
sns.pointplot(data=on_time_ratio, x='y_m', y='pay_on_time', color='red')
plt.xlabel('Year-Month')
plt.ylabel('Ratio of On-time Payments');
# default
plt.figure(figsize=(8, 5))
train['is_default'].value_counts(normalize=True).plot(kind='bar')
plt.xticks(rotation=90)
plt.xlabel('Default')
plt.ylabel('Ratio');<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code># set customer id and credit line as index
def set_data_index(df):
    """Set (customer id, credit line) as the data index."""
    try:
        df = df.set_index(keys=['customer_id', 'credit_line'])
    except:
        pass
    
    return df

train = set_data_index(train)

# convert categorical features into binary features
def categorical2binary(df):
    """Convert categorical columns into binary columns."""
    if 'gender_female' not in df.columns:
        df = pd.concat([df, pd.get_dummies(df[['gender', 'education', 'marital_status']])], axis=1)
    df = df.drop(columns=['gender', 'education', 'marital_status'], errors='ignore')
    
    return df

train = categorical2binary(train)

# normalization & standardization
def feature_norm_std_scaler(df, method='std', is_training=True, scaler=None):
    """Feature normalization or standardization."""
    if is_training:
        if method == 'std':
            scaler = StandardScaler()
            
        assert scaler is not None, 'Method argument not given or match...'
        df = df.drop(columns=['is_default'], errors='ignore')
        scaler.fit(df)
        scaled_data = pd.DataFrame(scaler.transform(df), columns=df.columns)
        
        return scaled_data, scaler
    
    else:
        assert scaler is not None, 'scaler object must be provided for test data...'
        scaled_data = pd.DataFrame(scaler.transform(df), columns=df.columns)
    
        return scaled_data

X_train_std_scale, scaler = feature_norm_std_scaler(train, method='std', is_training=True)
x_train, x_test, y_train, y_test = train_test_split(X_train_std_scale,
                                                    train['is_default'],
                                                    test_size=0.2,
                                                    random_state=42)
print('x train shape =', x_train.shape)
print('x test shape =', x_test.shape)
# configs
metrics_list = [
                'accuracy', 
                'balanced_accuracy', 
                #'top_k_accuracy',
                'precision', 
                'average_precision', 
                'recall',
                'f1', 
                #'f1_micro', 
                #'f1_macro', 
                #'f1_weighted', 
                #'f1_samples',
                'roc_auc',
                # 'roc_auc_ovr', 
                #'roc_auc_ovo', 
                #'roc_auc_ovr_weighted', 
                #'roc_auc_ovo_weighted',
                'jaccard',
                #'brier_score_loss', 
                #'neg_log_loss'
               ]
cv = 10
n_iter = 4<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 1: Logistic Regression<jupyter_code>lr = LogisticRegression(random_state=42, max_iter=5000)
lr_results = cross_validate(lr, 
                            X=x_train,
                            y=y_train,
                            scoring=metrics_list,
                            cv=cv)
lr_results<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 2: Decision Tree<jupyter_code>dt = DecisionTreeClassifier(random_state=42)
dt_results = cross_validate(dt, 
                            X=x_train,
                            y=y_train,
                            scoring=metrics_list,
                            cv=cv)
dt_results<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 3: Random Forest<jupyter_code>rf = RandomForestClassifier(random_state=42)
rf_results = cross_validate(rf, 
                            X=x_train,
                            y=y_train,
                            scoring=metrics_list,
                            cv=cv)
rf_results<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 4: Gradient Boosting<jupyter_code>gbc = GradientBoostingClassifier(random_state=42)
gbc_results = cross_validate(gbc, 
                             X=x_train,
                             y=y_train,
                             scoring=metrics_list,
                             cv=cv)
gbc_results<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 5: XGB<jupyter_code>xgb = XGBClassifier(random_state=42, use_label_encoder=False, booster='gbtree')
xgb_results = cross_validate(xgb, 
                             X=x_train,
                             y=y_train,
                             scoring=metrics_list,
                             cv=cv)
xgb_results<jupyter_output>[00:04:54] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[00:04:56] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[00:04:59] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[00:05:01] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old be[...]<jupyter_text>- Baseline Alternative 6: Light BGM<jupyter_code>lgbm = LGBMClassifier(random_state=42)
lgbm_results = cross_validate(lgbm, 
                              X=x_train,
                              y=y_train,
                              scoring=metrics_list,
                              cv=cv)
lgbm_results<jupyter_output><empty_output><jupyter_text>- Baseline Alternative 7: SVM<jupyter_code>svm = SVC(random_state=42)
svm_results = cross_validate(svm, 
                             X=x_train,
                             y=y_train,
                             scoring=metrics_list,
                             cv=cv)
svm_results
def results_dict2df(results, clf_name):
    """convert model performance results dictionary into data frame."""
    results = pd.DataFrame(results)
    num_cv = results.shape[0]
    
    return pd.concat([pd.DataFrame(np.array([[clf_name] * num_cv, np.arange(1, num_cv + 1)]).T, columns=['model', 'cv_idx']), 
                                   results], axis=1)

baseline_results = pd.DataFrame()
for model, results in zip(['lr', 'dt', 'rf', 'gbc', 'xgb', 'lgbm', 'svm'], [lr_results, dt_results, rf_results, gbc_results, xgb_results, lgbm_results, svm_results]):
    baseline_results = pd.concat([baseline_results, results_dict2df(results, clf_name=model)], axis=0)
baseline_results<jupyter_output><empty_output><jupyter_text>Model Performance Metrics Selection and Evaluation- on the one hand, since the train set global default rate is 22.19%, which is imbalanced, metrics that are threshold insensitive are prefered: *Average Precision, ROC-AUC*- on the other hand, metrics that consider True Positives and True Negatives at the same time, but separately, are prefered: *Balanced Accuracy, F1, Jaccard*  Model Baseline Selection<jupyter_code># max scores 
print('Cross Validation to Find the Best Baseline by Metric')
print('best model for Accuracy: {:s}, {:.2%}'.format(baseline_results.groupby(['model'])['test_accuracy'].mean().sort_values(ascending=False).index[0], 
                                                     baseline_results.groupby(['model'])['test_accuracy'].mean().sort_values(ascending=False)[0]))
print('best model for Balanced Accuracy: {:s}, {:.2%} *'.format(baseline_results.groupby(['model'])['test_balanced_accuracy'].mean().sort_values(ascending=False).index[0], 
                                                                baseline_results.groupby(['model'])['test_balanced_accuracy'].mean().sort_values(ascending=False)[0]))
print('best model for Precision: {:s}, {:.2%}'.format(baseline_results.groupby(['model'])['test_precision'].mean().sort_values(ascending=False).index[0], 
                                                      baseline_results.groupby(['model'])['test_precision'].mean().sort_values(ascending=False)[0]))
print('best model for Average Precision: {:s}, {:.2%} **'.format(baseline_results.groupby(['model'])['test_average_precision'].mean().sort_values(ascending=False).index[0], 
                                                                 baseline_results.groupby(['model'])['test_average_precision'].mean().sort_values(ascending=False)[0]))
print('best model for Recall: {:s}, {:.2%}'.format(baseline_results.groupby(['model'])['test_recall'].mean().sort_values(ascending=False).index[0], 
                                                   baseline_results.groupby(['model'])['test_recall'].mean().sort_values(ascending=False)[0]))
print('best model for F1: {:s}, {:.2%} *'.format(baseline_results.groupby(['model'])['test_f1'].mean().sort_values(ascending=False).index[0], 
                                                 baseline_results.groupby(['model'])['test_f1'].mean().sort_values(ascending=False)[0]))
print('best model for ROC-AUC: {:s}, {:.2%} **'.format(baseline_results.groupby(['model'])['test_roc_auc'].mean().sort_values(ascending=False).index[0], 
                                                       baseline_results.groupby(['model'])['test_roc_auc'].mean().sort_values(ascending=False)[0]))
print('best model for Jaccard: {:s}, {:.2%} *'.format(baseline_results.groupby(['model'])['test_jaccard'].mean().sort_values(ascending=False).index[0], 
                                                      baseline_results.groupby(['model'])['test_jaccard'].mean().sort_values(ascending=False)[0]))

# visualize test accuracy
fig, ((ax1, ax2), (ax3, ax4), (ax5, ax6), (ax7, ax8)) = plt.subplots(nrows=4, ncols=2, figsize=(10, 15))
sns.boxplot(data=baseline_results, x='model', y='test_accuracy', ax=ax1)
sns.boxplot(data=baseline_results, x='model', y='test_balanced_accuracy', ax=ax2)
sns.boxplot(data=baseline_results, x='model', y='test_precision', ax=ax3)
sns.boxplot(data=baseline_results, x='model', y='test_average_precision', ax=ax4)
sns.boxplot(data=baseline_results, x='model', y='test_recall', ax=ax5)
sns.boxplot(data=baseline_results, x='model', y='test_f1', ax=ax6)
sns.boxplot(data=baseline_results, x='model', y='test_roc_auc', ax=ax7)
sns.boxplot(data=baseline_results, x='model', y='test_jaccard', ax=ax8)
fig.tight_layout()

print('Based on 10-fold cross validation results, Gradient Boosting Classifier is selected as the baseline model.')
print('Average Precision and ROC-AUC will be used as the model performance metrics.')<jupyter_output>Based on 10-fold cross validation results, Gradient Boosting Classifier is selected as the baseline model.
Average Precision and ROC-AUC will be used as the model performance metrics.<jupyter_text>Hyperparameter Tuning on Gradient Boosting Classifier<jupyter_code>params = {'loss': ['deviance', 'exponential'],
          'learning_rate': [0.01, 0.1, 1],
          'n_estimators': [100, 200, 500],
          'subsample': [0.5, 0.8, 1], 
          'max_features': ['auto', 'sqrt', 'log2'],
          'criterion':['friedman_mse', 'mse'],
          }
cv = 10
n_iter = 1000
tuner = RandomizedSearchCV(estimator=gbc,
                           param_distributions=params,
                           n_iter=n_iter, 
                           scoring=['average_precision', 'roc_auc'],
                           cv=cv, 
                           verbose=1,
                           refit=False,
                           error_score=-1,
                           return_train_score=False,
                           n_jobs=-1)

tuner.fit(x_train, y_train)
average_precision_results = pd.DataFrame(np.array([tuner.cv_results_['split' + str(i) + '_test_average_precision'] for i in range(cv)]), 
                                         index=['cv_'+ str(i) for i in range(cv)], 
                                         columns=['params_' + str(i) for i in range(min(n_iter, len(tuner.cv_results_['params'])))])
average_precision_results
roc_auc_results = pd.DataFrame(np.array([tuner.cv_results_['split' + str(i) + '_test_roc_auc'] for i in range(cv)]), 
                               index=['cv_'+ str(i) for i in range(cv)], 
                               columns=['params_' + str(i) for i in range(min(n_iter, len(tuner.cv_results_['params'])))])
roc_auc_results
print('max Mean Average Precision =', np.round(max(average_precision_results.mean()), 3))
print('best params({:d}): {}'.format(np.argmax(average_precision_results.mean()), tuner.cv_results_['params'][np.argmax(average_precision_results.mean())]))
print('max Mean ROC-AUC =', np.round(max(roc_auc_results.mean()), 3))
print('best params({:d}): {}'.format(np.argmax(roc_auc_results.mean()), tuner.cv_results_['params'][np.argmax(roc_auc_results.mean())]))
# retrain the model on train set
model = GradientBoostingClassifier(loss='deviance', 
                                   n_estimators=100, 
                                   subsample=0.8,
                                   max_features='auto',
                                   learning_rate=0.1,
                                   criterion='mse')
model.fit(x_train, y_train)
pred_train = model.predict(x_train)
pred_test = model.predict(x_test)

# train scores
print('Train Set:')
print('Average Precision Score = {:3f}'.format(average_precision_score(y_true=y_train, y_score=pred_train)))
print('ROC-AUC = {:.2%}'.format(roc_auc_score(y_true=y_train, y_score=pred_train)))

# test scores
print('Test Set:')
print('Average Precision Score = {:3f}'.format(average_precision_score(y_true=y_test, y_score=pred_test)))
print('ROC-AUC = {:.2%}'.format(roc_auc_score(y_true=y_test, y_score=pred_test)))<jupyter_output>Train Set:
Average Precision Score = 0.400802
ROC-AUC = 66.49%
Test Set:
Average Precision Score = 0.397936
ROC-AUC = 66.32%<jupyter_text>Prediction<jupyter_code>def get_predictions(data, scaler, model, result='proba'):
    """Prediction path."""
    inputs = (
        data.pipe(assign_columns, is_training=False)
            .pipe(ordinal2categorical)
            .pipe(set_data_index)
            .pipe(categorical2binary)
            .pipe(feature_norm_std_scaler, is_training=False, scaler=scaler)
             )
    if result == 'proba':
        pred = model.predict_proba(inputs)
    else:
        pred = model.predict(inputs)
    
    return pred
# test predictions as default probabilities
pd.concat([test, pd.DataFrame(np.round(get_predictions(test, scaler, model, result='proba')[:, 1], 3), columns=['default_proba'])], axis=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14122539506515108, 'fraction_numerical': 0.020515663986692544, 'mean_word_length': 3.1721027064538516, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '177524', 'license': 'MIT', 'n_tokens_mistral': 6012, 'n_tokens_neox': 5563, 'n_words': 1112, 'path': 'credit-card-default-classification.ipynb', 'repo_name': 'ryq99/Use_Case_Backlogs', 'size': '18035'}