<jupyter_start><jupyter_text>Improving Models with Feature Engineering<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.metrics import accuracy_score, recall_score, precision_score

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Preparing credit card dataset<jupyter_code>default = pd.read_csv('../data/credit_card_default.csv', index_col="ID")
default.rename(columns=lambda x: x.lower(), inplace=True)
default.rename(columns={'pay_0':'pay_1','default payment next month':'default'}, inplace=True)

default['grad_school'] = (default['education'] == 1).astype(int)
default['university'] = (default['education'] == 2).astype(int)
default['high_school'] = (default['education'] == 3).astype(int)
default.drop('education', axis=1, inplace=True)

default['married_male'] = ((default['sex']==1) & (default['marriage'] == 1)).astype(int)
default['not_married_female'] = ((default['sex']==2) & (default['marriage'] != 1)).astype(int)
default.drop(['sex','marriage'], axis=1, inplace=True)

# For pay_n features if >0 then it means the customer was delayed on that month
pay_features = ['pay_' + str(i) for i in range(1,7)]
for p in pay_features:
    default[p] = (default[p] > 0).astype(int)
default.head()<jupyter_output><empty_output><jupyter_text>Feature engineering<jupyter_code># Bill amount minus payment
for i in range(1,7):
    i = str(i)
    new_var_name = 'bill_minus_pay' + i
    default[new_var_name] = default['bill_amt'+i] - default['pay_amt'+i]
    
# Reducing the 6 bill amount features to 1
bill_amt_features = ['bill_amt'+str(i) for i in range(1,7)]
bill_amt_pca = PCA(n_components=1)
default['bill_amt_new_feat'] = bill_amt_pca.fit_transform(default[bill_amt_features])[:,0]
default.drop(bill_amt_features, axis=1, inplace=True)

# Reducing the 5 pay_i features to 2
pay_features = ['pay_'+str(i) for i in range(2,7)]
pay_features_pca = PCA(n_components=2).fit_transform(default[pay_features])
default['new_pay1'] = pay_features_pca[:,0]
default['new_pay2'] = pay_features_pca[:,1]
default.drop(pay_features, axis=1, inplace=True)
money_features = ['limit_bal', 'pay_amt1', 'pay_amt2', 'pay_amt3', 'pay_amt4', 'pay_amt5', 'pay_amt6',
                  'bill_minus_pay1', 'bill_minus_pay2', 'bill_minus_pay3', 'bill_minus_pay4',
                  'bill_minus_pay5', 'bill_minus_pay6', 'bill_amt_new_feat']
default[money_features].var()
default[money_features] = default[money_features]/1000
default[money_features].var()<jupyter_output><empty_output><jupyter_text>Objects for modelling<jupyter_code>target_name = 'default'
X = default.drop('default', axis=1)
feature_names = X.columns
robust_scaler = RobustScaler()
X = robust_scaler.fit_transform(X)
y = default[target_name]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=125, stratify=y)<jupyter_output><empty_output><jupyter_text>Training new model<jupyter_code>parameter_grid = {'n_estimators': [25,50,100], 
                  'max_depth': [15,20,30],
                  'max_features': ['auto','sqrt']}

RF_classifier = GridSearchCV(RandomForestClassifier(random_state=12),
                             param_grid=parameter_grid, refit=True,
                             scoring='recall', cv=10, n_jobs=-1)

RF_classifier.fit(X_train, y_train)
y_pred_test = RF_classifier.predict(X_test)
test_accuracy = accuracy_score(y_pred=y_pred_test, y_true=y_test)
test_recall = recall_score(y_pred=y_pred_test, y_true=y_test)
print('Test Accuracy:', test_accuracy)
print('Test Recall:', test_recall)
RF_classifier.best_params_<jupyter_output><empty_output><jupyter_text>Random Forest classifier with threshold of 0.2<jupyter_code>y_pred_proba = RF_classifier.predict_proba(X_test)[:,1]
y_pred_test = (y_pred_proba >= 0.2).astype('int')
print("Recall: ", 100*round(recall_score(y_pred=y_pred_test, y_true=y_test),4))
print("Precision: ", 100*round(precision_score(y_pred=y_pred_test, y_true=y_test),4))<jupyter_output>Recall:  71.39
Precision:  37.38<jupyter_text>for the first model we had:- Recall:  68.9- Precision:  37.9<jupyter_code>fig, ax = plt.subplots(figsize=(8,6))
feature_importances = pd.Series(data=RF_classifier.best_estimator_.feature_importances_, index=feature_names)
feature_importances.sort_values().plot(kind='barh', ax=ax)
ax.set_xlabel('Importance')
ax.set_ylabel('Feature')
ax.set_title('Feature importances from Random Forest model')
ax.grid();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1403919897741798, 'fraction_numerical': 0.02194290583723903, 'mean_word_length': 7.28042328042328, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '446355', 'license': 'MIT', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1697, 'n_words': 289, 'path': 'Section3/Improving Models With Feature Engineering.ipynb', 'repo_name': 'PacktPublishing/Advanced-Predictive-Techniques-with-Scikit-Learn-and-TensorFlow', 'size': '4694'}