<jupyter_start><jupyter_text>Univariate stats feature selection<jupyter_code>import pandas as pd
from pandas_profiling import ProfileReport

loan_df = pd.read_csv('data/loan_data.csv',
                      parse_dates=['DATE_OF_BIRTH', 'DISBURSAL_DATE'],
                      infer_datetime_format=True)
# If we run ProfileReport once with minimal=True, it will not compute correlations after that
# ProfileReport(loan_df, minimal=True)
# we are sampling down so it runs faster
report = ProfileReport(loan_df.sample(10000, random_state=42))
# save as html so the notebook displays more easily
report.to_file('loan_df1.html')
loan_df.shape
for col in loan_df.columns:
    fraction_unique = loan_df[col].unique().shape[0] / loan_df.shape[0]
    if fraction_unique > 0.5:
        print(col)
loan_df.drop('UNIQUEID', axis=1, inplace=True)
# drop columns with too little variety
drop_cols = ['MOBILENO_AVL_FLAG']
pri_sec_cols = [c for c in loan_df.columns if c[:3] in ['PRI', 'SEC'] and c not in ['PRI_NO_ACCTS', 'PRI_OVERDUE_ACCTS']]
drop_cols.extend(pri_sec_cols)
loan_df.drop(columns=drop_cols, axis=1, inplace=True)
loan_df.shape
import matplotlib.pyplot as plt

f = plt.figure(figsize=(5.5, 5.5))
loan_df.corr().loc['LOAN_DEFAULT'][:-1].plot.barh()
plt.tight_layout()
from scipy.stats import pearsonr
from pandas.api.types import is_numeric_dtype

for c in loan_df.columns[:-1]:
    if is_numeric_dtype(loan_df[c]):
        correlation, pvalue = pearsonr(loan_df[c], loan_df['LOAN_DEFAULT'])            
        print(f'{c : <40}: {correlation : .4f}, significant: {pvalue <= 0.05}')
loan_df.drop(columns=['DRIVING_FLAG', 'DELINQUENT_ACCTS_IN_LAST_SIX_MONTHS'], axis=1, inplace=True)
report = ProfileReport(loan_df.sample(10000))
report.to_file('loan_df2.html')
import phik
loan_df_epoch_time = loan_df.copy()
loan_df_epoch_time['DATE_OF_BIRTH'] = (loan_df_epoch_time['DATE_OF_BIRTH'] - \
                                       pd.to_datetime('1-1-1970')).dt.total_seconds()
loan_df_epoch_time['DISBURSAL_DATE'] = (loan_df_epoch_time['DISBURSAL_DATE'] - pd.to_datetime('1-1-1970')).dt.total_seconds()

interval_columns = ['DISBURSED_AMOUNT', 'ASSET_COST', 'LTV', 'DATE_OF_BIRTH', 'DISBURSAL_DATE', 'PERFORM_CNS_SCORE',
                   'NEW_ACCTS_IN_LAST_SIX_MONTHS', 'NO_OF_INQUIRIES']

for c in loan_df_epoch_time.columns:
    if c not in interval_columns:
        loan_df_epoch_time[c] = loan_df_epoch_time[c].astype('category')
# WARNING: this takes a very long time to run
# loan_df_epoch_time.phik_matrix(interval_cols=interval_columns)
# this also takes a long time to run
# loan_df_epoch_time.significance_matrix(interval_cols=interval_columns)
phik_correlations = []
phik_significances = []
columns = loan_df_epoch_time.columns
y = loan_df_epoch_time['LOAN_DEFAULT']

for c in columns:
    x = loan_df_epoch_time[c]
    if c in interval_columns:
        phik_correlations.append(phik.phik_from_array(x, y, [c]))
        phik_significances.append(phik.significance.significance_from_array(x, y, [c])[0])
    else:
        phik_correlations.append(phik.phik_from_array(x, y))
        # currently need to set the drop over/underflow to false to avoid error. Should be fixed soon in an update
        phik_significances.append(phik.significance.significance_from_array(x, y, drop_overflow=False, drop_underflow=False)[0])
phik_df = pd.DataFrame({'phik': phik_correlations, 'p-value': phik_significances}, index=columns)
phik_df.sort_values(by='phik', ascending=False, inplace=True)
phik_df.iloc[1:].plot.bar(subplots=True, figsize=(5.5, 5.5))
plt.tight_layout()
phik_df<jupyter_output><empty_output><jupyter_text>Mutual information score and chi-squared<jupyter_code>from sklearn.metrics import normalized_mutual_info_score

loan_df.corr(method=normalized_mutual_info_score)
loan_df_epoch_time['EMPLOYMENT_TYPE'] = loan_df_epoch_time['EMPLOYMENT_TYPE'].astype('category')
from sklearn.feature_selection import mutual_info_classif

numeric_features = loan_df.select_dtypes(include=['number']).copy()
numeric_features.drop('LOAN_DEFAULT', axis=1, inplace=True)
list(zip(numeric_features.columns, mutual_info_classif(numeric_features, loan_df['LOAN_DEFAULT'])))
from sklearn.feature_selection import chi2

chi2(loan_df[['PAN_FLAG', 'STATE_ID']], loan_df['LOAN_DEFAULT'])
from sklearn.feature_selection import f_classif

f_classif(loan_df[['PERFORM_CNS_SCORE', 'PAN_FLAG', 'STATE_ID']], loan_df['LOAN_DEFAULT'])
from sklearn.feature_selection import SelectKBest

k_best = SelectKBest(f_classif, k=5).fit_transform(loan_df_epoch_time[interval_columns], loan_df_epoch_time['LOAN_DEFAULT'])
k_best.shape<jupyter_output><empty_output><jupyter_text>Feature engineering<jupyter_code>import re
# prototype regex
re.search(r'(\d+)yrs\s+(\d+)mon', '1yrs 11mon').groups()
def convert_date_spans(date_str):
    """
    Parses date spans of the form "1yrs 1mon"
    into the number of months as an integer.
    """
    yrs, mon = re.search(r'(\d+)yrs\s+(\d+)mon', date_str).groups()
    yrs, mon = int(yrs), int(mon)
    months = yrs * 12 + mon
    return months
import swifter

loan_df['AVERAGE_ACCT_AGE_MONTHS'] = \
    loan_df.swifter.apply(lambda x: \
                          convert_date_spans(x['AVERAGE_ACCT_AGE']),
                          axis=1)
loan_df['CREDIT_HISTORY_LENGTH_MONTHS'] = \
    loan_df.swifter.apply(lambda x: \
                          convert_date_spans(x['CREDIT_HISTORY_LENGTH']),
                          axis=1)
loan_df.drop(['AVERAGE_ACCT_AGE', 'CREDIT_HISTORY_LENGTH'], axis=1, inplace=True)
import seaborn as sns

sns.boxplot(data=loan_df, orient='h')
sns.boxplot(data=loan_df.drop(['DISBURSED_AMOUNT', 'ASSET_COST'], axis=1), orient='h')
sns.boxplot(data=loan_df[['DISBURSED_AMOUNT', 'ASSET_COST']], orient='h')
dropcols = ['DISBURSED_AMOUNT', 'ASSET_COST', 'PERFORM_CNS_SCORE', 'LOAN_DEFAULT'] + \
            [c for c in loan_df.columns if 'ID' in c or 'FLAG' in c]
sns.boxplot(data=loan_df.drop(dropcols, axis=1), orient='h')
loan_df['AVERAGE_ACCT_AGE_MONTHS'].plot.box()
f, ax = plt.subplots(2, 1, figsize=(5.5, 5.5))
loan_df['AVERAGE_ACCT_AGE_MONTHS'].plot.hist(bins=30, ax=ax[0], title='AVERAGE_ACCT_AGE_MONTHS')
loan_df['DISBURSED_AMOUNT'].plot.hist(bins=50, ax=ax[1], title='DISBURSED_AMOUNT')
plt.tight_layout()
loan_df['AVERAGE_ACCT_AGE_MONTHS'].plot.hist(bins=30)
loan_df['DISBURSED_AMOUNT'].plot.hist(bins=50)
import numpy as np

q3 = loan_df['DISBURSED_AMOUNT'].quantile(0.75)
q1 = loan_df['DISBURSED_AMOUNT'].quantile(0.25)
iqr = (q3 - q1)
outliers = np.where(loan_df['DISBURSED_AMOUNT'] > (q3 + 1.5 * iqr))[0]
1.5 * iqr + q3
loan_df['DISBURSED_AMOUNT'][outliers]
loan_df['DISBURSED_AMOUNT'].clip(upper=1.5 * iqr + q3)
loan_df['DISBURSED_AMOUNT'].clip(upper=1.5 * iqr + q3)[outliers]<jupyter_output><empty_output><jupyter_text>Combining multiple columns<jupyter_code>(loan_df['DISBURSAL_DATE'] - loan_df['DATE_OF_BIRTH']).dt.days
loan_df['AGE'] = (loan_df['DISBURSAL_DATE'] - loan_df['DATE_OF_BIRTH']) // 365
(loan_df['DISBURSAL_DATE'] - loan_df['DATE_OF_BIRTH']).dt.days
from dateutil import relativedelta

# this method of calculating age is more precise
def calculate_age_in_years(x):
    return relativedelta.relativedelta(
        x['DISBURSAL_DATE'],
        x['DATE_OF_BIRTH']
    ).years
    
loan_df['AGE'] = loan_df.swifter.apply(lambda x: calculate_age_in_years(x), axis=1)
f = plt.figure(figsize=(5.5, 5.5))
loan_df['AGE'].plot.hist(bins=50)<jupyter_output><empty_output><jupyter_text>Transforming numeric data Standardization<jupyter_code>from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
loan_df['standardized_age'] = scaler.\
    fit_transform(loan_df['AGE'].values.reshape(-1, 1))

f, ax = plt.subplots(2, 1, figsize=(5.5, 5.5))
loan_df['AGE'].plot.hist(ax=ax[0], title='AGE', bins=50)
loan_df['standardized_age'].\
    plot.hist(ax=ax[1], title='standardized_age', bins=50)
plt.tight_layout()
loan_df['standardized_age'].describe()<jupyter_output><empty_output><jupyter_text>Box-cox and Yeo-Johnson transforms<jupyter_code>from sklearn.preprocessing import PowerTransformer

pt = PowerTransformer()

loan_df['xform_age'] = pt.fit_transform(loan_df['AGE'].values.reshape(-1, 1))

f, ax = plt.subplots(2, 1, figsize=(5.5, 5.5))
loan_df['AGE'].plot.hist(ax=ax[0], title='AGE', bins=50)
loan_df['xform_age'].plot.hist(ax=ax[1], title='xform_age', bins=50)
plt.tight_layout()
pt.transform(loan_df['AGE'].values.reshape(-1, 1))
import pickle as pk

with open('age_pt.pk', 'wb') as f:
    pk.dump(pt, f)<jupyter_output><empty_output><jupyter_text>Extracting datetime features<jupyter_code>loan_df['DISBURSAL_DATE'].dt.dayofweek<jupyter_output><empty_output><jupyter_text>Binning<jupyter_code>from sklearn.preprocessing import KBinsDiscretizer

kbd = KBinsDiscretizer(n_bins=10, encode='ordinal')
loan_df['binned_disbursed_amount'] = kbd.fit_transform(
    loan_df['DISBURSED_AMOUNT'].values.reshape(-1, 1)).\
    astype('int')
loan_df['binned_disbursed_amount']<jupyter_output><empty_output><jupyter_text>One-hot encoding and label encoding<jupyter_code>loan_df['EMPLOYMENT_TYPE'].fillna('Self employed', inplace=True)
# double-check to make sure we only have 2 unique values
loan_df['EMPLOYMENT_TYPE'].unique()
loan_df['EMPLOYMENT_TYPE'] = \
    loan_df['EMPLOYMENT_TYPE'].\
    map(lambda x: 1 if x == 'Self employed' else 0)
# convert from object to int dtype
loan_df['EMPLOYMENT_TYPE'] = loan_df['EMPLOYMENT_TYPE'].astype('int')
loan_df['EMPLOYMENT_TYPE'].unique()
from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
loan_df['le_branch_id'] = le.fit_transform(loan_df['BRANCH_ID'])
loan_df['le_branch_id']
loan_df['BRANCH_ID'].unique().shape
loan_df['le_branch_id'].max()<jupyter_output><empty_output><jupyter_text>Simplifying categorical columns<jupyter_code>(loan_df['MANUFACTURER_ID'].value_counts().cumsum() / loan_df.shape[0])[:20]<jupyter_output><empty_output><jupyter_text>Simplifying categorical columns<jupyter_code>loan_df['MANUFACTURER_ID'].value_counts()
(loan_df['MANUFACTURER_ID'].value_counts().cumsum() / \
    loan_df.shape[0]).reset_index(drop=True).\
    plot(marker='.', figsize=(5.5, 5.5))
plt.xlabel('MANUFACTURER_ID')
plt.ylabel('cumulative percent of values')
plt.xticks(range(loan_df['MANUFACTURER_ID'].unique().shape[0]), \
           loan_df['MANUFACTURER_ID'].value_counts().index)
plt.tight_layout()
# any manufacterer ids not in the top 6 are set to 'other'
loan_df.loc[~loan_df['MANUFACTURER_ID'].isin([86, 45, 51, 48, 49, 120]), \
            'MANUFACTURER_ID'] = 'other'
# could also do this way
loan_df.loc[loan_df['MANUFACTURER_ID'].isin([67, 145, 153, 152]), \
            'MANUFACTURER_ID'] = 'other'
manufacturer_ohe = pd.get_dummies(
    loan_df['MANUFACTURER_ID'],
    prefix='MANU_ID',
    prefix_sep='='
).drop(['MANU_ID=other'], axis=1)
manufacturer_ohe
loan_df_ohe = pd.concat([loan_df, manufacturer_ohe], axis=1)
loan_df_ohe<jupyter_output><empty_output><jupyter_text>Dimensionality Reduction PCA<jupyter_code>from sklearn.decomposition import PCA

ss = StandardScaler()
scaled = ss.fit_transform(loan_df_epoch_time[interval_columns])
pca = PCA(random_state=42)
loan_pca = pca.fit_transform(scaled)
loan_pca.shape
idx = pca.explained_variance_ratio_.argsort()[::-1]
ticks = range(pca.n_components_)
f, ax = plt.subplots(2, 1, figsize=(5.5, 5.5))

ax[0].barh(ticks, pca.explained_variance_ratio_[idx])
ax[0].set_title('explained variance ratio')
ax[0].set_ylabel('pca component')
ax[0].set_yticks(ticks)

comp_idx = abs(pca.components_[0]).argsort()[::-1]
ax[1].barh(ticks, abs(pca.components_[0, comp_idx]))
plt.yticks(ticks, np.array(interval_columns)[comp_idx])
ax[1].set_title('PCA dim-0 components')

plt.tight_layout()
# we can look for an "elbow" in this plot. The only elbow is at component 7 (labeled "6" below).
# So we could throw out the last PCA dimension
plt.bar(ticks, pca.explained_variance_ratio_[idx].cumsum())
plt.title('cumulative explained variance ratio')
plt.xlabel('pca component')
# when clustering data, we can plot the PCA dimensions to make it easier to visualize. More on this in chapter 16.
plt.scatter(loan_pca[:, 0], loan_pca[:, 1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1441278207873497, 'fraction_numerical': 0.017789491022895735, 'mean_word_length': 6.223676383105294, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '607856', 'license': 'MIT', 'n_tokens_mistral': 4943, 'n_tokens_neox': 4632, 'n_words': 756, 'path': '10-Chapter-10/Chapter_10_code.ipynb', 'repo_name': 'DiegoMerino28/Practical-Data-Science-with-Python', 'size': '12142'}