<jupyter_start><jupyter_text>Script 4: - Explore categorical features' classes- Combine these classes- Remove features with zero importance score- Compare models after the above data cleaning<jupyter_code>import pandas as pd
import numpy as np

import matplotlib.pyplot as plt

from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import Normalizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import f1_score
from sklearn.metrics import auc
from sklearn.metrics import average_precision_score

from imblearn.over_sampling import SMOTE, ADASYN<jupyter_output>/anaconda3/lib/python3.6/site-packages/sklearn/externals/six.py:31: DeprecationWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).
  "(https://pypi.org/project/six/).", DeprecationWarning)<jupyter_text>Base Model<jupyter_code>df_modified = pd.read_csv('df_modified.csv')
display(df_modified.head())<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Change categorical variable
ordinal_encoder = OrdinalEncoder()
cat_encoded = ordinal_encoder.fit_transform(df_modified.select_dtypes('object'))
df_cat_encoded = pd.DataFrame(cat_encoded, columns = df_modified.select_dtypes('object').columns)

# Normalize numeric variables
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
df_num = df_modified.select_dtypes(include = numerics)
df_col = df_num.columns[2:]
transformer = Normalizer().fit(df_num.iloc[:,2:])
df_index = df_num.iloc[:,0:2]
df_num = pd.DataFrame(transformer.transform(df_num.iloc[:,2:]), columns = df_col)

# Combine Data
df_num = pd.concat([df_index, df_num], axis=1, sort = False)
df_modified = pd.concat([df_num, df_cat_encoded], axis=1, sort=False) 

display(df_modified.head())
display(df_modified.shape)<jupyter_output><empty_output><jupyter_text>Downsampling<jupyter_code># downsampling
# Count the number of Target 0 & 1
count0, count1 = df_modified.TARGET.value_counts()

# Split the Target 0 & 1
df_class0 = df_modified[df_modified['TARGET'] == 0]
df_class1 = df_modified[df_modified['TARGET'] == 1]

# Sample from Target 0
df_class0_resample = df_class0.sample(count1)

# Combine the Sampled Target 0 & Target 1
df_balance = pd.concat([df_class0_resample, df_class1], axis=0).reset_index(drop = True)

display(df_balance.head())
display(df_balance.shape)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code># Random Forest
# 1. Data Prepariation
Y = np.array(df_balance['TARGET'])
X = df_balance.drop('TARGET', axis = 1).iloc[:,1:]
feature = list(X.columns)
X = np.array(X) 

# 2. Split data into training and test set
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=42) 

# 3. Train the Random Forest on Training Data
rf = RandomForestClassifier(n_estimators = 1000, random_state = 42, bootstrap=True, n_jobs=-1, min_samples_split=20, min_samples_leaf=10)
rf.fit(X_train, Y_train)

Y_pred = rf.predict(X_test)
Y_pred_prob = rf.predict_proba(X_test)
# AUC Score
metrics.roc_auc_score(Y_test, Y_pred)
# Confusion Matrix
confusion_matrix(Y_test, Y_pred)
Y_positive_prob = Y_pred_prob[:,1]
precision1, recall1, _ = precision_recall_curve(Y_test, Y_positive_prob)
# Precision-Recall Curve
plt.step(recall1, precision1, color='g', alpha=0.2, where='post')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title("Precision-Recall Curve: Original Data")<jupyter_output><empty_output><jupyter_text>Import Data: Combined Data<jupyter_code>df_modified = pd.read_csv('Combined.csv')

display(df_modified.head())
display(df_modified.shape)
df_modified = df_modified.drop(df_modified.columns[0], axis=1)

display(df_modified.head())
display(df_modified.shape)<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Replace Infinity value
df_modified['AMT_CREDIT_SUM_RATIO'][df_modified['AMT_CREDIT_SUM_RATIO']>1000000] = df_modified['AMT_CREDIT_SUM_RATIO'].median()
def missingval(data):

    miss_tot = data.isnull().sum()
    miss_perc = data.isnull().sum()/len(data)
    miss_table = pd.concat([miss_tot,miss_perc],axis = 1, keys = ['miss_tot','miss_perc']).sort_values(by = 'miss_perc',ascending = False)
    miss_table_unique = miss_table.drop_duplicates('miss_tot')
    return miss_table,miss_table_unique
# missing variables
df_miss,df_miss_u = missingval(df_modified)
#drop missing data greater than 40%
df_modified.drop(df_miss[df_miss.miss_perc > 0.4].index,axis = 1,inplace = True)
#miss_app[miss_app.miss_perc > 0.4]
df_miss.head()
df_modified.STATUS_X_PERC.fillna(1,inplace = True)
df_modified.STATUS_C_PERC.fillna(0,inplace = True)
df_modified.STATUS_0_PERC.fillna(0,inplace = True)
df_modified.STATUS_1_PERC.fillna(0,inplace = True)
df_modified.STATUS_5_PERC.fillna(0,inplace = True)
df_modified.STATUS_2_TO_4_PERC.fillna(0,inplace = True)
# Use median to fill the rest of numeric feature
df_modified.fillna(df_modified.median(),inplace=True)
df_modified.fillna('XNA',inplace=True)
# Remove the TARGET_MAX
df_modified = df_modified.loc[:,~df_modified.columns.str.startswith('TARGET_')]

display(df_modified.head())
display(df_modified.shape)<jupyter_output><empty_output><jupyter_text>Down Sampling<jupyter_code># Count the number of Target 0 & 1
count0, count1 = df_modified.TARGET.value_counts()

# Split the Target 0 & 1
df_class0 = df_modified[df_modified['TARGET'] == 0]
df_class1 = df_modified[df_modified['TARGET'] == 1]

# Sample from Target 0
df_class0_resample = df_class0.sample(count1)

# Combine the Sampled Target 0 & Target 1
df_balance = pd.concat([df_class0_resample, df_class1], axis=0).reset_index(drop = True)

display(df_balance.head())
display(df_balance.shape)<jupyter_output><empty_output><jupyter_text>Base Random Forest:  Before Combining Class<jupyter_code>df_balance1 = df_balance.copy()<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Change categorical variable
ordinal_encoder = OrdinalEncoder()
cat_encoded = ordinal_encoder.fit_transform(df_balance1.select_dtypes('object'))
df_cat_encoded = pd.DataFrame(cat_encoded, columns = df_balance1.select_dtypes('object').columns)

# Normalize numeric variables
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
df_num = df_balance1.select_dtypes(include = numerics)
df_col = df_num.columns[2:]
transformer = Normalizer().fit(df_num.iloc[:,2:])
df_index = df_num.iloc[:,0:2]
df_num = pd.DataFrame(transformer.transform(df_num.iloc[:,2:]), columns = df_col)

# Combine Data
df_num = pd.concat([df_index, df_num], axis=1, sort = False)
df_balance1 = pd.concat([df_num, df_cat_encoded], axis=1, sort=False) 

display(df_balance1.head())
display(df_balance1.shape)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code># 1. Data Prepariation
Y = np.array(df_balance1['TARGET'])
X = df_balance1.drop('TARGET', axis = 1).iloc[:,1:]
feature = list(X.columns)
X = np.array(X) 

# 2. Split data into training and test set
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=42) 

# 3. Train the Random Forest on Training Data
rf = RandomForestClassifier(n_estimators = 1000, random_state = 42, bootstrap=True, n_jobs=-1, min_samples_split=20, min_samples_leaf=10)
rf.fit(X_train, Y_train)

Y_pred = rf.predict(X_test)
Y_pred_prob = rf.predict_proba(X_test)
# AUC Score
metrics.roc_auc_score(Y_test, Y_pred)
# Confusion Matrix
confusion_matrix(Y_test, Y_pred)
Y_positive_prob = Y_pred_prob[:,1]
precision2, recall2, _ = precision_recall_curve(Y_test, Y_positive_prob)
# Precision-Recall Curve
plt.step(recall2, precision2, color='b', alpha=0.2, where='post')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve: Combined Data')<jupyter_output><empty_output><jupyter_text>Remove Features with Zero Importance<jupyter_code>df_balance2 = df_balance.copy()
selected_feature = pd.read_csv('random_forest_selection 2.csv')
df_balance2 = df_balance2[[col for col in df_balance2.columns if col in selected_feature['0'].tolist()]]
df_balance2.head()<jupyter_output><empty_output><jupyter_text>Explore Categorical Feature's Class<jupyter_code># Select Categorical Features
df_cat = df_balance2.select_dtypes('object')
display(df_cat.columns)
display(df_cat.shape)<jupyter_output><empty_output><jupyter_text>Feature: NAME_TYPE_SUITE<jupyter_code>suite = df_modified[['SK_ID_CURR','TARGET','NAME_TYPE_SUITE']]
counts = pd.DataFrame(suite['NAME_TYPE_SUITE'].value_counts())
counts['Type'] = counts.index
#display(counts.head())

dist = suite.groupby(['TARGET','NAME_TYPE_SUITE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_TYPE_SUITE', right_on='Type')
dist = dist.rename(columns = {'NAME_TYPE_SUITE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 1<jupyter_code># NAME_TYPE_SUITE
# Children & Family
df_balance2['NAME_TYPE_SUITE'] = df_balance2['NAME_TYPE_SUITE'].replace({'Children':'Family'})<jupyter_output><empty_output><jupyter_text>NAME_INCOME_TYPE<jupyter_code>income = df_modified[['SK_ID_CURR','TARGET','NAME_INCOME_TYPE']]
counts = pd.DataFrame(income['NAME_INCOME_TYPE'].value_counts())
counts['Type'] = counts.index

dist = income.groupby(['TARGET','NAME_INCOME_TYPE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_INCOME_TYPE', right_on='Type')
dist = dist.rename(columns = {'NAME_INCOME_TYPE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 2: Inefficient<jupyter_code># NAME_INCOME_TYPE
# Maternity Leave & Unemployed
df_balance2['NAME_INCOME_TYPE'] = df_balance2['NAME_INCOME_TYPE'].replace({'Maternity leave':'Unemployed'})<jupyter_output><empty_output><jupyter_text>NAME_EDUCATION_TYPE<jupyter_code>education = df_modified[['SK_ID_CURR','TARGET','NAME_EDUCATION_TYPE']]
counts = pd.DataFrame(education['NAME_EDUCATION_TYPE'].value_counts())
counts['Type'] = counts.index

dist = education.groupby(['TARGET','NAME_EDUCATION_TYPE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_EDUCATION_TYPE', right_on='Type')
dist = dist.rename(columns = {'NAME_EDUCATION_TYPE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 3<jupyter_code># NAME_EDUCATION_TYPE
# Secondary / secondary special	 & Incomplete higher
df_balance2['NAME_EDUCATION_TYPE'] = df_balance2['NAME_EDUCATION_TYPE'].replace({'Secondary / secondary special':'Secondary', 'Incomplete higher':'Secondary'})<jupyter_output><empty_output><jupyter_text>NAME_FAMILY_STATUS<jupyter_code>family = df_modified[['SK_ID_CURR','TARGET','NAME_FAMILY_STATUS']]
counts = pd.DataFrame(family['NAME_FAMILY_STATUS'].value_counts())
counts['Type'] = counts.index

dist = family.groupby(['TARGET','NAME_FAMILY_STATUS']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_FAMILY_STATUS', right_on='Type')
dist = dist.rename(columns = {'NAME_FAMILY_STATUS_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 4<jupyter_code># NAME_FAMILY_STATUS
# Married & Civil marriage
df_balance2['NAME_FAMILY_STATUS'] = df_balance2['NAME_FAMILY_STATUS'].replace({'Civil marriage':'Married'})<jupyter_output><empty_output><jupyter_text>NAME_HOUSING_TYPE<jupyter_code>housing = df_modified[['SK_ID_CURR','TARGET','NAME_HOUSING_TYPE']]
counts = pd.DataFrame(housing['NAME_HOUSING_TYPE'].value_counts())
counts['Type'] = counts.index

dist = housing.groupby(['TARGET','NAME_HOUSING_TYPE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_HOUSING_TYPE', right_on='Type')
dist = dist.rename(columns = {'NAME_HOUSING_TYPE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 5: Inefficient<jupyter_code># NAME_HOUSING_TYPE
# Married & Civil marriage
#df_balance2['NAME_HOUSING_TYPE'] = df_balance2['NAME_HOUSING_TYPE'].replace({'Co-op apartment':'Apartment', 'House / apartment':'Apartment'})
df_balance2['NAME_HOUSING_TYPE'] = df_balance2['NAME_HOUSING_TYPE'].replace({'Office apartment':'Office', 'Municipal apartment':'Office'})<jupyter_output><empty_output><jupyter_text>OCCUPATION_TYPE<jupyter_code>occupation = df_modified[['SK_ID_CURR','TARGET','OCCUPATION_TYPE']]
counts = pd.DataFrame(occupation['OCCUPATION_TYPE'].value_counts())
counts['Type'] = counts.index

dist = occupation.groupby(['TARGET','OCCUPATION_TYPE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='OCCUPATION_TYPE', right_on='Type')
dist = dist.rename(columns = {'OCCUPATION_TYPE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>Combine 6: Inefficient<jupyter_code># OCCUPATION_TYPE
#df_balance2['OCCUPATION_TYPE'] = df_balance2['OCCUPATION_TYPE'].replace({'High skill tech staff':'Tech staff', 'IT staff':'Tech staff'})<jupyter_output><empty_output><jupyter_text>ORGANIZATION_TYPE<jupyter_code>organization = df_modified[['SK_ID_CURR','TARGET','ORGANIZATION_TYPE']]
counts = pd.DataFrame(organization['ORGANIZATION_TYPE'].value_counts())
counts['Type'] = counts.index

dist = organization.groupby(['TARGET','ORGANIZATION_TYPE']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='ORGANIZATION_TYPE', right_on='Type')
dist = dist.rename(columns = {'ORGANIZATION_TYPE_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
display(dist)<jupyter_output><empty_output><jupyter_text>NAME_PAYMENT_TYPE_Pre<jupyter_code>payment = df_modified[['SK_ID_CURR','TARGET','NAME_PAYMENT_TYPE_Pre']]
counts = pd.DataFrame(payment['NAME_PAYMENT_TYPE_Pre'].value_counts())
counts['Type'] = counts.index

dist = payment.groupby(['TARGET','NAME_PAYMENT_TYPE_Pre']).size().reset_index(name="Time")
dist = pd.merge(dist, counts, left_on='NAME_PAYMENT_TYPE_Pre', right_on='Type')
dist = dist.rename(columns = {'NAME_PAYMENT_TYPE_Pre_y': 'Total'})
dist['Contribution'] = dist['Time'] / dist['Total']
#display(dist)<jupyter_output><empty_output><jupyter_text>Random Forest: Combine Class<jupyter_code>df_balance2 = df_balance.copy()<jupyter_output><empty_output><jupyter_text>Combine Class<jupyter_code># Combine 1: Efficient
# NAME_TYPE_SUITE
# Children & Family
df_balance2['NAME_TYPE_SUITE'] = df_balance2['NAME_TYPE_SUITE'].replace({'Children':'Family'})<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Change categorical variable
ordinal_encoder = OrdinalEncoder()
cat_encoded = ordinal_encoder.fit_transform(df_balance2.select_dtypes('object'))
df_cat_encoded = pd.DataFrame(cat_encoded, columns = df_balance2.select_dtypes('object').columns)

# Normalize numeric variables
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
df_num = df_balance2.select_dtypes(include = numerics)
df_col = df_num.columns[2:]
transformer = Normalizer().fit(df_num.iloc[:,2:])
df_index = df_num.iloc[:,0:2]
df_num = pd.DataFrame(transformer.transform(df_num.iloc[:,2:]),columns=df_col)

# Combine Data
df_num= pd.concat([df_index, df_num], axis=1, sort=False)
df_balance2 = pd.concat([df_num, df_cat_encoded], axis=1, sort=False) 

display(df_balance2.head())
display(df_balance2.shape)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code># 1. Data Prepariation
Y = np.array(df_balance2['TARGET'])
X = df_balance2.drop('TARGET', axis = 1).iloc[:,1:]
feature = list(X.columns)
X = np.array(X) 

# 2. Split data into training and test set
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=42) 

# 3. Train the Random Forest on Training Data
rf = RandomForestClassifier(n_estimators = 1000, random_state = 42, bootstrap=True, n_jobs=-1, min_samples_split=20, min_samples_leaf=10)
rf.fit(X_train, Y_train)

Y_pred = rf.predict(X_test)
Y_pred_prob = rf.predict_proba(X_test)

# AUC Score
metrics.roc_auc_score(Y_test, Y_pred)
# Confusion Matrix
confusion_matrix(Y_test, Y_pred)
Y_positive_prob = Y_pred_prob[:,1]
precision3, recall3, _ = precision_recall_curve(Y_test, Y_positive_prob)
# plot no skill
plt.plot([0, 1], [0.5, 0.5], linestyle='--')

# Precision-Recall Curve
plt.step(recall3, precision3, color='r', alpha=0.2, where='post', label='Combined')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve: Combined Data')<jupyter_output><empty_output><jupyter_text>Up Sampling<jupyter_code>selected_feature = pd.read_csv('random_forest_selection 2.csv')
df_modified = df_modified[[col for col in df_modified.columns if col in selected_feature['0'].tolist()]]
df_modified = df_modified.loc[:,~df_modified.columns.str.startswith('TARGET_')]

df_modified.head()
df_modified['NAME_TYPE_SUITE'] = df_modified['NAME_TYPE_SUITE'].replace({'Children':'Family'})
# Change categorical variable
ordinal_encoder = OrdinalEncoder()
cat_encoded = ordinal_encoder.fit_transform(df_modified.select_dtypes('object'))
df_cat_encoded = pd.DataFrame(cat_encoded, columns = df_modified.select_dtypes('object').columns)

# Normalize numeric variables
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
df_num = df_modified.select_dtypes(include = numerics)
df_col = df_num.columns[2:]
transformer = Normalizer().fit(df_num.iloc[:,2:])
df_index = df_num.iloc[:,0:2]
df_num = pd.DataFrame(transformer.transform(df_num.iloc[:,2:]),columns=df_col)

# Combine Data
df_num= pd.concat([df_index, df_num], axis=1, sort=False)
df_modified = pd.concat([df_num, df_cat_encoded], axis=1, sort=False) 

display(df_modified.head())
# 1. Data Prepariation: denoting X and Y variable
X = df_modified.drop('TARGET', axis = 1).iloc[:,1:]
Y = df_modified['TARGET']
feature = list(X.columns)

# 2. Split data into training and test set: 30% of data is test data, 70% of data is training 
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=42)
X_resampled, Y_resampled = SMOTE().fit_resample(X_train, Y_train)
X_resampled = pd.DataFrame(X_resampled,columns = X.columns)
Y_resampled = pd.DataFrame(Y_resampled,columns = ['TARGET'])
# 3. Train the Random Forest on Training Data
rf = RandomForestClassifier(n_estimators = 100, random_state = 12,bootstrap=True,n_jobs=-1,class_weight='balanced',criterion='entropy' )
rf.fit(X_resampled, Y_resampled)

Y_pred = rf.predict(X_test)
Y_pred_prob = rf.predict_proba(X_test)

# AUC Score
metrics.accuracy_score(Y_test, Y_pred)
# Confusion Matrix
confusion_matrix(Y_test, Y_pred)
Y_positive_prob = Y_pred_prob[:,1]
precision4, recall4, _ = precision_recall_curve(Y_test, Y_positive_prob)
# plot no skill
plt.plot([0, 1], [0.5, 0.5], linestyle='--')

# Precision-Recall Curve
plt.step(recall4, precision4, color='y', alpha=0.2, where='post', label='Combined')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve: Up Sampling')<jupyter_output><empty_output><jupyter_text>Model Comparison<jupyter_code>plt.figure(figsize=(10,7))
# plot no skill
plt.plot([0, 1], [0.5, 0.5], linestyle='--', label='No Skill')

# Precision-Recall Curve
plt.step(recall1, precision1, color='g', alpha=0.2, where='post', label='Base Model')
plt.step(recall3, precision3, color='r', alpha=0.2, where='post', label='Improved Model')

plt.legend(loc='upper right')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14523472278429786, 'fraction_numerical': 0.017604208822339134, 'mean_word_length': 9.14315033350436, 'pattern_counts': {'":': 0, '<': 132, '<?xml version=': 0, '>': 135, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '33', 'dir': 'jupyter-structured-clean-dedup', 'id': '482264', 'license': 'MIT', 'n_tokens_mistral': 7550, 'n_tokens_neox': 7184, 'n_words': 1155, 'path': 'src/STA160_4 Data Cleaning.ipynb', 'repo_name': 'ashleyqqqq/Home_Credit_Analysis', 'size': '19768'}