<jupyter_start><jupyter_text>Upload, Initial Exploration<jupyter_code>df = pd.read_csv('application_data_updated.csv')
# Drop SK_ID_CURR column, not needed since we're not combining
df = df.drop('SK_ID_CURR', axis = 1)
print(df.shape)
df.head()
df.dtypes<jupyter_output><empty_output><jupyter_text>Cleaning + Dealing with Missing Data<jupyter_code>null_columns = df.columns[df.isnull().any()]
df[null_columns].isnull().sum()
# AMT_ANNUITY: drop if NA? hard to impute these, especially gien context of problem
df['AMT_ANNUITY'].value_counts()
# AMT_GOODS_PRICE: For consumer loans it is the price of the goods for which the loan is given
# Impute nulls with median, since the NA ones are those without consumer loans
df['AMT_GOODS_PRICE'].fillna((df['AMT_GOODS_PRICE'].median()), inplace=True)
# NAME_TYPE_SUITE: who was accompanying client when he was appplying for loan
# Change the null values to "Unaccompanied"
df['NAME_TYPE_SUITE'].fillna('Unaccompanied', inplace = True)
# CNT_FAM_MEMBERS: change nulls to 0
df['CNT_FAM_MEMBERS'].describe()
df.loc[df['CNT_FAM_MEMBERS'].isnull()] = 0
# DAYS_LAST_PHONE_CHANGE: change nulls to 0
df['DAYS_LAST_PHONE_CHANGE'].value_counts()
df.loc[df['DAYS_LAST_PHONE_CHANGE'].isnull()] = 0
# High correlation between 30 and 60 for social circle - just use 60
social_subset = df.loc[:, ['OBS_30_CNT_SOCIAL_CIRCLE', 'OBS_60_CNT_SOCIAL_CIRCLE', \
                           'DEF_30_CNT_SOCIAL_CIRCLE', 'DEF_60_CNT_SOCIAL_CIRCLE']]
social_subset.corr()

df = df.drop(['OBS_30_CNT_SOCIAL_CIRCLE', 'DEF_30_CNT_SOCIAL_CIRCLE'], axis = 1)
df.loc[df['OBS_60_CNT_SOCIAL_CIRCLE'].isnull()] = 0
df.loc[df['DEF_60_CNT_SOCIAL_CIRCLE'].isnull()] = 0
# Change amt_req_credit_bureau
df['AMT_REQ_CREDIT_BUREAU'] = df['AMT_REQ_CREDIT_BUREAU_HOUR'] + df['AMT_REQ_CREDIT_BUREAU_DAY'] + \
                            df['AMT_REQ_CREDIT_BUREAU_WEEK'] + df['AMT_REQ_CREDIT_BUREAU_MON'] + \
                            df['AMT_REQ_CREDIT_BUREAU_QRT'] + df['AMT_REQ_CREDIT_BUREAU_YEAR']
# Remove old columns
df.drop(['AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_WEEK', \
         'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR'], axis = 1, inplace = True)
# Value counts are really skewed
df['AMT_REQ_CREDIT_BUREAU'].value_counts()
# Impute nulls with median, because of skewness (and this column doesn't seem that interesting)
df['AMT_REQ_CREDIT_BUREAU'].fillna((df['AMT_REQ_CREDIT_BUREAU'].median()), inplace=True)
# Check to see how many are null
null_columns = df.columns[df.isnull().any()]
df[null_columns].isnull().sum()
# Final step: remove all rows without, since these are probably important
# Drop occupation too?
df = df.dropna(subset = ['AMT_ANNUITY', 'EXT_SOURCE_2'])
df.shape
# Check to make sure data types weren't messed up during the cleaning process
# df.dtypes<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code># None really have that strong of a correlation with target...sad
df.corr()<jupyter_output><empty_output><jupyter_text>Target Analysis, Preprocessing<jupyter_code># TARGET = our predicted value, 1 = default, 0 = no default
y = df['TARGET']
print(y.mean())
print(y.value_counts())
# Columns: Count_fam_members, income, credit, annuity, contract type, gender, education
X = df[['CNT_FAM_MEMBERS', 'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'CODE_GENDER', 'NAME_CONTRACT_TYPE', 
       'NAME_EDUCATION_TYPE']]
X.dtypes
X = pd.get_dummies(X, drop_first = True)
X.dtypes<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12326438084443185, 'fraction_numerical': 0.009634457353357892, 'mean_word_length': 6.323651452282157, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '627333', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1349, 'n_words': 272, 'path': 'Application Data Initial Cleaning.ipynb', 'repo_name': 'CMU-IDS-2020/a3-nathan-juliette-a3', 'size': '3529'}