<jupyter_start><jupyter_text>Data Exploration and Cleanup - how do they relate to output?<jupyter_code># To Visualize the Correlation Between Label and Features
# Modify Target Output from Text to 0 and 1
# Loan Status text to 0/1 Integer

# df['loan_status'].value_counts()
# df['year'] = df['date'].apply(lambda date: date.year)
df['loan_repaid'] = df['loan_status'].apply(lambda status: status=='Fully Paid')
df['loan_repaid'] = df['loan_status'].apply(lambda x : 1 if (x == 'Fully Paid') else 0)
df['loan_repaid'] = df['loan_status'].map({'Fully Paid':1,'Charged Off':0})
df.corr()['loan_repaid'].sort_values().drop('loan_repaid').plot(kind='bar')
# visualize nulls
# need to remove missing data or fill missing data
# remove any unnecessary or repetitive features
# convert categorical string features to dummy variables
df.head()
print('Length of dataframe =', len(df))
# Missing Data Analysis
# Number of missing:
df.isnull().sum()
# Percentage missing:
df.isnull().sum()/len(df)*100

pd.DataFrame([df.isnull().sum(), (df.isnull().sum()/len(df)*100) ], index=['Number of Null','Percent Null (%)']).transpose()
# Missing Data Analysis
# Null heatmap
plt.figure(figsize=(10,5))
sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='viridis')
# looks like there are 396030 entries
# and a number of them have null
"""
emp_title               373103 non-null object
emp_length              377729 non-null object
title                   394275 non-null object
revol_util              395754 non-null float64
mort_acc                358235 non-null float64
pub_rec_bankruptcies    395495 non-null float64
"""

# are we going to drop it or fill it with something? 
# depends if it is interesting or not
df['emp_title'].nunique() # 173000 unique employment titles
df['emp_title'].value_counts() 
# future: could map to high income job or low income jobs
# for now: just drop it
# Drop emp_title
df = df.drop('emp_title', axis = 1)
# Next one: emp_length
# get a list of unique values and then plot them by count, plot will be ordered

df['emp_length'].value_counts()
# df['emp_length'].isna().sum() # 18301
"""
10+ years    126041
2 years       35827
< 1 year      31725
3 years       31665
5 years       26495
1 year        25882
4 years       23952
6 years       20841
7 years       20819
8 years       19168
9 years       15314
NaN 18301
"""

# run this to get a list:
sorted(df['emp_length'].dropna().unique())
# ['1 year','10+ years','2 years','3 years','4 years','5 years','6 years','7 years','8 years','9 years','< 1 year']

# we run the above and take above output list and paste it here, and then we re-order it manually:
emp_length_order = ['< 1 year', '1 year',
 '2 years',
 '3 years',
 '4 years',
 '5 years',
 '6 years',
 '7 years',
 '8 years',
 '9 years',
'10+ years']

# count plot seaborn
plt.figure(figsize=(10,5))
sns.countplot(x='emp_length', data=df, order=emp_length_order) # looks like most borrowers have been working a long time
# redo above chart with hue to differentiate folks who default
plt.figure(figsize=(10,5))
emp_length_order = ['< 1 year', '1 year',
 '2 years',
 '3 years',
 '4 years',
 '5 years',
 '6 years',
 '7 years',
 '8 years',
 '9 years',
'10+ years']
sns.countplot(x='emp_length', data=df, hue='loan_status', order=emp_length_order)

# chart does not say much - do ratio analysis
# ratio analysis
# want to build percent of people per category

# first filter on loan_status, then groupby emp_length, then count to get each number of the columns (above chart)

df[df['loan_status']=='Charged Off'].groupby('emp_length').count()
df[df['loan_status']=='Fully Paid'].groupby('emp_length').count()
# first filter on loan_status, then groupby emp_length, then count to get each number of the columns (above chart)
# second grab column loan_status
emp_charged_off = df[df['loan_status']=='Charged Off'].groupby('emp_length').count()['loan_status']
emp_fully_paid = df[df['loan_status']=='Fully Paid'].groupby('emp_length').count()['loan_status']
emp_ratio = emp_charged_off/(emp_charged_off+emp_fully_paid) # note this is percent of total rather than A:B it is A / (A+B)
emp_ratio_df = pd.DataFrame([emp_charged_off, emp_fully_paid, emp_ratio],
                          index=['Charged Off', 'Fully Paid', 'Charged Off Percent']
                         )

# here is the previous calculation
def emp_length_extraction(term_text):
    if term_text == '0':
        return 0
    else:
        temp_num = term_text.split()[0]
        if temp_num == '<':
            return 6
        elif temp_num == '10+':
            return 120
        else:
            return int(temp_num)*12
# df['emp_length'].fillna(value='0', inplace=True)    
# df['emp_length_values'] = df['emp_length'].apply(emp_length_extraction)
# df['emp_length_values'].value_counts()

emp_ratio.transpose().plot(kind='bar') 
emp_ratio_df.transpose()
# looks like 20% regardless of emp_length, so not useful info, so do not keep the feature
# drop the feature
df.drop(['emp_length'], axis=1, inplace=True)
# looked at title vs purpose (same info)
df = df.drop('title', axis=1)<jupyter_output><empty_output><jupyter_text>FILLING MISSING DATA<jupyter_code># mort_acc has 10% missing data

# so, what do we do with this column?
# if you dropna then you drop 10% of data - is not very good
# if you drop the feature then you have to be careful not to lose good info...

# so if we fill with value, what is a reasonable way to fill in the data? look for other features are highly correlated

df.corr()['mort_acc'].sort_values()
# looks like a positive correlation with "total_acc" feature

# next step: 
# use the average of mortgage accounts per total accounts to fill in the missing data
# so first analyze total account information
df.groupby('total_acc').mean()

# take this total account info and focus on number of mortgage accounts
df.groupby('total_acc').mean()['mort_acc']

# this will be the lookup table used to fill in mort_acc:
# fill in mort_acc missing values with a mapping table you created above
missing_data_map_total_acc = df.groupby('total_acc').mean()['mort_acc']
# how to use the mapping table:
missing_data_map_total_acc[2.0] # returns 0.0 like the above

# this is a helper function that checks if NaN, if it is then return the mapping table output, otherwise return mort_acc #
def fill_mort_acc(total_acc, mort_acc):
    '''
    Accepts the total_acc and mort_acc values for the row.
    Checks if the mort_acc is NaN , if so, it returns the avg mort_acc value
    for the corresponding total_acc value for that row.
    
    total_acc_avg here should be a Series or dictionary containing the mapping of the
    groupby averages of mort_acc per total_acc values.
    '''
    if np.isnan(mort_acc):
        return missing_data_map_total_acc[total_acc] # this maps to the table above
    else:
        return mort_acc

# this executes the filling function which uses the mapping table function, axis=1 because column
# helpful: https://stackoverflow.com/questions/13331698/how-to-apply-a-function-to-two-columns-of-pandas-dataframe
df['mort_acc'] = df.apply(lambda x: fill_mort_acc(x['total_acc'],x['mort_acc']), axis=1)
df.isnull().sum()
# 2 more nulls to investigate

# i use fillna=0 but solutions use dropna because small number of rows
"""
df['revol_util'].fillna(value=0,inplace=True)
df['mort_acc'].fillna(value=0,inplace=True)
df['pub_rec_bankruptcies'].fillna(value=0,inplace=True)
"""
# drop empty values for 'revol_util' and 'pub_rec_bankruptcies' 
# do not need to drop the feature but just drop the rows that have NaN
df = df.dropna()

# my own model: 
# drop all the other columns
# df.drop(['home_ownership','address','emp_title','title','purpose', 'earliest_cr_line','initial_list_status'], axis=1, inplace=True)
# should be all zero now
df.isnull().sum()  # df.isna().sum() might work too but isnull seems more comprehensive<jupyter_output><empty_output><jupyter_text>Categorical Data and String Data<jupyter_code># First step: determine which features are not values
df.select_dtypes(['object']).columns
# term
# remove months text and convert to integer 36 and 60
# method 1 = create manual mapping table
# method 2 = grab 2 characters
df['term'].value_counts()
"""
36 months
60 months
"""

def month_extraction(term_text):
    return int(term_text.split()[0])

# another way = df['term'] = df['term'].apply(lambda term: int(term[:3]))
df['term'] = df['term'].apply(month_extraction)

# run this to check if did it properly:
# df['term'].value_counts()
# grade
# grade already covered by sub_grade so drop it
df = df.drop('grade', axis=1)

# sub_grade (first run - not used but example code here anyway)
# convert sub_grade to dummy variables using a mapping function
# this might be wrong because not really a continous variable?

# sub grade might be better than grade
df['sub_grade'].value_counts()
"""
B3    26655
B4    25601
C1    23662
C2    22580
B2    22495
B5    22085
C3    21221
C4    20280
B1    19182
A5    18526
C5    18244
F5     1397
G1     1058
G2      754
G3      552
G4      374
G5      316
"""

# my way (not used in second run): 
def grade_extraction(term_text):
    grade_letter = term_text[0]
    if grade_letter == 'A':
        return int(term_text[1])
    elif grade_letter == 'B':
        return 10 + int(term_text[1])
    elif grade_letter == 'C':
        return 20 + int(term_text[1])
    elif grade_letter == 'D':
        return 30 + int(term_text[1])
    elif grade_letter == 'E':
        return 40 + int(term_text[1])
    elif grade_letter == 'F':
        return 50 + int(term_text[1])
    else:
        return 60 + int(term_text[1])
    
df['sub_grade_values'] = df['sub_grade'].apply(grade_extraction)

# drop columns because not used in second run:
df.drop(['sub_grade_values'], axis=1, inplace=True)
# sub_grade (second run)
# convert to dummy variables using one hot encoding

# use pandas dummy variables to add new columns to original dataframe, drop original sub_grade column 
# and add drop_first - do not need to duplicate information
# A/B/C --> _/1/0 because A is implicitly encoded
dummies = pd.get_dummies(df['sub_grade'], drop_first=True)

# REMOVE the sub_grade column THEN concat to original dataframe
df = pd.concat([df.drop('sub_grade', axis=1), dummies], axis=1)
# more dummy variables:
# verification_status, appliction_type, initial_list_status, purpose
# mass create dummy variables since low unique values

df['purpose'].value_counts() # only 15 unique values -> 15 unique columns

# use pandas dummy variables to add new columns to original dataframe, drop original column 
dummies = pd.get_dummies(df[['verification_status', 'application_type', 'initial_list_status', 'purpose']], drop_first=True)

# REMOVE the sub_grade column THEN concat to original dataframe
df = pd.concat([df.drop(['verification_status', 'application_type', 'initial_list_status', 'purpose'], axis=1), 
                dummies], axis=1)
# home_ownership
# we want to replace (NONE and ANY) into OTHER
# or custom replace function or mapping function

df['home_ownership'].value_counts()
"""
MORTGAGE    198022
RENT        159395
OWN          37660
OTHER          110
NONE            29
ANY              3
"""

# we want to replace (NONE and ANY) into OTHER
# or custom replace function or mapping function
df['home_ownership'] = df['home_ownership'].replace(['NONE','ANY'], 'OTHER')

df['home_ownership'].value_counts()

# use pandas dummy variables to add new columns to original dataframe, drop original column 
dummies = pd.get_dummies(df['home_ownership'], drop_first=True)

# REMOVE the sub_grade column THEN concat to original dataframe
df = pd.concat([df.drop('home_ownership', axis=1), dummies], axis=1)

# See what columns we got
# df.columns
# Address
# Extract the zip code

df['zip_code'] = df['address'].apply(lambda address:address[-5:])
df['zip_code'].value_counts() # luckily we only have 20 here

# use pandas dummy variables to add new columns to original dataframe, drop original column 
dummies = pd.get_dummies(df['zip_code'], drop_first=True)

# REMOVE the sub_grade column THEN concat to original dataframe
df = pd.concat([df.drop('zip_code', axis=1), dummies], axis=1)

df = df.drop('address', axis=1)
# issue_d
# does this feature help to show which will default?
# course thinks this is data leakage - but maybe in reality vintage matters

# date conversion (if needed)
df['issue_d'].value_counts()
df['issue_d'] = pd.to_datetime(df['issue_d'])
# date is not useful by itself, so extract year
df['issue_d'] = df['issue_d'].apply(lambda date: date.year) # df['issue_d'].apply(lambda date: date.month)

# will stick to course workbook:
df.drop(['issue_d'], axis=1, inplace=True)

# earliest_cr_line
# does this feature help to show which will default?
# this is a historical time stamp feature - extract the year
# no need to convert to dummy variable because year can be treated as a continuous data type

# date conversion (if needed)
df['earliest_cr_line'].value_counts()
df['earliest_cr_line'] = pd.to_datetime(df['earliest_cr_line'])
# date is not useful by itself, so extract year
# df['earliest_cr_line'] = df['earliest_cr_line'].apply(lambda date: int(date[-4:])) # last 4 letters approach
df['earliest_cr_line'] = df['earliest_cr_line'].apply(lambda date: date.year) # df['issue_d'].apply(lambda date: date.month)

# df['earliest_cr_line'].value_counts()
# len(df['earliest_cr_line']) # 396030<jupyter_output><empty_output><jupyter_text>DATA Cleaned - Now DATA PREPROCESSING (TRAIN/TEST/SPLIT)<jupyter_code>df.columns
df.select_dtypes(['object']).columns
# Remove old text output since we already have loan_repaid as 0/1
df = df.drop('loan_status', axis=1)
# Train Test Split
# df = df.sample(frac=0.1,random_state=101) # if many entries, can sample 10% of the rows for speed/memory issues
X = df.drop('loan_repaid', axis=1).values
y = df['loan_repaid'].values

# create two numpy arrays from pd dataframes
from sklearn.model_selection import train_test_split
# train_test_split()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=101) # 20% will be test set

# normalize the data and fit the scaler
from sklearn.preprocessing import MinMaxScaler

# create instance of scaler
scaler = MinMaxScaler()
# only fit to the training set (to avoid data leakage)
scaler.fit(X_train)
# overwrite the X_train and X_test with the scaled values
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
# Tensorflow Model for Classification
# THIRD MODEL (callback with dropout) - see other models (Model 1=no callback, Model 2 = with callback no dropout)
# runs a little bit more than the Model 2 because of the randomly stopped updating neurons
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,Dropout

# Can use this to check: # X_train.shape[1] is number of features, in this case = 30
print('X_train shape: ', X_train.shape) # (316175 rows, 78 columns representing the features) 
first_layer_neurons = X_train.shape[1]

# Build Neural Network
model = Sequential()

# Layer 1 - recommended to use number of features
# model.add(Dense(78, activation='relu', input_shape=(78,)))
model.add(Dense(first_layer_neurons, activation='relu', input_shape=(first_layer_neurons,))) # relu = rectified linear unit y = max(0, x)
model.add(Dropout(0.2)) 
# % Neurons Randomly Dropped Each time = means 0.2*100=20 neurons will not have weights or biases updated. 
# Ex) Dropout = 1 drops 100% of neurons, if dropout = 0 then drops 0% of neurons

# Layer 2 - reduce half
model.add(Dense(39, activation='relu'))
model.add(Dropout(0.2)) 

# Layer 3 - reduce half
model.add(Dense(19, activation='relu'))
model.add(Dropout(0.2)) 

# Final Layer - Binary classification model so output must be binary: use sigmoid
model.add(Dense(units=1, activation='sigmoid')) # output is 0 or 1

# Binary classification model
model.compile(loss='binary_crossentropy', optimizer='adam')
# see the model: print the string summary of the network 
model.summary()

# another way to visualize the model
# from keras.utils.vis_utils import plot_model
from tensorflow.keras.utils import plot_model
plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True, expand_nested=True)

# this graphic requires the first layer to have input_shape: model.add(Dense(19, activation='relu', input_shape=(19,)))
from tensorflow.keras.callbacks import EarlyStopping
# min = stop when quantity monitored has stopped decreasing (minimize validation loss)
# max = stop when quantity monitored has stopped increasing (maximize accuracy)
# patience = how many epochs to wait before stopping despite hitting
early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)

# Now we fit the model to 600 epochs to demonstrate the overfitting
# Also included the validation data
# model.fit(x=X_train, y=y_train, epochs=600,validation_data=(X_test,y_test), callbacks=[early_stop])

# course used epochs=30
model.fit(x=X_train, y=y_train, epochs=50,batch_size=256,
          validation_data=(X_test,y_test),
          callbacks=[early_stop]
         )<jupyter_output>Train on 316175 samples, validate on 79044 samples
Epoch 1/50
316175/316175 [==============================] - 18s 56us/sample - loss: 0.2984 - val_loss: 0.2644
Epoch 2/50
316175/316175 [==============================] - 16s 50us/sample - loss: 0.2655 - val_loss: 0.2627
Epoch 3/50
316175/316175 [==============================] - 16s 51us/sample - loss: 0.2626 - val_loss: 0.2624
Epoch 4/50
316175/316175 [==============================] - 16s 51us/sample - loss: 0.2620 - val_loss: 0.2619
Epoch 5/50
316175/316175 [==============================] - 15s 49us/sample - loss: 0.2610 - val_loss: 0.2616
Epoch 6/50
316175/316175 [==============================] - 16s 52us/sample - loss: 0.2602 - val_loss: 0.2615
Epoch 7/50
316175/316175 [==============================] - 14s 43us/sample - loss: 0.2600 - val_loss: 0.2613
Epoch 8/50
316175/316175 [==============================] - 13s 41us/sample - loss: 0.2593 - val_loss: 0.2617
Epoch 9/50
316175/316175 [==============================] - 16s 51us/[...]<jupyter_text>EVALUATE MODEL PERFORMANCE AND SAVE/LOAD MODEL<jupyter_code># Evaluate Model Performance - Plot Validation Loss vs Training Loss
# look for flattening of errors - glad to see the early stop did its job
model_loss = pd.DataFrame(model.history.history)
model_loss.plot()
plt.title('Model Loss - Validation vs Training - with Early Stop')

# performance for classification problems
from sklearn.metrics import classification_report, confusion_matrix

# create predictions based on model 
# predictions = model.predict(X_test)
predictions = model.predict_classes(X_test)

# compare predictions to actuals
print(classification_report(y_test, predictions))
print(confusion_matrix(y_test,predictions))

# https://en.wikipedia.org/wiki/Precision_and_recall
# Precision = Accuracy of Positive Values = TP / (TP + FP)
# Recall = Accuracy overall = TP / all

# Accuracy is how much you got right
# Do not forget that most of the data was re-paid (imbalanced data set to begin with)
# Imbalanced Dataset assessment 
# Re Assess y_test or the true data
df['loan_repaid'].value_counts() # 317696 Repaid and 77523 not repaid
317696 / len(df)   # notice that 80% of my points were already repaid itself based on test data set, so 80% should be baseline

# hence 90% accuracy is not necessarily good because baseline is 80%. We need more context.
# so TRUE NOTIFICATION use F1 score of 0.6 as your metric to improve
# ways to improve:
    # 1) add layers
    # 2) add more neurons
    # 3) try other algorithms
    # 4) add more epochs
# use model to make a prediction on a single person

# select random existing person
import random
random.seed(101)
random_int = random.randint(0,len(df))
new_customer = df.drop('loan_repaid',axis=1).iloc[random_int]

# this is a pandas series
new_customer

# model requires it in a numpy array:
new_customer.values.reshape(1,78)

# recall, the model requires scaled input data
new_customer = scaler.transform(new_customer.values.reshape(1,78))
new_customer
# prediction using model # predictions = model.predict_classes(X_test)
predictions = model.predict_classes(new_customer)
predictions
# so did we get it right?
# pull the new_customer without the drop: new_customer = df.drop('loan_repaid',axis=1).iloc[random_int]
new_customer_orig = df.iloc[random_int]
new_customer_orig['loan_repaid']
# save the model down, after all this work
from tensorflow.keras.models import load_model
model.save('my_TF_lending_club_model1.h5')
# load the model
later_model = load_model('my_TF_lending_club_model1.h5')

# run predict - this is wrong:
later_model.predict(new_customer)
# run predict - the right way:
later_model.predict_classes(new_customer)<jupyter_output><empty_output><jupyter_text>TENSORBOARD<jupyter_code># Tensorboard
# https://www.tensorflow.org/tensorboard/get_started
from tensorflow.keras.callbacks import EarlyStopping,TensorBoard

# Unique log for each time you run the model
from datetime import datetime
timestamp = datetime.now().strftime("%Y-%m-%d--%H%M-%S")
timestamp
# double backslash? yes
pwd
# Log folder
# Windows: use "logs\\fit"
# MacOS/Linux: use "logs\fit"
import os
print('Current Directory: ', os.getcwd())
log_directory = 'logs\\fit'
log_directory = log_directory + '\\' + timestamp
log_directory
board = TensorBoard(log_dir=log_directory, # where stuff saved
                   histogram_freq=1,       # after freq=1, so each epoch, get weights, weights and histogram
                   write_graph=True,
                   write_images=True,
                   update_freq='epoch',
                   profile_batch=2,
                   embeddings_freq=1)



first_layer_neurons = X_train.shape[1]
# Build Neural Network
model2 = Sequential()
# Layer 1 - recommended to use number of features
# model.add(Dense(78, activation='relu', input_shape=(78,)))
model2.add(Dense(first_layer_neurons, activation='relu', input_shape=(first_layer_neurons,))) # relu = rectified linear unit y = max(0, x)
model2.add(Dropout(0.2)) 
# % Neurons Randomly Dropped Each time = means 0.2*100=20 neurons will not have weights or biases updated. 
# Ex) Dropout = 1 drops 100% of neurons, if dropout = 0 then drops 0% of neurons
# Layer 2 - reduce half
model2.add(Dense(39, activation='relu'))
model2.add(Dropout(0.2)) 
# Layer 3 - reduce half
model2.add(Dense(19, activation='relu'))
model2.add(Dropout(0.2)) 
# Final Layer - Binary classification model so output must be binary: use sigmoid
model2.add(Dense(units=1, activation='sigmoid')) # output is 0 or 1
# Binary classification model
model2.compile(loss='binary_crossentropy', optimizer='adam')

# add "board" to callbacks=[early_stop, board]

early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)

# course used epochs=30
model2.fit(x=X_train, y=y_train, epochs=50,batch_size=256,
          validation_data=(X_test,y_test),
          callbacks=[early_stop, board]
         )
# run tensorboard visualization
# open http://localhost:6006/

# Two important pieces: 1) where did you save the log files and 2) where are you located
print(log_directory)
# then run command line in anaconda prompt

# conda activate [env]
# cd to the folder
# then: "tensorboard --logdir logs\fit"
# C:\A\Projects\Python\Python - matplotlib_VC>         tensorboard --logdir\logs\fit\2020-04-27--1607-04

# now go back to http://localhost:6006/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12156362863819843, 'fraction_numerical': 0.037773528786913105, 'mean_word_length': 4.626583791537175, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '455174', 'license': 'MIT', 'n_tokens_mistral': 8111, 'n_tokens_neox': 7385, 'n_words': 2437, 'path': 'vp1_TensorFlow_Project_LendingClubData.ipynb', 'repo_name': 'arcticv/python', 'size': '23535'}