<jupyter_start><jupyter_text>SMOTE Analysis<jupyter_code>smt = SMOTE()
x_train, y_train = smt.fit_sample(X_train, Y_train)
np.bincount(Y_train)
np.bincount(y_train)
dataset.action_taken.value_counts().plot(kind='bar', title='Count');
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
X_test = sc.transform(X_test)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>rand_class_smote = RandomForestClassifier(n_estimators = 8,random_state=2)
rand_class_smote.fit(x_train, y_train)
y_pred_smote = rand_class_smote.predict(X_test)
y_pred_train_smote = rand_class_smote.predict(x_train)
print("Test")
print('Accuracy:',accuracy_score(Y_test,y_pred_smote))
print('\n')
print('Precision of 0:',precision_score(Y_test,y_pred_smote,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_smote,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_smote,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_smote,pos_label=1))

print("------------------------")
print("Train")
print('Accuracy:',accuracy_score(y_train,y_pred_train_smote))

conf_matrix = confusion_matrix(Y_test, y_pred_smote)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
print(confusion_matrix(Y_test,y_pred_log,labels=[0,1]))
# save the model to disk
#joblib_file_random_forest = "rand_class_smote.pkl"
#joblib.dump(rand_class_smote, joblib_file_random_forest)
print(classification_report(Y_test,y_pred_smote))<jupyter_output>precision    recall  f1-score   support

           0       0.34      0.30      0.32      2808
           1       1.00      1.00      1.00    419568

    accuracy                           0.99    422376
   macro avg       0.67      0.65      0.66    422376
weighted avg       0.99      0.99      0.99    422376<jupyter_text>Logistic Regression<jupyter_code>log_reg = LogisticRegression()
log_reg.fit(x_train, y_train)
y_pred_log = log_reg.predict(X_test)
y_pred_train_log = log_reg.predict(x_train)
print('Test')
print('Accuracy',accuracy_score(y_pred_log,Y_test))

print('Precision of 0:',precision_score(Y_test,y_pred_log,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_log,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_log,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_log,pos_label=1))

print('Recall',recall_score(y_pred_log,Y_test))
print('--------------------------')
print('Train')
print('Accuracy',accuracy_score(y_pred_train_log,y_train))
conf_matrix = confusion_matrix(Y_test, y_pred_log)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
# save the model to disk
#joblib_file_logistic = "log_reg.pkl"
#joblib.dump(log_reg, joblib_file_logistic)
print(classification_report(Y_test,y_pred_log))<jupyter_output>precision    recall  f1-score   support

           0       0.17      1.00      0.30      2808
           1       1.00      0.97      0.98    419568

    accuracy                           0.97    422376
   macro avg       0.59      0.98      0.64    422376
weighted avg       0.99      0.97      0.98    422376<jupyter_text>Naive Bayes<jupyter_code>naive_bayes = GaussianNB()
naive_bayes.fit(x_train,y_train)
y_pred_naive = naive_bayes.predict(X_test)
y_pred_train_naive = naive_bayes.predict(x_train)
print('Test')
print('Accuracy',accuracy_score(y_pred_naive,Y_test))
print('Precision of 0:',precision_score(Y_test,y_pred_naive,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_naive,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_naive,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_naive,pos_label=1))

print('--------------------------')
print('Train')
print('Accuracy',accuracy_score(y_pred_train_naive,y_train))
conf_matrix = confusion_matrix(Y_test, y_pred_naive)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
# save the model to disk
#joblib_file_naive_bayes = "naive_bayes.pkl"
#joblib.dump(naive_bayes, joblib_file_naive_bayes)
print(classification_report(Y_test,y_pred_naive))<jupyter_output>precision    recall  f1-score   support

           0       0.18      0.97      0.30      2808
           1       1.00      0.97      0.98    419568

    accuracy                           0.97    422376
   macro avg       0.59      0.97      0.64    422376
weighted avg       0.99      0.97      0.98    422376<jupyter_text>LDA<jupyter_code>lda = LinearDiscriminantAnalysis()
lda.fit(x_train,y_train)
y_pred_lda = lda.predict(X_test)
y_pred_train_lda = lda.predict(x_train)
print('Test')
print('Accuracy',accuracy_score(y_pred_lda,Y_test))
print('Precision of 0:',precision_score(Y_test,y_pred_lda,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_lda,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_lda,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_lda,pos_label=1))

print('--------------------------')
print('Train')
print('Accuracy',accuracy_score(y_pred_train_lda,y_train))
conf_matrix = confusion_matrix(Y_test, y_pred_lda)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
print(classification_report(Y_test,y_pred_lda))
# save the model to disk
#joblib_file_lda = "lda.pkl"
#joblib.dump(lda, joblib_file_lda)<jupyter_output><empty_output><jupyter_text>XG Boost<jupyter_code>from xgboost import XGBClassifier

xg_boost = XGBClassifier()
xg_boost.fit(x_train,y_train)
y_pred_xg = xg_boost.predict(X_test)
y_pred_xg_train = xg_boost.predict(x_train)
print('Accuracy test',accuracy_score(y_pred_xg,Y_test))
print('Precision of 0:',precision_score(Y_test,y_pred_xg,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_xg,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_xg,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_xg,pos_label=1))

print('Recall',recall_score(y_pred_xg,Y_test))
print('Accuracy Train',accuracy_score(y_pred_xg_train,y_train))
conf_matrix = confusion_matrix(Y_test, y_pred_xg)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
# save the model to disk
#joblib_file_xgboost = "xgboost.pkl"
#joblib.dump(xg_boost, joblib_file_xgboost)
print(classification_report(Y_test,y_pred_xg))<jupyter_output>precision    recall  f1-score   support

           0       0.22      0.85      0.35      2808
           1       1.00      0.98      0.99    419568

    accuracy                           0.98    422376
   macro avg       0.61      0.92      0.67    422376
weighted avg       0.99      0.98      0.98    422376<jupyter_text>Adaboost<jupyter_code>ada_boost = AdaBoostClassifier(DecisionTreeClassifier(max_depth=10),n_estimators=12)
ada_boost.fit(x_train, y_train)
y_pred_ada = ada_boost.predict(X_test)
y_pred_ada_train = ada_boost.predict(x_train)
print('Test')
print('Accuracy test',accuracy_score(y_pred_ada,Y_test))
print('Precision of 0:',precision_score(Y_test,y_pred_ada,pos_label=0))
print('Precision of 1:',precision_score(Y_test,y_pred_ada,pos_label=1))
print('\n')
print('Recall of 0:',recall_score(Y_test,y_pred_ada,pos_label=0))
print('Recall of 1:',recall_score(Y_test,y_pred_ada,pos_label=1))

print('--------------------------')
print('Train')
print('Accuracy',accuracy_score(y_pred_ada_train,y_train))
conf_matrix = confusion_matrix(Y_test, y_pred_ada)
plot_confusion_matrix(conf_matrix,cmap='tab10',colorbar=True)
print(conf_matrix)
#joblib_file_adaboost = "adaboost.pkl"
#joblib.dump(ada_boost, joblib_file_adaboost)
print(classification_report(Y_test,y_pred_ada))
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt

classifier_predictions = [y_pred_smote,y_pred_log,y_pred_naive,y_pred_lda,y_pred_xg,y_pred_ada]
classifier_names = ["Random Forest","Logistic Regression",'Naive Bayes','LDA','XGBoost','AdaBoost']
for i in range(len(classifier_predictions)):
    fpr,tpr,thresholds = roc_curve(Y_test,classifier_predictions[i])
    plt.plot(fpr,tpr,label= classifier_names[i])
    plt.xlabel("False Positive Rate(FPR)")
    plt.ylabel("True Positive Rate(TPR)")
    plt.legend()
    print("AUC for ",classifier_names[i]," = ",auc(fpr,tpr))

from sklearn.metrics import precision_recall_curve

classifier_predictions = [y_pred_smote,y_pred_log,y_pred_naive,y_pred_lda,y_pred_xg,y_pred_ada]
classifier_names = ["Random Forest","Logistic Regression",'Naive Bayes','LDA','XGBoost','AdaBoost']

for i in range(len(classifier_predictions)):
    precision, recall, thresholds = precision_recall_curve(Y_test,classifier_predictions[i])
    plt.plot(recall,precision,label= classifier_names[i])
    plt.xlabel("Recall")
    plt.ylabel("Precision")
    plt.legend()
    print("Precision Recall score for ",classifier_names[i]," = ",(average_precision_score(Y_test,classifier_predictions[i])))<jupyter_output>Precision Recall score for  Random Forest  =  0.9953053169980692
Precision Recall score for  Logistic Regression  =  0.999778965826201
Precision Recall score for  Naive Bayes  =  0.9996221361838308
Precision Recall score for  LDA  =  0.9997113230129927
Precision Recall score for  XGBoost  =  0.9988736029621913
Precision Recall score for  AdaBoost  =  0.9952760842851813<jupyter_text>Selected top features that we got from Lasso and Random Forest and run our model<jupyter_code>features = ['aus_1','applicant_credit_score_type','loan_term','purchaser_type','property_value','loan_purpose','income','applicant_age','loan_amount','manufactured_home_land_property_interest']
X_new = df[features]
Y_new = df['action_taken']
X_train_new, X_test_new, Y_train_new, Y_test_new = train_test_split(X_new, Y_new, test_size=0.25) 
smt = SMOTE()
x_train_new, y_train_new = smt.fit_sample(X_train_new, Y_train_new)
np.bincount(Y_train_new)
np.bincount(y_train_new)

sc = StandardScaler()
x_train_new = sc.fit_transform(x_train_new)
X_test_new = sc.transform(X_test_new)
log_reg_top = LogisticRegression()
log_reg_top.fit(x_train_new, y_train_new)
y_pred_log_top = log_reg_top.predict(X_test_new)
y_pred_train_log_top = log_reg_top.predict(x_train_new)
print('Accuracy Test',accuracy_score(y_pred_log_top,Y_test_new))
print('Accuracy Train',accuracy_score(y_pred_train_log_top,y_train_new))
print(classification_report(y_pred_log_top,Y_test_new))
rand_class_top = RandomForestClassifier(n_estimators = 8,random_state=2)
rand_class_top.fit(x_train_new, y_train_new)
y_pred_rand_top = rand_class_top.predict(X_test_new)
y_pred_rand_train_top = rand_class_top.predict(x_train_new)
print('Accuracy Test',accuracy_score(y_pred_rand_top,Y_test_new))
print('Accuracy Train',accuracy_score(y_pred_rand_train_top,y_train_new))
print(classification_report(y_pred_rand_top,Y_test_new))
ada_boost_top = AdaBoostClassifier(DecisionTreeClassifier(max_depth=10),n_estimators=12)
ada_boost_top.fit(x_train_new, y_train_new)
y_pred_ada_top = ada_boost_top.predict(X_test_new)
y_pred_ada_train_top = ada_boost_top.predict(x_train_new)
print('Accuracy Test',accuracy_score(y_pred_ada_top,Y_test_new))
print('Accuracy Train',accuracy_score(y_pred_ada_train_top,y_train_new))
print(classification_report(y_pred_ada_top,Y_test_new))
from sklearn.metrics import precision_recall_curve

classifier_predictions = [y_pred_rand_top,y_pred_log_top,y_pred_ada_top]
classifier_names = ["Random Forest","Logistic Regression",'AdaBoost']

for i in range(len(classifier_predictions)):
    precision, recall, thresholds = precision_recall_curve(Y_test_new,classifier_predictions[i])
    plt.plot(recall,precision,label= classifier_names[i])
    plt.xlabel("Recall")
    plt.ylabel("Precision")
    plt.legend()
    print("Precision Recall score for ",classifier_names[i]," = ",(average_precision_score(Y_test_new,classifier_predictions[i])))<jupyter_output>Precision Recall score for  Random Forest  =  0.9956264833943154
Precision Recall score for  Logistic Regression  =  0.9997575425387186
Precision Recall score for  AdaBoost  =  0.998075044641125
{'fraction_non_alphanumeric': 0.12430939226519337, 'fraction_numerical': 0.04260840448685752, 'mean_word_length': 6.253794778384942, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '150144', 'license': 'MIT', 'n_tokens_mistral': 5025, 'n_tokens_neox': 4622, 'n_words': 605, 'path': 'SourceCode/Modeling/Loan_Prediction_Oversampling.ipynb', 'repo_name': 'varun-jagadeesh/Loan-Prediction-Analysis', 'size': '11946'}