<jupyter_start><jupyter_text>Non-trained model<jupyter_code>y_pred = train_df['Credit_History']
display_result(label, y_pred)

y_test_pred = [1]*367 #test_df['Credit_History']
write_to_csv('baseline.csv', testing_data['Loan_ID'], y_test_pred, 'baseline')<jupyter_output>Classification Accuracy: 0.7703583061889251
Confusion matrix:
[[ 95  97]
 [ 44 378]]
baseline<jupyter_text>LR Model<jupyter_code>m1_clf = LogisticRegression(C=1)

m1_clf.fit(X_train, y_train)

y_pred = m1_clf.predict(X_train)
display_result(y_true_=y_train, y_pred_=y_pred)

y_pred = m1_clf.predict(X_test)
display_result(y_true_=y_test, y_pred_=y_pred)

# linear model is underfitting, since the dataset is not linearly separatable<jupyter_output>Classification Accuracy: 0.7735507246376812
Confusion matrix:
[[ 85  87]
 [ 38 342]]
Classification Accuracy: 0.7580645161290323
Confusion matrix:
[[ 7 13]
 [ 2 40]]<jupyter_text>Decision Tree<jupyter_code>dt_clf = DecisionTreeClassifier()

dt_clf.fit(X_train, y_train)

# Model is overfitting
y_train_pred = dt_clf.predict(X_train)
display_result(y_true_=y_train, y_pred_=y_train_pred)

y_test_pred = dt_clf.predict(X_test)
display_result(y_true_=y_test, y_pred_=y_test_pred)

#Tunning parameter to prevent overfitting
parameters = {'max_depth': np.arange(3,9,1),
              'min_samples_leaf':np.arange(3,30,2)}
gs_clf = GridSearchCV(dt_clf, parameters, n_jobs=-1, cv=5, refit=True)
gs_clf.fit(X_train, y_train)
gs_clf.grid_scores_

best_dt_clf = gs_clf.best_estimator_
print(best_dt_clf)
best_dt_pred = best_dt_clf.predict(X_test)
display_result(y_true_=y_test, y_pred_=best_dt_pred)

test_dt_pred = best_dt_clf.predict(test_df)
write_to_csv('dt_tuned.csv', testing_data['Loan_ID'], test_dt_pred, 'tuned dt model')
m2_clf = RandomForestClassifier(n_estimators=100)

m2_clf.fit(X_train, y_train)

print(m2_clf)

y_train_pred = m2_clf.predict(X_train)
display_result(y_true_=y_train, y_pred_=y_train_pred)

y_test_pred = m2_clf.predict(X_test)
display_result(y_true_=y_test, y_pred_=y_test_pred)<jupyter_output>RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
            max_depth=None, max_features='auto', max_leaf_nodes=None,
            min_samples_leaf=1, min_samples_split=2,
            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,
            oob_score=False, random_state=None, verbose=0,
            warm_start=False)
Classification Accuracy: 1.0
Confusion matrix:
[[172   0]
 [  0 380]]
Classification Accuracy: 0.6935483870967742
Confusion matrix:
[[ 7 13]
 [ 6 36]]<jupyter_text>Gradient Boosting Tree<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier  #GBM algorithm
from sklearn import cross_validation, metrics   #Additional scklearn functions
from sklearn.grid_search import GridSearchCV   #Perforing grid search

#Tunning parameter to prevent overfitting
min_samples_split = 30
min_samples_leaf = 15
max_depth = 5
# max_features = 'sqrt'
subsample = 0.6

gbt_clf = GradientBoostingClassifier(min_samples_split=min_samples_split, 
                                     min_samples_leaf=min_samples_leaf, 
                                     max_depth=max_depth, 
#                                      max_features=max_features, 
                                     learning_rate=0.1,
                                     subsample=subsample, random_state=10)

parameters={'n_estimators':np.arange(5,80,5)}
gbt_search_1 = GridSearchCV(estimator=gbt_clf, param_grid=parameters, scoring='accuracy',n_jobs=4,iid=False, cv=10)
gbt_search_1.fit(X_train, y_train)
gbt_search_1.grid_scores_
# gbt_search_1.best_estimator_
n_estimators = 20
learning_rate = 0.1
param_test2 = {'max_depth':np.arange(3,8,1), 
               'min_samples_split':np.arange(10,40,5)}
gbt_search_2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,
                                                                  n_estimators=20, 
                                                                  max_features='sqrt', 
                                                                  subsample=0.8, 
                                                                  random_state=10),
                           param_grid = param_test2,
                           scoring='accuracy',
                           n_jobs=4,iid=False, cv=10)
gbt_search_2.fit(X_train, y_train)
gbt_search_2.grid_scores_, gbt_search_2.best_params_, gbt_search_2.best_score_
max_depth = 5
min_samples_split= 30
param_test3 = {#'min_samples_split':np.arange(10,80,5), 
               'min_samples_leaf':np.arange(5,30,5)}
gbt_search_3 = GridSearchCV(GradientBoostingClassifier(learning_rate=0.1,
                                                        n_estimators=20, 
                                                        max_features='sqrt', 
                                                        subsample=0.8, 
                                                        random_state=10,
                                                        max_depth = 5,
                                                        min_samples_split= 30),
                             param_grid = param_test3, scoring='accuracy',n_jobs=4,iid=False, cv=10)
gbt_search_3.fit(X_train, y_train)
gbt_search_3.grid_scores_, gbt_search_3.best_params_, gbt_search_3.best_score_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09107876073161628, 'fraction_numerical': 0.0425531914893617, 'mean_word_length': 2.8223965763195435, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '127763', 'license': 'Apache-2.0', 'n_tokens_mistral': 1991, 'n_tokens_neox': 1827, 'n_words': 259, 'path': 'LoanPrediction/notebooks/Loan_Prediction_Explore.ipynb', 'repo_name': 'Zhenxingzhang/AnalyticsVidhya', 'size': '5358'}