<jupyter_start><jupyter_text>Supervised Classification Learning Algorithms Decision Trees  Importing Necessary Libraries<jupyter_code>%matplotlib inline
import sklearn
import matplotlib
import numpy as np
import pandas as pd
from sklearn.base import clone
from sklearn import set_config
import matplotlib.pyplot as plt
from sklearn import preprocessing
from statistics import mean, stdev
from IPython.display import display
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score, StratifiedKFold, StratifiedShuffleSplit, validation_curve, learning_curve
from sklearn.metrics import roc_curve, precision_score, accuracy_score, average_precision_score, recall_score, f1_score, classification_report, confusion_matrix<jupyter_output><empty_output><jupyter_text>Setting Graph Defaults<jupyter_code># Specify printing and plot dimensions 
pd.options.display.max_columns = None
matplotlib.rc('figure', figsize=[10,5])

# Initializing Random Seed to make experiments reproducible
np.random.seed(1126)

# setting config to print all parameters
set_config(print_changed_only=False)<jupyter_output><empty_output><jupyter_text>Uploading and Pre-process Data<jupyter_code># Upload Adult Dataset
adultdata = pd.read_csv('adult.csv',names=['age', 'workclass', 'fnlwgt','education', 'education_num', 'marital_status', 'occupation', 'relationship', 'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country'], index_col=False )
# print('--------------------------------- adultdata ---------------------------------')
# print(adultdata.head())
# print()
# print('-------------- adultdata info -------------')
# adultdata.info()
# print()
# print(adultdata.head())

# Upload credit Dataset
creditdata = pd.read_csv('creditdata.csv')
creditdata = creditdata.iloc[: , 1:]
newheader = list(creditdata.loc[0])
creditdata = creditdata.drop(labels=0, axis=0)
creditdata.columns = newheader
creditdata.reset_index(inplace=True)
# print('--------------------------------- creditdata ---------------------------------')
# print(creditdata.head())
# print()
# print('-------------- creditdata info -------------')
# creditdata.info()
# print()<jupyter_output><empty_output><jupyter_text>Assigning Features and Target variables<jupyter_code>#split Adult dataset in features and target variable
adult_feature_cols = ['age', 'workclass', 'fnlwgt', 'marital_status', 'occupation', 'relationship', 'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country']
adultX = adultdata.reindex(columns =adult_feature_cols) # Features
objcols = list(adultX.columns[adultX.dtypes.eq('object')])
# encode categorical variables
le = preprocessing.LabelEncoder()
# use df.apply() to apply le.fit_transform to all string columns
adultX[objcols] = adultX[objcols].apply(le.fit_transform)
intcols = list(adultX.columns[adultX.dtypes.eq('int64')])
adultX[intcols] = adultX[intcols].astype(float)
# print('adultX head')
# print(adultX.head())
# print(adultX.info())
# print()
adultY = adultdata.education # Target variable
# print( 'AdultY Value Counts before replacement')
# print(adultY.value_counts())
# print()
# All the college graduates = 1, all the non-college graduates = -1
adultY = adultY.replace(['Bachelors', 'Masters', 'Doctorate'], 1,regex=True)
adultY = adultY.replace(['Some-college', '11th', 'HS-grad', 'Prof-school', 'Assoc-acdm', 'Assoc-voc', '9th', '7th-8th', '12th', '1st-4th', '10th', '5th-6th', 'Preschool'], -1,regex=True)
# print( 'AdultY Value Counts after replacement')
# print(adultY.value_counts())
# print()

#split Credit dataset in features and target variable
credit_feature_cols = ['LIMIT_BAL','SEX','EDUCATION','MARRIAGE','AGE','PAY_0','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6','BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6','PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']
creditX = creditdata.reindex(columns =credit_feature_cols) # Features
# print('creditX head')
# print(creditX.head())
# print(creditX.info())
# print()
creditY = creditdata.reindex(columns =['default payment next month']) # Target variable
# print( 'CreditY Value Counts before replacement')
# print(creditY.value_counts())
# print()
# All the credit non-defaulters = 1, all the credit defaulters= -1
creditY = creditY.replace(['0','1'], [1,-1],regex=True)
# print( 'CreditY Value Counts after replacement')
# print(creditY.value_counts())
# print()<jupyter_output><empty_output><jupyter_text>Splitting the Datasets<jupyter_code># Splitting data into Train and Test sets
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=10)

# Splitting Adult Dataset
sss.get_n_splits(adultX, adultY)
for train_index, test_index in sss.split(adultX, adultY):
    adultx_train, adultx_test = adultX.loc[train_index], adultX.loc[test_index]
    adulty_train, adulty_test = adultY.loc[train_index], adultY.loc[test_index]

# Splitting Credit Dataset
sss.get_n_splits(creditX, creditY)
for train_index, test_index in sss.split(creditX, creditY):
    creditx_train, creditx_test = creditX.loc[train_index], creditX.loc[test_index]
    credity_train, credity_test = creditY.loc[train_index], creditY.loc[test_index]<jupyter_output><empty_output><jupyter_text>Building a Simple Decision Tree & Making Predictions<jupyter_code># DT Classifier object
clf = DecisionTreeClassifier(ccp_alpha=0.0001, criterion='gini')

# Decision Tree for Adult Data Set
clf.fit(adultx_train, adulty_train)
adulty_predtrain = clf.predict(adultx_train)
adulty_predtest = clf.predict(adultx_test)

# Decision Tree for Credit Data Set
clf.fit(creditx_train, credity_train)
credity_predtrain = clf.predict(creditx_train)
credity_predtest = clf.predict(creditx_test)<jupyter_output><empty_output><jupyter_text>Decision Tree Predictions Accuracy Measurements<jupyter_code># DT Adult Data Set Predictions
print("Adult Train Accuracy:", accuracy_score(adulty_train, adulty_predtrain))
print("Adult Pred Train Mean:",adulty_predtrain.mean())
print("Adult Pred Train f1 Score:", f1_score(adulty_train, adulty_predtrain, average='macro'))
print()


# DT Credit Data Set Predictions
print("Credit Train Accuracy:", accuracy_score(credity_train, credity_predtrain))
print("Credit Pred Train Mean:",credity_predtrain.mean())
print("Credit Pred Train f1 Score:", f1_score(credity_train, credity_predtrain, average='macro'))
print()<jupyter_output>Adult Train Accuracy: 0.8466292997542998
Adult Pred Train Mean: -0.6177825552825553
Adult Pred Train f1 Score: 0.769357657200151

Credit Train Accuracy: 0.8875833333333333
Credit Pred Train Mean: 0.70175
Credit Pred Train f1 Score: 0.8137316288388912<jupyter_text>Decision Tree Cross-Validation Scores (Simple)<jupyter_code># Adult CV Scores
adulttrain_cv_scores = cross_val_score(clf, adultx_train, adulty_train, scoring="f1_macro", cv=10)
print("Adult Train CV scores:", adulttrain_cv_scores)
print("Adult Train CV scores average:", adulttrain_cv_scores.mean())
print()

# Credit CV Scores
credittrain_cv_scores = cross_val_score(clf, creditx_train, credity_train, scoring="f1_macro", cv=10)
print("Credit Train CV scores:", credittrain_cv_scores)
print("Credit Train CV scores average:", credittrain_cv_scores.mean())
print()<jupyter_output>Adult Train CV scores: [0.70630482 0.71344082 0.7310472  0.70349316 0.69359322 0.69933899
 0.72674018 0.73326082 0.73420435 0.70829196]
Adult Train CV scores average: 0.7149715523822572

Credit Train CV scores: [0.6316138  0.64490478 0.63648671 0.65363044 0.63451297 0.62537618
 0.65996651 0.66141072 0.63929866 0.65301864]
Credit Train CV scores average: 0.6440219405005368<jupyter_text>Decision Tree with Straified Kfold Cross Validation<jupyter_code># Create StratifiedKFold object.
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
lst_accu_stratified = []

# StratfiedKfold on Adult Training Dataset
for train_index, test_index in skf.split(adultx_train, adulty_train):
    x_train_fold, x_test_fold = adultx_train.iloc[train_index], adultx_train.iloc[test_index]
    y_train_fold, y_test_fold = adulty_train.iloc[train_index], adulty_train.iloc[test_index]
    clf.fit(x_train_fold, y_train_fold)
    lst_accu_stratified.append(clf.score(x_test_fold, y_test_fold))

print('List of possible accuracies on Adult Training Dataset :', lst_accu_stratified)
print('\nMaximum Accuracy That can be obtained from this model is:',
      max(lst_accu_stratified))
print('\nMinimum Accuracy:',
      min(lst_accu_stratified))
print('\nAverage Accuracy:',
      mean(lst_accu_stratified))
print('\nStandard Deviation is:', stdev(lst_accu_stratified))

# StratfiedKfold on Adult Training Dataset
for train_index, test_index in skf.split(creditx_train, credity_train):
    x_train_fold, x_test_fold = creditx_train.iloc[train_index], creditx_train.iloc[test_index]
    y_train_fold, y_test_fold = credity_train.iloc[train_index], credity_train.iloc[test_index]
    clf.fit(x_train_fold, y_train_fold)
    lst_accu_stratified.append(clf.score(x_test_fold, y_test_fold))

print('\nList of possible accuracy on Credit Training Dataset :', lst_accu_stratified)
print('\nMaximum Accuracy That can be obtained from this model is:',
      max(lst_accu_stratified))
print('\nMinimum Accuracy:',
      min(lst_accu_stratified))
print('\nAverage Accuracy:',
      mean(lst_accu_stratified))
print('\nStandard Deviation is:', stdev(lst_accu_stratified))<jupyter_output>List of possible accuracies on Adult Training Dataset : [0.8122840690978886, 0.8103646833013436, 0.8122840690978886, 0.8119001919385797, 0.8049904030710173, 0.8103646833013436, 0.8130518234165067, 0.8234165067178503, 0.8033794162826421, 0.8168202764976958]

Maximum Accuracy That can be obtained from this model is: 0.8234165067178503

Minimum Accuracy: 0.8033794162826421

Average Accuracy: 0.8118856122722756

Standard Deviation is: 0.005612644102860724

List of possible accuracy on Credit Training Dataset : [0.8122840690978886, 0.8103646833013436, 0.8122840690978886, 0.8119001919385797, 0.8049904030710173, 0.8103646833013436, 0.8130518234165067, 0.8234165067178503, 0.8033794162826421, 0.8168202764976958, 0.78, 0.7808333333333334, 0.7895833333333333, 0.7741666666666667, 0.7825, 0.785, 0.76625, 0.79125, 0.7720833333333333, 0.7733333333333333]

Maximum Accuracy That can be obtained from this model is: 0.8234165067178503

Minimum Accuracy: 0.76625

Average Accuracy: 0.7956928061361378

Stan[...]<jupyter_text>Using Grid Search to find best estimators with stratified Kfold<jupyter_code># Using Gridsearch on Adult Dataset
param_grid = {
    'criterion': ['gini', 'entropy'],
    'ccp_alpha': [0.00001, 0.0001, 0.00012, 0.00015, 0.00017]
}
search_f1 = GridSearchCV(DecisionTreeClassifier(), param_grid, cv=10, scoring='f1_macro', n_jobs=-1)
search_f1.fit(adultx_train, adulty_train)
print('\n Adult Dataset Best Estimator: ',search_f1.best_estimator_)
print('\n Adult Dataset Score: ',search_f1.best_score_)

# Using Gridsearch on Credit Dataset
param_grid = {
    'criterion': ['gini', 'entropy'],
    'ccp_alpha': [0.00001, 0.0001, 0.00012, 0.00015, 0.00017]
}
search_f1 = GridSearchCV(DecisionTreeClassifier(), param_grid, cv=10, scoring='f1_macro', n_jobs=-1)
search_f1.fit(creditx_train, credity_train)
print('\n credit Dataset Best Estimator: ',search_f1.best_estimator_)
print('\n credit Dataset Score: ',search_f1.best_score_)<jupyter_output>Adult Dataset Best Estimator:  DecisionTreeClassifier(ccp_alpha=0.00015, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')

 Adult Dataset Score:  0.7293412516514702

 credit Dataset Best Estimator:  DecisionTreeClassifier(ccp_alpha=0.00017, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')

 credit Dataset Score:  0.6756886210702538<jupyter_text>Create validation curve helper function<jupyter_code>def plot_val_curve(X, y, param_range,  param_name , estimator, title):
    train_scores, test_scores = validation_curve(
    estimator, X, y, param_name= param_name, param_range=param_range,
    scoring='f1_macro', n_jobs=-1,verbose=1, cv=10)

    train_scores_mean = np.mean(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    plt.title(title)
    plt.xlabel(param_name)
    plt.ylabel("Score")
    plt.ylim(0.0, 1.1)

    plt.plot(param_range, train_scores_mean , "r-+", linewidth=2, label="Training score")
    plt.plot(param_range, test_scores_mean, "b-+", linewidth=2, label="Cross-validation score")
    plt.title(title)
    plt.legend()
    plt.grid(color='gray', linestyle='-', linewidth=0.5, alpha=0.3)
#     plt.gca().invert_xaxis()
    
    return plt<jupyter_output><empty_output><jupyter_text>Plot Validation curves to identify best ccp_alpha parameters for pruning<jupyter_code># Plot ccp_alpha validation curve on adult dataset
aX, ay = adultx_train, adulty_train
param_range = np.linspace(0.0, 0.0005, 10)
param_name="ccp_alpha"
title = "Adult Dataset F1_macro score Complexity Curve on " + param_name
estimator =  DecisionTreeClassifier()
plot_val_curve(aX, ay, param_range,  param_name , estimator,title)
plt.show()

# Plot ccp_alpha validation curve on Credit dataset
cX, cy = creditx_train, credity_train
title = "Credit Dataset F1_macro score Complexity Curve on " + param_name
plot_val_curve(cX, cy, param_range,  param_name , estimator,title)

plt.show()<jupyter_output>[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    1.9s finished<jupyter_text>Creating Learning Curve Plot Helper Function<jupyter_code>def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None,
                        n_jobs=None, train_sizes=np.linspace(.1, 1.0, 5)):
    
    if axes is None:
        _, axes = plt.subplots(1, 3, figsize=(20, 5))

    axes[0].set_title(title)
    if ylim is not None:
        axes[0].set_ylim(*ylim)
    axes[0].set_xlabel("Training instances")
    axes[0].set_ylabel("Score")
    
    train_sizes, train_scores, test_scores, fit_times, _ = \
    learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs,
                   train_sizes=train_sizes,
                   return_times=True)
        
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    fit_times_mean = np.mean(fit_times, axis=1)
    fit_times_std = np.std(fit_times, axis=1)

    # Plot learning curve
    axes[0].grid()
    axes[0].fill_between(train_sizes, train_scores_mean - train_scores_std,
                         train_scores_mean + train_scores_std, alpha=0.1,
                         color="r")
    axes[0].fill_between(train_sizes, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1,
                         color="g")
    axes[0].plot(train_sizes, train_scores_mean, 'o-', color="r",
                 label="Training score")
    axes[0].plot(train_sizes, test_scores_mean, 'o-', color="g",
                 label="Cross-validation score")
    axes[0].legend(loc="best")

    # Plot n_samples vs fit_times
    axes[1].grid()
    axes[1].plot(train_sizes, fit_times_mean, 'o-')
    axes[1].fill_between(train_sizes, fit_times_mean - fit_times_std,
                         fit_times_mean + fit_times_std, alpha=0.1)
    axes[1].set_xlabel("Training instances")
    axes[1].set_ylabel("fit_times")
    axes[1].set_title("Scalability of the model")

    # Plot fit_time vs score
    axes[2].grid()
    axes[2].plot(fit_times_mean, test_scores_mean, 'o-')
    axes[2].fill_between(fit_times_mean, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1)
    axes[2].set_xlabel("fit_times")
    axes[2].set_ylabel("Score")
    axes[2].set_title("Performance of the model")

    return plt<jupyter_output><empty_output><jupyter_text>Plot Learning curves with simple DT classifier<jupyter_code>fig, axes = plt.subplots(3, 2, figsize=(10, 15))

X, y = adultx_train, adulty_train

title = "Adult Dataset Learning Curves"
# Cross validation with 100 iterations to get smoother mean test and train
# score curves, each time with 20% data randomly selected as a validation set.
# cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
estimator = DecisionTreeClassifier()
plot_learning_curve(estimator, title, X, y, axes=axes[:, 0], ylim=(0.7, 1.01),
                    cv=cv , n_jobs=4, )

X, y = creditx_train, credity_train

title = "Credit Dataset Learning Curves"
# SVC is more expensive so we do a lower number of CV iterations:
# cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
estimator = DecisionTreeClassifier()
plot_learning_curve(estimator, title, X, y, axes=axes[:, 1], ylim=(0.7, 1.01),
                    cv=cv, n_jobs=4)

plt.show()<jupyter_output><empty_output><jupyter_text>Plot Learning curves with Gridsearch best estimator and StratifiedKfold<jupyter_code>fig, axes = plt.subplots(3, 2, figsize=(10, 15))

X, y = adultx_train, adulty_train

title = "Adult Dataset Learning Curves"
# Cross validation with 100 iterations to get smoother mean test and train
# score curves, each time with 20% data randomly selected as a validation set.
# cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
estimator = DecisionTreeClassifier(ccp_alpha=0.00015, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')
plot_learning_curve(estimator, title, X, y, axes=axes[:, 0], ylim=(0.7, 1.01),
                    cv=cv , n_jobs=4)

X, y = creditx_train, credity_train

title = "Credit Dataset Learning Curves"
# SVC is more expensive so we do a lower number of CV iterations:
# cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
estimator = DecisionTreeClassifier(ccp_alpha=0.00017, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')
plot_learning_curve(estimator, title, X, y, axes=axes[:, 1], ylim=(0.7, 1.01),
                    cv=cv, n_jobs=4)

plt.show()<jupyter_output><empty_output><jupyter_text>Apply Best Model on test data<jupyter_code># Make Predictions on Adult Dataset
print("\n Adult Dataset Predictions Results")
best_adult_model = DecisionTreeClassifier(ccp_alpha=0.00015, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')
best_adult_model.fit(adultx_train, adulty_train)
adulty_predictions = best_adult_model.predict(adultx_test)
print(classification_report(adulty_test, adulty_predictions,digits=5))
print(confusion_matrix(adulty_test, adulty_predictions))

# Make Predictions on Credit Dataset
print("\n Credit Dataset Predictions Results")
best_credit_model = DecisionTreeClassifier(ccp_alpha=0.00017, class_weight=None, criterion='gini',
                       max_depth=None, max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, random_state=None,
                       splitter='best')
best_credit_model.fit(creditx_train, credity_train)
credity_predictions = best_credit_model.predict(creditx_test)
print(classification_report(credity_test, credity_predictions,digits=5))
print(confusion_matrix(credity_test, credity_predictions))<jupyter_output>Adult Dataset Predictions Results
              precision    recall  f1-score   support

          -1    0.86254   0.91964   0.89018      5015
           1    0.65437   0.50935   0.57282      1498

    accuracy                        0.82527      6513
   macro avg    0.75846   0.71449   0.73150      6513
weighted avg    0.81466   0.82527   0.81718      6513

[[4612  403]
 [ 735  763]]

 Credit Dataset Predictions Results
              precision    recall  f1-score   support

          -1    0.61955   0.35343   0.45010      1327
           1    0.83635   0.93837   0.88443      4673

    accuracy                        0.80900      6000
   macro avg    0.72795   0.64590   0.66726      6000
weighted avg    0.78840   0.80900   0.78837      6000

[[ 469  858]
 [ 288 4385]]
{'fraction_non_alphanumeric': 0.11712328767123288, 'fraction_numerical': 0.06940639269406393, 'mean_word_length': 4.494480682388359, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '413419', 'license': 'Apache-2.0', 'n_tokens_mistral': 8664, 'n_tokens_neox': 7494, 'n_words': 1548, 'path': 'Supervised Learning/DecisionTree.ipynb', 'repo_name': 'cindynyoumsigit/MachineLearning', 'size': '21900'}