<jupyter_start><jupyter_text>Implementing binary decision trees<jupyter_code>import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Load the lending club dataset<jupyter_code>loans = pd.read_csv('lending-club-data.csv')
loans['safe_loans'] = loans['bad_loans'].apply(lambda x: +1 if x==0 else -1)
loans.drop('bad_loans', axis=1, inplace=True)
features = ['grade',              # grade of the loan
            'term',               # the term of the loan
            'home_ownership',     # home_ownership status: own, mortgage or rent
            'emp_length',         # number of years of employment
           ]
target = 'safe_loans'
loans = loans[features + [target]]
loans.head()
safe_loans_raw = loans[loans[target] == 1]
risky_loans_raw = loans[loans[target] == -1]

# Since there are less risky loans than safe loans, find the ratio of the sizes
# and use that percentage to undersample the safe loans.
percentage = len(risky_loans_raw)/float(len(safe_loans_raw))
safe_loans = safe_loans_raw.sample(frac = percentage, random_state = 1)
risky_loans = risky_loans_raw
loans_data = risky_loans.append(safe_loans)

print("Percentage of safe loans                 :", len(safe_loans) / float(len(loans_data)))
print("Percentage of risky loans                :", len(risky_loans) / float(len(loans_data)))
print("Total number of loans in our new dataset :", len(loans_data))
loans_data = risky_loans.append(safe_loans)
def onehot_transform(X, names=None,prefix_sep='.'):
    dummies_X = pd.get_dummies(X,prefix_sep=prefix_sep)
    if names is None:
        return dummies_X, dummies_X.columns.values
    else:
        return pd.DataFrame(dummies_X, columns=names).fillna(0)
loans_data,names = onehot_transform(loans_data,)
loans_data.head()
features = loans_data.columns.tolist()
features.remove('safe_loans')  # Remove the response variable
features
print("Number of features (after binarizing categorical variables) = %s" % len(features))
print("Total number of grade.A loans : %s" % loans_data['grade.A'].sum())
print("Expexted answer               : 6422")
train_idx = pd.read_json('module-5-assignment-2-train-idx.json',typ='series').values
test_idx = pd.read_json('module-5-assignment-2-test-idx.json',typ='series').values
train_data, test_data = loans.iloc[train_idx],loans.iloc[test_idx]
train_data = onehot_transform(train_data,names,)
test_data = onehot_transform(test_data,names,)<jupyter_output><empty_output><jupyter_text>Decision tree implementation<jupyter_code>def intermediate_node_num_mistakes(labels_in_node):
    # Corner case: If labels_in_node is empty, return 0
    if len(labels_in_node) == 0:
        return 0
    safe_loans = sum(labels_in_node==1)
    risky_loans = sum(labels_in_node==-1)
    return min(safe_loans, risky_loans)
# Test case 1
example_labels = np.array([-1, -1, 1, 1, 1])
if intermediate_node_num_mistakes(example_labels) == 2:
    print('Test passed!')
else:
    print('Test 1 failed... try again!')

# Test case 2
example_labels = np.array([-1, -1, 1, 1, 1, 1, 1])
if intermediate_node_num_mistakes(example_labels) == 2:
    print('Test passed!')
else:
    print('Test 2 failed... try again!')
    
# Test case 3
example_labels = np.array([-1, -1, -1, -1, -1, 1, 1])
if intermediate_node_num_mistakes(example_labels) == 2:
    print('Test passed!')
else:
    print('Test 3 failed... try again!')<jupyter_output>Test passed!
Test passed!
Test passed!<jupyter_text>Function to pick best feature to split on<jupyter_code>def best_splitting_feature(data, features, target):
    best_feature = None
    best_error =10
    num_data_points = float(len(data))
    for feature in features:
        left_split = data[data[feature]==0]
        right_split = data[data[feature]==1]
        left_mistakes = intermediate_node_num_mistakes(left_split[target])
        right_mistakes = intermediate_node_num_mistakes(right_split[target])
        error = (left_mistakes+right_mistakes)/num_data_points
        if error < best_error:
            best_error = error
            best_feature = feature
    return best_feature
if best_splitting_feature(train_data, features, 'safe_loans') == 'term. 36 months':
    print('Test passed!')
else:
    print('Test failed... try again!')<jupyter_output>Test passed!<jupyter_text>Building the tree<jupyter_code>def create_leaf(target_values):
    leaf = {
        'spliting_feature': None,
        'left': None,
        'right': None,
        'is_leaf': True
    }
    num_ones = len(target_values[target_values == +1])
    num_minus_ones = len(target_values[target_values == -1])
    
    # For the leaf node, set the prediction to be the majority class.
    # Store the predicted class (1 or -1) in leaf['prediction']
    if num_ones > num_minus_ones:
        leaf['prediction'] = 1         ## YOUR CODE HERE
    else:
        leaf['prediction'] = -1        ## YOUR CODE HERE
    return leaf
def decision_tree_create(data, features, target, current_depth = 0, max_depth = 10):
    remaining_features = features[:] # Make a copy of the features.
    
    target_values = data[target]
    print("--------------------------------------------------------------------")
    print("Subtree, depth = %s (%s data points)." % (current_depth, len(target_values)))
    

    # Stopping condition 1
    # (Check if there are mistakes at current node.
    # Recall you wrote a function intermediate_node_num_mistakes to compute this.)
    if intermediate_node_num_mistakes(target_values) == 0:  ## YOUR CODE HERE
        print("Stopping condition 1 reached.")     
        # If not mistakes at current node, make current node a leaf node
        return create_leaf(target_values)
    
    # Stopping condition 2 (check if there are remaining features to consider splitting on)
    if remaining_features == None:   ## YOUR CODE HERE
        print("Stopping condition 2 reached.")    
        # If there are no remaining features to consider, make current node a leaf node
        return create_leaf(target_values)    
    
    # Additional stopping condition (limit tree depth)
    if current_depth >= max_depth:  ## YOUR CODE HERE
        print("Reached maximum depth. Stopping for now.")
        # If the max tree depth has been reached, make current node a leaf node
        return create_leaf(target_values)

    # Find the best splitting feature (recall the function best_splitting_feature implemented above)
    ## YOUR CODE HERE
    splitting_feature = best_splitting_feature(data,remaining_features,target)
    
    # Split on the best feature that we found. 
    left_split = data[data[splitting_feature] == 0]
    right_split = data[data[splitting_feature] == 1]      ## YOUR CODE HERE
    remaining_features.remove(splitting_feature)
    print("Split on feature %s. (%s, %s)" % (\
                      splitting_feature, len(left_split), len(right_split)))
    
    # Create a leaf node if the split is "perfect"
    if len(left_split) == len(data):
        print("Creating leaf node.")
        return create_leaf(left_split[target])
    if len(right_split) == len(data):
        print("Creating leaf node.")
        ## YOUR CODE HERE
        return create_leaf(right_split[target])
        
    # Repeat (recurse) on left and right subtrees
    left_tree = decision_tree_create(left_split, remaining_features, target, current_depth + 1, max_depth)        
    ## YOUR CODE HERE
    right_tree = decision_tree_create(right_split, remaining_features, target, current_depth + 1, max_depth)

    return {'is_leaf'          : False, 
            'prediction'       : None,
            'splitting_feature': splitting_feature,
            'left'             : left_tree, 
            'right'            : right_tree}
def count_nodes(tree):
    if tree['is_leaf']:
        return 1
    return 1 + count_nodes(tree['left']) + count_nodes(tree['right'])
small_data_decision_tree = decision_tree_create(train_data, features, 'safe_loans', max_depth = 3)
if count_nodes(small_data_decision_tree) == 13:
    print('Test passed!')
else:
    print('Test failed... try again!')
    print('Number of nodes found                :', count_nodes(small_data_decision_tree))
    print('Number of nodes that should be there : 13')<jupyter_output>--------------------------------------------------------------------
Subtree, depth = 0 (37224 data points).
Split on feature term. 36 months. (9223, 28001)
--------------------------------------------------------------------
Subtree, depth = 1 (9223 data points).
Split on feature grade.A. (9122, 101)
--------------------------------------------------------------------
Subtree, depth = 2 (9122 data points).
Split on feature grade.B. (8074, 1048)
--------------------------------------------------------------------
Subtree, depth = 3 (8074 data points).
Reached maximum depth. Stopping for now.
--------------------------------------------------------------------
Subtree, depth = 3 (1048 data points).
Reached maximum depth. Stopping for now.
--------------------------------------------------------------------
Subtree, depth = 2 (101 data points).
Split on feature emp_length.< 1 year. (90, 11)
--------------------------------------------------------------------
Subtree, depth = 3 (90 data p[...]<jupyter_text>Build the  tree<jupyter_code># Make sure to cap the depth at 6 by using max_depth = 6
my_decision_tree = decision_tree_create(train_data, features, 'safe_loans', max_depth = 6)<jupyter_output>--------------------------------------------------------------------
Subtree, depth = 0 (37224 data points).
Split on feature term. 36 months. (9223, 28001)
--------------------------------------------------------------------
Subtree, depth = 1 (9223 data points).
Split on feature grade.A. (9122, 101)
--------------------------------------------------------------------
Subtree, depth = 2 (9122 data points).
Split on feature grade.B. (8074, 1048)
--------------------------------------------------------------------
Subtree, depth = 3 (8074 data points).
Split on feature grade.C. (5884, 2190)
--------------------------------------------------------------------
Subtree, depth = 4 (5884 data points).
Split on feature grade.D. (3826, 2058)
--------------------------------------------------------------------
Subtree, depth = 5 (3826 data points).
Split on feature grade.E. (1693, 2133)
--------------------------------------------------------------------
Subtree, depth = 6 (1693 data points).
R[...]<jupyter_text>Making predictions with a decision tree<jupyter_code>def classify(tree, x, annotate = False):   
    # if the node is a leaf node.
    if tree['is_leaf']:
        if annotate: 
            print("At leaf, predicting %s" % tree['prediction'])
        return tree['prediction'] 
    else:
        # split on feature.
        split_feature_value = x[tree['splitting_feature']]
        if annotate: 
            print("Split on %s = %s" % (tree['splitting_feature'], split_feature_value))
        if split_feature_value == 0:
            return classify(tree['left'], x, annotate)
        else:
               ### YOUR CODE HERE
            return classify(tree['right'], x, annotate)
test_data.iloc[0]
print('Predicted class: %s ' % classify(my_decision_tree, test_data.iloc[0]))
classify(my_decision_tree, test_data.iloc[0], annotate=True)
def evaluate_classification_error(tree, data):
    # Apply the classify(tree, x) to each row in your data
    prediction = [classify(tree,data.iloc[i]) for i in range(0,data.shape[0])]
    
    # Once you've made the predictions, calculate the classification error and return it
    ## YOUR CODE HERE
    return float(sum(prediction!=data['safe_loans']))/len(data)
evaluate_classification_error(my_decision_tree, test_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18729239417426113, 'fraction_numerical': 0.020356017375010648, 'mean_word_length': 2.5155688622754493, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '508051', 'license': 'MIT', 'n_tokens_mistral': 3625, 'n_tokens_neox': 3377, 'n_words': 955, 'path': 'Machine_Learning_WashingTon/Classification/Week3 Decision Tress/Implementing binary decision trees.ipynb', 'repo_name': 'PerpetualSmile/-Coursera', 'size': '11741'}