<jupyter_start><jupyter_text>Tree-based models - Exercises<jupyter_code>import warnings
warnings.filterwarnings("ignore")

import hashlib

import pandas as pd
import numpy as np

from utils.utils import *<jupyter_output><empty_output><jupyter_text>Exercise 1 - Decision Trees 1.1 Gini impurityUsed by the CART algorithm for classification, Gini impurity is an alternative to entropy.As entropy, it is a way to measure node homogeneity. Hence, it can be used to identify promissing splits.Take $p$ as the probability of the positive class, i.e., the proportion of positive cases in the set:$$I_G(p)= 1 - p^2 - (1-p)^2$$It measures how often a randomly chosen element from the set would be incorrectly labeled.<jupyter_code>def gini(p):
    # YOUR CODE HERE
    raise NotImplementedError()
    
    
data = make_data()
expected_hash_1 = '5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9'
assert hashlib.sha256(str(gini(p=0)).encode('utf-8')).hexdigest() == expected_hash_1

expected_hash_2 = '0061765517977143a7ed83b494b172f02af75de1c3cdde6f969effdef053ee8c'
assert hashlib.sha256(str(gini(p=1/6)).encode('utf-8')).hexdigest() == expected_hash_2

expected_hash_3 = '683487f42b8f523b6df156fbc42818868de8d2b70ad48f801df4df36d07b40f2'
assert hashlib.sha256(str(gini(p=1/3)).encode('utf-8')).hexdigest() == expected_hash_3

expected_hash_4 = 'd2cbad71ff333de67d07ec676e352ab7f38248eb69c942950157220607c55e84'
assert hashlib.sha256(str(gini(p=1/2)).encode('utf-8')).hexdigest() == expected_hash_4<jupyter_output><empty_output><jupyter_text>1.2 Applying the Gini  1.2.1 Single nodeCompute the impurity of a node that includes all instances where $x_i^{Windy}$ is `'true'`.(Note that `'true'` is a string.)<jupyter_code># single_node_gini = ...
def compute_probability(node):
    # YOUR CODE HERE
    raise NotImplementedError()

# single_node_gini = ...
# YOUR CODE HERE
raise NotImplementedError()
expected_hash_5 = 'd2cbad71ff333de67d07ec676e352ab7f38248eb69c942950157220607c55e84'
assert hashlib.sha256(str(single_node_gini).encode('utf-8')).hexdigest() == expected_hash_5<jupyter_output><empty_output><jupyter_text>1.2.2 Single featureWrite a function to compute the mean impurity of branching on a given feature.(Hint: compute the impurity of each resulting node and average the results.)<jupyter_code>from statistics import mean


def mean_impurity(data, feature_name):
    # YOUR CODE HERE
    raise NotImplementedError()
assert mean_impurity(data, 'Temperature') == 0.4398148148148148

expected_hash_6 = '60bb8f6c52a2aca85b1a3ae08e71f97dd3fa8603b55abefb09ebb0edfe5294d1'
assert hashlib.sha256(str(mean_impurity(data, 'Outlook')).encode('utf-8')).hexdigest() == expected_hash_6

expected_hash_7 = '1bd499c4b5bbae82a4f4a4efe3950b803e7dc64764c56115810e696db1b28afe'
assert hashlib.sha256(str(mean_impurity(data, 'Humidity')).encode('utf-8')).hexdigest() == expected_hash_7

expected_hash_8 = 'ccb9ffbc97edf0207e9b20dd9331f576984b45f178aa756421dd16508dd984fc'
assert hashlib.sha256(str(mean_impurity(data, 'Windy')).encode('utf-8')).hexdigest() == expected_hash_8<jupyter_output><empty_output><jupyter_text>1.3 Worst splitUse the `mean_impurity()` function above to identify the **least promising** feature.(Uncomment the **correct option**.)<jupyter_code># worst_split = 'Temperature'
# worst_split = 'Outlook'
# worst_split = 'Humidity'
# worst_split = 'Windy'

# YOUR CODE HERE
raise NotImplementedError()
expected_hash_9 = 'cdcaccab2e98e50c7fd550e5be166e6c260312708903794361d1815222ed0975'
assert hashlib.sha256(str(worst_split).encode('utf-8')).hexdigest() == expected_hash_9<jupyter_output><empty_output><jupyter_text>Exercise 2 - Random Forests  2.1 BaggingUncomment the **incorrect statement**.<jupyter_code># incorrect_statement_1 = 'Bagging helps to deal with overfitting, which is a big risk when using Decision Trees.'
# incorrect_statement_1 = 'Bagging involves averaging the predictions of multiple independent models.'
# incorrect_statement_1 = 'Bagging involves creating multiple data sets by sampling observations.'
# incorrect_statement_1 = 'Bagging involves creating multiple data sets by sampling columns.'

# YOUR CODE HERE
raise NotImplementedError()
expected_hash_10 = '1fb9060eefe647e998b5868f95d4930673fa3645ed0120176f6867f6c43ecf71'
assert hashlib.sha256(str(incorrect_statement_1).encode('utf-8')).hexdigest() == expected_hash_10<jupyter_output><empty_output><jupyter_text>2.2 Random forestsUncomment the **incorrect statement**.<jupyter_code># incorrect_statement_2 = 'Random forests rely on bootstrapping to build several training sets.'
# incorrect_statement_2 = 'Random forests rely on random feature selection before each split.'
# incorrect_statement_2 = 'Random forests aggregates the predictions of multiple models running in parallel.'
# incorrect_statement_2 = 'Random forests aggregates the predictions of multiple models running sequentially.'

# YOUR CODE HERE
raise NotImplementedError()
expected_hash_11 = '3a36af9faaf10138801ffef535b24bfe0a34217160a886dacdb3bd68b7777067'
assert hashlib.sha256(str(incorrect_statement_2).encode('utf-8')).hexdigest() == expected_hash_11<jupyter_output><empty_output><jupyter_text>3 Gradient boostingUncomment the **incorrect statement**.<jupyter_code># incorrect_statement_3 = 'Gradient boosting fits individual trees sequentially.'
# incorrect_statement_3 = 'Gradient boosting fits individual trees on the predictions from the previous tree.'
# incorrect_statement_3 = 'Gradient boosting fits individual trees on the pseudo-residuals of the previous tree.'
# incorrect_statement_3 = 'Gradient boosting can be used to optimize a differentiable cost function.'

# YOUR CODE HERE
raise NotImplementedError()
expected_hash_12 = '4000dc46debb11c42c4d44c82eb50120463cad49ae182b5130c9e276104570bd'
assert hashlib.sha256(str(incorrect_statement_3).encode('utf-8')).hexdigest() == expected_hash_12<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09446419502285425, 'fraction_numerical': 0.10512950736414424, 'mean_word_length': 7.88421052631579, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '430634', 'license': 'MIT', 'n_tokens_mistral': 2343, 'n_tokens_neox': 2001, 'n_words': 437, 'path': 'stats-279/SLU13 - Tree-Based Models/Exercise notebook.ipynb', 'repo_name': 'hershaw/stats-279', 'size': '5907'}