<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load Data2&nbsp;&nbsp;Train Test Split3&nbsp;&nbsp;Model3.1&nbsp;&nbsp;OOB: Out of Bag3.2&nbsp;&nbsp;Results4&nbsp;&nbsp;Plotting the Tree5&nbsp;&nbsp;Variable Importance5.1&nbsp;&nbsp;Method 15.2&nbsp;&nbsp;Method 26&nbsp;&nbsp;Tree Depth7&nbsp;&nbsp;Cross Validation  Random Forest*"Random forest builds multiple decision trees and merges them together to get a more accurate and stable prediction."*Credit: - https://towardsdatascience.com/random-forest-in-python-24d0893d51c0- https://www.blopig.com/blog/2017/07/using-random-forests-in-python-with-scikit-learn/- https://chrisalbon.com/machine_learning/trees_and_forests/random_forest_classifier_example/- https://towardsdatascience.com/the-random-forest-algorithm-d457d499ffcd<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets

from sklearn.model_selection import cross_val_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import recall_score

# python version 
import sys
print('python vers:', sys.version[:31])<jupyter_output>python vers: 3.6.5 |Anaconda custom (64-bit)<jupyter_text>Load Data<jupyter_code># load iris
iris = datasets.load_iris()

# convert to pandas df 
df = pd.DataFrame(iris.data, columns=iris.feature_names)

df.head()
labels = pd.Categorical.from_codes(iris.target, iris.target_names)
labels[:10]
labels_onehot = pd.get_dummies(labels)
labels_onehot.head()<jupyter_output><empty_output><jupyter_text>Train Test Split<jupyter_code>from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(df[iris.feature_names], labels_onehot, test_size=0.5, stratify=iris.target, random_state=0)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=10, random_state=0)#oob_score=True
rf.fit(x_train, y_train)
from sklearn.metrics import accuracy_score
predicted = rf.predict(x_test)
accuracy = accuracy_score(y_test, predicted)
# print(f'Out-of-bag score estimate: {rf.oob_score_:.3}')
print(f'Mean accuracy score: {accuracy:.3}')<jupyter_output>Mean accuracy score: 0.933<jupyter_text>OOB: Out of Bag<jupyter_code>from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=10, random_state=0,oob_score=True)
rf.fit(x_train, y_train)
from sklearn.metrics import accuracy_score
predicted = rf.predict(x_test)
accuracy = accuracy_score(y_test, predicted)
print(f'Out-of-bag score estimate: {rf.oob_score_:.3}')
print(f'Mean accuracy score: {accuracy:.3}')<jupyter_output>Out-of-bag score estimate: 0.964
Mean accuracy score: 0.933<jupyter_text>Results<jupyter_code>rf.predict(x_test)[:5]
rf.predict_proba(x_test)[0][:5]
# from sklearn.metrics import confusion_matrix
# cm = pd.DataFrame(confusion_matrix(y_test, predicted), columns=iris.target_names, index=iris.target_names)
# sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>Plotting the Tree<jupyter_code># Import tools needed for visualization
from sklearn.tree import export_graphviz
import pydot
# Pull out one tree from the forest
tree = rf.estimators_[5]

feature_list = iris.feature_names

# Export the image to a dot file
export_graphviz(tree, out_file = 'tree.dot', feature_names = feature_list, rounded = True, precision = 1)
# Use dot file to create a graph
(graph, ) = pydot.graph_from_dot_file('tree.dot')
# Write graph to a png file
graph.write_png('tree.png')<jupyter_output><empty_output><jupyter_text>Variable Importance  Method 1<jupyter_code># Get numerical feature importances
importances = list(rf.feature_importances_)

# List of tuples with variable and importance
feature_importances = [(feature, round(importance, 2)) for feature, importance in zip(feature_list, importances)]

# Sort the feature importances by most important first
feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)
# Print out the feature and importances 
[print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importances];<jupyter_output>Variable: petal width (cm)     Importance: 0.54
Variable: petal length (cm)    Importance: 0.26
Variable: sepal length (cm)    Importance: 0.12
Variable: sepal width (cm)     Importance: 0.08<jupyter_text>Method 2<jupyter_code>feature_importances = pd.DataFrame(rf.feature_importances_,
                                   index = x_train.columns,
                                    columns=['importance']).sort_values('importance', ascending=False)

feature_importances<jupyter_output><empty_output><jupyter_text>Tree Depth<jupyter_code>def dectree_max_depth(tree):
    n_nodes = tree.node_count
    children_left = tree.children_left
    children_right = tree.children_right

    def walk(node_id):
        if (children_left[node_id] != children_right[node_id]):
            left_max = 1 + walk(children_left[node_id])
            right_max = 1 + walk(children_right[node_id])
            return max(left_max, right_max)
        else: # leaf
            return 1

    root_node_id = 0
    return walk(root_node_id)

[dectree_max_depth(t.tree_) for t in rf.estimators_]<jupyter_output><empty_output><jupyter_text>Cross Validation<jupyter_code>from sklearn.model_selection import cross_val_score
scores = cross_val_score(rf, x_train,y_train, cv=5)
scores
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
scores = cross_val_score(rf, x_train,y_train, cv=5 ,scoring = 'roc_auc')
scores<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10820764643984566, 'fraction_numerical': 0.018940722553490004, 'mean_word_length': 6.264968152866242, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '664138', 'license': 'MIT', 'n_tokens_mistral': 2047, 'n_tokens_neox': 1946, 'n_words': 406, 'path': '*Machine_Learning/Trees/Random_Forest.ipynb', 'repo_name': 'jasjung/Machine_Learnin', 'size': '5702'}