<jupyter_start><jupyter_text>Making a tree diagram<jupyter_code>fig = plt.figure(figsize=(10, 4))
ax = fig.add_axes([0, 0, 0.8, 1], frameon=False, xticks=[], yticks=[])
ax.set_title('Example Decision Tree: Covid severity', size=24)

def text(ax, x, y, t, size=20, **kwargs):
    ax.text(x, y, t,
            ha='center', va='center', size=size,
            bbox=dict(boxstyle='round', ec='k', fc='w'), **kwargs)

text(ax, 0.5, 0.9, "sugar level of the patient?", 20)
text(ax, 0.3, 0.6, "pre existing heart disease?", 18)
text(ax, 0.7, 0.6, "high cholestrol?", 18)
text(ax, 0.12, 0.3, "high LDL?", 14)
text(ax, 0.38, 0.3, "high HDL?", 14)
text(ax, 0.62, 0.3, "diabetes?", 14)
text(ax, 0.88, 0.3, "high blood pressure", 14)

text(ax, 0.4, 0.75, "> 160", 12, alpha=0.4)
text(ax, 0.6, 0.75, "< 160", 12, alpha=0.4)

text(ax, 0.21, 0.45, "yes", 12, alpha=0.4)
text(ax, 0.34, 0.45, "no", 12, alpha=0.4)

text(ax, 0.66, 0.45, "yes", 12, alpha=0.4)
text(ax, 0.79, 0.45, "no", 12, alpha=0.4)

ax.plot([0.3, 0.5, 0.7], [0.6, 0.9, 0.6], '-k')
ax.plot([0.12, 0.3, 0.38], [0.3, 0.6, 0.3], '-k')
ax.plot([0.62, 0.7, 0.88], [0.3, 0.6, 0.3], '-k')
ax.plot([0.0, 0.12, 0.20], [0.0, 0.3, 0.0], '--k')
ax.plot([0.28, 0.38, 0.48], [0.0, 0.3, 0.0], '--k')
ax.plot([0.52, 0.62, 0.72], [0.0, 0.3, 0.0], '--k')
ax.plot([0.8, 0.88, 1.0], [0.0, 0.3, 0.0], '--k')
ax.axis([0, 1, 0, 1])

# fig.savefig('figures/05.08-decision-tree.png')<jupyter_output><empty_output><jupyter_text>visualizing a clustered dataset<jupyter_code>import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs

X, y = make_blobs(n_samples=300, centers=4,
                  random_state=0, cluster_std=1.0)
plt.scatter(X[:, 0], X[:, 1], c=y, s=30, zorder=1, cmap='viridis');<jupyter_output><empty_output><jupyter_text>Lets run decision tree classifier on this data<jupyter_code>from sklearn.tree import DecisionTreeClassifier
model= DecisionTreeClassifier(max_depth=3)
model.fit(X, y)
import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from ipywidgets import interact
xlim = None
ylim = None
ax = plt.gca()
    
# Plot the training points
ax.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap='viridis',
            clim=(y.min(), y.max()), zorder=3)
ax.axis('tight')
ax.axis('off')
if xlim is None:
    xlim = ax.get_xlim()
if ylim is None:
    ylim = ax.get_ylim()

# fit the estimator
model.fit(X, y)
xx, yy = np.meshgrid(np.linspace(*xlim, num=200),
                      np.linspace(*ylim, num=200))
Z = model.predict(np.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
n_classes = len(np.unique(y))
Z = Z.reshape(xx.shape)
contours = ax.contourf(xx, yy, Z, alpha=0.3,
                        levels=np.arange(n_classes + 1) - 0.5,
                        cmap='viridis',
                        zorder=1)

ax.set(xlim=xlim, ylim=ylim)<jupyter_output><empty_output><jupyter_text>Putting things in helper functions<jupyter_code>def visualize_tree(estimator, X, y, boundaries=True,
                   xlim=None, ylim=None, ax=None):
    ax = ax or plt.gca()
    
    # Plot the training points
    ax.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap='viridis',
               clim=(y.min(), y.max()), zorder=3)
    ax.axis('tight')
    ax.axis('off')
    if xlim is None:
        xlim = ax.get_xlim()
    if ylim is None:
        ylim = ax.get_ylim()
    
    # fit the estimator
    estimator.fit(X, y)
    xx, yy = np.meshgrid(np.linspace(*xlim, num=200),
                         np.linspace(*ylim, num=200))
    Z = estimator.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    n_classes = len(np.unique(y))
    Z = Z.reshape(xx.shape)
    contours = ax.contourf(xx, yy, Z, alpha=0.3,
                           levels=np.arange(n_classes + 1) - 0.5,
                           cmap='viridis',
                           zorder=1)

    ax.set(xlim=xlim, ylim=ylim)
    
    # Plot the decision boundaries
    def plot_boundaries(i, xlim, ylim):
        if i >= 0:
            tree = estimator.tree_
        
            if tree.feature[i] == 0:
                ax.plot([tree.threshold[i], tree.threshold[i]], ylim, '-k', zorder=2)
                plot_boundaries(tree.children_left[i],
                                [xlim[0], tree.threshold[i]], ylim)
                plot_boundaries(tree.children_right[i],
                                [tree.threshold[i], xlim[1]], ylim)
        
            elif tree.feature[i] == 1:
                ax.plot(xlim, [tree.threshold[i], tree.threshold[i]], '-k', zorder=2)
                plot_boundaries(tree.children_left[i], xlim,
                                [ylim[0], tree.threshold[i]])
                plot_boundaries(tree.children_right[i], xlim,
                                [tree.threshold[i], ylim[1]])
            
    if boundaries:
        plot_boundaries(0, xlim, ylim)


def plot_tree_interactive(X, y):
    def interactive_tree(depth=5):
        clf = DecisionTreeClassifier(max_depth=depth, random_state=0)
        visualize_tree(clf, X, y)

    return interact(interactive_tree, depth=(1, 10))


def randomized_tree_interactive(X, y):
    N = int(0.75 * X.shape[0])
    
    xlim = (X[:, 0].min(), X[:, 0].max())
    ylim = (X[:, 1].min(), X[:, 1].max())
    
    def fit_randomized_tree(random_state=0):
        clf = DecisionTreeClassifier(max_depth=15)
        i = np.arange(len(y))
        rng = np.random.RandomState(random_state)
        rng.shuffle(i)
        visualize_tree(clf, X[i[:N]], y[i[:N]], boundaries=False,
                       xlim=xlim, ylim=ylim)
    
    interact(fit_randomized_tree, random_state=[0, 100]);
visualize_tree(model, X, y)<jupyter_output><empty_output><jupyter_text>Plotting for multiple depths<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_blobs

        
fig, ax = plt.subplots(1, 4, figsize=(16, 3))
fig.subplots_adjust(left=0.02)

X, y = make_blobs(n_samples=300, centers=4,
                  random_state=0, cluster_std=1.0)

for axi, depth in zip(ax, range(1, 5)):
    model = DecisionTreeClassifier(max_depth=depth)
    visualize_tree(model, X, y, ax=axi)
    axi.set_title('depth = {0}'.format(depth))

# fig.savefig('figures/05.08-decision-tree-levels.png')
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_blobs

        
fig, ax = plt.subplots(1, 4, figsize=(16, 3))
# fig.subplots_adjust(left=0.02, right=0.98)

X, y = make_blobs(n_samples=300, centers=4,
                  random_state=0, cluster_std=1.0)

for axi, depth in zip(ax, range(1, 5)):
    model = DecisionTreeClassifier(max_depth=depth)
    visualize_tree(model, X, y, ax=axi)
    axi.set_title('depth = {0}'.format(depth))

# fig.savefig('figures/05.08-decision-tree-levels.png')
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_blobs
X, y = make_blobs(n_samples=300, centers=4,
                  random_state=0, cluster_std=1.0)
        

fig, ax = plt.subplots(2, 3, figsize=(16, 10))
fig.subplots_adjust(left=0.02, right=0.98, wspace=0.1)

for axi, depth in zip(ax.ravel(), range(1, 7)):
    model = DecisionTreeClassifier(max_depth=depth)
    visualize_tree(model, X, y, ax=axi)
    axi.set_title('depth = {0}'.format(depth))<jupyter_output><empty_output><jupyter_text>Different boundaries for different data sample<jupyter_code>model = DecisionTreeClassifier(max_depth=10)

fig, ax = plt.subplots(1, 2, figsize=(16, 6))
fig.subplots_adjust(left=0.0625, right=0.95, wspace=0.1)
visualize_tree(model, X[::2], y[::2], boundaries=False, ax=ax[0])
visualize_tree(model, X[1::2], y[1::2], boundaries=False, ax=ax[1])<jupyter_output><empty_output><jupyter_text>Interactive visualization<jupyter_code>from ipywidgets import interact

@interact(depth=(1, 10))
def tree_output(depth=5):
    clf = DecisionTreeClassifier(max_depth=depth, random_state=0)
    visualize_tree(clf, X, y)<jupyter_output><empty_output><jupyter_text>Good Starting Models - Random Forest<jupyter_code>from ipywidgets import interact
from sklearn.ensemble import RandomForestClassifier
@interact(n_estimators=[1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
def visualize_bagging(n_estimators=1):
    ax = plt.gca()
    
    # Plot the training points
    ax.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap='rainbow',
               zorder=3)
    ax.axis('tight')
    ax.axis('off')
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    
    # fit the estimator
    model = RandomForestClassifier(n_estimators=n_estimators, random_state=0)
    model.fit(X, y)
    xx, yy = np.meshgrid(np.linspace(*xlim, num=200),
                         np.linspace(*ylim, num=200))
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)

    # Create a color plot with the results
    n_classes = len(np.unique(y))
    contours = ax.contourf(xx, yy, Z, alpha=0.3,
                           levels=np.arange(n_classes + 1) - 0.5,
                           cmap='rainbow',
                           zorder=1)

    ax.set(xlim=xlim, ylim=ylim)<jupyter_output><empty_output><jupyter_text>Model Selection<jupyter_code>! wget --header="Host: storage.googleapis.com" --header="User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" --header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" --header="Accept-Language: en-US,en;q=0.9" --header="Referer: https://www.kaggle.com/" "https://storage.googleapis.com/kagglesdsdata/competitions/4526/34463/train.json.zip?GoogleAccessId=<EMAIL>&Expires=1621999154&Signature=Hl760lyS2gZ%2B5HmLikeTvbGFaxfkjSKTz8QuOlWrHnwEW6sLO39kSGFm54FOwkXT6YoPSKxYY08aFekAF9sRnHdF7M%2F1qlikXl17eRr4CfTmugTIEHADQNApqX%2FbeBma6TjsCNWWwavDcmMF0M5GyDBLJMPt%2BI7oT%2BWuoBRoM0lX9TFf7adVY8aECGmHxenSLW0lP8kj4XB%2B6iaOWPu9DpuzhZg5hqzwlswmu2%2FtQTEMwh0HMXqCxH6SBpGIWo6BX7VKy8s%2BDD4am8tZ1fkwR9lXL82SmftAZQnUnfvdZ0l6lFqi3TuPslG%2FZqEkuGFnMoJZFg6x0ViMvm1RIBvckw%3D%3D&response-content-disposition=attachment%3B+filename%3Dtrain.json.zip" -c -O 'train.json.zip'
! unzip train.json.zip
import pandas as pd
df = pd.read_json('train.json')
df.info()
import pandas as pd
from sklearn.model_selection import train_test_split
X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=0.2)
y_train = pd.Series(y_train)
y_validation = pd.Series(y_validation)

((y_train.value_counts() / len(y_train))[:4]).T
((y_validation.value_counts() / len(y_validation))[:4]).T
from sklearn.model_selection import train_test_split
X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=0.2, stratify=y)<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning<jupyter_code>from sklearn import datasets
iris = datasets.load_iris()
import pandas as pd
df = pd.DataFrame(iris.data, columns=iris.feature_names)
df['target'] = iris.target
df['target'] = df['target'].apply(lambda x: iris.target_names[x])
df.head()
%%time
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
estimator = RandomForestClassifier()
params = { 
    'n_estimators': [5, 10, 20, 40, 100, 150],
    'max_features': ['auto', 'sqrt', 'log2'],
    'max_depth' : [4,5,6,7,8],
    'criterion' :['gini', 'entropy']
}
clf = GridSearchCV(estimator, params, cv= 5, return_train_score=False)
clf.fit(iris.data, iris.target)
import pandas as pd
gs_results = pd.DataFrame(clf.cv_results_)
gs_results
clf.best_estimator_
clf.best_params_
clf.best_score_
%time
from sklearn.model_selection import RandomizedSearchCV
rs = RandomizedSearchCV(estimator, params, 
                        cv=5, return_train_score=False, 
                        n_iter=10)
rs.fit(iris.data, iris.target)
pd.DataFrame(rs.cv_results_)
rs.best_score_
rs.best_params_
rs.best_estimator_<jupyter_output><empty_output><jupyter_text>Automating Model Selection<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
params = {
    'svm': {
        'model': svm.SVC(gamma='auto'),
        'params': {
            'C': [1, 10, 20],
            'kernel': ['rbf', 'linear']
        }
    },
    'random_forest': {
        'model': RandomForestClassifier(),
        'params': {
            'n_estimators': [1, 10, 20]
        }
    },
    'logistic_regression': {
        'model': LogisticRegression(solver='liblinear', multi_class='auto'),
        'params': {
            'C': [1, 5, 15]
        }
    }
}
all_scores = {'model_name': [], 'best_score': [], 'best_params': []}
for model_name, model_data in params.items():
  clf = GridSearchCV(model_data['model'], model_data['params'], cv=5, return_train_score=False)
  clf.fit(iris.data, iris.target)
  all_scores['model_name'].append(model_name)
  all_scores['best_score'].append(clf.best_score_)
  all_scores['best_params'].append(clf.best_params_)
import pandas as pd
pd.DataFrame(all_scores)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15676003947468306, 'fraction_numerical': 0.04645866545206103, 'mean_word_length': 3.593444909344491, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '559248', 'license': 'MIT', 'n_tokens_mistral': 5533, 'n_tokens_neox': 5121, 'n_words': 949, 'path': 'Chapter 05/Chapter_5.ipynb', 'repo_name': 'bpbpublications/Applied-Machine-Learning-Solutions-with-Python', 'size': '13212'}