<jupyter_start><jupyter_text>Bias-variance trade-offMeasure bias and variance for a tiny subsample of a test set. We avg the error from each prediction and if test set is too big not seeing very high variance due to avg smoothing.<jupyter_code>import numpy as np
import pandas as pd

from sklearn.linear_model import LinearRegression, Ridge, Lasso, LogisticRegression
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.datasets import load_boston, load_iris, load_wine, load_digits, \
                             load_breast_cancer, load_diabetes
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, precision_score, recall_score

from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.metrics import mean_absolute_error

import matplotlib.pyplot as plt
%config InlineBackend.figure_format = 'retina'

from sklearn import tree
from dtreeviz.trees import *<jupyter_output><empty_output><jupyter_text>Regression<jupyter_code># can't use too many records as model.score() will be average of many values
# which reduces variance; we're trying to highlight not hide that variance
df_rent = pd.read_csv("data/rent-ideal.csv", nrows=4000)
X = df_rent[['bedrooms','bathrooms','latitude','longitude']]
y = df_rent['price']
X.head()
def rent():
    # can't use too many records as model.score() will be average of many values
    # which reduces variance; we're trying to highlight not hide that variance
    df_rent = pd.read_csv("data/rent-ideal.csv")
    X = df_rent[['bedrooms','bathrooms','latitude','longitude']].values
    y = df_rent['price'].values

    # shuffle
    idx = np.random.permutation(len(X))
    X = X[idx]
    y = y[idx]
    return X, y

def testobs(X, y, nchunks = 200, ntests=5):
    n = len(X)
    chunk_size = n // nchunks

    # grab say just 5 records; too many and average r2 over test set shows too little variance
    if chunk_size<ntests:
        raise "Oops"
    X_test, y_test = X[0:ntests], y[0:ntests]
    return X_test, y_test, chunk_size
nchunks = 50
X, y = rent()
X_test, y_test, chunk_size = testobs(X, y, nchunks=nchunks, ntests=10)
leaf_sizes = range(1,100)

all_test_scores = np.zeros(shape=(len(leaf_sizes), nchunks-1)) # first chunk is test set
for i,min_samples_leaf in enumerate(leaf_sizes):
#     print(f"min_samples_leaf={min_samples_leaf}: ", end='')
    test_scores = []
    for j in range(1,nchunks): # first chunk is test set; skip
#         print(j,' ', end='')
        X_train = X[j*chunk_size:j*chunk_size+chunk_size]
        y_train = y[j*chunk_size:j*chunk_size+chunk_size]
        t = DecisionTreeRegressor(min_samples_leaf=min_samples_leaf)
        t.fit(X_train, y_train)
        s = t.score(X_test, y_test)
        test_scores.append(s)

    all_test_scores[i] = test_scores
    avg = np.mean(test_scores)
    std = np.std(test_scores)
#     print(avg, std)
test_means = np.mean(all_test_scores, axis=1)
test_variances = np.std(all_test_scores, axis=1)
figsize = (4,2.8)
fig, ax = plt.subplots(1, 1, figsize=figsize)

a = ax.plot(leaf_sizes, 1-test_means, lw=.75, marker='.', markersize=3, label="test set $1-R^2$")

plt.fill_between(leaf_sizes, 
                 1-test_means - test_variances,
                 1-test_means + test_variances,
                 color='gray', alpha=0.2)

ax2 = ax.twinx()
b = ax2.plot(leaf_sizes, test_variances, lw=1, label="test set $\sigma$", c='orange')
ax2.yaxis.label.set_color('orange')
#ax2.set_ylabel("Test $\sigma$")
ax2.tick_params(axis='y', colors='orange')
ax2.set_ylim(-0.002,0.3)

ax.set_xlabel("min_samples_leaf")
ax.set_ylabel("1-$R^2$")
ax.set_ylim(-.05,1.0)
lines = a+b
labels = [l.get_label() for l in lines]
ax.legend(lines,labels,loc='upper center')

ax.text(5,-.04,f"ntrain={len(X_train)}, ntest={len(X_test)}, ntrials={nchunks}")

ax.set_title("NYC Rent decision tree bias-stddev vs leaf size", fontsize=10)<jupyter_output><empty_output><jupyter_text>Classification<jupyter_code>digits = load_digits()

df_digits = pd.DataFrame(data=digits.data)
df_digits['digit'] = digits.target
df_digits = df_digits.sample(n=len(df_digits))
X = df_digits.drop('digit', axis=1)
y = df_digits['digit']

nchunks = 10
X_test, y_test, chunk_size = testobs(X, y, nchunks=nchunks, ntests=100)
leaf_sizes = range(1,100+1)

all_test_scores = np.zeros(shape=(len(leaf_sizes), nchunks-1)) # first chunk is test set
for i,min_samples_leaf in enumerate(leaf_sizes):
#     print(f"min_samples_leaf={min_samples_leaf}: ", end='')
    test_scores = []
    for j in range(1,nchunks): # first chunk is test set; skip
#         print(j,' ', end='')
        X_train = X[j*chunk_size:j*chunk_size+chunk_size]
        y_train = y[j*chunk_size:j*chunk_size+chunk_size]
        t = DecisionTreeClassifier(min_samples_leaf=min_samples_leaf)
        t.fit(X_train, y_train)
        s = t.score(X_test, y_test)
        test_scores.append(s)

    all_test_scores[i] = test_scores
    avg = np.mean(test_scores)
    std = np.std(test_scores)
#     print(avg, std)
test_means = np.mean(all_test_scores, axis=1)
test_variances = np.std(all_test_scores, axis=1)
figsize = (4,2.8)
fig, ax = plt.subplots(1, 1, figsize=figsize)

a = ax.plot(leaf_sizes, 1-test_means, lw=.75, marker='.', markersize=3, label="test set $1-R^2$")

plt.fill_between(leaf_sizes, 
                 1-test_means - test_variances,
                 1-test_means + test_variances,
                 color='gray', alpha=0.2)

ax2 = ax.twinx()
b = ax2.plot(leaf_sizes, test_variances, lw=1, label="test set $\sigma$", c='orange')
ax2.yaxis.label.set_color('orange')
#ax2.set_ylabel("Test $\sigma$")
ax2.tick_params(axis='y', colors='orange')
ax2.set_ylim(-0.002,0.25)

ax.set_xlabel("min_samples_leaf")
ax.set_ylabel("1-$R^2$")
ax.set_ylim(-.05,1.0)
lines = a+b
labels = [l.get_label() for l in lines]
ax.legend(lines,labels,loc='center right')

ax.text(5,-0.04,f"ntrain={len(X_train)}, ntest={len(X_test)}, ntrials={nchunks}")

ax.set_title("MNIST decision tree bias-stddev vs leaf size", fontsize=10)<jupyter_output><empty_output><jupyter_text>Error sources diagrams<jupyter_code>X = np.linspace(0,4,30)
y = (X-2)**2
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10)
ax.set_title("Perfect $y = f(X) = (X-2)^2$ data")
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10)
for i in [3,9,13,21,24,28]:
    ax.scatter([X[i]],[y[i]+np.random.rand()],s=10,c='orange')
ax.set_title("Inconsistent $y = f(X) = (X-2)^2$ data")
X = np.linspace(0,4,30)
y = (X-2)**2 + np.random.random(size=len(X))*1.5
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10,c='#2679B2')
ax.set_title("Noisy $y = f(X) = (X-2)^2 + \epsilon$ data")
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from dtreeviz.trees import *

X = np.linspace(0,4,30).reshape(-1,1)
y = (X-2)**2

dt = DecisionTreeRegressor(max_depth=4)
dt.fit(X, y)

fig, ax = plt.subplots(1,1, figsize=(4,2))
rtreeviz_univar(dt, X, y, 'x', 'y', ax=ax)
ax.set_title("Underfit (shallow) decision tree")
X = np.linspace(0,4,30)
y = (X-2)**2 + np.random.random(size=X.shape[0])*1.5
X = X.reshape(-1,1)

dt = DecisionTreeRegressor(max_depth=4)
dt.fit(X, y)
fig, ax = plt.subplots(1,1, figsize=(4,2))
rtreeviz_univar(dt, X, y, 'x', 'y', ax=ax)
ax.set_title("Noisy data (shallow) decision tree")
dt = DecisionTreeRegressor()
dt.fit(X, y)

fig, ax = plt.subplots(1,1, figsize=(4,2))
rtreeviz_univar(dt, X, y, 'x', 'y', ax=ax)
ax.set_title("Noisy data (full) decision tree")
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,(X-2)**2,s=10,c='#2679B2')
ax.plot([0,4], [np.mean(y)]*2, c='red',lw=1)
ax.set_title("Underfit")
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10,c='#2679B2')
ax.plot([0,4], [np.mean(y)]*2, c='red',lw=1)
ax.set_title("Underfit")
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10,c='#2679B2')
ax.plot(X,(X-2)**2+.75, c='red',lw=1)
ax.set_title("Balanced model")
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10,c='#2679B2')
ax.plot(X,y,c='red',lw=1)
ax.set_title("Overfit model")
# do overfitting on noisy data
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline

X = np.linspace(0,4,30)
y = (X-2)**2 + np.random.random(size=len(X))

model = Pipeline([('poly', PolynomialFeatures(degree=27)),
                  ('linear', LinearRegression(fit_intercept=False))])
# fit to an order-3 polynomial data
#y = 3 - 2 * X + X ** 2 - X ** 3
model = model.fit(X[:, np.newaxis], y)
coef = model.named_steps['linear'].coef_
coef
fig, ax = plt.subplots(1,1, figsize=(4,2))
ax.scatter(X,y,s=10,c='#2679B2')
#ax.plot(X,coef[0]+coef[1]*X + coef[2]*X**2 + coef[3]*X**3)
ax.plot(X,model.predict(X.reshape(-1,1)), c='red', lw=1)
ax.set_title("Overfit degree 27 polynomial on noisy data", fontsize=10)
X = np.linspace(0,4,30)
ntrials = 35
fig, ax = plt.subplots(1,1, figsize=(4,2))
ys = np.zeros(shape=(ntrials,len(X)))
for i in range(ntrials):
    y = (X-2)**2 + np.random.random(size=len(X))*2.0
    ys[i] = y
#    ax.scatter(X,y,s=.25,c='#2679B2')
    ax.plot(X,y,c='#2679B2',lw=.25, alpha=.5)

y_avg = np.mean(ys,axis=0)
#ax.scatter(X,y,s=.5,c='#2679B2')
ax.plot(X,y_avg,c='red',lw=1.5)

#ax.set_title("Overfit model")
ax.set_title("Ensemble of overfit models")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17094472254766216, 'fraction_numerical': 0.036425604430716796, 'mean_word_length': 4.946801773274224, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '360665', 'license': 'MIT', 'n_tokens_mistral': 4044, 'n_tokens_neox': 3859, 'n_words': 744, 'path': 'notebooks/assessment/bias-variance.ipynb', 'repo_name': 'dashbrookhart/msds621', 'size': '9389'}