<jupyter_start><jupyter_text>How to use bias and variance to identify overfitting and underfitting?  Import library<jupyter_code>import random
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.tree import DecisionTreeRegressor

from mlxtend.evaluate import bias_variance_decomp
from sklearn.pipeline import make_pipeline<jupyter_output><empty_output><jupyter_text>Step1. generate fack data  Circle Parametric form[Reference](https://en.wikipedia.org/wiki/Circle) ${\displaystyle x=a+r\,\cos t}$${\displaystyle y=b+r\,\sin t}$<jupyter_code>rad = 0.4  # radius
thk = 0.2  # thickness
n_data_points = 30
#  create semi circle fack data by random parameter
r = np.random.rand(n_data_points)*thk+rad
a = np.random.rand(n_data_points)*np.pi

# Circle Parametric form
x = rad + thk + r*np.cos(a)
y = rad + thk + r*np.sin(a)

plt.scatter(x, y, marker='.', linewidths=0.1, s=150)

plt.title('Generate semi circle fack data', fontsize=12)
# plt.savefig("semi_circle_fack_data.png", bbox_inches='tight')
plt.show()
data = pd.DataFrame([x, y]).T
data.columns=['x', 'y']
data.head()<jupyter_output><empty_output><jupyter_text>Step2. split data<jupyter_code>x = np.array(data['x'])
y = np.array(data['y'])
# split the data
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.33, random_state=20210410)
X_train, y_train
X_test, y_test
plt.scatter(X_train, y_train, marker='.', linewidths=0.1, s=150)
plt.scatter(X_test, y_test, marker='.', linewidths=0.1, s=150)
plt.title('Blue is training data. Orange is test data.', fontsize=12)
plt.show()
# define the model
lr_model = LinearRegression()
X_train = X_train.reshape(-1, 1)
X_test = X_test.reshape(-1, 1)<jupyter_output><empty_output><jupyter_text>Step3. training model  Linear Regression<jupyter_code>lr_model.fit(X_train, y_train)
x_sample = np.linspace(min(data['x']), max(data['y']), 25)
y_lr_predict = lr_model.predict(x_sample[:, np.newaxis])<jupyter_output><empty_output><jupyter_text>Decision Tree Regressor<jupyter_code>tree_model = DecisionTreeRegressor(random_state=20210410)
tree_model.fit(X_train, y_train)
y_tree_predict = tree_model.predict(x_sample[:, np.newaxis])<jupyter_output><empty_output><jupyter_text>Polynomial Basis Model  degree=2<jupyter_code>poly_model_degree2 = make_pipeline(PolynomialFeatures(degree=2), LinearRegression())
poly_model_degree2.fit(X_train, y_train)
y_poly_degree2_predict = poly_model_degree2.predict(x_sample[:, np.newaxis])<jupyter_output><empty_output><jupyter_text>degree=8<jupyter_code>poly_model_degree27 = make_pipeline(PolynomialFeatures(degree=8), LinearRegression())
poly_model_degree27.fit(X_train, y_train)
y_poly_degree27_predict = poly_model_degree27.predict(x_sample[:, np.newaxis])
fig, axs = plt.subplots(2, 2, figsize=(18,10))
fig.suptitle('Model training status', fontsize=30)

for row, col, title in zip([0,0,1,1], [0,1,0,1], ['Linear Regression: Underfitting',
                                                  'Decision Tree Regressor: Overfitting',
                                                  'Polynomial Basis Model(degree=2): The best model',
                                                  'Polynomial Basis Model(degree=8): Bad model']):
    
    axs[row, col].scatter(X_train, y_train, marker='.', linewidths=0.1, s=150)
    axs[row, col].scatter(X_test, y_test, marker='.', linewidths=0.1, s=150)
    axs[row, col].set_title(title, fontsize=20)

    
axs[0, 0].plot(x_sample, y_lr_predict, c='black')
axs[0, 1].plot(x_sample, y_tree_predict, c='red')
axs[1, 0].plot(x_sample, y_poly_degree2_predict, c='green')
axs[1, 1].plot(x_sample, y_poly_degree27_predict, c='brown')


for ax in axs.flat:
    ax.set(xlabel='X', ylabel='Y')
    
# # Hide x labels and tick labels for top plots and y ticks for right plots.
# for ax in axs.flat:
#     ax.label_outer()

plt.tight_layout()
# plt.show()<jupyter_output><empty_output><jupyter_text>Step4. find out bias and variance [Reference](http://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/)  Underfitting: low variance and high bias<jupyter_code># estimate bias and variance
avg_expected_loss, avg_bias, avg_var = bias_variance_decomp(
        lr_model, X_train, y_train, X_test, y_test,
        loss='mse', 
        random_seed=20210410)

# summarize results
print('Average expected loss(MSE): %.3f' % avg_expected_loss)
print('Average bias: %.3f' % avg_bias)
print('Average variance: %.3f' % avg_var)<jupyter_output>Average expected loss(MSE): 0.070
Average bias: 0.068
Average variance: 0.002<jupyter_text>Overfitting: low bias and heigh varience<jupyter_code>avg_expected_loss, avg_bias, avg_var = bias_variance_decomp(
        tree_model, X_train, y_train, X_test, y_test, 
        loss='mse',
        random_seed=20210410)

# summarize results
print('Average expected loss(MSE): %.3f' % avg_expected_loss)
print('Average bias: %.3f' % avg_bias)
print('Average variance: %.3f' % avg_var)<jupyter_output>Average expected loss(MSE): 0.015
Average bias: 0.007
Average variance: 0.009<jupyter_text>The best model<jupyter_code># estimate bias and variance
avg_expected_loss, avg_bias, avg_var = bias_variance_decomp(
        poly_model_degree2, X_train, y_train, X_test, y_test, 
        loss='mse',
        random_seed=20210410)

# summarize results
print('Average expected loss(MSE): %.3f' % avg_expected_loss)
print('Average bias: %.3f' % avg_bias)
print('Average variance: %.3f' % avg_var)<jupyter_output>Average expected loss(MSE): 0.006
Average bias: 0.004
Average variance: 0.002<jupyter_text>Bad training model<jupyter_code># estimate bias and variance
avg_expected_loss, avg_bias, avg_var = bias_variance_decomp(
        poly_model_degree27, X_train, y_train, X_test, y_test, 
        loss='mse',
        random_seed=20210410)

# summarize results
print('Average expected loss(MSE): %.3f' % avg_expected_loss)
print('Average bias: %.3f' % avg_bias)
print('Average variance: %.3f' % avg_var)<jupyter_output>Average expected loss(MSE): 140.643
Average bias: 1.412
Average variance: 139.231
{'fraction_non_alphanumeric': 0.12768708582511717, 'fraction_numerical': 0.03394213673832229, 'mean_word_length': 5.472803347280335, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '382574', 'license': 'MIT', 'n_tokens_mistral': 2425, 'n_tokens_neox': 2256, 'n_words': 464, 'path': 'model_performance/ML_bias_and_varience.ipynb', 'repo_name': 'EricTsai83/machine_learning', 'size': '6187'}