<jupyter_start><jupyter_text>Ridge RegressionRidge regression (AKA Tychonoff regularization, AKA $L^2$ penalization) is a penalized version of linear regression.StatTools implements ridge regression by providing the `Ridge` estimator class in its `glm` module.<jupyter_code>import numpy as np
from sklearn import datasets
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="ticks", palette="muted", color_codes=True)

from stattools.glm import LinearRegression, Ridge
# Set NumPy random number generator seed for replicability
np.random.seed(100)<jupyter_output><empty_output><jupyter_text>Load the Diabetes Dataset<jupyter_code>data = datasets.load_diabetes()

# Shuffle data
n = len(data.target)
idx = np.random.permutation(n)
x = data.data[idx, :]
y = data.target[idx]

# 50/50 training/testing split
x_train = x[:(n // 2), :]
x_test = x[(n // 2):, :]
y_train = y[:(n // 2)]
y_test = y[(n // 2):]<jupyter_output><empty_output><jupyter_text>Fit a Classical Linear Regression Model<jupyter_code>%%time
model = LinearRegression().fit(x_train, y_train)<jupyter_output>CPU times: user 1.41 ms, sys: 1.04 ms, total: 2.45 ms
Wall time: 1.52 ms<jupyter_text>Report Mean Squared Error<jupyter_code>mse_train = model.mse(x_train, y_train)
mse_test = model.mse(x_test, y_test)
mse_full = model.mse(x, y)
print(f"Training MSE:   {mse_train:.3f}")
print(f"Testing MSE:    {mse_test:.3f}")
print(f"Full data MSE:  {mse_full:.3f}")<jupyter_output>Training MSE:   2500.425
Testing MSE:    3320.616
Full data MSE:  2910.521<jupyter_text>Fit a Ridge Regression Model<jupyter_code>%%time
model = Ridge(lam=0.1)
model.fit(x_train, y_train)<jupyter_output>CPU times: user 1.52 ms, sys: 1.2 ms, total: 2.72 ms
Wall time: 1.67 ms<jupyter_text>Report Mean Squared Error<jupyter_code>mse_train = model.mse(x_train, y_train)
mse_test = model.mse(x_test, y_test)
mse_full = model.mse(x, y)
print(f"Training MSE:   {mse_train:.3f}")
print(f"Testing MSE:    {mse_test:.3f}")
print(f"Full data MSE:  {mse_full:.3f}")<jupyter_output>Training MSE:   2500.525
Testing MSE:    3319.699
Full data MSE:  2910.112<jupyter_text>Regularization Path<jupyter_code>lam_min = np.exp(-4)
lam_max = np.exp(8)

lambdas = np.geomspace(start=lam_min, stop=lam_max, num=100)
path = np.empty((len(lambdas), len(data.feature_names)))
mses = np.empty(len(lambdas))
for i, lam in enumerate(lambdas):
    model_ = Ridge(lam=lam).fit(x_train, y_train)
    mses[i] = model_.mse(x_test, y_test)
    path[i, :] = model_._coef

plt.figure()
for j in range(len(data.feature_names)):
    plt.plot(-np.log(lambdas), path[:, j], ls="--")

plt.title("Regularization Path")
plt.xlabel("$-\log(\lambda)$")
plt.ylabel("Standardized Coefficient")
plt.xlim((-np.log(lam_max), -np.log(lam_min)))
plt.show()
plt.close()
plt.figure()
plt.plot(-np.log(lambdas), mses)
plt.title("MSE Along Regularization Path")
plt.xlabel("$-\log(\lambda)$")
plt.ylabel("Mean Squared Error")
plt.xlim((-np.log(lam_max), -np.log(lam_min)))
plt.show()
plt.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15359152598477324, 'fraction_numerical': 0.029791459781529295, 'mean_word_length': 5.9631336405529956, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '185420', 'license': 'MIT', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1216, 'n_words': 253, 'path': 'examples/Ridge Regression.ipynb', 'repo_name': 'artemmavrin/SLTools', 'size': '3021'}