<jupyter_start><jupyter_text>Initial Debugging NotebookTest out basic regression loss functions, link functions, and general BoostModel implementation using a scikit-learn LinearRegression model as the booster. Each run below should result in decreasing loss over model iterations.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler, scale
from sklearn.datasets import make_regression
from sklearn.base import clone as clone_model

from genestboost import BoostedModel
from genestboost.link_functions import *
from genestboost.loss_functions import *

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Create Fake Dataset for Regression<jupyter_code>X, y = make_regression(n_samples=10000,
                       n_features=30,
                       n_informative=30,
                       noise=1.0,
                       random_state=0)
X = scale(X)
y = scale(y)<jupyter_output><empty_output><jupyter_text>Test 1: Identity Link with Least Squares Loss<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=LeastSquaresLoss(),
                     model_callback=LinearRegression)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 2: Identity Link with Least Squares Loss and Newton Weights<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=LeastSquaresLoss(),
                     model_callback=LinearRegression,
                     weights="newton")
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 2: Identity Link with Absolute Loss<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=AbsoluteLoss(),
                     model_callback=LinearRegression,
                     alpha=0.1)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 3: Identity Link with Log Cosh Loss<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=LogCoshLoss(),
                     model_callback=LinearRegression,
                     alpha=0.1)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 4: Identity Link with Log Cosh Loss and Newton Weights<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=LogCoshLoss(),
                     model_callback=LinearRegression,
                     alpha=0.25,
                     weights="newton")
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 5: Identity Link with Quantile Loss - p = 0.5 (should approximately match AbsoluteLoss above)<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=QuantileLoss(p=0.5),
                     model_callback=LinearRegression,
                     alpha=0.1)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");<jupyter_output><empty_output><jupyter_text>Test 6: Identity Link with Quantile Loss - p = 0.95<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=QuantileLoss(p=0.95),
                     model_callback=LinearRegression,
                     alpha=0.25)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");
model.predict(X)[:10]   # should be greater than below predicted<jupyter_output><empty_output><jupyter_text>Test 7: Identity Link with Quantile Loss - p = 0.05<jupyter_code>model = BoostedModel(link=IdentityLink(),
                     loss=QuantileLoss(p=0.05),
                     model_callback=LinearRegression,
                     alpha=0.25)
model.fit(X, y, iterations=100)

# plot loss to make sure it is decreasing
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot(pd.DataFrame(model.get_loss_history()).iloc[:, 0])
ax.set_title("Mean Loss vs. Model Iteration")
ax.set_xlabel("Model Iteration")
ax.set_ylabel("Mean Loss");
model.predict(X)[:10]   # should be less than above predicted<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11547619047619048, 'fraction_numerical': 0.02295918367346939, 'mean_word_length': 3.9378673383711167, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '404955', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1868, 'n_words': 420, 'path': 'notebooks/debugging/sklr_regression_debugging_notebook.ipynb', 'repo_name': 'btcross26/forward_stagewise_regression', 'size': '5880'}