<jupyter_start><jupyter_text>Linear models<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn as sk
from sklearn.linear_model import LinearRegression
from string import ascii_lowercase as letters<jupyter_output><empty_output><jupyter_text>Make some data<jupyter_code>n = 1000
p = 10
X = np.random.standard_normal((n,p))
X.shape
A = np.random.random((p,1))
A
y = X @ A
y.shape<jupyter_output><empty_output><jupyter_text>Too easy<jupyter_code>model = LinearRegression().fit(X, y)
model
X_test = np.random.standard_normal((n,p))
y_test = X_test @ A
from sklearn.metrics import r2_score
y_pred = model.predict(X_test)
r2_score(y_test, y_pred)
plt.scatter(y_test, y_pred, color='#3033ff30')
plt.show()<jupyter_output><empty_output><jupyter_text>Adding Noise<jupyter_code>noise = 1/2
X_train = X + np.random.normal(loc=0, scale=noise, size=(n,p))
y_train = y + np.random.normal(loc=0, scale=noise, size=(n,1))
model = LinearRegression().fit(X_train, y_train)
X_test_noisy = X_test + np.random.normal(loc=0, scale=noise, size=(n,p))
y_test_noisy = y_test + np.random.normal(loc=0, scale=noise, size=(n,1))
y_pred = model.predict(X_test_noisy)
r2_score(y_test, y_pred)
plt.scatter(y_test_noisy, y_pred, color='#3033ff30')
plt.show()
import itertools
def learning_curve(A, noise=1/3):
    p = A.shape[0]
    results = []

    n_train_seq = itertools.chain.from_iterable(itertools.repeat(x, 10) for x in range(20, 500, 20))
    
    for n in n_train_seq:
        X = np.random.standard_normal((n,p))
        y = X @ A

        X_train = X + np.random.normal(loc=0, scale=noise, size=(n,p))
        y_train = y + np.random.normal(loc=0, scale=noise, size=(n,1))

        model = LinearRegression().fit(X_train, y_train)

        n_test = 1000
        X_test = np.random.standard_normal((n_test,p))
        y_test = X_test @ A
        
        X_test_noisy = X_test + np.random.normal(loc=0, scale=noise, size=(n_test,p))
        y_test_noisy = y_test + np.random.normal(loc=0, scale=noise, size=(n_test,1))

        y_pred = model.predict(X_test_noisy)

        results.append((n, r2_score(y_test_noisy, y_pred)))
    return np.array(results)
lc = learning_curve(A, noise=1/3)
from sklearn.linear_model import Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
X_lc = lc[:,0:1]
y_lc = lc[:,1]
degree = 3
lc_model = make_pipeline(PolynomialFeatures(degree), Ridge())
lc_model.fit(X_lc, y=y_lc)
lc_y_plot = lc_model.predict(X_lc)
plt.scatter(lc[:,0], lc[:,1], color='#3033ff30')
plt.plot(lc[:,0], lc_y_plot, color='teal', linewidth=2,
         label="degree %d" % degree)
plt.title('r-squared as a function of n')
plt.show()<jupyter_output><empty_output><jupyter_text>Transformed featuresLet's make the problem harder. Let's say there are 10 true features that are linearly related with our target variable. We don't necessarily get to observe those, but we can measure 10 other features. These might be combinations of the original features with more or less noise added. Some variable are totally hidden.<jupyter_code>def tr(v, extra_noise):
    a,b,c,d,e,f,g,h,i,j = v
    super_noisy = np.random.normal(loc=0, scale=extra_noise, size=None)
    return (a+b, b*c, (c + d + e)/3, d + i/10, e, f, g+super_noisy, h + i/5, h + c/3, 0)
noise = 1/5
X_tr_train = np.apply_along_axis(tr, axis=1, arr=X, extra_noise=2) + np.random.normal(loc=0, scale=noise, size=(n,p))
model = LinearRegression().fit(X_tr_train, y_train)
X_tr_test = np.apply_along_axis(tr, axis=1, arr=X_test, extra_noise=1) + np.random.normal(loc=0, scale=noise, size=(n,p))
y_pred = model.predict(X_tr_test)
r2_score(y_test, y_pred)
plt.scatter(y_test, y_pred, color='#3033ff30')
plt.show()
model.coef_
A
from pandas.plotting import scatter_matrix
df = pd.DataFrame(X_tr_train, columns=list(letters[:10]))
df['y'] = y_train
df.shape
df.head()
x = scatter_matrix(df, alpha = 0.2, figsize = (6, 6), diagonal = 'kde')
df = pd.DataFrame(X, columns=list(letters[:10]))
df['y'] = y_train
df.shape
x = scatter_matrix(df, alpha = 0.2, figsize = (6, 6), diagonal = 'kde')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14465105047090074, 'fraction_numerical': 0.025839169282781935, 'mean_word_length': 5.172876304023845, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '520227', 'license': 'Apache-2.0', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1625, 'n_words': 347, 'path': 'notebooks/linear_model.ipynb', 'repo_name': 'cbare/Etudes', 'size': '4141'}