<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pylab as plt
#matplotlib inline
ORDER = 8
def make_fake_data(N, sigma=0.1):
  xtrain = np.random.normal(size=N)
  ytrain = xtrain + sigma * np.random.normal(size=N)
  xtest = np.random.uniform()
  ytest = xtest + sigma * np.random.normal()
  return xtrain, ytrain, xtest, ytest
def design_matrix(xs):
  A = np.vstack([xs ** k for k in range(ORDER + 1)]).T
  return A

def predict(xtrain, ytrain, xtest):
  pars = np.linalg.lstsq(design_matrix(xtrain), ytrain, rcond=1e-12)[0]
  prediction = design_matrix(xtest) @ pars
  return prediction, pars

def resid(xtrain, ytrain, xtest, ytest):
  return ytest - predict(xtrain, ytrain, xtest)[0]
np.random.seed(42)
xtr, ytr, xte, yte = make_fake_data(5)
ypr, pars = predict(xtr, ytr, xte)
plt.plot(xtr, ytr, "ko")
plt.plot(xte, yte, "ko", alpha=0.5)
plt.text(xte, yte, " test object")
plt.plot(xte, ypr, "ro")
plt.text(xte, ypr, " prediction at polynomial order {:d}".format(ORDER))
def estimate_mse_with_trials(Ntrain, Ntrial):
    dys = np.array([resid(*make_fake_data(Ntrain)) for t in range(Ntrial)])
    return np.median(dys ** 2)
print(estimate_mse_with_trials(3, 128))
Ns = np.arange(ORDER // 2, ORDER * 2 + 1)
mses = [estimate_mse_with_trials(N, 8192) for N in Ns]
plt.axvline(ORDER + 1)
plt.plot(Ns, mses, "ko")
plt.ylabel("median squared error polynomial prediction")
plt.xlabel("size of training set N")
plt.title("polynomial order {:d}".format(ORDER))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15425531914893617, 'fraction_numerical': 0.01795212765957447, 'mean_word_length': 5.270833333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '548578', 'license': 'MIT', 'n_tokens_mistral': 651, 'n_tokens_neox': 614, 'n_words': 145, 'path': 'ipynb/double_descent.ipynb', 'repo_name': 'davidwhogg/GenerativeVsDiscriminative', 'size': '1504'}