<jupyter_start><jupyter_text>Make dataset with x^D, x^(D-1), ..., x^0<jupyter_code>def make_poly(x, D):
    N = len(x)
    X = np.empty((N, D+1))  # +1 for bias
    for d in range(D+1):
        X[:, d] = x**d
        if d > 1:
            # Normalize
            X[:, d] = (X[:, d] - X[:, d].mean()) / X[:, d].std()
    return X
def f(x):
    return np.sin(x)

x_axis = np.linspace(-np.pi, np.pi, 100)
y_axis = f(x_axis)

plt.plot(x_axis, y_axis)<jupyter_output><empty_output><jupyter_text>Define Dataset X with N points (25 points)<jupyter_code>X = np.linspace(-np.pi, np.pi, N)
np.random.shuffle(X)
f_X = f(X)

X_poly = make_poly(X, MAX_POLY)
train_scores = np.zeros((NUM_DATASETS, MAX_POLY))
test_scores = np.zeros((NUM_DATASETS, MAX_POLY))
train_predictions = np.zeros((N_train, NUM_DATASETS, MAX_POLY))
prediction_curves = np.zeros((100, NUM_DATASETS, MAX_POLY))
model = LinearRegression()
for k in range(NUM_DATASETS):
    Y = f_X + np.random.randn(N) * NOISE_VARIANCE
    
    X_train = X_poly[:N_train]
    Y_train = Y[:N_train]
    
    X_test = X_poly[N_train:]
    Y_test = Y[N_train:]
    
    for d in range(MAX_POLY):
        model.fit(X_train[:, :d+2], Y_train)
        predictions = model.predict(X_poly[:, :d+2])
        
        x_axis_poly = make_poly(x_axis, d+1)
        prediction_axis = model.predict(x_axis_poly)
        
        prediction_curves[:, k, d] = prediction_axis
        
        train_prediction = predictions[:N_train]
        test_prediction = predictions[N_train:]
        
        train_predictions[:, k, d] = train_prediction
        
        train_score = mse(train_prediction, Y_train)
        test_score = mse(test_prediction, Y_test)
        
        train_scores[k, d] = train_score
        test_scores[k, d] = test_score<jupyter_output><empty_output><jupyter_text>Show all prediction curves for each polynomial degree along with mean curve<jupyter_code>for d in range(MAX_POLY):
    for k in range(NUM_DATASETS):
        plt.plot(x_axis, prediction_curves[:, k, d], color='green', alpha=0.5)
    plt.plot(x_axis, prediction_curves[:, :, d].mean(axis=1), color='blue', linewidth=2.0)
    plt.title('All curves for degree = %s' % (d + 1))<jupyter_output><empty_output><jupyter_text>Calculate squared bias<jupyter_code>avg_train_prediction = np.zeros((N_train, MAX_POLY))
squared_bias = np.zeros(MAX_POLY)
f_X_train = f_X[:N_train]
for d in range(MAX_POLY):
    for i in range(N_train):
        avg_train_prediction[i, d] = train_predictions[i, :, d].mean()
    squared_bias[d] = ((avg_train_prediction[:, d] - f_X_train) ** 2).mean()<jupyter_output><empty_output><jupyter_text>Calculate variance<jupyter_code>variances = np.zeros((N_train, MAX_POLY))
for d in range(MAX_POLY):
  for i in range(N_train):
    delta = train_predictions[i,:,d] - avg_train_prediction[i,d]
    variances[i,d] = delta.dot(delta) / N
variance = variances.mean(axis=0)<jupyter_output><empty_output><jupyter_text>Make bias-variance plots<jupyter_code>degrees = np.arange(MAX_POLY) + 1
best_degree = np.argmin(test_scores.mean(axis=0)) + 1
plt.plot(degrees, squared_bias, label='squared bias')
plt.plot(degrees, variance, label='variance')
plt.plot(degrees, test_scores.mean(axis=0), label='test scores')
plt.plot(degrees, squared_bias + variance, label='squared bias + variance')
plt.axvline(x=best_degree, linestyle='--', label='best complexity')
plt.legend()<jupyter_output><empty_output><jupyter_text>Train score vs test score<jupyter_code>plt.plot(degrees, train_scores.mean(axis=0), label='train scores')
plt.plot(degrees, test_scores.mean(axis=0), label='test scores')
plt.axvline(x=best_degree, linestyle='--', label='best complexity')
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1484079870480302, 'fraction_numerical': 0.00836481381543443, 'mean_word_length': 4.380261248185777, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '662341', 'license': 'Apache-2.0', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1442, 'n_words': 261, 'path': 'supervised_ensemble/bias_variance.ipynb', 'repo_name': 'opplieam/Udemy-Lazy', 'size': '3706'}