<jupyter_start><jupyter_text>What are the "pieces" needed to "learn" a line that matches the curve above- loss (cost) function- derivatices (some form of adjustment)- parameters<jupyter_code># Hyperparameters
num_epochs = 100
learning_rate = 0.1

# Model parameters
m = torch.randn(1, requires_grad=True)
b = torch.zeros(1, requires_grad=True)

# Torch utils
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD([m, b], lr=learning_rate)

# Train
for epoch in range(num_epochs):
    
    yhat = m * x_train + b
    
    optimizer.zero_grad()
    loss = criterion(yhat, y_train)
    loss.backward()
    optimizer.step()

plot_regression_data(model=lambda x: m * x + b, MSE=loss.item())
# Hyperparameters
num_epochs = 100
learning_rate = 0.1

# Model parameters
w2 = torch.randn(1, requires_grad=True)
w1 = torch.randn(1, requires_grad=True)
w0 = torch.zeros(1, requires_grad=True)

def model(x):
    return w2 * x * x + w1 * x + w0

# Torch utils
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD([w0, w1, w2], lr=learning_rate)

# Train
for epoch in range(num_epochs):
    
    yhat = model(x_train)
    
    optimizer.zero_grad()
    loss = criterion(yhat, y_train)
    loss.backward()
    optimizer.step()

plot_regression_data(model=model, MSE=loss.item())<jupyter_output><empty_output><jupyter_text>$$y = mx + b$$ $$y = w_2 x^2 + w_1 x^1 + w_0$$<jupyter_code># Hyperparameters
learning_rate = 0.1
num_epochs = 1000

# Model parameters
degrees = 50  # 3, 4, 16, 32, 64, 128
powers = torch.arange(degrees + 1)
x_poly = x_train.unsqueeze(-1).pow(powers)
params = torch.randn(degrees + 1, requires_grad=True)

# Torch utils
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD([params], lr=learning_rate)

# Regression
for epoch in range(num_epochs):
    # Model
    yhat = x_poly @ params

    # Update parameters
    optimizer.zero_grad()
    loss = criterion(yhat, y_train)
    loss.backward()
    optimizer.step()

plot_regression_data(lambda x: x @ params, poly_deg=degrees, MSE=loss.item())
params.round()
params = ((x_poly.T @ x_poly).inverse() @ x_poly.T) @ y_train
mse = torch.nn.functional.mse_loss(x_poly @ params, y_train)
plot_regression_data(lambda x: x @ params, poly_deg=degrees, MSE=mse)
# params
params
# Hyperparameters
learning_rate = 0.01
num_epochs = 10000
regularization =  1e-1

# Model parameters
model = torch.nn.Sequential(
    torch.nn.Linear(1, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 1),
)

# Torch utils
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(
    model.parameters(), lr=learning_rate, weight_decay=regularization
)

# Training
for epoch in progress_bar(range(num_epochs)):
    # Model
    yhat = model(x_train.unsqueeze(-1))

    # Update parameters
    optimizer.zero_grad()
    loss = criterion(yhat.squeeze(), y_train)
    loss.backward()
    optimizer.step()

plot_regression_data(model, loss.item())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1329804829639431, 'fraction_numerical': 0.027125372146873966, 'mean_word_length': 4.259130434782609, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '222153', 'license': 'CC0-1.0', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1156, 'n_words': 243, 'path': 'lectures/l15-inclass.ipynb', 'repo_name': 'davidd-55/cs152fa21', 'size': '3023'}