<jupyter_start><jupyter_text>Generate Data<jupyter_code># import dependencies
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# define seed & sample size
SEED = 1234
NUM_SAMPLES = 50
# set seed for reproducibility
np.random.seed(SEED)
# generate synthetic data
def generate_data(num_samples):
  """Generate dummy data for linear regression."""
  X = np.array(range(num_samples))
  random_noise = np.random.uniform(-10,20, size=num_samples)
  y = 3.5*X + random_noise # add some noise
  return X, y
# generate random (linear) data
X, y = generate_data(num_samples=NUM_SAMPLES)
data = np.vstack([X, y]).T
print(data[:5])
# load into a pandas dataframe
df = pd.DataFrame(data, columns=['X', 'y'])
X = df[['X']].values
y = df[['y']].values
df.head()
# scatter plot
plt.title("Generated data")
plt.scatter(x=df['X'], y=df['y'])
plt.show()<jupyter_output><empty_output><jupyter_text>Linear Reg. with Numpy  Split Data<jupyter_code># split percentages
TRAIN_SIZE =0.7
VAL_SIZE = 0.15
TEST_SIZE = 0.15
# shuffle data. P.S - never shuffle X & y differently to avoid mismatch
indices = list(range(NUM_SAMPLES))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# split indices
train_start = 0
train_end = int(0.7*NUM_SAMPLES)
val_start = train_end
val_end = int((TRAIN_SIZE+VAL_SIZE)*NUM_SAMPLES)
test_start = val_end
# split dataset
X_train = X[train_start:train_end]
y_train = y[train_start:train_end]
X_val = X[val_start:val_end]
y_val = y[val_start:val_end]
X_test = X[test_start:]
y_test = y[test_start:]
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
print(f"X_val: {X_val.shape}, y_val: {y_val.shape}")
print(f"X_test: {X_test.shape}, y_test: {y_test.shape}")<jupyter_output>X_train: (35, 1), y_train: (35, 1)
X_val: (7, 1), y_val: (7, 1)
X_test: (8, 1), y_test: (8, 1)<jupyter_text>Standardize Dataset<jupyter_code># standardize data function
def standardize_data(data, mean, std):
  """Standardize dataset to zero mean & unit variance."""
  return (data - mean)/std
# determine means and stds
X_mean = np.mean(X_train)
X_std = np.std(X_train)
y_mean = np.mean(y_train)
y_std = np.std(y_train)<jupyter_output><empty_output><jupyter_text>We need to treat the validation and test sets as if they were hidden datasets. So we only use the train set to determine the mean and std to avoid biasing our training process.<jupyter_code># standardize data
X_train = standardize_data(X_train, X_mean, X_std)
y_train = standardize_data(y_train, y_mean, y_std)
X_val = standardize_data(X_val, X_mean, X_std)
y_val = standardize_data(y_val, y_mean, y_std)
X_test = standardize_data(X_test, X_mean, X_std)
y_test = standardize_data(y_test, y_mean, y_std)
# Check: means should be ~0 and std should be ~1
print(f"mean: {np.mean(X_test, axis=0)[0]:.1f}, std: {np.std(X_test, axis=0)[0]:.1f}")
print(f"mean: {np.mean(y_test, axis=0)[0]:.1f}, std: {np.std(y_test, axis=0)[0]:.1f}")<jupyter_output>mean: -0.6, std: 0.9
mean: -0.6, std: 0.9<jupyter_text>Weights<jupyter_code># model dimesions
INPUT_DIM = X_train.shape[1] # X is 1-dimensional
OUTPUT_DIM = y_train.shape[1] # y is 1-dimensional
# initialize random weights
W = 0.01 * np.random.randn(INPUT_DIM, OUTPUT_DIM)
b = np.zeros((1,1))
print(f"W: {W.shape}")
print(f"b: {b.shape}")<jupyter_output>W: (1, 1)
b: (1, 1)<jupyter_text>Model<jupyter_code># forward pass [Nx1] · [1x1] = [Nx1]
y_pred = np.dot(X_train, W) + b
print(f"y_pred: {y_pred.shape}")<jupyter_output>y_pred: (35, 1)<jupyter_text>Loss<jupyter_code># calculate loss using mean squared error (MSE)
N = len(y_train)
loss = (1/N) * np.sum((y_train - y_pred)**2)
print(f"loss: {loss:.2f}")<jupyter_output>loss: 0.99<jupyter_text>Gradient<jupyter_code># backpropagation
dW = -(2/N) * np.sum((y_train - y_pred) * X_train)
db = -(2/N) * np.sum((y_train - y_pred) * 1)<jupyter_output><empty_output><jupyter_text>Update Weight<jupyter_code># define learning rate hyperparameter
LEARNING_RATE = 1e-1
# update weights
W += -LEARNING_RATE * dW
b += -LEARNING_RATE * db<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># define number of training epochs
NUM_EPOCHS = 100
# initialize random weights
W = 0.01 * np.random.randn(INPUT_DIM, OUTPUT_DIM)
b = np.zeros((1,1))

# training loop
for epoch_num in range(NUM_EPOCHS):
  # forward pass [Nx1] · [1x1] = [Nx1]
  y_pred = np.dot(X_train, W) + b

  # calculate loss
  loss = (1/len(y_train)) * np.sum((y_train - y_pred)**2)

  # show training update & progress
  if epoch_num%10 == 0:
    print(f"Epoch: {epoch_num}, loss: {loss:.3f}")

  # backpropagation
  dW = -(2/N) * np.sum((y_train - y_pred) * X_train)
  db = -(2/N) * np.sum((y_train - y_pred) * 1)

  # update weight
  W += -LEARNING_RATE * dW
  b += -LEARNING_RATE * db<jupyter_output>Epoch: 0, loss: 0.990
Epoch: 10, loss: 0.039
Epoch: 20, loss: 0.028
Epoch: 30, loss: 0.028
Epoch: 40, loss: 0.028
Epoch: 50, loss: 0.028
Epoch: 60, loss: 0.028
Epoch: 70, loss: 0.028
Epoch: 80, loss: 0.028
Epoch: 90, loss: 0.028<jupyter_text>Evaluation<jupyter_code># prediction
pred_train = W*X_train + b
pred_test = W*X_test + b
# train & test MSE
train_mse = np.mean((y_train - y_pred) **2)
test_mse = np.mean((y_test - pred_test) **2)
print(f"train_MSE: {train_mse:.2f}, test_MSE: {test_mse:.2f}")
# plot figure size
plt.figure(figsize=(15,5))

# plot train data
plt.subplot(1,2,1)
plt.title("Train")
plt.scatter(X_train, y_train, label='y_train')
plt.plot(X_train, pred_train, color='red', linewidth=1, linestyle='-', label='model')
plt.legend(loc='lower right')

# plot test data
plt.subplot(1,2,2)
plt.title("Test")
plt.scatter(X_test, y_test, label='y_test')
plt.plot(X_test, pred_test, color='red', linewidth=1, linestyle='-', label='model')
plt.legend(loc='lower right')

# show plots
plt.show()<jupyter_output><empty_output><jupyter_text>Interpretability<jupyter_code># unscaled weights
W_unscaled = W * (y_std/X_std)
b_unscaled = b * y_std + y_mean - np.sum(W_unscaled*X_mean)
print("[actual] y = 3.5X + noise")
print(f"[model] y_hat = {W_unscaled[0][0]:.1f}X + {b_unscaled[0][0]:.1f}")<jupyter_output>[actual] y = 3.5X + noise
[model] y_hat = 3.4X + 7.8<jupyter_text>Linear Reg. with PyTorch  Set up<jupyter_code># import dependeencies
import torch
# set seed for reproducibility
torch.manual_seed(SEED)<jupyter_output><empty_output><jupyter_text>Split Data<jupyter_code># use sklearn split function
from sklearn.model_selection import  train_test_split
# define split sizes
TRAIN_SIZE = 0.7
VAL_SIZE = 0.15
TEST_SIZE = 0.15
# split train dataset
X_train, X_, y_train, y_ = train_test_split(X, y, train_size=TRAIN_SIZE)
print(f"train: {len(X_train)} ({(len(X_train) / len(X)):.2f})\n"
      f"remaining: {len(X_)} ({(len(X_) / len(X)):.2f})")
# split test datasets
X_val, X_test, y_val, y_test = train_test_split(X_, y_, train_size=0.5)
print(f"trian: {len(X_train)} ({len(X_train) / len(X):.2f})\n"
      f"val: {len(X_val)} ({len(X_val) / len(X):.2f})\n"
      f"test: {len(X_test)} ({len(X_test) / len(X):.2f})")<jupyter_output>trian: 35 (0.70)
val: 7 (0.14)
test: 8 (0.16)<jupyter_text>Standardize Data<jupyter_code># standardize using sklearn StandardScaler
from sklearn.preprocessing import StandardScaler
# standardize the daaset (mean=0, std=1) using training data
X_scaler = StandardScaler().fit(X_train)
y_scaler = StandardScaler().fit(y_train)
# apply scaler on training and test data
X_train = X_scaler.transform(X_train)
y_train = y_scaler.transform(y_train).ravel().reshape(-1,1)
X_val = X_scaler.transform(X_val)
y_val = y_scaler.transform(y_val).ravel().reshape(-1,1)
X_test = X_scaler.transform(X_test)
y_test = y_scaler.transform(y_test).ravel().reshape(-1,1)
# check: (means should be ~0 and std should be ~1)
print(f"mean: {np.mean(X_test, axis=0)[0]:.1f}, std: {np.std(X_test, axis=0)[0]:.1f}")
print(f"mean: {np.mean(y_test, axis=0)[0]:.1f}, std: {np.std(y_test, axis=0)[0]:.1f}")<jupyter_output>mean: -0.2, std: 1.2
mean: -0.2, std: 1.1<jupyter_text>Weights<jupyter_code># using pytorch linear layers
from torch import nn
# define hyperparameters
LEARNING_RATE = 1e-1
INPUT_DIM = X_train.shape[1] # X is 1-dimensional
OUTPUT_DIM = y_train.shape[1] # y is 1-dimensional
NUM_EPOCHS = 100
# inputs & model dimesions
N = 3 # number of sample
x = torch.randn(N, INPUT_DIM)
print("x.shape: ", x.shape)
print("x.numpy(): \n", x.numpy())
# weights
m = nn.Linear(INPUT_DIM, OUTPUT_DIM)
print("m: ", m)
print(f"weights ({m.weight.shape}): {m.weight[0][0]:.2f}")
print(f"bias ({m.bias.shape}): {m.bias[0]:.2f}")
# forward pass
z = m(x)
print("z.shape: ", z.shape)
print("z.detach: \n", z.detach().numpy())<jupyter_output>z.shape:  torch.Size([3, 1])
z.detach: 
 [[-0.32104054]
 [-0.19719592]
 [-0.68869597]]<jupyter_text>Model<jupyter_code># build model class
class LinearRegression(nn.Module):
  """Linear regression class."""

  def __init__(self, input_dim, output_dim):
    """initialize LinearRegression class."""
    super(LinearRegression, self).__init__()
    self.fc1 = nn.Linear(input_dim, output_dim)

  def forward(self, x_in):
    y_pred = self.fc1(x_in)
    return y_pred
# initialize model
model = LinearRegression(input_dim=INPUT_DIM, output_dim=OUTPUT_DIM)
print("model parameters: \n", model.named_parameters)<jupyter_output>model parameters: 
 <bound method Module.named_parameters of LinearRegression(
  (fc1): Linear(in_features=1, out_features=1, bias=True)
)><jupyter_text>Model Loss<jupyter_code># using pytorch MSELoss
loss_fn = nn.MSELoss()
y_pred = torch.Tensor([0., 0., 1., 1.])
y_true = torch.Tensor([1., 1., 1., 0.])
loss = loss_fn(y_pred, y_true)
print("Loss: ", loss.numpy())<jupyter_output>Loss:  0.75<jupyter_text>Optimizer<jupyter_code># using adam optimizer
from torch.optim import  Adam
# optimizer
optimizer = Adam(model.parameters(), lr=LEARNING_RATE)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># convert data to tensors
X_train = torch.Tensor(X_train)
y_train = torch.Tensor(y_train)
X_val = torch.Tensor(X_val)
y_val = torch.Tensor(y_val)
X_test = torch.Tensor(X_test)
y_test = torch.Tensor(y_test)
# train model
for epoch in range(NUM_EPOCHS):
  # forward pass
  y_pred = model(X_train)

  # model loss
  loss = loss_fn(y_pred, y_train)

  # zero all gradients
  optimizer.zero_grad()

  # backward pass
  loss.backward()

  # update weights
  optimizer.step()

  # update model performance
  if epoch%20==0:
    print(f"Epoch: {epoch} | loss: {loss:.2f}")<jupyter_output>Epoch: 0 | loss: 0.03
Epoch: 20 | loss: 0.03
Epoch: 40 | loss: 0.03
Epoch: 60 | loss: 0.03
Epoch: 80 | loss: 0.03<jupyter_text>Evaluation<jupyter_code># predictions
pred_train = model(X_train)
pred_test = model(X_test) 
# performance
train_error = loss_fn(pred_train, y_train)
test_error = loss_fn(pred_test, y_test)
print(f"train_error: {train_error:.2f}")
print(f"test_error: {test_error:.2f}")
# inspect with figure size
plt.figure(figsize=(15,5))

# plot train size
plt.subplot(1,2,1)
plt.title("Train")
plt.scatter(X_train, y_train, label="y_train")
plt.plot(X_train, pred_train.detach().numpy(), color="red", linewidth=1, linestyle='-', label="model")
plt.legend(loc="lower right")

# plot test data
plt.subplot(1,2,2)
plt.title("Test")
plt.scatter(X_test, y_test, label="y_test")
plt.plot(X_test, pred_test.detach().numpy(), color="red", linewidth=1, linestyle="-", label="model")
plt.legend(loc="lower right")

# show plots
plt.show()<jupyter_output><empty_output><jupyter_text>Inference<jupyter_code># feed in new data
sample_indices = [10,15,25]
X_infer = np.array(sample_indices, dtype=np.float32)
X_infer = torch.Tensor(X_scaler.transform(X_infer.reshape(-1,1)))
# un-standardize predictions
pred_infer = model(X_infer).detach().numpy() * np.sqrt(y_scaler.var_) + y_scaler.mean_
for i, index in enumerate(sample_indices):
  print(f"{df.iloc[index]['y']:.2f} (actual) → {pred_infer[i][0]:.2f} (predicted)")<jupyter_output>35.73 (actual) → 42.36 (predicted)
59.34 (actual) → 59.38 (predicted)
97.04 (actual) → 93.42 (predicted)<jupyter_text>Interpretability<jupyter_code># un-standardize coefficients
W = model.fc1.weight.data.numpy()[0][0]
b = model.fc1.bias.data.numpy()[0]
W_unscaled = W * (y_scaler.scale_/X_scaler.scale_)
b_unscaled = b * y_scaler.scale_ + y_scaler.mean_ - np.sum(W_unscaled*X_scaler.mean_)
print("[actual y = 3.5X + noise")
print(f"[model] y_hat = {W_unscaled[0]:.1f}X + {b_unscaled[0]:.1f}")<jupyter_output>[actual y = 3.5X + noise
[model] y_hat = 3.4X + 8.3<jupyter_text>Regularization<jupyter_code># adjust weight decay hyperparameter
L2_LAMBDA = 10e-2<jupyter_output><empty_output><jupyter_text>initialize modelmodel = LinearRegression(input_dim=INPUT_DIM, output_dim=OUTPUT_DIM)<jupyter_code># optimizer (w / L2 regularization)
optimizer = Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=L2_LAMBDA)
# training model
for epoch in range(NUM_EPOCHS):

  #  forward pass
  y_pred = model(X_train)

  # loss
  loss = loss_fn(y_pred, y_train)

  # zero all gradients
  optimizer.zero_grad()

  # backward pass
  loss.backward()

  # update weights
  optimizer.step()

  # update model performance output
  if epoch%20==0:
    print(f"Epoch: {epoch} | loss: {loss:.2f}")
# predictions
pred_train = model(X_train)
pred_test = model(X_test)
# performance
train_error = loss_fn(pred_train, y_train)
test_error = loss_fn(pred_test, y_test)
print(f"train_error: {train_error:.2f}")
print(f"test_error: {test_error:.2f}")<jupyter_output>train_error: 0.03
test_error: 0.03
{'fraction_non_alphanumeric': 0.16033723793180632, 'fraction_numerical': 0.032156979780646126, 'mean_word_length': 5.937888198757764, 'pattern_counts': {'":': 0, '<': 86, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '292778', 'license': 'MIT', 'n_tokens_mistral': 5839, 'n_tokens_neox': 5547, 'n_words': 1130, 'path': 'Linear_Regression.ipynb', 'repo_name': 'callezenwaka/machine-learning', 'size': '13403'}