<jupyter_start><jupyter_text>Refs:https://github.com/deep-learning-with-pytorch/dlwpt-code<jupyter_code>import numpy as np
import torch
import torch.optim as optim
import torch.nn as nn<jupyter_output><empty_output><jupyter_text>Neural Networks<jupyter_code>model = nn.Sequential(
            nn.Linear(1, 13),
            nn.Tanh(),
            nn.Linear(13, 1)) 
model<jupyter_output><empty_output><jupyter_text>Inspecting the parameters<jupyter_code>## shapes of parameters
[param.shape for param in model.parameters()]
## names of parameters
for name, param in model.named_parameters():
    print(name, param.shape)
## name of each module
from collections import OrderedDict

model = nn.Sequential(OrderedDict([
    ('hidden_linear', nn.Linear(1, 13)),
    ('hidden_activation', nn.Tanh()),
    ('output_linear', nn.Linear(13, 1))
]))

model
for name, param in model.named_parameters():
    print(name, param.shape)
model.output_linear.bias<jupyter_output><empty_output><jupyter_text>Training neural networks<jupyter_code>x = [35.7, 55.9, 58.2, 81.9, 56.3, 48.9, 33.9, 21.8, 48.4, 60.4, 68.4]
y = [0.5,  14.0, 15.0, 28.0, 11.0,  8.0,  3.0, -4.0,  6.0, 13.0, 21.0]

x = torch.tensor(x)
y = torch.tensor(y)

## normalize:
x = 0.1*x 

x = x.reshape((-1,1))
y = y.reshape((-1,1))

print(x.shape, y.shape)
## Splitting the dataset
n_samples = x.shape[0]
n_val = int(0.2 * n_samples)

i_shuffled = torch.randperm(n_samples)

i_train = i_shuffled[:-n_val]
i_val = i_shuffled[-n_val:]

i_train, i_val

x_train = x[i_train]
y_train = y[i_train]

x_val = x[i_val]
y_val = y[i_val]
def training_loop(n_epochs, optimizer, model, loss_fn, x_train, y_train, x_val, y_val):
    for epoch in range(1, n_epochs + 1):
        y_train_p = model(x_train)
        loss_train = loss_fn(y_train_p, y_train)
                             
        y_val_p = model(x_val)
        loss_val = loss_fn(y_val_p, y_val)
        
        optimizer.zero_grad()
        loss_train.backward()
        optimizer.step()

        if epoch <= 3 or epoch % 500 == 0:
            print(f"Epoch {epoch}, Training loss {loss_train.item():.4f},"
                  f" Validation loss {loss_val.item():.4f}")
loss_fn = nn.MSELoss()
optimizer = optim.SGD(model.parameters(), lr= 1e-3)

training_loop(n_epochs = 3000, optimizer = optimizer, model = model, loss_fn = loss_fn,
              x_train = x_train, y_train = y_train, x_val = x_val, y_val = y_val)
print('y_pred:', model(x_val))
print('y_actual:', y_val)
print('check w.grad of hidden layer:')
print(model.hidden_linear.weight.grad)<jupyter_output>check w.grad of hidden layer:
tensor([[ 1.7610e+01],
        [-6.8478e-03],
        [ 1.0071e+00],
        [ 5.2210e-03],
        [-1.3800e+01],
        [ 2.0804e+01],
        [ 2.0887e+01],
        [-2.0675e+00],
        [-1.5381e-02],
        [ 7.8584e-01],
        [ 1.2547e-03],
        [-2.2070e+01],
        [ 2.0565e+01]])
{'fraction_non_alphanumeric': 0.15600138360428917, 'fraction_numerical': 0.06502940159114494, 'mean_word_length': 3.1373390557939915, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '552139', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1167, 'n_words': 225, 'path': 'Tutorial/.ipynb_checkpoints/c6_neural_networks-checkpoint.ipynb', 'repo_name': 'danhtaihoang/pytorch-deeplearning', 'size': '2891'}