<jupyter_start><jupyter_text>Refs:https://github.com/deep-learning-with-pytorch/dlwpt-code<jupyter_code>import numpy as np
import torch<jupyter_output><empty_output><jupyter_text>Optimizers<jupyter_code>x = [35.7, 55.9, 58.2, 81.9, 56.3, 48.9, 33.9, 21.8, 48.4, 60.4, 68.4]
y = [0.5,  14.0, 15.0, 28.0, 11.0,  8.0,  3.0, -4.0,  6.0, 13.0, 21.0]

x = torch.tensor(x)
y = torch.tensor(y)

#x = 0.1*x # normalize
x_norm = 0.1*x
def model(x, w, b):
    return w * x + b
def loss_fn(y_p, y):
    squared_diffs = (y_p - y)**2
    return squared_diffs.mean()
import torch.optim as optim
dir(optim)
def training_loop(n_epochs, optimizer, params, x, y):
    
    for epoch in range(1, n_epochs + 1):
        y_p = model(x, *params) 
        loss = loss_fn(y_p, y)
        
        ## reset gradients to zero
        optimizer.zero_grad()
        
        ## calculate gradients
        loss.backward()  
        
        ## update params: params -= learning_rate * params.grad
        optimizer.step() 

        if epoch % 500 == 0:
            print('Epoch %d, Loss %f' % (epoch, float(loss)))
            
    return params
params = torch.tensor([1.0, 0.0], requires_grad=True)

learning_rate = 1e-2
optimizer = optim.SGD([params], lr=learning_rate)

training_loop(n_epochs = 5000, params = params,
              optimizer = optimizer, x = x_norm, y = y)<jupyter_output>Epoch 500, Loss 7.860115
Epoch 1000, Loss 3.828538
Epoch 1500, Loss 3.092191
Epoch 2000, Loss 2.957698
Epoch 2500, Loss 2.933134
Epoch 3000, Loss 2.928648
Epoch 3500, Loss 2.927830
Epoch 4000, Loss 2.927679
Epoch 4500, Loss 2.927652
Epoch 5000, Loss 2.927647
{'fraction_non_alphanumeric': 0.13880320789636028, 'fraction_numerical': 0.11782850092535473, 'mean_word_length': 2.807511737089202, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '667603', 'license': 'MIT', 'n_tokens_mistral': 783, 'n_tokens_neox': 654, 'n_words': 161, 'path': 'Tutorial/.ipynb_checkpoints/c5_optimizers-checkpoint.ipynb', 'repo_name': 'danhtaihoang/pytorch-deeplearning', 'size': '1621'}