<jupyter_start><jupyter_text>$$ \mathbf{s}_t \leftarrow \mathbf{s}_{t-1} + \mathbf{g}_t \odot \mathbf{g}_t $$$$ \mathbf{x}_t \leftarrow \mathbf{x}_{t-1} - \frac{\eta}{\sqrt{\mathbf{s}_t + \epsilon}} \odot \mathbf{g}_t $$<jupyter_code>%matplotlib inline
import MyD2l as d2l
import math
import torch
def adagrad_2d(x1, x2, s1, s2):
    g1, g2, eps = 0.2 * x1, 4 * x2, 1e-6
    s1 += g1 ** 2
    s2 += g2 ** 2
    x1 -= eta / math.sqrt(s1 + eps) * g1
    x2 -= eta / math.sqrt(s2 + eps) * g2
    return x1, x2, s1, s2

def f_2d(x1, x2):
    return 0.1 * x1 ** 2 + 2 * x2 ** 2

eta = 0.4
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
eta = 2
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
features, labels = d2l.get_data_ch7()

def init_adagrad_states():
    s_w = torch.zeros(features.shape[1], 1)
    s_b = torch.zeros(1)
    return s_w, s_b

def adagrad(params, states, hyperparams):
    eps = 1e-6
    for p, s in zip(params, states):
        s[:] += p.grad ** 2
        p[:] -= hyperparams['lr'] * p.grad / (s + eps).sqrt()
        p.grad.zero_()
d2l.train_ch7(adagrad, init_adagrad_states(), {'lr': 0.1}, features, labels)
d2l.train_ch7(adagrad, init_adagrad_states(), {'lr': 1}, features, labels)
d2l.train_nn_ch7(torch.optim.Adagrad, {'lr': 0.1}, features, labels)
d2l.train_nn_ch7(torch.optim.Adagrad, {'lr': 1}, features, labels)<jupyter_output>loss: 0.484505, 1.007340 sec per epoch
{'fraction_non_alphanumeric': 0.18481375358166188, 'fraction_numerical': 0.0680515759312321, 'mean_word_length': 3.565359477124183, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '312517', 'license': 'Apache-2.0', 'n_tokens_mistral': 678, 'n_tokens_neox': 651, 'n_words': 137, 'path': 'Chapter 7/AdaGrad.ipynb', 'repo_name': 'Ricinix/dive_into_deep_learning_pytorch', 'size': '1396'}