<jupyter_start><jupyter_text>Momentum Standard gradient descent.<jupyter_code>%matplotlib inline
import math
import matplotlib.pyplot as plt
import numpy as np
eta = 0.4

def f_2d(x1, x2):
    return 0.1 * x1 ** 2 + 2 * x2 ** 2

def gd_2d(x1, x2, s1, s2):
    return (x1 - eta * 0.2 * x1, x2 - eta * 4 * x2, 0, 0)

def train_2d(trainer):  
    x1, x2, s1, s2 = -5, -2, 0, 0 
    results = [(x1, x2)]
    for i in range(20):
        x1, x2, s1, s2 = trainer(x1, x2, s1, s2)
        results.append((x1, x2))
    print('epoch %d, x1 %f, x2 %f' % (i + 1, x1, x2))
    return results

def show_trace_2d(f, results):  
    plt.plot(*zip(*results), '-o', color='#ff7f0e')
    x1, x2 = np.meshgrid(np.arange(-5.5, 1.0, 0.1), np.arange(-3.0, 1.0, 0.1))
    plt.contour(x1, x2, f(x1, x2), colors='#1f77b4')
    plt.xlabel('x1')
    plt.ylabel('x2')

show_trace_2d(f_2d, train_2d(gd_2d))<jupyter_output>epoch 20, x1 -0.943467, x2 -0.000073<jupyter_text>MomentumStochastic gradient descent with momentum remembers the update $\Delta w$ at each iteration, and determines the next update as a linear combination of the gradient and the previous update.<jupyter_code>def momentum_2d(x1, x2, v1, v2):
    v1 = gamma * v1 + eta * 0.2 * x1
    v2 = gamma * v2 + eta * 4 * x2
    return x1 - v1, x2 - v2, v1, v2

eta, gamma = 0.4, 0.5
show_trace_2d(f_2d, train_2d(momentum_2d))<jupyter_output>epoch 20, x1 -0.062843, x2 0.001202<jupyter_text>GD with a large learning rate<jupyter_code>eta = 0.6
show_trace_2d(f_2d, train_2d(gd_2d))<jupyter_output>epoch 20, x1 -0.387814, x2 -1673.365109<jupyter_text>Same learning rate for momentum.<jupyter_code>show_trace_2d(f_2d, train_2d(momentum_2d))<jupyter_output>epoch 20, x1 0.007188, x2 0.002553<jupyter_text>Adagrad<jupyter_code>def adagrad_2d(x1, x2, s1, s2):
    # The first two terms are the independent variable gradients
    g1, g2, eps = 0.2 * x1, 4 * x2, 1e-6
    s1 += g1 ** 2
    s2 += g2 ** 2
    x1 -= eta / math.sqrt(s1 + eps) * g1
    x2 -= eta / math.sqrt(s2 + eps) * g2
    return x1, x2, s1, s2

eta = 0.4
show_trace_2d(f_2d, train_2d(adagrad_2d))<jupyter_output>epoch 20, x1 -2.382563, x2 -0.158591<jupyter_text>Use a much larger learning rate.<jupyter_code>eta = 2
show_trace_2d(f_2d, train_2d(adagrad_2d))<jupyter_output>epoch 20, x1 -0.002295, x2 -0.000000<jupyter_text>RMSProp<jupyter_code>def rmsprop_2d(x1, x2, s1, s2):
    g1, g2, eps = 0.2 * x1, 4 * x2, 1e-6
    s1 = gamma * s1 + (1 - gamma) * g1 ** 2
    s2 = gamma * s2 + (1 - gamma) * g2 ** 2
    x1 -= eta / math.sqrt(s1 + eps) * g1
    x2 -= eta / math.sqrt(s2 + eps) * g2
    return x1, x2, s1, s2


eta, gamma = 0.4, 0.9
show_trace_2d(f_2d, train_2d(rmsprop_2d))<jupyter_output>epoch 20, x1 -0.010599, x2 0.000000
{'fraction_non_alphanumeric': 0.15031238515251746, 'fraction_numerical': 0.1231165012862918, 'mean_word_length': 3.476973684210526, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '617333', 'license': 'Apache-2.0', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1312, 'n_words': 311, 'path': 'L26 Momentum, AdaGrad, RMSProp, Adam/L26_2_Momentum,_Adagrad,_RMSProb_in_Python.ipynb', 'repo_name': 'JiahongChen/d2l-pytorch-implementation', 'size': '2721'}