<jupyter_start><jupyter_text>Super Simple RNN  NOTEBOOK DEPRECATEDCheck this notebook instead: [here](../../BasicNN/0010_FC_1Layer.html) Count "1" in linear RNN<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pdb
train_x = np.array([[[1], [0], [0]],    # dim-0 is training examples in batch - here T=5
                    [[0], [1], [0]],    # dim-1 is across time steps - here 3
                    [[1], [1], [0]],    # dim-2 is width of features vector - here 1
                    [[1], [0], [1]],
                    [[1], [1], [1]]], dtype=np.float64)
train_y = np.array([[1],                # we ignore time steps, thus only 2 dimensions
                    [1],
                    [2],
                    [2],
                    [3]], dtype=np.float64)

s0 = np.array([[0]], dtype=np.float64)
W_hh = np.array([[1]], dtype=np.float64)  # correct answer is W_hh == [[1]]
W_xh = np.array([[1]], dtype=np.float64)  # correct answer is W_xh == [[1]]
train_y
def fwd(x, Wxh, Whh):
    assert x.ndim==3 and x.shape[1:]==(3, 1)
    
    x_t1 = x[:,0,:]
    x_t2 = x[:,1,:]
    x_t3 = x[:,2,:]
        
    s0 = np.zeros([len(x), 1])   # [batch_size, nb_state_neurons]
    s1 = s0 @ Whh + x_t1 @ Wxh
    s2 = s1 @ Whh + x_t2 @ Wxh
    s3 = s2 @ Whh + x_t3 @ Wxh
    y_hat = s3
    return y_hat

def back(x, y, Wxh, Whh):
    assert x.ndim==3 and x.shape[1:]==(3, 1)
    assert y.ndim==2 and y.shape[1:]==(1,)
    
    # Forward
    x_t1 = x[:,0,:]
    x_t2 = x[:,1,:]
    x_t3 = x[:,2,:]
    s1 = s0 @ Whh + x_t1 @ Wxh
    s2 = s1 @ Whh + x_t2 @ Wxh
    s3 = s2 @ Whh + x_t3 @ Wxh
    y_hat = s3
    
    # Backward Whh
    err = (y-y_hat)
    der_t1 = s0
    der_t2 = der_t1 @ Whh + s1
    der_t3 = der_t2 @ Whh + s2
    dWhh = -np.sum(err * der_t3, keepdims=True) / len(x)
    
    # Backward Wxh
    der_t1 = x_t1
    der_t2 = der_t1 @ Whh + x_t2
    der_t3 = der_t2 @ Whh + x_t3
    dWxh = -np.sum(err * der_t3, keepdims=True) / len(x)
    
    return dWxh, dWhh

def mse(x, y, Wxh, Whh):
    y_hat = fwd(x, Wxh, Whh)
    return 0.5 * np.mean((y-y_hat)**2)
np.arange(10)[::-1]<jupyter_output><empty_output><jupyter_text>Numerical Gradient Check<jupyter_code>def ngrad(x, y, Wxh, Whh):
    dWhh = np.zeros_like(Whh)
    dWxh = np.zeros_like(Wxh)
    eps = 1e-4
    
    for r in range(len(Whh)):
        for c in range(Whh.shape[1]):
            Whh_pls = Whh.copy()
            Whh_min = Whh.copy()
            
            Whh_pls[r, c] += eps
            Whh_min[r, c] -= eps
            
            l_pls = mse(x, y, Wxh, Whh_pls)
            l_min = mse(x, y, Wxh, Whh_min)
            
            dWhh[r, c] = (l_pls - l_min) / (2*eps)
            
    for r in range(len(Wxh)):
        for c in range(Wxh.shape[1]):
            Wxh_pls = Wxh.copy()
            Wxh_min = Wxh.copy()
            
            Wxh_pls[r, c] += eps
            Wxh_min[r, c] -= eps
            
            l_pls = mse(x, y, Wxh_pls, Whh)
            l_min = mse(x, y, Wxh_min, Whh)
            
            dWxh[r, c] = (l_pls - l_min) / (2*eps)
    
    return dWxh, dWhh<jupyter_output><empty_output><jupyter_text>**Init to some random values**<jupyter_code>W_hh = np.array([[0.95]], dtype=np.float64)  # correct answer is W_hh == [[1]]
W_xh = np.array([[0.95]], dtype=np.float64)  # correct answer is W_xh == [[1]]
fwd(train_x, W_xh, W_hh)<jupyter_output><empty_output><jupyter_text>**W_xh check**<jupyter_code>dW_xh, _ = back(train_x, train_y, W_xh, W_hh)
dW_xh
gW_xh, _ = ngrad(train_x, train_y, W_xh, W_hh)
gW_xh
assert np.allclose(dW_xh, gW_xh)<jupyter_output><empty_output><jupyter_text>**W_hh check**<jupyter_code>_, dW_hh = back(train_x, train_y, W_xh, W_hh)
dW_hh
_, gW_hh = ngrad(train_x, train_y, W_xh, W_hh)
gW_hh
assert np.allclose(dW_hh, gW_hh)<jupyter_output><empty_output><jupyter_text>**Train both check**<jupyter_code>W_hh = np.array([[0.95]], dtype=np.float64)  # correct answer is W_hh == [[1]]
W_xh = np.array([[0.95]], dtype=np.float64)  # correct answer is W_xh == [[1]]

W_hh = np.random.normal(0.0, 1.0, [1,1])
W_xh = np.random.normal(0.0, 1.0, [1,1])

print('W_xh', W_xh)
print('W_hh', W_hh)
total_iter = 0
log_iter = []
log_loss = []
lr = 0.01
for i in range(500):
    dW_xh, dW_hh = back(train_x, train_y, W_xh, W_hh)
    W_xh += -lr * dW_xh
    W_hh += -lr * dW_hh
    
    loss = mse(train_x, train_y, W_xh, W_hh)
    
    log_iter.append(total_iter)
    log_loss.append(loss)
    total_iter += 1
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(log_iter, log_loss)
plt.show()
fwd(train_x, W_xh, W_hh)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1655759162303665, 'fraction_numerical': 0.037303664921465966, 'mean_word_length': 2.349159970781592, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '294295', 'license': 'MIT', 'n_tokens_mistral': 2164, 'n_tokens_neox': 2108, 'n_words': 431, 'path': 'Udacity_DL_Nanodegree/031 RNN Super Basics/RNN Super Basics.ipynb', 'repo_name': 'marcinbogdanski/ai-sketchpad', 'size': '4584'}