<jupyter_start><jupyter_text>Variables<jupyter_code>from __future__ import print_function
from mxnet import nd, autograd
import numpy as np
import matplotlib
%matplotlib inline
import matplotlib.pyplot as plt
from datetime import date
date.today()
author = "lambdaofgod. based on https://github.com/Kyubyong/pytorch_exercises"
mx.__version__
np.__version__<jupyter_output><empty_output><jupyter_text>NOTE on notation    _x, _y, _z, ...: NumPy 0-d or 1-d arrays    _X, _Y, _Z, ...: NumPy 2-d or higer dimensional arrays    x, y, z, ...: 0-d or 1-d tensors    X, Y, Z, ...: 2-d or higher dimensional tensors  Variables  Q0. Create a variable `X` of the size (3, 2), filled with 1's.<jupyter_code>X = nd.ones(shape=(3, 2))
print(X)<jupyter_output>[[ 1.  1.]
 [ 1.  1.]
 [ 1.  1.]]
<NDArray 3x2 @cpu(0)><jupyter_text>Q1. Complete the code.<jupyter_code># Create a trainable variable `w` of scalar 10.
w = ??? 
w.???()

gs, ws, grads = [], [], []
for i in range(10):
    with autograd.record():
      y = w ** 2 + 2
    
    # apply backpropagation to y.
    y.???
    
    w[:] -= 0.01 * w.grad
    
    gs.append(i)
    ws.append(w[0].asscalar())
    grads.append(w.grad[0].asscalar())

plt.figure(figsize=(15,6))
ax=plt.subplot(1, 2, 1)
ax.scatter(gs, ws, c="b", label="w")
ax.legend(loc="upper right")
ax=plt.subplot(1, 2, 2)
plt.scatter(gs, grads, c="r", label="gradient")
plt.legend(loc="upper left")
plt.show()<jupyter_output><empty_output><jupyter_text>Q3. Complete the code. This is adapted from `http://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_autograd.html`.<jupyter_code># Untrainable variables
# Create untrainable variables X and Y.
X = nd.random_normal(shape=(64, 1000))
Y = nd.random_normal(shape=(64, 10))
X.attach_grad()
Y.attach_grad()

# Trainable Variables
w1 = nd.random_normal(shape=(1000, 100))
w2 = nd.random_normal(shape=(100, 10))
w1.attach_grad()
w2.attach_grad()

# Parameters
n_epochs = 500
learning_rate = 1e-6
log_interval = 10

losses = []
for t in range(n_epochs):
    with autograd.record():
      Y_pred = (
        nd.dot(
          nd.dot(X, w1)
            .clip(a_min=0, a_max=np.inf),
        w2)
      )

      # Apply L2 loss to Y pred and Y.
      loss = ???
    
    losses.append(loss[0].asscalar())

    loss.backward()
    
    # Apply gradient descent updates
    w1 -= learning_rate * ???
    w2 -= learning_rate * ???

    if (t + 1) % log_interval == 0:
        print("Epoch {:03d}/{:03d}: loss {:.5f}".format(
                t + 1, n_epochs, loss[0].asscalar()))

# plot
plt.figure()
plt.plot(losses, label="loss")
plt.legend()
plt.show()<jupyter_output>Epoch 010/500: loss 1685365.62500
Epoch 020/500: loss 219796.64062
Epoch 030/500: loss 65765.21875
Epoch 040/500: loss 24334.04492
Epoch 050/500: loss 10217.20898
Epoch 060/500: loss 4673.43701
Epoch 070/500: loss 2283.03809
Epoch 080/500: loss 1174.62915
Epoch 090/500: loss 630.70221
Epoch 100/500: loss 350.97342
Epoch 110/500: loss 201.12633
Epoch 120/500: loss 118.22195
Epoch 130/500: loss 71.00894
Epoch 140/500: loss 43.42177
Epoch 150/500: loss 26.96246
Epoch 160/500: loss 16.96135
Epoch 170/500: loss 10.79233
Epoch 180/500: loss 6.93480
Epoch 190/500: loss 4.49499
Epoch 200/500: loss 2.93557
Epoch 210/500: loss 1.93008
Epoch 220/500: loss 1.27650
Epoch 230/500: loss 0.84876
Epoch 240/500: loss 0.56709
Epoch 250/500: loss 0.38045
Epoch 260/500: loss 0.25627
Epoch 270/500: loss 0.17317
Epoch 280/500: loss 0.11744
Epoch 290/500: loss 0.07987
Epoch 300/500: loss 0.05447
Epoch 310/500: loss 0.03726
Epoch 320/500: loss 0.02558
Epoch 330/500: loss 0.01763
Epoch 340/500: loss 0.01220
Epo[...]
{'fraction_non_alphanumeric': 0.14683195592286502, 'fraction_numerical': 0.15013774104683195, 'mean_word_length': 3.5217932752179326, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '476674', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1468, 'n_words': 379, 'path': 'Variables.ipynb', 'repo_name': 'lambdaofgod/mxnet-exercises', 'size': '3630'}