<jupyter_start><jupyter_text>Weight Decay<jupyter_code>%matplotlib inline
import d2l
from mxnet import autograd, gluon, init, nd
from mxnet.gluon import data as gdata, loss as gloss, nn<jupyter_output><empty_output><jupyter_text>High-dimensional Linear Regression$$y = 0.05 + \sum_{i = 1}^d 0.01 x_i + \epsilon \text{ where }\epsilon \sim \mathcal{N}(0, 0.01)$$<jupyter_code>n_train, n_test, num_inputs = 20, 100, 200
true_w, true_b = nd.ones((num_inputs, 1)) * 0.01, 0.05

features = nd.random.normal(shape=(n_train + n_test, num_inputs))
labels = nd.dot(features, true_w) + true_b
labels += nd.random.normal(scale=0.01, shape=labels.shape)
train_features, test_features = features[:n_train, :], features[n_train:, :]
train_labels, test_labels = labels[:n_train], labels[n_train:]<jupyter_output><empty_output><jupyter_text>Define Training and Testing<jupyter_code>batch_size, num_epochs, lr = 1, 100, 0.003
train_iter = gdata.DataLoader(gdata.ArrayDataset(
    train_features, train_labels), batch_size, shuffle=True)<jupyter_output><empty_output><jupyter_text>Weight Decay in Gluon<jupyter_code>def fit_and_plot_gluon(wd):
    net = nn.Sequential()
    net.add(nn.Dense(1))
    net.initialize(init.Normal(sigma=1))
    # The weight parameter has been decayed. Weight names generally end with "weight".
    trainer_w = gluon.Trainer(net.collect_params('.*weight'), 'sgd',
                              {'learning_rate': lr, 'wd': wd})
    # The bias parameter has not decayed. Bias names generally end with "bias".
    trainer_b = gluon.Trainer(net.collect_params('.*bias'), 'sgd',
                              {'learning_rate': lr})
    train_ls, test_ls = [], []
    for _ in range(num_epochs):
        for X, y in train_iter:
            with autograd.record():
                l = loss(net(X), y)
            l.backward()
            # Call the step function on each of the two Trainer instances to update the weight and bias separately.
            trainer_w.step(batch_size)
            trainer_b.step(batch_size)
        evaluate = lambda X, y: loss(net(X), y).mean().asscalar()
        train_ls.append(evaluate(train_features, train_labels))
        test_ls.append(evaluate(test_features, test_labels))
    d2l.semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',
                range(1, num_epochs + 1), test_ls, ['train', 'test'], [8, 4])
    print('L2 norm of w:', net[0].weight.data().norm().asscalar())<jupyter_output><empty_output><jupyter_text>Using Weight Decay<jupyter_code>fit_and_plot_gluon(wd=0)<jupyter_output><empty_output><jupyter_text>Training without Regularization<jupyter_code>fit_and_plot_gluon(wd=5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13360778443113772, 'fraction_numerical': 0.01908682634730539, 'mean_word_length': 4.23091976516634, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '173653', 'license': 'Apache-2.0', 'n_tokens_mistral': 985, 'n_tokens_neox': 944, 'n_words': 193, 'path': '03_machine_learning/.ipynb_checkpoints/weight-decay-checkpoint.ipynb', 'repo_name': 'eric-haibin-lin/JSALT19-GluonNLP', 'size': '2672'}