<jupyter_start><jupyter_text>权重衰减权重衰减是最广泛使用的正则化的技术之一<jupyter_code>import sys
sys.path.append('..')
%matplotlib inline
import mindspore
import numpy as np
from mindspore import nn
import mindspore.ops as ops
from d2l import mindspore as d2l<jupyter_output><empty_output><jupyter_text>像以前一样生成一些数据$$y = 0.05 + \sum_{i = 1}^d 0.01 x_i + \epsilon \text{ where }\epsilon \sim \mathcal{N}(0, 0.01^2)$$<jupyter_code>n_train, n_test, num_inputs, batch_size = 20, 100, 200, 5
true_w, true_b = np.ones((num_inputs, 1)) * 0.01, 0.05
train_data = d2l.synthetic_data(true_w, true_b, n_train)
train_dataset = d2l.load_array(train_data, batch_size)
test_data = d2l.synthetic_data(true_w, true_b, n_test)
test_dataset = d2l.load_array(test_data, batch_size, is_train=False)<jupyter_output><empty_output><jupyter_text>定义$L_2$范数惩罚<jupyter_code>def l2_penalty(w):
    return ops.Pow()(w, 2).sum() / 2<jupyter_output><empty_output><jupyter_text>定义模型<jupyter_code>from mindspore import Parameter
from mindspore.common.initializer import initializer, Normal, Zero

class Net(nn.Cell):
    def __init__(self, num_inputs, lambd=0):
        super().__init__()
        self.w = Parameter(initializer(Normal(1, 0), (num_inputs, 1), mindspore.float32))
        self.b = Parameter(initializer(Zero(), (1,), mindspore.float32))
        self.lambd = lambd

    def construct(self, x, y):
        y_hat = d2l.linreg(x, self.w, self.b)
        loss = d2l.squared_loss(y_hat, y)
        if self.training:
            loss += self.lambd * l2_penalty(self.w)
        return loss<jupyter_output><empty_output><jupyter_text>定义训练代码实现<jupyter_code>import mindspore.numpy as mnp

def train(lambd):
    num_epochs, lr = 100, 0.003
    net = Net(num_inputs, lambd)
    net.set_train()
    optim = d2l.SGD(lr, batch_size, net.trainable_params())
    train = d2l.Train(net, optim)
    animator = d2l.Animator(xlabel='epochs', ylabel='loss', yscale='log',
                            xlim=[5, num_epochs], legend=['train', 'test'])
    for epoch in range(num_epochs):
        for X, y in train_dataset.create_tuple_iterator():
            loss = train(X, y)
        if (epoch + 1) % 5 == 0:
            animator.add(epoch + 1, (d2l.evaluate_loss(net, train_dataset),
                                     d2l.evaluate_loss(net, test_dataset)))
    print('w的L2范数是：', mnp.norm(net.w).asnumpy())<jupyter_output><empty_output><jupyter_text>忽略正则化直接训练<jupyter_code>from mindspore import context
train(lambd=0)<jupyter_output>w的L2范数是： 12.237418<jupyter_text>使用权重衰减<jupyter_code>train(lambd=3)<jupyter_output>w的L2范数是： 0.38045686<jupyter_text>简洁实现<jupyter_code>def train_concise(wd):
    num_epochs, lr = 100, 0.003
    net = nn.SequentialCell([nn.Dense(num_inputs, 1, weight_init=Normal(1, 0))])
    loss = nn.MSELoss()
    net_with_loss = nn.WithLossCell(net, loss)
    # mindspore的SGD不支持对单个parameter做weight_decay
    optim = nn.SGD(net.trainable_params(), learning_rate=lr, weight_decay=wd)
    train = nn.TrainOneStepCell(net_with_loss, optim)
    animator = d2l.Animator(xlabel='epochs', ylabel='loss', yscale='log',
                            xlim=[5, num_epochs], legend=['train', 'test'])
    for epoch in range(num_epochs):
        for X, y in train_dataset.create_tuple_iterator():
            l = train(X, y)
        if (epoch + 1) % 5 == 0:
            animator.add(epoch + 1, (d2l.evaluate_loss(net_with_loss, train_dataset),
                                     d2l.evaluate_loss(net_with_loss, test_dataset)))
    print('w的L2范数：', mnp.norm(net[0].weight).asnumpy())<jupyter_output><empty_output><jupyter_text>这些图看起来和我们从零开始实现权重衰减时的图相同<jupyter_code>train_concise(0)
train_concise(3)<jupyter_output>w的L2范数： 0.35746962
{'fraction_non_alphanumeric': 0.13340593520283148, 'fraction_numerical': 0.032943098284780835, 'mean_word_length': 4.095700416088766, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '559299', 'license': 'MIT', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1543, 'n_words': 233, 'path': 'chapter_04_multilayer-perceptrons/4_weight-decay.ipynb', 'repo_name': 'lvyufeng/d2l-mindspore', 'size': '3673'}