<jupyter_start><jupyter_text>多 GPU 训练<jupyter_code>import d2l
from mxnet import autograd, gluon, init, np, npx
from mxnet.gluon import nn
npx.set_np()<jupyter_output><empty_output><jupyter_text>在多 GPU 上初始模型参数。<jupyter_code>net = d2l.resnet18(10)
ctx = d2l.try_all_gpus()
net.initialize(init=init.Normal(sigma=0.01), ctx=ctx)<jupyter_output><empty_output><jupyter_text>验证。<jupyter_code>x = np.random.uniform(size=(4, 1, 28, 28))
gpu_x = gluon.utils.split_and_load(x, ctx)
net(gpu_x[0]), net(gpu_x[1])<jupyter_output><empty_output><jupyter_text>在多 GPU 上计算精度。<jupyter_code>def evaluate_accuracy_gpus(net, data_iter):
    # 查看所在的所有设备
    ctx_list = list(net.collect_params().values())[0].list_ctx()
    metric = d2l.Accumulator(2)  # 分类正确的样本数，总样本数。
    for features, labels in data_iter:
        Xs, ys = d2l.split_batch(features, labels, ctx_list)
        pys = [net(X) for X in Xs]  # 并行执行。
        metric.add(sum(float(d2l.accuracy(py, y)) for py, y in zip(pys, ys)), 
                   labels.size)
    return metric[0]/metric[1]<jupyter_output><empty_output><jupyter_text>训练函数。<jupyter_code>def train(num_gpus, batch_size, lr):
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
    ctx_list = [d2l.try_gpu(i) for i in range(num_gpus)]
    net.initialize(init=init.Normal(sigma=0.01),
                   ctx=ctx_list, force_reinit=True)
    trainer = gluon.Trainer(
        net.collect_params(), 'sgd', {'learning_rate': lr})
    loss = gluon.loss.SoftmaxCrossEntropyLoss()
    timer, num_epochs = d2l.Timer(), 10
    animator = d2l.Animator('epoch', 'test acc', xlim=[1, num_epochs])
    for epoch in range(num_epochs):
        timer.start()
        for features, labels in train_iter:
            Xs, ys = d2l.split_batch(features, labels, ctx_list)
            with autograd.record():
                ls = [loss(net(X), y) for X, y in zip(Xs, ys)]
            for l in ls:
                l.backward()
            trainer.step(batch_size)
        npx.waitall()
        timer.stop()
        animator.add(epoch+1, (evaluate_accuracy_gpus(net, test_iter),))
    print('test acc: %.2f, %.1f sec/epoch on %s' % (
        animator.Y[0][-1], timer.avg(), ctx_list))<jupyter_output><empty_output><jupyter_text>使用一个 GPU。<jupyter_code>train(num_gpus=1, batch_size=256, lr=0.1)<jupyter_output>test acc: 0.93, 13.2 sec/epoch on [gpu(0)]<jupyter_text>使用两个 GPU。<jupyter_code>train(num_gpus=2, batch_size=512, lr=0.2)<jupyter_output>test acc: 0.92, 6.8 sec/epoch on [gpu(0), gpu(1)]
{'fraction_non_alphanumeric': 0.1474487746082764, 'fraction_numerical': 0.02691844114102049, 'mean_word_length': 3.970059880239521, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '387476', 'license': 'Apache-2.0', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1061, 'n_words': 170, 'path': 'notebooks-3/2-multiple-gpus.ipynb', 'repo_name': 'jhlholly/d2l-1day-notebooks-zh', 'size': '2489'}