<jupyter_start><jupyter_text>自注意力和位置编码<jupyter_code>import sys
sys.path.append('..')
import math
import mindspore
import numpy as np
import mindspore.numpy as mnp
import mindspore.nn as nn
from d2l import mindspore as d2l<jupyter_output><empty_output><jupyter_text>自注意力<jupyter_code>num_hiddens, num_heads = 100, 5
attention = d2l.MultiHeadAttention(num_hiddens, num_hiddens, num_hiddens,
                                   num_hiddens, num_heads, 0.5)
attention.set_train(False)
batch_size, num_queries, valid_lens = 2, 4, mindspore.Tensor([3, 2], mindspore.int32)
X = mnp.ones((batch_size, num_queries, num_hiddens))
attention(X, X, X, valid_lens)[0].shape<jupyter_output><empty_output><jupyter_text>位置编码<jupyter_code>class PositionalEncoding(nn.Cell):
    """位置编码"""
    def __init__(self, num_hiddens, dropout, max_len=1000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(1 - dropout)
        self.P = mnp.zeros((1, max_len, num_hiddens))
        X = mnp.arange(max_len, dtype=mindspore.float32).reshape(
            -1, 1) / mnp.power(mindspore.Tensor(10000, mindspore.int32), mnp.arange(
            0, num_hiddens, 2, dtype=mindspore.float32) / num_hiddens)
        self.P[:, :, 0::2] = mnp.sin(X)
        self.P[:, :, 1::2] = mnp.cos(X)

    def construct(self, X):
        X = X + self.P[:, :X.shape[1], :]
        return self.dropout(X)<jupyter_output><empty_output><jupyter_text>行代表词元在序列中的位置，列代表位置编码的不同维度<jupyter_code>encoding_dim, num_steps = 32, 60
pos_encoding = PositionalEncoding(encoding_dim, 0.)
pos_encoding.set_train(False)
X = pos_encoding(mnp.zeros((1, num_steps, encoding_dim)))
P = pos_encoding.P[:, :X.shape[1], :]
d2l.plot(mnp.arange(num_steps).asnumpy(), P[0, :, 6:10].T.asnumpy(), xlabel='Row (position)',
         figsize=(6, 2.5), legend=["Col %d" % d for d in np.arange(6, 10)])<jupyter_output>[WARNING] KERNEL(882468,7eff72988740,python):2021-11-24-12:17:57.010.249 [mindspore/ccsrc/backend/kernel_compiler/gpu/gpu_kernel_factory.cc:96] ReducePrecision] Kernel [TensorScatterUpdate] does not support int64, cast input 1 to int32.
[WARNING] PRE_ACT(882468,7eff72988740,python):2021-11-24-12:17:57.010.406 [mindspore/ccsrc/backend/optimizer/gpu/reduce_precision_fusion.cc:83] Run] Reduce precision for [TensorScatterUpdate] input 1
[WARNING] KERNEL(882468,7eff72988740,python):2021-11-24-12:17:57.022.569 [mindspore/ccsrc/backend/kernel_compiler/gpu/gpu_kernel_factory.cc:96] ReducePrecision] Kernel [TensorScatterUpdate] does not support int64, cast input 1 to int32.
[WARNING] PRE_ACT(882468,7eff72988740,python):2021-11-24-12:17:57.022.689 [mindspore/ccsrc/backend/optimizer/gpu/reduce_precision_fusion.cc:83] Run] Reduce precision for [TensorScatterUpdate] input 1<jupyter_text>二进制表示<jupyter_code>for i in range(8):
    print(f'{i} in binary is {i:>03b}')<jupyter_output>0 in binary is 000
1 in binary is 001
2 in binary is 010
3 in binary is 011
4 in binary is 100
5 in binary is 101
6 in binary is 110
7 in binary is 111<jupyter_text>在编码维度上降低频率<jupyter_code>P = mnp.expand_dims(mnp.expand_dims(P[0, :, :], 0), 0)
d2l.show_heatmaps(P, xlabel='Column (encoding dimension)',
                  ylabel='Row (position)', figsize=(3.5, 4), cmap='Blues')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13995693632728393, 'fraction_numerical': 0.08059058751153492, 'mean_word_length': 5.170777988614801, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '373938', 'license': 'MIT', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1328, 'n_words': 233, 'path': 'chapter_10_attention_mechanisms/4_self-attention-and-positional-encoding.ipynb', 'repo_name': 'lvyufeng/d2l-mindspore', 'size': '3251'}