<jupyter_start><jupyter_text>Text Sentiment Classification: Using Recurrent Neural Networks with Self-Attention<jupyter_code>import d2l
import mxnet as mx
from mxnet import gluon, init, nd
from mxnet.gluon import data as gdata, loss as gloss, nn, rnn, utils as gutils
import gluonnlp as nlp
import os
import tarfile<jupyter_output><empty_output><jupyter_text>Text Sentiment Classification Data  Reading Data<jupyter_code>vocab, train_iter, test_iter = d2l.load_data_imdb(batch_size=64)<jupyter_output><empty_output><jupyter_text>Print the shape of the first mini-batch of data and the number of mini-batches in the training set.<jupyter_code>for X, y in train_iter:
    print('X', X.shape, 'y', y.shape)
    break
'# batches:', len(train_iter)<jupyter_output>X (64, 500) y (64,)<jupyter_text>Use a Recurrent Neural Network Model with Self Attention<jupyter_code>class SelfAttention(nn.HybridBlock):
    def __init__(self, num_atention_units, num_attention_channels, **kwargs):
        super(SelfAttention, self).__init__(**kwargs)
        with self.name_scope():
            self.proj_query = nn.Dense(num_atention_units, activation='tanh', flatten=False)
            self.parametric_key = nn.Dense(num_attention_channels, activation=None, flatten=False)

    def hybrid_forward(self, F, query):
        # query shape: [batch_size, seq_len, embedding_width]
        # projected query shape: [batch_size, seq_len, num_atention_units]
        query = self.proj_query(query)
        # scores shape: [batch_size, seq_len, attention_channels]
        scores = self.parametric_key(query)

        # attention_weights shape: [batch_size,  att_hops, seq_len]
        attention_weights = F.softmax(F.transpose(scores, axes=(0, 2, 1)), axis=-1)
        # output shape [batch_size, att_hops, embedding_width]
        output = F.batch_dot(attention_weights, query)

        return output, attention_weights
class AttentiveBiLSTM(nn.HybridBlock):
    """Lin et al.: A Structured Self-Attentive Sentence Embedding. ICLR 2017"""
    def __init__(self, vocab_len, embed_size, num_hiddens, num_layers,
                 num_attention_units, num_attention_channels, **kwargs):
        super(AttentiveBiLSTM, self).__init__(**kwargs)
        with self.name_scope():
            self.embedding = nn.Embedding(vocab_len, embed_size)
            self.encoder = rnn.LSTM(num_hiddens, num_layers=num_layers, bidirectional=True)
            self.attention = SelfAttention(num_attention_units, num_attention_channels)
            self.decoder = nn.Dense(2)

    def hybrid_forward(self, F, inputs):
        # The shape of inputs is (batch size, number of words). Because LSTM
        # needs to use sequence as the first dimension, the input is
        # transformed and the word feature is then extracted. The output shape
        # is (number of words, batch size, word vector dimension).
        embeddings = self.embedding(F.transpose(inputs))
        # The shape of states is (number of words, batch size, 2 * number of
        # hidden units).
        states = self.encoder(embeddings)
        context_vec, att_weights = self.attention(F.transpose(states, (1, 0, 2)))
        
        outputs = self.decoder(F.flatten(context_vec))
        return outputs<jupyter_output><empty_output><jupyter_text>Create the model.<jupyter_code>embed_size, num_hiddens, num_layers, ctx = 100, 100, 2, d2l.try_all_gpus()
natt_unit, natt_channel = 500, 2

net = AttentiveBiLSTM(len(vocab), embed_size, num_hiddens, num_layers,
                            natt_unit, natt_channel)
net.initialize(init.Xavier(), ctx=ctx)
net.hybridize()
net<jupyter_output><empty_output><jupyter_text>Load Pre-trained Word Vectors<jupyter_code>glove_embedding = nlp.embedding.create('glove', source='glove.6B.100d')
idx_to_vec = glove_embedding[vocab.idx_to_token]
idx_to_vec.shape<jupyter_output><empty_output><jupyter_text>Use these word vectors as feature vectors for each word in the reviews.<jupyter_code>net.embedding.weight.set_data(idx_to_vec)
net.embedding.collect_params().setattr('grad_req', 'null')<jupyter_output><empty_output><jupyter_text>Train and Evaluate the Model<jupyter_code>lr, num_epochs = 0.01, 5
trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr})
loss = gloss.SoftmaxCrossEntropyLoss()
d2l.train(train_iter, test_iter, net, loss, trainer, ctx, num_epochs)<jupyter_output>training on [gpu(0)]
epoch 1, loss 0.4950, train acc 0.751, test acc 0.831, time 42.9 sec
epoch 2, loss 0.3545, train acc 0.846, test acc 0.854, time 42.8 sec
epoch 3, loss 0.3073, train acc 0.870, test acc 0.868, time 42.7 sec
epoch 4, loss 0.2863, train acc 0.880, test acc 0.872, time 42.5 sec
epoch 5, loss 0.2465, train acc 0.899, test acc 0.873, time 42.8 sec<jupyter_text>Define the prediction function.<jupyter_code>def predict_sentiment(net, vocab, sentence):
    sentence = nd.array(vocab[sentence.split()], ctx=d2l.try_gpu())
    label = nd.argmax(net(sentence.reshape((1, -1))), axis=1)
    return 'positive' if label.asscalar() == 1 else 'negative'<jupyter_output><empty_output><jupyter_text>Then, use the trained model to classify the sentiments of two simple sentences.<jupyter_code>predict_sentiment(net, vocab, 'this movie is so great')
predict_sentiment(net, vocab, 'this movie is so bad')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10564044519901905, 'fraction_numerical': 0.02565553669119034, 'mean_word_length': 4.546025104602511, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '288066', 'license': 'Apache-2.0', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1773, 'n_words': 446, 'path': '07_attention/sentiment-analysis-attention.ipynb', 'repo_name': 'eric-haibin-lin/JSALT19-GluonNLP', 'size': '5301'}