<jupyter_start><jupyter_text>Model contructionSelf attention layer, weighted cross entropy, and whole model<jupyter_code># custom attention layer
# in this class, we want to implement the operation:
# softmax(W_2 * tanh(W_1 * H))
# where H is the word embedding of the whole sentence, of shape (num_of_word, embed_size)
class SelfAttention(nn.HybridBlock):
    def __init__(self, att_unit, att_hops, **kwargs):
        super(SelfAttention, self).__init__(**kwargs)
        with self.name_scope():
            # this layer is tanh(w_1 * H), the att_unit corresponds to d_a in the essay
            self.ut_dense = nn.Dense(att_unit, activation='relu', flatten=False)
            # this layer implements the multiple hops
            self.et_dense = nn.Dense(att_hops, activation=None, flatten=False)

    def hybrid_forward(self, F, x): # F is the backend which implements the tensor operation
        # x shape: [batch_size, seq_len, embedding_width]
        # ut shape: [batch_size, seq_len, att_unit]
        ut = self.ut_dense(x) # batch_size * seq_len [* embed_size * embed_size *] att_unit
        # et shape: [batch_size, seq_len, att_hops]
        et = self.et_dense(ut)# batch_size * seq_len [* att_unit * att_unit *] att_hops

        # att shape: [batch_size,  att_hops, seq_len]
        # softmax is performed along the seq_len dimension
        att = F.softmax(F.transpose(et, axes=(0, 2, 1)), axis=-1)
        # output shape [batch_size, att_hops, embedding_width]
        output = F.batch_dot(att, x)
        # output is the weighted matrix representation of the matrix
        # att is the weighted vector we use as attention
        return output, att
    
# d_a = 20, hops = 5
print(SelfAttention(20, 5))
class WeightedSoftmaxCE(nn.HybridBlock):
    def __init__(self, sparse_label=True, from_logits=False,  **kwargs):
        super(WeightedSoftmaxCE, self).__init__(**kwargs)
        with self.name_scope():
            self.sparse_label = sparse_label
            self.from_logits = from_logits

    def hybrid_forward(self, F, pred, label, class_weight, depth=None):
        if self.sparse_label:
            label = F.reshape(label, shape=(-1, ))
            label = F.one_hot(label, depth)
        if not self.from_logits:
            pred = F.log_softmax(pred, -1)

        weight_label = F.broadcast_mul(label, class_weight)
        loss = -F.sum(pred * weight_label, axis=-1)

        # return F.mean(loss, axis=0, exclude=True)
        return loss
class SelfAttentiveBiLSTM(nn.HybridBlock):
    def __init__(self, vocab_len, embsize, nhidden, nlayers, natt_unit, natt_hops, \
                 nfc, nclass, # these two params are not used currrently
                 drop_prob, pool_way, prune_p=None, prune_q=None, **kwargs):
        super(SelfAttentiveBiLSTM, self).__init__(**kwargs)
        with self.name_scope():
            # now we switch back to shared layers
            self.embedding_layer = nn.Embedding(vocab_len, embsize)
            
            self.bilstm = rnn.LSTM(nhidden, num_layers=nlayers, dropout=drop_prob, \
                                   bidirectional=True)
            
            self.att_encoder = SelfAttention(natt_unit, natt_hops)
            self.dense = nn.Dense(nfc, activation='relu')
            # this layer is used to output the final class
            self.output_layer = nn.Dense(nclass)
            
            self.dense_p, self.dense_q = None, None
            if all([prune_p, prune_q]):
                self.dense_p = nn.Dense(prune_p, activation='relu', flatten=False)
                self.dense_q = nn.Dense(prune_q, activation='relu', flatten=False)

            self.drop_prob = drop_prob
            self.pool_way = pool_way

    def hybrid_forward(self, F, inp):
        # inp_embed size: [batch, seq_len, embed_size]
        inp_embed = self.embedding_layer(inp)
        # rnn requires the first dimension to be the time steps
        h_output = self.bilstm(F.transpose(inp_embed, axes=(1, 0, 2)))
        # att_output: [batch, att_hops, emsize]
        output, att = self.att_encoder(F.transpose(h_output, axes=(1, 0, 2)))
        '''
        FIXME: now this code will only work with flatten
        '''
        dense_input = None
        if self.pool_way == 'flatten':
            dense_input = F.Dropout(F.flatten(output), self.drop_prob)
        else:
            raise NotImplemented
        '''
        elif self.pool_way == 'mean':
            dense_input = F.Dropout(F.mean(att_output, axis=1), self.drop_prob)
        elif self.pool_way == 'prune' and all([self.dense_p, self.dense_q]):
            # p_section: [batch, att_hops, prune_p]
            p_section = self.dense_p(att_output)
            # q_section: [batch, emsize, prune_q]
            q_section = self.dense_q(F.transpose(att_output, axes=(0, 2, 1)))
            dense_input = F.Dropout(F.concat(F.flatten(p_section), F.flatten(q_section), dim=-1), self.drop_prob)
        '''
        dense_out = self.dense(dense_input)
        output = self.output_layer(F.Dropout(dense_out, self.drop_prob))

        return output, att
vocab_len = len(vocab)
emsize = 300   # word embedding size
nhidden = 200    # lstm hidden_dim
nlayers = 1     # lstm layers
natt_unit = 200     # the hidden_units of attention layer
natt_hops = 10    # the channels of attention
nfc = 128  # last dense layer size
nclass = 72 # we have 72 emoji in total

drop_prob = 0.
pool_way = 'flatten'    # # The way to handle M
prune_p = None
prune_q = None


try:
    assert(False)
    model = gluon.nn.SymbolBlock.imports("model/model-symbol.json", ['data'], \
                                         "model/model-0001.params", ctx=ctx)
    print('use saved model params to start')
except:
    model = SelfAttentiveBiLSTM(vocab_len, emsize, nhidden, nlayers,
                            natt_unit, natt_hops, nfc, nclass,
                            drop_prob, pool_way, prune_p, prune_q)

    print('initialize a new model')
    model.initialize(init=init.Xavier(), ctx=ctx)
    model.hybridize()

    # Attach a pre-trained glove word vector to the embedding layer
    model.embedding_layer.weight.set_data(vocab.embedding.idx_to_vec)
    # fixed the embedding layer
    model.embedding_layer.collect_params().setattr('grad_req', 'null')

print(ctx, model)
vocab.embedding.idx_to_vec[vocab.embedding.token_to_idx['i']]<jupyter_output><empty_output><jupyter_text>Training helpersCalculate loss, one epoch computation and top function for train and valid<jupyter_code>def calculate_loss(x, y, model, loss, class_weight, penal_coeff):
    pred, att = model(x)
    y = nd.array(y.asnumpy().astype('int32')).as_in_context(ctx)
    if loss_name in ['sce', 'l1', 'l2']:
        l = loss(pred, y)
    elif loss_name == 'wsce':
        l = loss(pred, y, class_weight, class_weight.shape[0])
    else:
        raise NotImplemented
    # penalty, now we have two att's
    diversity_penalty = nd.batch_dot(att, nd.transpose(att, axes=(0, 2, 1))) - \
                        nd.eye(att.shape[1], ctx=att.context)
    l = l + penal_coeff * diversity_penalty.norm(axis=(1, 2))

    return pred, l
def one_epoch(data_iter, model, loss, trainer, ctx, is_train, epoch,
              penal_coeff=0.0, clip=None, class_weight=None, loss_name='sce'):

    loss_val = 0.
    total_pred = []
    total_true = []
    n_batch = 0

    for batch_x, batch_y in data_iter:
        batch_x = batch_x.as_in_context(ctx)
        batch_y = batch_y.as_in_context(ctx)

        if is_train:
            with autograd.record():
                batch_pred, l = calculate_loss(batch_x, batch_y, model, \
                                               loss, class_weight, penal_coeff)

            # backward calculate
            l.backward()

            # clip gradient
            clip_params = [p.data() for p in model.collect_params().values()]
            if clip is not None:
                norm = nd.array([0.0], ctx)
                for param in clip_params:
                    if param.grad is not None:
                        norm += (param.grad ** 2).sum()
                norm = norm.sqrt().asscalar()
                if norm > clip:
                    for param in clip_params:
                        if param.grad is not None:
                            param.grad[:] *= clip / norm

            # update parmas
            trainer.step(batch_x.shape[0])

        else:
            batch_pred, l = calculate_loss(batch_x, batch_y, model, \
                                           loss, class_weight, penal_coeff)

        # keep result for metric
        batch_pred = nd.argmax(nd.softmax(batch_pred, axis=1), axis=1).asnumpy()
        batch_true = np.reshape(batch_y.asnumpy(), (-1, ))
        total_pred.extend(batch_pred.tolist())
        total_true.extend(batch_true.tolist())
        
        batch_loss = l.mean().asscalar()

        n_batch += 1
        loss_val += batch_loss

        # check the result of traing phase
        if is_train and n_batch % 400 == 0:
            print('epoch %d, batch %d, batch_train_loss %.4f, batch_train_acc %.3f' %
                  (epoch, n_batch, batch_loss, accuracy_score(batch_true, batch_pred)))

    # metric
    F1 = f1_score(np.array(total_true), np.array(total_pred), average='weighted')
    acc = accuracy_score(np.array(total_true), np.array(total_pred))
    loss_val /= n_batch

    if is_train:
        print('epoch %d, learning_rate %.5f \n\t train_loss %.4f, acc_train %.3f, F1_train %.3f, ' %
              (epoch, trainer.learning_rate, loss_val, acc, F1))
        # declay lr
        if epoch % 3 == 0:
            trainer.set_learning_rate(trainer.learning_rate * 0.9)
    else:
        print('\t valid_loss %.4f, acc_valid %.3f, F1_valid %.3f, ' % (loss_val, acc, F1))
def train_valid(data_iter_train, data_iter_valid, model, loss, trainer, ctx, nepochs,
                penal_coeff=0.0, clip=None, class_weight=None, loss_name='sce'):

    for epoch in range(1, nepochs+1):
        start = time.time()
        # train
        is_train = True
        one_epoch(data_iter_train, model, loss, trainer, ctx, is_train,
                  epoch, penal_coeff, clip, class_weight, loss_name)

        # valid
        is_train = False
        one_epoch(data_iter_valid, model, loss, trainer, ctx, is_train,
                  epoch, penal_coeff, clip, class_weight, loss_name)
        end = time.time()
        print('time %.2f sec' % (end-start))
        print("*"*100)<jupyter_output><empty_output><jupyter_text>TrainNow we will train this model. To handle data inbalance, we first set an estimated weight of the labels.<jupyter_code>from util import get_weight
weight_list = get_weight(DATA_FOLDER, LABEL_FILE)
# print(weight_list)
class_weight = None
loss_name = 'sce'
optim = 'adam'
lr = 0.001
penal_coeff = 0.0003
clip = .5
nepochs = 5

trainer = gluon.Trainer(model.collect_params(), optim, {'learning_rate': lr})

if loss_name == 'sce':
    loss = gluon.loss.SoftmaxCrossEntropyLoss()
elif loss_name == 'wsce':
    loss = WeightedSoftmaxCE()
    # the value of class_weight is obtained by counting data in advance. It can be seen as a hyperparameter.
    class_weight = nd.array(weight_list, ctx=ctx)
elif loss_name == 'l1':
    loss = gluon.loss.L1Loss()
elif loss_name == 'l2':
    loss = gluon.loss.L2Loss()
# train and valid
print(ctx)
train_valid(train_dataloader, valid_dataloader, model, loss, \
            trainer, ctx, nepochs, penal_coeff=penal_coeff, \
            clip=clip, class_weight=class_weight, loss_name=loss_name)
model.export("model/self-att-179", epoch=1)
TEST_DATA = 'test.csv'
test_df = pd.read_csv(DATA_FOLDER+TEST_DATA, header=None, sep='\t')

model = gluon.nn.SymbolBlock.imports("model/self-att-207-symbol.json", ['data'], \
                                     "model/self-att-207-0001.params", ctx=ctx)
predictions = []
start = time.time()
for _, tweet in test_df.iterrows():
    token = vocab[seg.cut(tweet[1])]
    if len(token)==0:
        token = [0.]
    inp = nd.array(token, ctx=ctx).reshape(1,-1)
    pred, _ = model(inp)
    pred = nd.argmax(pred, axis=1).asscalar()
    predictions.append(int(pred))
    if len(predictions)%2000==0:
        ckpt = time.time()
        print('current pred len %d, time %.2fs' % (len(predictions), ckpt-start))
        start = ckpt
submit = pd.DataFrame({'Expected': predictions})
submit.to_csv('submission.csv', sep=',', index_label='ID')<jupyter_output>current pred len 2000, time 9.44s
current pred len 4000, time 9.23s
current pred len 6000, time 9.37s
current pred len 8000, time 9.25s
current pred len 10000, time 9.29s
current pred len 12000, time 7.66s
current pred len 14000, time 6.21s
current pred len 16000, time 8.77s
current pred len 18000, time 9.16s
current pred len 20000, time 8.67s
current pred len 22000, time 9.03s
current pred len 24000, time 9.26s
current pred len 26000, time 9.28s
current pred len 28000, time 8.79s
current pred len 30000, time 9.34s
current pred len 32000, time 9.34s
current pred len 34000, time 9.42s
current pred len 36000, time 9.38s
current pred len 38000, time 9.30s
current pred len 40000, time 9.38s
current pred len 42000, time 9.17s
current pred len 44000, time 9.31s
current pred len 46000, time 9.31s
current pred len 48000, time 9.22s
current pred len 50000, time 9.25s
current pred len 52000, time 9.16s
current pred len 54000, time 9.40s
current pred len 56000, time 9.21s
current pred len 58000, [...]
{'fraction_non_alphanumeric': 0.11005890686749684, 'fraction_numerical': 0.02773842368205205, 'mean_word_length': 2.6219281663516067, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '529976', 'license': 'Apache-2.0', 'n_tokens_mistral': 4728, 'n_tokens_neox': 4390, 'n_words': 1141, 'path': 'self-att.ipynb', 'repo_name': 'zhiming-xu/weibo-emoji-predict', 'size': '13411'}