<jupyter_start><jupyter_text>序列逆置 （加注意力的seq2seq）使用attentive sequence to sequence 模型将一个字符串序列逆置。例如 `OIMESIQFIQ` 逆置成 `QIFQISEMIO`(下图来自网络，是一个加attentino的sequence to sequence 模型示意图)![attentive seq2seq](./seq2seq-attn.jpg)<jupyter_code>import numpy as np
import tensorflow as tf
import collections
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import layers, optimizers, datasets
import os,sys,tqdm<jupyter_output><empty_output><jupyter_text>玩具序列数据生成生成只包含[A-Z]的字符串，并且将encoder输入以及decoder输入以及decoder输出准备好（转成index）<jupyter_code>import random
import string

def randomString(stringLength):
    """Generate a random string with the combination of lowercase and uppercase letters """

    letters = string.ascii_uppercase
    return ''.join(random.choice(letters) for i in range(stringLength))

def get_batch(batch_size, length):
    batched_examples = [randomString(length) for i in range(batch_size)]
    enc_x = [[ord(ch)-ord('A')+1 for ch in list(exp)] for exp in batched_examples]
    y = [[o for o in reversed(e_idx)] for e_idx in enc_x]
    dec_x = [[0]+e_idx[:-1] for e_idx in y]
    return (batched_examples, tf.constant(enc_x, dtype=tf.int32), 
            tf.constant(dec_x, dtype=tf.int32), tf.constant(y, dtype=tf.int32))
print(get_batch(2, 10))<jupyter_output>(['GLSUIOUHPB', 'QLXIKLZQBX'], <tf.Tensor: id=25627, shape=(2, 10), dtype=int32, numpy=
array([[ 7, 12, 19, 21,  9, 15, 21,  8, 16,  2],
       [17, 12, 24,  9, 11, 12, 26, 17,  2, 24]], dtype=int32)>, <tf.Tensor: id=25628, shape=(2, 10), dtype=int32, numpy=
array([[ 0,  2, 16,  8, 21, 15,  9, 21, 19, 12],
       [ 0, 24,  2, 17, 26, 12, 11,  9, 24, 12]], dtype=int32)>, <tf.Tensor: id=25629, shape=(2, 10), dtype=int32, numpy=
array([[ 2, 16,  8, 21, 15,  9, 21, 19, 12,  7],
       [24,  2, 17, 26, 12, 11,  9, 24, 12, 17]], dtype=int32)>)<jupyter_text>建立sequence to sequence 模型完成两空，模型搭建以及单步解码逻辑<jupyter_code>class mySeq2SeqModel(keras.Model):
    def __init__(self):
        super(mySeq2SeqModel, self).__init__()
        self.v_sz=27
        self.hidden = 128
        self.embed_layer = tf.keras.layers.Embedding(self.v_sz, 64, 
                                                    batch_input_shape=[None, None])
        
        self.encoder_cell = tf.keras.layers.SimpleRNNCell(self.hidden)
        self.decoder_cell = tf.keras.layers.SimpleRNNCell(self.hidden)
        
        self.encoder = tf.keras.layers.RNN(self.encoder_cell, 
                                           return_sequences=True, return_state=True)
        self.decoder = tf.keras.layers.RNN(self.decoder_cell, 
                                           return_sequences=True, return_state=True)
        self.dense_attn = tf.keras.layers.Dense(self.hidden)
        self.dense = tf.keras.layers.Dense(self.v_sz)
        
        
    @tf.function
    def call(self, enc_ids, dec_ids):
        '''
        todo
        
        完成带attention机制的 sequence2sequence 模型的搭建，模块已经在`__init__`函数中定义好，
        用双线性attention，或者自己改一下`__init__`函数做加性attention
        '''
        return logits
    
    
    @tf.function
    def encode(self, enc_ids):
        enc_emb = self.embed_layer(enc_ids) # shape(b_sz, len, emb_sz)
        enc_out, enc_state = self.encoder(enc_emb)
        return enc_out, [enc_out[:, -1, :], enc_state]
    
    def get_next_token(self, x, state, enc_out):
        '''
        shape(x) = [b_sz,] 
        '''
    
        '''
        todo
        参考sequence_reversal-exercise, 自己构建单步解码逻辑'''
        return out, state<jupyter_output><empty_output><jupyter_text>Loss函数以及训练逻辑<jupyter_code>@tf.function
def compute_loss(logits, labels):
    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=labels)
    losses = tf.reduce_mean(losses)
    return losses

@tf.function
def train_one_step(model, optimizer, enc_x, dec_x, y):
    with tf.GradientTape() as tape:
        logits = model(enc_x, dec_x)
        loss = compute_loss(logits, y)

    # compute gradient
    grads = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(grads, model.trainable_variables))
    return loss

def train(model, optimizer, seqlen):
    loss = 0.0
    accuracy = 0.0
    for step in range(2000):
        batched_examples, enc_x, dec_x, y = get_batch(32, seqlen)
        loss = train_one_step(model, optimizer, enc_x, dec_x, y)
        if step % 500 == 0:
            print('step', step, ': loss', loss.numpy())
    return loss<jupyter_output><empty_output><jupyter_text>训练迭代<jupyter_code>optimizer = optimizers.Adam(0.0005)
model = mySeq2SeqModel()
train(model, optimizer, seqlen=20)<jupyter_output>step 0 : loss 3.3069763
step 500 : loss 1.3605067
step 1000 : loss 0.2817158
step 1500 : loss 0.105303034<jupyter_text>测试模型逆置能力首先要先对输入的一个字符串进行encode，然后在用decoder解码出逆置的字符串测试阶段跟训练阶段的区别在于，在训练的时候decoder的输入是给定的，而在预测的时候我们需要一步步生成下一步的decoder的输入<jupyter_code>def sequence_reversal():
    def decode(init_state, steps, enc_out):
        b_sz = tf.shape(init_state[0])[0]
        cur_token = tf.zeros(shape=[b_sz], dtype=tf.int32)
        state = init_state
        collect = []
        for i in range(steps):
            cur_token, state = model.get_next_token(cur_token, state, enc_out)
            collect.append(tf.expand_dims(cur_token, axis=-1))
        out = tf.concat(collect, axis=-1).numpy()
        out = [''.join([chr(idx+ord('A')-1) for idx in exp]) for exp in out]
        return out
    
    batched_examples, enc_x, _, _ = get_batch(32, 20)
    enc_out, state = model.encode(enc_x)
    return decode(state, enc_x.get_shape()[-1], enc_out), batched_examples

def is_reverse(seq, rev_seq):
    rev_seq_rev = ''.join([i for i in reversed(list(rev_seq))])
    if seq == rev_seq_rev:
        return True
    else:
        return False
print([is_reverse(*item) for item in list(zip(*sequence_reversal()))])
print(list(zip(*sequence_reversal())))<jupyter_output>[False, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, False, False, True, True, True, True, True, True, True]
[('LGNVPEUAIFNLDIRQPMEY', 'YEMPQRIDLNFIAUEPVNGL'), ('ZFKNSOQGHINBCDJMOQXO', 'OXQOMJDCBNIHGQOSNKFZ'), ('ZXGDWLBRJMDSNYXIEMKJ', 'JKMEIXYNSDMJRBLWDGXZ'), ('KSEOFXPRMGUBHSJWIGYC', 'CYGIWJSHBUGMRPXFOESK'), ('HHMACLLSVMESLETLYJLZ', 'ZLJYLTELSEMVSLLCAMHH'), ('PVIABJTRCEQZFSJZLKQP', 'PQKLZJSFZQECRTJBAIVP'), ('PTUYDMQCQKZWBNCJRZBG', 'GBZRJCNBWZKQCQMDYUTP'), ('ZFZEWFPTWQFFSNNQLOFW', 'WFOLQNNSFFQWTPFWEZFZ'), ('NAHVTGYNHHEXOKMOEDRN', 'NRDEOMKOXEHHNYGTVHAN'), ('CTDDCDXRMIQDKCHODXVU', 'UVXDOHCKDQIMRXDCDDTL'), ('QWMRORZQQQTSJEHNVSZN', 'NZSVNHEJSTQQQZRORMWQ'), ('HTLTKWDCPXYCOREQGOUA', 'AUOGQEROCYXPCDWKTLTH'), ('DIQEGEZTNCFGNHHCXEDE', 'EDEXCHHNGFCNTZEGEQIJ'), ('HLGMERZQBOYYDQWMKKVE', 'EVKKMWQDYYOBQZREMGLH'), ('MCUCLWJZZVBMJFTNMAMU', 'UMAMNTFJMBVZZJWLCUCY'), ('UJIKEJYXYBBUSNIDOMBL', 'LBMODINSUBBYXYJEKIJU'), ('[...]
{'fraction_non_alphanumeric': 0.13136146150500216, 'fraction_numerical': 0.0352327098738582, 'mean_word_length': 3.806968641114983, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '148717', 'license': 'Apache-2.0', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2844, 'n_words': 496, 'path': 'assignments/nndl-exercise/chap7-seq2seq-and-attention/sequence_reversal_with_attention-exercise.ipynb', 'repo_name': 'Cannizza-zzk/DL_review', 'size': '6897'}