<jupyter_start><jupyter_text>Keras 自定义层示意<jupyter_code>from keras import backend as K
from keras.engine.topology import Layer

class MyLayer(Layer):

    def __init__(self, output_dim, **kwargs):
        self.output_dim = output_dim
        super(MyLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        # 为该层创建一个可训练的权重
#         self.kernel = self.add_weight(name='kernel', 
#                                       shape=(input_shape[1], self.output_dim),
#                                       initializer='uniform',
#                                       trainable=True)
        self.kernel = self.add_weight(name='atteention_weight', 
                                      shape=(input_shape[1], self.output_dim),
                                      initializer='uniform',
                                      trainable=True)
        super(MyLayer, self).build(input_shape)  # 一定要在最后调用它

    def call(self, x):
        return K.dot(x, self.kernel)

    def compute_output_shape(self, input_shape):
        return (input_shape[0], self.output_dim)<jupyter_output><empty_output><jupyter_text>Attention layer<jupyter_code>def attention_3d_block(inputs, single_attention_vector=False):
    # 如果上一层是LSTM，需要return_sequences=True
    # inputs.shape = (batch_size, time_steps, input_dim) = (?, 5, 128)
    time_steps = np.shape(inputs)[1]
    input_dim = np.shape(inputs)[2]
    a = Permute((2, 1))(inputs) # shape = (?, 128, 5)
    a = Dense(time_steps, activation='softmax')(a) # shape = (?, 128, 5)
    if single_attention_vector:
        a = Lambda(lambda x: K.mean(x, axis=1))(a)
        a = RepeatVector(input_dim)(a)

    a_probs = Permute((2, 1))(a) # shape = (?, 5, 128)
    print('a_probs维度：',np.shape(a_probs))
    # 乘上了attention权重，但是并没有求和，好像影响不大
    # 如果分类任务，进行Flatten展开就可以了
    # element-wise
    output_attention_mul = Multiply()([inputs, a_probs]) # shape = (?, 5, 128)
    return output_attention_mul
class AttentionLayer(Layer):
    def __init__(self, **kwargs):
        super(AttentionLayer, self).__init__(** kwargs)

    def build(self, input_shape):
        assert len(input_shape)==3
        # W.shape = (time_steps, time_steps)
        self.W = self.add_weight(name='att_weight', 
                                 shape=(input_shape[1], input_shape[1]),
                                 initializer='uniform',
                                 trainable=True)
        self.b = self.add_weight(name='att_bias', 
                                 shape=(input_shape[1],),
                                 initializer='uniform',
                                 trainable=True)
        super(AttentionLayer, self).build(input_shape)

    def call(self, inputs):
        # inputs.shape = (batch_size, time_steps, seq_len)
        x = K.permute_dimensions(inputs, (0, 2, 1))
        # x.shape = (batch_size, seq_len, time_steps)
        a = K.softmax(K.tanh(K.dot(x, self.W) + self.b))
        outputs = K.permute_dimensions(a * x, (0, 2, 1))
        outputs = K.sum(outputs, axis=1)
        return outputs

    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[2]
# Attention GRU network       
class AttLayer(Layer):
    def __init__(self, **kwargs):
        self.init = initializers.get('normal')
        #self.input_spec = [InputSpec(ndim=3)]
        super(AttLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape)==3
        #self.W = self.init((input_shape[-1],1))
        self.W = self.init((input_shape[-1],))
        #self.input_spec = [InputSpec(shape=input_shape)]
        self.trainable_weights = [self.W]
        super(AttLayer, self).build(input_shape)  # be sure you call this somewhere!

    def call(self, x, mask=None):
        eij = K.tanh(K.dot(x, self.W))
        
        ai = K.exp(eij)
        weights = ai/K.sum(ai, axis=1).dimshuffle(0,'x')
        
        weighted_input = x*weights.dimshuffle(0,1,'x')
        return weighted_input.sum(axis=1)

    def get_output_shape_for(self, input_shape):
        return (input_shape[0], input_shape[-1])
encoder_input = Input(shape=(n_step, n_class)) # [batch_size, n_step, n_lcass]
decoder_input = Input(shape=(n_step, n_class))

encoder = LSTM(n_hidden, return_sequences=True, return_state=True)
# dropout
encoder_output, state_h, state_c = encoder(encoder_input)
encoder_state = [state_h, state_c]

decoder = LSTM(n_hidden, return_sequences=True)
decoder_output = decoder(decoder_input, initial_state = encoder_state)

# attention = attention_3d_block(())
attention_output = attention_3d_block(decoder_output)

dense = Dense(n_class, activation='softmax')
outputs = dense(attention_output)

model = Model([encoder_input, decoder_input], outputs)
model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['acc'])
model.fit([input_batch, output_batch], target_batch, epochs=100)
predict_batch = [np.eye(n_class)[[word_dict[n] for n in 'P P P P P'.split()]]]
result = model.predict([input_batch, np.array(predict_batch)])
result = np.argmax(result, axis=2)
print(sentences[0].split(), '->', [number_dict[n] for n in result[0]])<jupyter_output>['ich', 'mochte', 'ein', 'bier', 'P'] -> ['i', 'want', 'a', 'beer', 'E']
{'fraction_non_alphanumeric': 0.1369548584544759, 'fraction_numerical': 0.01224177505738332, 'mean_word_length': 2.867603550295858, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '617223', 'license': 'MIT', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1766, 'n_words': 307, 'path': '4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Keras.ipynb', 'repo_name': 'nocater/nlp-tutorial', 'size': '5228'}