def attention(inputs, attention_size, l2_reg_lambda):
    '\n    Attention mechanism layer.\n    :param inputs: outputs of RNN/Bi-RNN layer (not final state)\n    :param attention_size: linear size of attention weights\n    :return: outputs of the passed RNN/Bi-RNN reduced with attention vector\n    '
    if isinstance(inputs, tuple):
        inputs = tf.concat(2, inputs)
    sequence_length = inputs.get_shape()[1].value
    hidden_size = inputs.get_shape()[2].value
    W_omega = tf.get_variable('W_omega', initializer=tf.random_normal([hidden_size, attention_size], stddev=0.1))
    b_omega = tf.get_variable('b_omega', initializer=tf.random_normal([attention_size], stddev=0.1))
    u_omega = tf.get_variable('u_omega', initializer=tf.random_normal([attention_size], stddev=0.1))
    v = tf.tanh((tf.matmul(tf.reshape(inputs, [(- 1), hidden_size]), W_omega) + tf.reshape(b_omega, [1, (- 1)])))
    vu = tf.matmul(v, tf.reshape(u_omega, [(- 1), 1]))
    exps = tf.reshape(tf.exp(vu), [(- 1), sequence_length])
    alphas = (exps / tf.reshape(tf.reduce_sum(exps, 1), [(- 1), 1]))
    output = tf.reduce_sum((inputs * tf.reshape(alphas, [(- 1), sequence_length, 1])), 1)
    return output
Attention mechanism layer.
:param inputs: outputs of RNN/Bi-RNN layer (not final state)
:param attention_size: linear size of attention weights
:return: outputs of the passed RNN/Bi-RNN reduced with attention vector
def attention(inputs, attention_size, l2_reg_lambda):
    '\n    Attention mechanism layer.\n    :param inputs: outputs of RNN/Bi-RNN layer (not final state)\n    :param attention_size: linear size of attention weights\n    :return: outputs of the passed RNN/Bi-RNN reduced with attention vector\n    '
    if isinstance(inputs, tuple):
        inputs = tf.concat(2, inputs)
    sequence_length = inputs.get_shape()[1].value
    hidden_size = inputs.get_shape()[2].value
    W_omega = tf.get_variable('W_omega', initializer=tf.random_normal([hidden_size, attention_size], stddev=0.1))
    b_omega = tf.get_variable('b_omega', initializer=tf.random_normal([attention_size], stddev=0.1))
    u_omega = tf.get_variable('u_omega', initializer=tf.random_normal([attention_size], stddev=0.1))
    v = tf.tanh((tf.matmul(tf.reshape(inputs, [(- 1), hidden_size]), W_omega) + tf.reshape(b_omega, [1, (- 1)])))
    vu = tf.matmul(v, tf.reshape(u_omega, [(- 1), 1]))
    exps = tf.reshape(tf.exp(vu), [(- 1), sequence_length])
    alphas = (exps / tf.reshape(tf.reduce_sum(exps, 1), [(- 1), 1]))
    output = tf.reduce_sum((inputs * tf.reshape(alphas, [(- 1), sequence_length, 1])), 1)
    return output