<jupyter_start><jupyter_text>TensorFlow Attention Layer Referencehttps://www.tensorflow.org/api_docs/python/tf/keras/layers/Attention<jupyter_code>import numpy as np
import tensorflow as tf
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn
import matplotlib.pyplot as plt
inputs = tf.random.normal([32, 10, 8])
lstm = tf.keras.layers.LSTM(4)
output = lstm(inputs)
print(output.shape)<jupyter_output>(32, 4)<jupyter_text>Args:      query: Query tensor of shape `[batch_size, Tq, dim]`.            key: Key tensor of shape `[batch_size, Tv, dim]`.Output shape:      Attention outputs of shape `[batch_size, Tq, dim]`.<jupyter_code>batch_size = 6
Tq = Tv = 10
# base_vector = np.arange(0, Tq)
base_vector = np.linspace(1, 100, Tq)
a = np.asarray(batch_size * [base_vector])
b = np.zeros((batch_size, Tv))
for i in range(batch_size):
    b[i, :] = np.concatenate([base_vector[i:], base_vector[0:i]])
print(b.shape)
print(b)
dim = 1
query_seq_encoding = tf.constant(np.expand_dims(a, axis=-1), dtype=tf.float32)
value_seq_encoding = tf.constant(np.expand_dims(b, axis=-1), dtype=tf.float32)
query_value_attention_seq = tf.keras.layers.Attention()([query_seq_encoding, value_seq_encoding])
scores = math_ops.matmul(query_seq_encoding, value_seq_encoding, transpose_b=True)
plt.matshow(scores[0,:])
plt.matshow(scores[1,:])
plt.matshow(scores[2,:])
plt.matshow(scores[3,:])
weights = nn.softmax(scores)
plt.matshow(weights[0,:])
plt.matshow(weights[1,:])
plt.matshow(weights[2,:])
plt.matshow(weights[3,:])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1539448364336113, 'fraction_numerical': 0.021167415009621552, 'mean_word_length': 6.572815533980583, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '451635', 'license': 'MIT', 'n_tokens_mistral': 636, 'n_tokens_neox': 607, 'n_words': 110, 'path': 'keras_basics/tf.keras.layers.Attention_test.ipynb', 'repo_name': 'KellyHwong/deeplearning-basics', 'size': '1559'}