<jupyter_start><jupyter_text>$$h_t = \tanh (Wh + Ux)$$$$h_t = ReLU (Ux)$$<jupyter_code>inputs = tf.random_normal([2, 4, 2])

outputs = model(inputs)
outputs.shape
outputs.numpy()
model.trainable_variables
opt = tf.train.AdamOptimizer()
for i in range(10):
  with tf.GradientTape() as tape:
    inputs = tf.random_normal([1, 4, 2])
    outputs = model(inputs)
    
    loss = tf.losses.mean_squared_error(labels=tf.ones(outputs.get_shape()),
                                        predictions=outputs)
  grad = tape.gradient(loss, model.variables)
  opt.apply_gradients(zip(grad, model.variables))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1467304625199362, 'fraction_numerical': 0.012759170653907496, 'mean_word_length': 3.868217054263566, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '390984', 'license': 'Apache-2.0', 'n_tokens_mistral': 234, 'n_tokens_neox': 221, 'n_words': 38, 'path': 'tf.version.2/04.rnn/00.rnn.custom.layers.implementation.ipynb', 'repo_name': 'ilguyi/tensorflow.tutorials', 'size': '627'}