<jupyter_start><jupyter_text>https://www.tensorflow.org/guide/autodiff<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf<jupyter_output><empty_output><jupyter_text>Computing gradientsTo differentiate automatically, TensorFlow needs to remember what operations happen in what order during the forward pass. Then, during the backward pass, TensorFlow traverses this list of operations in reverse order to compute gradients.<jupyter_code>x = tf.Variable(4.0)

with tf.GradientTape() as tape:
    y = x**2
y
dy_dx = tape.gradient(y, x)

dy_dx
w = tf.Variable(tf.random.normal((4, 2)))

w
b = tf.Variable(tf.ones(2, dtype=tf.float32))

b
x = tf.Variable([[10., 20., 30., 40.]], dtype=tf.float32)

x
with tf.GradientTape(persistent=True) as tape:
    y = tf.matmul(x, w) + b
    
    loss = tf.reduce_mean(y**2)
[dl_dw, dl_db] = tape.gradient(loss, [w, b])<jupyter_output><empty_output><jupyter_text>The gradient with respect to each source has the shape of the source<jupyter_code>dl_dw
dl_db
layer = tf.keras.layers.Dense(2, activation='relu')

x = tf.constant([[10., 20., 30.]])
with tf.GradientTape() as tape:
    y = layer(x)
    
    loss = tf.reduce_sum(y**2)

grad = tape.gradient(loss, layer.trainable_variables)
grad
for var, g in zip(layer.trainable_variables, grad):
    print(f'{var.name}, shape: {g.shape}')<jupyter_output>dense/kernel:0, shape: (3, 2)
dense/bias:0, shape: (2,)<jupyter_text>Gradients are calculated only with respect to trainable variables Trainable variable, the value associated with this will be updated during the training process<jupyter_code>x1 = tf.Variable(5.0)

x1<jupyter_output><empty_output><jupyter_text>Trainable has been explicitly set to false<jupyter_code>x2 = tf.Variable(5.0, trainable=False)

x2<jupyter_output><empty_output><jupyter_text>Tensor, not a variable. Gradients are not calculated on Tensors<jupyter_code>x3 = tf.add(x1, x2)

x3
x4 = tf.constant(5.0)

x4
with tf.GradientTape() as tape:
    y = (x1**2) + (x2**2) + (x3**2) + (x4**2)

grad = tape.gradient(y, [x1, x2, x3, x4])

grad<jupyter_output><empty_output><jupyter_text>Watch constants to calculate gradients with respect to themtf.GradientTape provides hooks that give the user control over what is or is not watched. To record gradients with respect to a tf.Tensor, you need to call GradientTape.watch(x)<jupyter_code>x1 = tf.constant(5.0)

x2 = tf.Variable(3.0)
with tf.GradientTape() as tape:
    tape.watch(x1)
    
    y = (x1**2) + (x2**2)
[dy_dx1, dy_dx2] = tape.gradient(y, [x1, x2])

dy_dx1, dy_dx2
with tf.GradientTape(watch_accessed_variables=False) as tape:
    tape.watch(x1)
    
    y = (x1**2) + (x2**2)
[dy_dx1, dy_dx2] = tape.gradient(y, [x1, x2])

dy_dx1, dy_dx2<jupyter_output><empty_output><jupyter_text>Gradient tape records operations as they occurConditionals are naturally handled. The gradient only connects to the variable that was used.<jupyter_code>x = tf.constant(1.0)
x1 = tf.Variable(5.0)
x2 = tf.Variable(3.0)
with tf.GradientTape(persistent=True) as tape:
    tape.watch(x)

    if x > 0.0:
        result = x1**2
    else:
        result = x2**2 

dx1, dx2 = tape.gradient(result, [x1, x2])

dx1, dx2
x = tf.constant(-1.0)
x1 = tf.Variable(5.0)
x2 = tf.Variable(3.0)
with tf.GradientTape(persistent=True) as tape:
    tape.watch(x)

    if x > 0.0:
        result = x1**2
    else:
        result = x2**2 

dx1, dx2 = tape.gradient(result, [x1, x2])

dx1, dx2
x = tf.Variable(2.)
y = tf.Variable(3.)

with tf.GradientTape() as tape:
    z = y * y

    dy_dx = tape.gradient(z, x)
    
print(dy_dx)<jupyter_output>None
{'fraction_non_alphanumeric': 0.13318534961154274, 'fraction_numerical': 0.03607103218645949, 'mean_word_length': 4.453857791225416, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '74737', 'license': 'MIT', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1396, 'n_words': 352, 'path': 'notebooks/tensor01/05. GradientTape.ipynb', 'repo_name': 'mmercan/study_python', 'size': '3604'}