<jupyter_start><jupyter_text>Gradient Descent<jupyter_code>import numpy as np
import scipy as sc
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D<jupyter_output><empty_output><jupyter_text>Function definition and visualization $F(x, y) = sin(\frac{1}{2}x^2 - \frac{1}{4}y^2 + 3)cos(2x + 1 - e^y)$<jupyter_code>func = lambda th: np.sin(((1 / 2) * (th[0] ** 2)) - ((1 / 4) * (th[0] ** 2)) + 3) * np.cos((2 * th[0]) + 1 - (np.e ** th[1]))
res = 100
x = np.linspace(-2, 2, res)
y = np.linspace(-2, 2, res)
x, y = np.meshgrid(x, y)
z = func([x, y])
fig = plt.figure(figsize=(15,10))
ax1 = fig.add_subplot(221, projection='3d')
surface_plot = ax1.plot_surface(x, y, z, cmap='winter')

ax2 = fig.add_subplot(222)
contour_plot = ax2.contourf(x, y, z, 100, cmap='winter')
plt.colorbar(contour_plot, ax=ax2)

plt.show()<jupyter_output><empty_output><jupyter_text>Gradient descent calculation<jupyter_code># Set random start point
theta = (np.random.rand(2) * 4) - 2

plt.contourf(x, y, z, 100, cmap='winter')
plt.colorbar()
plt.plot(theta[0], theta[1], 'o', c='red')
plt.show()
# Derivative ratio
h = 0.001
# Learning rate
lr = 0.001
# Set a gradient variable
gradient = np.zeros(len(theta))

# Plot contour and start point
plt.contourf(x, y, z, 100, cmap='winter')
plt.colorbar()
plt.plot(theta[0], theta[1], 'o', c='white')

_theta = np.copy(theta)
for _ in range (100000):
    for it, th in enumerate(theta):
        _theta = np.copy(theta)
        # Derivative calculation
        _theta[it] = _theta[it] + h
        deriv = (func(_theta) - func(theta)) / h
        # Update gradient
        gradient[it] = deriv
    
    # Update theta with the learning rate
    theta = theta - (lr * gradient)
    
    if (_ % 100 == 0):
        # Plot some intermediate points
        plt.plot(theta[0], theta[1], '.', c='red') 
    
    
# Plot final point
plt.plot(theta[0], theta[1], 'o', c='black')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17770929635336416, 'fraction_numerical': 0.04365690806368772, 'mean_word_length': 3.8944723618090453, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '623041', 'license': 'MIT', 'n_tokens_mistral': 846, 'n_tokens_neox': 779, 'n_words': 185, 'path': 'notebooks/from_scratch/gradient_descent.ipynb', 'repo_name': 'lcambero/ml-playground', 'size': '1947'}