<jupyter_start><jupyter_text>Gradient descent for other functionsGradient descent works well for other functions, even if they are non-convex; gradient descent, if it converges, converges to a local minimum. Lets look at the function $$f(a,b) = \sin(a)\cos(b).$$ We would like to find parameters $(a,b)$ that minimize the function $f$. The gradients are:$$\frac{\partial f}{\partial a} = \cos(a)\cos(b),$$$$\frac{\partial f}{\partial b} = -\sin(a)\sin(b).$$<jupyter_code># Code source: <NAME> and <NAME>.

# Python Notebook Commands
%matplotlib inline
%reload_ext autoreload
%load_ext autoreload
%autoreload 2

# Numerical Libraries
import numpy as np
import matplotlib.pyplot as plt

from matplotlib import rcParams
rcParams['figure.figsize'] = (10, 5)

# IPython Libraries
import IPython
import ipywidgets
from ipywidgets import interact, interactive, interact_manual


# Custom Libraries
from utilities.load_data import polynomial_data
from utilities import plot_helpers
from utilities.util import gradient_descent
class sincos(object):
    structured = False
    def __init__(self):
        pass
    
    @property
    def number_samples(self):
        return 0

    def loss(self, w, *args):
        return np.sin(w[0]) * np.cos(w[1])

    def gradient(self, w, *args):
        return np.array([np.cos(w[0]) * np.cos(w[1]), -np.sin(w[0]) * np.sin(w[1])])
rcParams['figure.figsize'] = (10, 5)   # Change this if figures look ugly. 
rcParams['font.size'] = 16   # Change this if figures look ugly. 

eta_widget = ipywidgets.FloatSlider(value=4, min=1e-1, max=10, step=1e-1, readout_format='.1f', 
                                    description='Learning rate:', style={'description_width': 'initial'}, 
                                    continuous_update=False)
n_iter_widget = ipywidgets.IntSlider(value=10, min=5, max=50, step=1, description='Number of iterations:',
                                     style={'description_width': 'initial'}, continuous_update=False)

lr_widget = ipywidgets.RadioButtons(options=['Bold driver', 'AdaGrad', 'Annealing', 'None'], value='None',
                                    description='Learning rate heuristics:', style={'description_width': 'initial'})

a0_widget = ipywidgets.FloatSlider(value=-0.9, min=-3, max=3, step=.1, readout_format='.1f', description='w_0:',
                                   style={'description_width': 'initial'}, continuous_update=False)

b0_widget = ipywidgets.FloatSlider(value=1.1, min=-3, max=3, step=.1, readout_format='.1f', description='w_1:',
                                   style={'description_width': 'initial'}, continuous_update=False)
def optimize_sincos(eta, n_iter, learning_rate_scheduling, a0, b0):
    loss_function = sincos()
    w0 = np.array([a0, b0])
    
    if learning_rate_scheduling == 'None':
        learning_rate_scheduling = None

    opts = {'eta0': eta,
            'n_iter': n_iter,
            'learning_rate_scheduling': learning_rate_scheduling
            }
    trajectory, _ = gradient_descent(w0, loss_function, opts=opts)
    contourplot = plt.subplot(111)
    dataplot = None
    contour_opts = {'x_label': '$w_0$', 'y_label': '$w_1$', 'title': 'Weight trajectory', 'legend': False}
    plot_opts = {'contour_opts': contour_opts}

    plot_helpers.linear_regression_progression(np.array([]), np.array([]), trajectory, np.array([]), 
                                               loss_function.loss,
                                               contourplot, dataplot, options=plot_opts)


interact_manual(optimize_sincos, eta=eta_widget, n_iter=n_iter_widget, learning_rate_scheduling=lr_widget,
                a0=a0_widget, b0=b0_widget);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13017591339648174, 'fraction_numerical': 0.016779431664411367, 'mean_word_length': 3.2877030162412995, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '572856', 'license': 'MIT', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1136, 'n_words': 270, 'path': 'iml2020/examples/Gradient Descent on Other Functions.ipynb', 'repo_name': 'AriJordan/IML_project', 'size': '3711'}