<jupyter_start><jupyter_text>Code from [kmkolasinski/deep-learning-notes](https://github.com/kmkolasinski/deep-learning-notes/blob/master/max-normed-optimizer/notebooks/Solving_quadratic_equation.ipynb)  Generate dataset<jupyter_code>import sklearn.datasets as sk_datasets
# dimensionality of the problem
ndims = 100
# Q [ndims, ndims] symetric positive define matrix
Q = sk_datasets.make_spd_matrix(ndims).astype(np.float32)
# b random vector of shape [ndims, 1]
b = np.random.randn(ndims, 1).astype(np.float32)
# w0 initial state of the unknown variable, vector of shape [ndims, 1]
w0 = np.random.randn(ndims, 1).astype(np.float32)<jupyter_output><empty_output><jupyter_text>Loss function<jupyter_code>weights = tf.get_variable(
        name='w', 
        dtype=tf.float32, 
        initializer=w0)
def L(w0, Q, b, weights):
    """Define loss function"""
    # 0.5 x^T Q x + b^T x
    loss = 0.5 * tf.matmul(tf.transpose(weights), tf.matmul(Q, weights)) \
         - tf.matmul(tf.transpose(b), weights)
    
    return tf.reshape(loss, (1,))
lr = 0.1
history = []
max_iter = 1500

sgd = SGD(lr=lr, momentum=0.9, nesterov=True)
optimizer = NormalizedOptimizer(sgd, normalization='l2')
loss = L(w0, Q, b, weights)
train_op = optimizer.get_updates(loss, [weights])
sess.run(tf.global_variables_initializer())
for i in range(max_iter):
    loss_value, _ = sess.run([loss, train_op])
    history.append(loss_value)
    
    if i % 99 == 0:
        print("Iteration %d: Loss = %0.4f" % (i + 1, loss_value))
history = np.array(history)
plt.figure(figsize=(20, 7))
plt.ylabel('loss')
plt.xlabel('step')
plt.plot(history[:, 0], 'r-', lw=3)
plt.xlim([0, max_iter])<jupyter_output><empty_output><jupyter_text>Comparison with other optimizers and different learning rates<jupyter_code>def fit(optimizer, scope):
    """Solve quadratic problem with given optimizer.
    Returns loss history.
    """
    history = []
    max_iter = 1000
    with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
        weights = tf.get_variable(
                            name='w-%s' % (scope), 
                            dtype=tf.float32, 
                            initializer=w0)
        loss = L(w0, Q, b, weights)
        update_op = optimizer.get_updates(loss, [weights])
        sess.run(tf.global_variables_initializer())    
        
    for i in range(max_iter):
        loss_value, _= sess.run([loss, update_op])     
        history.append([i, loss_value])   
    history = np.array(history)
    return history

from keras.optimizers import Adam, RMSprop
from collections import OrderedDict
comparisons = OrderedDict()
min_value, max_value = 0, 0
for lr_np in [0.001, 0.005, 0.01, 0.03, 0.05, 0.1]:
    print('lr=%0.3f' % (lr_np))
    
    sgd = NormalizedOptimizer(SGD(lr_np, momentum=0.9, nesterov=True), normalization='std')
    rmsprop = RMSprop(lr=lr_np, rho=0.9)
    adam = Adam(lr=lr_np)
    
    hist_nsgd = fit(sgd, 'NSGD-%0.3f' % (lr_np))
    hist_adam = fit(adam, 'Adam-%0.3f' % (lr_np))
    hist_momentum = fit(rmsprop, 'Momentum-%0.3f' % (lr_np))
    
    comparisons[lr_np] = OrderedDict()
    comparisons[lr_np]['NSGD'] = hist_nsgd
    comparisons[lr_np]['Adam'] = hist_adam
    comparisons[lr_np]['Momentum'] = hist_momentum
    
    # keep min max for plotting reasons
    min_value = min(np.min(hist_nsgd), min_value)
    max_value = max(np.max(hist_nsgd), max_value)
plt.figure(figsize=(20, 10))
plt.subplots_adjust(hspace=0.35)

for k, (lr, hists) in enumerate(comparisons.items()):    
    plt.subplot(len( comparisons.items())//2, 2, k + 1)
    plt.text(10, min_value+30, '%s) lr=%0.3f' % (chr(ord("a") + k), lr), size=20)
    for opt, hist in hists.items():        
        plt.plot(hist[:, 0], hist[:, 1], label=opt, lw=3)
    plt.legend(loc='upper right', fontsize=14)
    plt.ylim([min_value, 100])
    plt.xlim([0, 1000])
    plt.ylabel('loss')
    plt.xlabel('step')<jupyter_output><empty_output><jupyter_text>Dependency on initial choice of learning rate<jupyter_code>from matplotlib import cm
plt.figure(figsize=(20, 5))
for s, opt in enumerate(['NSGD', 'Adam', 'Momentum']):
    plt.subplot(1, 3, s + 1)        
    plt.text(15, min_value+0.02*abs(min_value), '%s) %s' % (chr(ord("a") + s), opt), size=20)
    for k, (lr, hists) in enumerate(comparisons.items()):  
        label = 'lr = %0.3f' % lr if hists[opt][-100:, 1].max() < 0 else 'lr = %0.3f (not converged)' % (l4)
        if hists[opt][-100:, 1].max() < 0:
            plt.plot(hists[opt][:, 0], hists[opt][:, 1], label=label, lw=3, alpha=0.8)
        else:
            plt.plot(hists[opt][:, 0], hists[opt][:, 1]*0, label=label, alpha=0.0)
    plt.legend(loc='upper right', fontsize=14)
    plt.ylim([min_value, 100])    
    plt.xlim([0, 1000])
    plt.xlabel('step')
    if s == 0: plt.ylabel('loss')<jupyter_output><empty_output><jupyter_text>Normalizing other optimizers (ADAM)<jupyter_code>comparisons = OrderedDict()
min_value, max_value = 0, 0
for lr_np in [0.001, 0.005, 0.01, 0.03, 0.05, 0.1]:
    print('lr=%0.3f' % (lr_np))
    
    normalized_adam = NormalizedOptimizer(Adam(lr_np), normalization='std')
    rmsprop = RMSprop(lr=lr_np, rho=0.9)
    adam = Adam(lr=lr_np)
    
    hist_nsgd = fit(sgd, 'NADAM-%0.3f' % (lr_np))
    hist_adam = fit(adam, 'Adam-%0.3f' % (lr_np))
    hist_momentum = fit(rmsprop, 'Momentum-%0.3f' % (lr_np))
    
    comparisons[lr_np] = OrderedDict()
    comparisons[lr_np]['NADAM'] = hist_nsgd
    comparisons[lr_np]['Adam'] = hist_adam
    comparisons[lr_np]['Momentum'] = hist_momentum
    
    # keep min max for plotting reasons
    min_value = min(np.min(hist_nsgd), min_value)
    max_value = max(np.max(hist_nsgd), max_value)
plt.figure(figsize=(20, 10))
plt.subplots_adjust(hspace=0.35)

for k, (lr, hists) in enumerate(comparisons.items()):    
    plt.subplot(len( comparisons.items())//2, 2, k + 1)
    plt.text(10, min_value+30, '%s) lr=%0.3f' % (chr(ord("a") + k), lr), size=20)
    for opt, hist in hists.items():        
        plt.plot(hist[:, 0], hist[:, 1], label=opt, lw=3)
    plt.legend(loc='upper right', fontsize=14)
    plt.ylim([min_value, 100])
    plt.xlim([0, 1000])
    plt.ylabel('loss')
    plt.xlabel('step')<jupyter_output><empty_output><jupyter_text>Benefits of NormalizedOptimizer wrapper class- Since this is a wrapper class over all optimizers, no matter which optimizer is chosen as the base Optimizer, we can perform any kind of normalization. - It is also serializable with `model.save()` and can be loaded with `load_model()` while supplying the `custom_objects` parameter.- It will monkey patch only the optimizer object that it wraps, and not the original class itself, so you can use multiple optimizers and wrap only those you want.- **Normalized Adam is very useful for fast convergence, even with high learning rates !**  Dependency on initial choice of learning rate<jupyter_code>plt.figure(figsize=(20, 5))
for s, opt in enumerate(['NADAM', 'Adam', 'Momentum']):
    plt.subplot(1, 3, s + 1)        
    plt.text(15, min_value+0.02*abs(min_value), '%s) %s' % (chr(ord("a") + s), opt), size=20)
    for k, (lr, hists) in enumerate(comparisons.items()):  
        label = 'lr = %0.3f' % lr if hists[opt][-100:, 1].max() < 0 else 'lr = %0.3f (not converged)' % (l4)
        if hists[opt][-100:, 1].max() < 0:
            plt.plot(hists[opt][:, 0], hists[opt][:, 1], label=label, lw=3, alpha=0.8)
        else:
            plt.plot(hists[opt][:, 0], hists[opt][:, 1]*0, label=label, alpha=0.0)
    plt.legend(loc='upper right', fontsize=14)
    plt.ylim([min_value, 100])    
    plt.xlim([0, 1000])
    plt.xlabel('step')
    if s == 0: plt.ylabel('loss')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1634891616610081, 'fraction_numerical': 0.0363019065030034, 'mean_word_length': 3.6277945619335346, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '301446', 'license': 'MIT', 'n_tokens_mistral': 3112, 'n_tokens_neox': 2911, 'n_words': 638, 'path': 'notebooks/simple_optimization.ipynb', 'repo_name': 'titu1994/keras-normalized-optimizers', 'size': '7658'}