<jupyter_start><jupyter_text>Using Fashion MNIST to describe the use of following optimizers -* SGD* Adam* Adagrad* Adadelta* RMSProp* NAG<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import keras # Don't import keras from tensorflow like (from tensorflow import keras)
import seaborn as sns
# Reading train and test data
train_data = pd.read_csv('fashion-mnist_train.csv')
test_data = pd.read_csv('fashion-mnist_test.csv')
# Class names
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
# Making validation data out of test data
val_data = test_data.iloc[:5000,:]
test_data = test_data.iloc[5000:,:]
# Fetching the labels
train_labels = train_data.label
val_labels = val_data.label
test_labels = test_data.label
# Reshaping training data
train_images = train_data.iloc[:,1:].values.reshape(60000, 28, 28)
train_images.shape
# Reshaping validation data
val_images = val_data.iloc[:,1:].values.reshape(5000, 28, 28)
val_images.shape
# Reshaping test data
test_images = test_data.iloc[:,1:].values.reshape(5000, 28, 28)
test_images.shape
# Scaling data in the range of 0-1
train_images = train_images/255.0
val_images = val_images/255.0
test_images = test_images/255.0
# Defining multi-layer perceptron model with 1 hidden layer having 256 neurons and output layer with 10 possible answers
model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)), # Perform conversion of higher dimensional data (here, 2-D) to 1-D data.
    keras.layers.Dense(256, activation=tf.nn.relu), # Hidden layer with 256 neurons and ReLU activation function
    keras.layers.Dense(10, activation=tf.nn.softmax) # Output layer with softmax activation function 
])                                                   # which gives final output in terms of probability. 
import re
def models_with_different_optimizers(list_of_optimizers):    
    
    for i in range(len(list_of_optimizers)):        
        model.compile(loss='sparse_categorical_crossentropy',
                  optimizer=list_of_optimizers[i], # Learning rate and momentum can be passed inside optimizer
                  metrics=['accuracy'])

        # Traning the model and writing log files for TensorBoard in distinct directories        
        val = re.search('optimizers\..*\so', str(list_of_optimizers[i])).group(0)[11:][:-2] # Fetching optimizer name
        logdir = r'optims\\' + val # Each log file needs to be written in a distinct directory. (Mandatory)
        
        # Writing graph will take time. Hence, keeping it False.
        cb = keras.callbacks.TensorBoard(log_dir=logdir, write_graph=False) 
        epochs = 5
        print('Building model using '+ val + ' optimizer')
        history = model.fit(train_images, train_labels, epochs=epochs, 
                           validation_data=(val_images, val_labels),
                           callbacks=[cb])
        print('Model built sucessfully.')
        print('')
optims = [keras.optimizers.Adam(), keras.optimizers.Adadelta(), 
          keras.optimizers.Adagrad(), keras.optimizers.RMSprop(), 
          keras.optimizers.SGD()]
models_with_different_optimizers(optims)<jupyter_output>Building model using Adam optimizer
Train on 60000 samples, validate on 5000 samples
Epoch 1/5
60000/60000 [==============================] - 12s 204us/step - loss: 0.2835 - acc: 0.8981 - val_loss: 0.3651 - val_acc: 0.8754
Epoch 2/5
60000/60000 [==============================] - 11s 182us/step - loss: 0.2540 - acc: 0.9072 - val_loss: 0.2941 - val_acc: 0.8908
Epoch 3/5
60000/60000 [==============================] - 11s 181us/step - loss: 0.2372 - acc: 0.9117 - val_loss: 0.3281 - val_acc: 0.8846
Epoch 4/5
60000/60000 [==============================] - 11s 183us/step - loss: 0.2263 - acc: 0.9151 - val_loss: 0.3072 - val_acc: 0.8910
Epoch 5/5
60000/60000 [==============================] - 11s 188us/step - loss: 0.2168 - acc: 0.9193 - val_loss: 0.3289 - val_acc: 0.8832
Model built sucessfully.

Building model using Adadelta optimizer
Train on 60000 samples, validate on 5000 samples
Epoch 1/5
60000/60000 [==============================] - 12s 207us/step - loss: 0.1996 - acc: 0.9264 - val_los[...]
{'fraction_non_alphanumeric': 0.14596491228070174, 'fraction_numerical': 0.07134502923976609, 'mean_word_length': 3.583065380493033, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '356755', 'license': 'MIT', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1353, 'n_words': 380, 'path': 'base/Various optimization methods.ipynb', 'repo_name': 'vivekec/deepLearning', 'size': '4275'}