<jupyter_start><jupyter_text>**Importing libraries**<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.utils import np_utils<jupyter_output><empty_output><jupyter_text>**Loading data**<jupyter_code>from keras.datasets import mnist
data = mnist.load_data()<jupyter_output><empty_output><jupyter_text>**Dividing Train/Test data**<jupyter_code>(X_train, y_train), (X_test, y_test) = data
X_train[0].shape
X_train.shape<jupyter_output><empty_output><jupyter_text>**Plotting values in X_train**<jupyter_code>fig = plt.figure()
for i in range(4):
  plt.subplot(2,4,i+1)
  plt.tight_layout()
  plt.imshow(X_train[i], cmap='gray', interpolation='none')
  plt.title("Digit: {}".format(y_train[i]))
  plt.xticks([])
  plt.yticks([])
fig<jupyter_output><empty_output><jupyter_text>Reshaping and normalizing data<jupyter_code># let's print the shape before we reshape and normalize
print("X_train shape", X_train.shape)
print("y_train shape", y_train.shape)
print("X_test shape", X_test.shape)
print("y_test shape", y_test.shape)

# building the input vector from the 28x28 pixels
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

# normalizing the data to help with the training
X_train /= 255
X_test /= 255

# print the final input shape ready for training
print("Train matrix shape", X_train.shape)
print("Test matrix shape", X_test.shape)

# one-hot encoding using keras' numpy-related utilities
n_classes = 10
print("Shape before one-hot encoding: ", y_train.shape)
Y_train = np_utils.to_categorical(y_train, n_classes)
Y_test = np_utils.to_categorical(y_test, n_classes)
print("Shape after one-hot encoding: ", Y_train.shape)<jupyter_output>X_train shape (60000, 28, 28)
y_train shape (60000,)
X_test shape (10000, 28, 28)
y_test shape (10000,)
Train matrix shape (60000, 784)
Test matrix shape (10000, 784)
Shape before one-hot encoding:  (60000,)
Shape after one-hot encoding:  (60000, 10)<jupyter_text>**Building a dense network**<jupyter_code># Importing the required Keras modules containing model and layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Dropout
import tensorflow as tf
from keras.callbacks import LearningRateScheduler
from keras.callbacks import EarlyStopping

epochs = 75
learning_rate = 0.002
decay_rate = learning_rate / epochs
input_dim = X_train.shape[1]

num_classes = 10


def dense_model_adam():
  # Creating a Sequential Model and adding the layers
  model = Sequential()
  model.add(Dense(40, activation=tf.nn.relu))
  model.add(Flatten())
  model.add(Dense(20,activation=tf.nn.softmax))
  model.add(Dropout(0.2))
  model.add(Dense(10,activation=tf.nn.sigmoid))

  model.add(Dense(num_classes,activation=tf.nn.softmax))

  opt = tf.keras.optimizers.RMSprop(learning_rate=0.002)

  model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')

  return model
def exp_learning(epochs):
  lrate = learning_rate * np.exp(-decay_rate*epochs)
  return lrate

exp_model = dense_model_adam()
lr_rate = LearningRateScheduler(exp_learning)


es = EarlyStopping(monitor='val_loss', patience=25)

checkpoint_filepath = 'weights-improvement_rms.hdf5'

model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=True,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True)

callbacks_list = [lr_rate, es]
exp_history = exp_model.fit(X_train, Y_train,
                        batch_size=128,
                        epochs=epochs,
                        callbacks=callbacks_list,
                        validation_data=(X_test, Y_test))<jupyter_output>Epoch 1/75
469/469 [==============================] - 2s 3ms/step - loss: 1.9273 - accuracy: 0.3389 - val_loss: 1.4691 - val_accuracy: 0.4605
Epoch 2/75
469/469 [==============================] - 1s 3ms/step - loss: 1.3693 - accuracy: 0.4548 - val_loss: 0.9971 - val_accuracy: 0.5954
Epoch 3/75
469/469 [==============================] - 1s 3ms/step - loss: 1.1537 - accuracy: 0.5011 - val_loss: 0.8250 - val_accuracy: 0.6344
Epoch 4/75
469/469 [==============================] - 1s 3ms/step - loss: 1.0702 - accuracy: 0.5324 - val_loss: 0.7465 - val_accuracy: 0.6963
Epoch 5/75
469/469 [==============================] - 1s 3ms/step - loss: 0.9949 - accuracy: 0.5841 - val_loss: 0.6624 - val_accuracy: 0.7599
Epoch 6/75
469/469 [==============================] - 1s 3ms/step - loss: 0.8720 - accuracy: 0.6384 - val_loss: 0.5599 - val_accuracy: 0.7739
Epoch 7/75
469/469 [==============================] - 1s 3ms/step - loss: 0.7598 - accuracy: 0.6679 - val_loss: 0.4988 - val_accuracy: 0.7691
Epoch [...]<jupyter_text>Accuracy is approximately 0.97 when the model is trained over the training data and receives around 0.96 validation accuracy<jupyter_code>#PLot for the model with constant learning rate
# Plot the loss function
fig, ax = plt.subplots(1, 1, figsize=(10,6))
ax.plot(np.sqrt(exp_history.history['loss']), 'r', label='train')
ax.plot(np.sqrt(exp_history.history['val_loss']), 'b' ,label='val')
ax.set_xlabel(r'Epoch', fontsize=20)
ax.set_ylabel(r'Loss', fontsize=20)
ax.legend()
ax.tick_params(labelsize=20)<jupyter_output><empty_output><jupyter_text>**What is the value of lr when loss shoots up?** At 0.1  **Using RMSProp Optimizer**<jupyter_code>def dense_model_rms():
  # Creating a Sequential Model and adding the layers
  model = Sequential()
  model.add(Dense(40, activation=tf.nn.relu))
  model.add(Flatten())
  model.add(Dense(20,activation=tf.nn.softmax))
  model.add(Dropout(0.2))
  model.add(Dense(10,activation=tf.nn.sigmoid))

  model.add(Dense(num_classes,activation=tf.nn.softmax))

  rms_opt = tf.keras.optimizers.RMSprop(learning_rate=0.002)

  model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=rms_opt)

  return model


exp_model = dense_model_rms()
lr_rate = LearningRateScheduler(exp_learning)


es = EarlyStopping(monitor='val_loss', patience=25)

checkpoint_filepath = 'weights-improvement_rms.hdf5'

model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=True,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True)

callbacks_list = [lr_rate,es]
exp_history = exp_model.fit(X_train, Y_train,
                        batch_size=128,
                        epochs=epochs,
                        callbacks=callbacks_list,
                        verbose=1,
                        validation_data=(X_test, Y_test))<jupyter_output>Epoch 1/75
469/469 [==============================] - 2s 3ms/step - loss: 1.8931 - accuracy: 0.4210 - val_loss: 1.3640 - val_accuracy: 0.5627
Epoch 2/75
469/469 [==============================] - 1s 3ms/step - loss: 1.2683 - accuracy: 0.4765 - val_loss: 0.9201 - val_accuracy: 0.5803
Epoch 3/75
469/469 [==============================] - 1s 3ms/step - loss: 1.1252 - accuracy: 0.4907 - val_loss: 0.8149 - val_accuracy: 0.6092
Epoch 4/75
469/469 [==============================] - 1s 3ms/step - loss: 1.0524 - accuracy: 0.5147 - val_loss: 0.7736 - val_accuracy: 0.6221
Epoch 5/75
469/469 [==============================] - 1s 3ms/step - loss: 0.9464 - accuracy: 0.6388 - val_loss: 0.6432 - val_accuracy: 0.8254
Epoch 6/75
469/469 [==============================] - 1s 3ms/step - loss: 0.7670 - accuracy: 0.7369 - val_loss: 0.4828 - val_accuracy: 0.8907
Epoch 7/75
469/469 [==============================] - 1s 3ms/step - loss: 0.6048 - accuracy: 0.8139 - val_loss: 0.3397 - val_accuracy: 0.9496
Epoch [...]<jupyter_text>Accuracy is approximately 0.95 when the model is trained over the training data and receives around 0.96 validation accuracy<jupyter_code>#PLot for the model with constant learning rate
# Plot the loss function
fig, ax = plt.subplots(1, 1, figsize=(10,6))
ax.plot(np.sqrt(exp_history.history['loss']), 'r', label='train')
ax.plot(np.sqrt(exp_history.history['val_loss']), 'b' ,label='val')
ax.set_xlabel(r'Epoch', fontsize=20)
ax.set_ylabel(r'Loss', fontsize=20)
ax.legend()
ax.tick_params(labelsize=20)<jupyter_output><empty_output><jupyter_text>**Using SGD Optimizer**<jupyter_code>def dense_model_sgd():
  # Creating a Sequential Model and adding the layers
  model = Sequential()
  model.add(Dense(40, activation=tf.nn.relu))
  model.add(Flatten())
  model.add(Dense(20,activation=tf.nn.softmax))
  model.add(Dropout(0.2))
  model.add(Dense(10,activation=tf.nn.sigmoid))

  model.add(Dense(num_classes,activation=tf.nn.softmax))

  sgd_opt = tf.keras.optimizers.SGD(learning_rate=0.002)

  model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=sgd_opt)

  return model


exp_model = dense_model_rms()
lr_rate = LearningRateScheduler(exp_learning)


es = EarlyStopping(monitor='val_loss', patience=50)

checkpoint_filepath = 'weights-improvement_rms.hdf5'

model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=True,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True)

callbacks_list = [lr_rate,es]
exp_history = exp_model.fit(X_train, Y_train,
                        batch_size=128,
                        epochs=100,
                        callbacks=callbacks_list,
                        verbose=1,
                        validation_data=(X_test, Y_test))<jupyter_output>Epoch 1/75
469/469 [==============================] - 2s 3ms/step - loss: 2.0066 - accuracy: 0.2874 - val_loss: 1.6141 - val_accuracy: 0.3981
Epoch 2/75
469/469 [==============================] - 1s 3ms/step - loss: 1.5598 - accuracy: 0.3476 - val_loss: 1.2938 - val_accuracy: 0.4132
Epoch 3/75
469/469 [==============================] - 1s 3ms/step - loss: 1.4533 - accuracy: 0.3688 - val_loss: 1.2026 - val_accuracy: 0.5537
Epoch 4/75
469/469 [==============================] - 1s 3ms/step - loss: 1.2866 - accuracy: 0.4976 - val_loss: 1.0027 - val_accuracy: 0.6784
Epoch 5/75
469/469 [==============================] - 1s 3ms/step - loss: 1.0407 - accuracy: 0.6349 - val_loss: 0.7692 - val_accuracy: 0.7800
Epoch 6/75
469/469 [==============================] - 1s 3ms/step - loss: 0.8402 - accuracy: 0.7103 - val_loss: 0.5938 - val_accuracy: 0.8214
Epoch 7/75
469/469 [==============================] - 1s 3ms/step - loss: 0.7203 - accuracy: 0.7424 - val_loss: 0.4987 - val_accuracy: 0.8543
Epoch [...]<jupyter_text>Accuracy is approximately 0.97 when the model is trained over the training data and receives around 0.96 validation accuracy<jupyter_code>#PLot for the model with constant learning rate
# Plot the loss function
fig, ax = plt.subplots(1, 1, figsize=(10,6))
ax.plot(np.sqrt(exp_history.history['loss']), 'r', label='train')
ax.plot(np.sqrt(exp_history.history['val_loss']), 'b' ,label='val')
ax.set_xlabel(r'Epoch', fontsize=20)
ax.set_ylabel(r'Loss', fontsize=20)
ax.legend()
ax.tick_params(labelsize=20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1815199562602515, 'fraction_numerical': 0.07882267176963732, 'mean_word_length': 4.616683725690891, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '173519', 'license': 'MIT', 'n_tokens_mistral': 4581, 'n_tokens_neox': 3917, 'n_words': 818, 'path': 'Assignments/Assignment_5.ipynb', 'repo_name': 'niranjana1997/RIT-DSCI-633-FDS', 'size': '10974'}