<jupyter_start><jupyter_text>Step 0 - import NN libraries<jupyter_code>import numpy as np
import pandas as pd 
import math
import matplotlib.cm as cm 
import matplotlib.pyplot as plt
import seaborn as sns
import os
from keras.layers import Conv2D, MaxPool2D
from keras.layers import MaxPooling2D
from keras.models import Sequential
from keras.callbacks import ReduceLROnPlateau
from keras.layers import Dense , Dropout , Lambda, Flatten
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.utils import plot_model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.optimizers import RMSprop
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.callbacks import LearningRateScheduler
from keras import  backend as K
from keras.preprocessing.image import ImageDataGenerator
from subprocess import check_output
import tensorflow as tf<jupyter_output><empty_output><jupyter_text>Step 1 - Load the data<jupyter_code>from keras.datasets import mnist #importing the mnist datset from keras
(x_train, y_train),(x_test, y_test) = mnist.load_data() #splitting the data as test and train
assert x_train.shape == (60000, 28, 28)
assert x_test.shape == (10000, 28, 28)
assert y_train.shape == (60000,)
assert y_test.shape == (10000,)
unique, counts = np.unique(y_train, return_counts=True)
print("Train labels: ", dict(zip(unique, counts))) # displaying the count of unique train.  labels
unique, counts = np.unique(y_test, return_counts=True)
print("\nTest labels: ", dict(zip(unique, counts))) # displaying the count of unique tets labels
#displaying a bar graph of counts of each label
y_value=np.zeros((1,10))
for i in range (10):
    y_value[0,i-1]= np.count_nonzero(y_test==i)
y_value=y_value.ravel()
x_value=[0,1,2,3,4,5,6,7,8,9]
plt.xlabel('label')
plt.ylabel('count')
plt.bar(x_value,y_value,0.7,color='y')
print('Train Data Dimensions: x=%s, y=%s' % (x_train.shape, y_train.shape)) #printing the dimensions of the train and test data 
print('Test Data Dimensions: x=%s, y=%s' % (x_test.shape, y_test.shape))

for i in range(9): #displaying 9 images from the train data
	plt.subplot(330 + 1 + i) 	#defining the subplot
	plt.imshow(x_train[i], cmap=plt.get_cmap('gray')) #plotting the raw pixel data
plt.show() #displaying the train data
num_labels = len(np.unique(y_train))
print(num_labels) #printing the number of unique numbers present<jupyter_output>10<jupyter_text>Step 2 - Data pre-processing<jupyter_code>y_train = to_categorical(y_train, num_classes=10) #one hot encoding
y_test = to_categorical(y_test, num_classes=10)
# image dimensions (assumed square)
image_size = x_train.shape[1]
input_size = image_size * image_size
input_size
# resize and normalize
x_train = x_train.astype('float32') / 255
x_train = x_train.reshape(-1,28,28,1)
x_test = x_test.astype('float32') / 255
x_test=x_test.reshape(-1,28,28,1)
print(x_train.shape)
print(x_test.shape)
# network parameters
batch_size = 128
hidden_units = 256
dropout = 0.45<jupyter_output><empty_output><jupyter_text>Building a simple dense network, using sigmoid, relu, and softmax as activation functions<jupyter_code>model = Sequential()
#
model.add(Conv2D(filters = 8, kernel_size = (5,5),padding = 'Same', 
                 activation ='relu', input_shape = (28,28,1)))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(Dropout(0.25))
#
model.add(Conv2D(filters = 16, kernel_size = (3,3),padding = 'Same', 
                 activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
model.add(Dropout(0.25))
# fully connected
model.add(Flatten())
model.add(Dense(256, activation = "sigmoid"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.summary()
# Defining the Adams optimizer
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # dimesion reduction
        rotation_range=5,  # randomly rotate images in the range 5 degrees
        zoom_range = 0.1, # Randomly zoom image 10%
        width_shift_range=0.1,  # randomly shift images horizontally 10%
        height_shift_range=0.1,  # randomly shift images vertically 10%
        horizontal_flip=False,  # randomly flip images
        vertical_flip=False)  # randomly flip images
datagen.fit(x_train)
# Compile the model
model.compile(optimizer = adam , loss = "categorical_crossentropy", metrics=["accuracy"])
#fiiting the model
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
                              epochs = 10, validation_data = (x_test,y_test), steps_per_epoch=x_train.shape[0] // batch_size)
#displaying the accuracy
score = model.evaluate(x_test,y_test)
print("Accuracy:",score[1]*100)<jupyter_output>313/313 [==============================] - 1s 4ms/step - loss: 0.0463 - accuracy: 0.9854
Accuracy: 98.54000210762024<jupyter_text>Plotting loss as a function of number of epoch<jupyter_code># Plot the loss and accuracy curves for training and validation 
plt.plot(history.history['val_loss'], color='b', label="validation loss")
plt.title("Test Loss")
plt.xlabel("Number of Epochs")
plt.ylabel("Loss")
plt.legend()
plt.show()
# confusion matrix
import seaborn as sns
# Predict the values from the validation dataset
y_pred = model.predict(x_test)
# Convert predictions classes to one hot vectors 
y_pred_classes = np.argmax(y_pred,axis = 1) 
# Convert validation observations to one hot vectors
y_true = np.argmax(y_test,axis = 1) 
# compute the confusion matrix
confusion_mtx = confusion_matrix(y_true, y_pred_classes) 
# plot the confusion matrix
f,ax = plt.subplots(figsize=(8, 8))
sns.heatmap(confusion_mtx, annot=True, linewidths=0.01,cmap="Greens",linecolor="gray", fmt= '.1f',ax=ax)
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.title("Confusion Matrix")
plt.show()
#the loss function
loss_fn = tf.keras.losses.MeanSquaredError(reduction='none')
loss_fn(tf.ones((2, 2,)), tf.zeros((2, 2)))
#setting the callbacks and earlystop in the model
checkpoint_path= r"/n" #setting a local path in the collab
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
checkpoint = ModelCheckpoint(checkpoint_path,monitor='val_loss',mode='min',save_best_only=True,verbose=1)
callbacks_list = [checkpoint]
keras_callbacks = [EarlyStopping(monitor='val_loss', patience=30, mode='min', min_delta=0.0001),
      ModelCheckpoint(checkpoint_path, monitor='val_loss', save_best_only=True, mode='min')]
earlystop = EarlyStopping(monitor = 'val_loss',min_delta = 0,patience = 3, verbose = 1,restore_best_weights = True)
model.fit(x_train, y_train,batch_size=batch_size, epochs=100, validation_split=0.2,callbacks=[earlystop,checkpoint])

#printing the latest test and train accuracy
train_acc=model.evaluate(x_train, y_train,verbose=0)
test_acc=model.evaluate(x_test, y_test,verbose=0)
print('Train Accuracy: %.3f, Test Accuracy: %.3f' % (train_acc[1]*100, test_acc[1]*100))<jupyter_output>Train Accuracy: 99.343, Test Accuracy: 98.990<jupyter_text>Compiling losses, using various optimizers<jupyter_code>#using adagrad optimizer
adagrad = tf.keras.optimizers.Adagrad(learning_rate=0.001,initial_accumulator_value=0.1,epsilon=1e-07,name="Adagrad")
model.compile(optimizer = adagrad , loss = "categorical_crossentropy", metrics=["accuracy"])
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
                              epochs = 5, validation_data = (x_test,y_test), steps_per_epoch=x_train.shape[0] // batch_size)
#using RMSprop optimizer
RMSprop = tf.keras.optimizers.RMSprop(learning_rate=0.001,rho=0.9, momentum=0.0, epsilon=1e-07, centered=False,name='RMSprop') 
model.compile(optimizer = adagrad , loss = "categorical_crossentropy", metrics=["accuracy"])
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
                              epochs = 5, validation_data = (x_test,y_test), steps_per_epoch=x_train.shape[0] // batch_size)
#using SGD optimizer
SGD=tf.keras.optimizers.SGD(learning_rate=0.01,momentum=0.0,nesterov=False, name="SGD")
model.compile(optimizer = adagrad , loss = "categorical_crossentropy", metrics=["accuracy"])
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
                              epochs = 5, validation_data = (x_test,y_test), steps_per_epoch=x_train.shape[0] // batch_size)

class LearningRateDecay:
	def plot(self, epochs, title="Learning Rate Schedule"):
		# compute the set of learning rates for each corresponding
		# epoch
		lrs = [self(i) for i in epochs]
		# the learning rate schedule
		plt.style.use("ggplot")
		plt.figure()
		plt.plot(epochs, lrs)
		plt.title(title)
		plt.xlabel("Epoch #")
		plt.ylabel("Learning Rate")
def my_learning_rate(epoch, lrate):
	return lrate
lrs = LearningRateScheduler(my_learning_rate)
history=model.fit(x_train, y_train,batch_size=batch_size, epochs=5, validation_split=0.2,callbacks=[earlystop,checkpoint,lrs])


# learning rate decay
def decay_lrate(initial_lrate, decay, iteration):
	return initial_lrate * (1.0 / (1.0 + decay * iteration))
decays = [1E-1, 1E-2, 1E-3, 1E-4]
lrate = 0.01
n_updates = 200
for decay in decays:
	# calculate learning rates for updates
	lrates = [decay_lrate(lrate, decay, i) for i in range(n_updates)]
	# plot result
	plt.plot(lrates, label=str(decay))
plt.legend()
plt.show()
SGD=tf.keras.optimizers.SGD(learning_rate=0.01,momentum=0.0,nesterov=False, name="SGD")
model.compile(optimizer = adagrad , loss = "categorical_crossentropy", metrics=["accuracy"])
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
                              epochs = 5, validation_data = (x_test,y_test), steps_per_epoch=x_train.shape[0] // batch_size)<jupyter_output><empty_output><jupyter_text>Learning rate<jupyter_code>def exp_decay(epoch):
    lr = learningRate * np.exp(-decay_rate*epoch)
    return lrate
lr_rate = LearningRateScheduler(exp_decay)
checkpoint_path=r"\n"
checkpoint = ModelCheckpoint(checkpoint_path,monitor='val_loss',mode='min',save_best_only=True,verbose=1)
keras_callbacks = [EarlyStopping(monitor='val_loss', patience=30, mode='min', min_delta=0.0001),
      ModelCheckpoint(checkpoint_path, monitor='val_loss', save_best_only=True, mode='min')]
earlystop = EarlyStopping(monitor = 'val_loss',min_delta = 0,patience = 3, verbose = 1,restore_best_weights = True)
model.fit(x_train, y_train,batch_size=batch_size, epochs=100, validation_split=0.2,callbacks=[earlystop,checkpoint,lr_rate])

plt.plot(history.history['loss'], label='Loss')
plt.plot(lr, label='Learning Rate')
plt.title('lrate='+str(lrate), pad=-50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12485169298165556, 'fraction_numerical': 0.02993520124121566, 'mean_word_length': 6.010876519513755, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '647019', 'license': 'MIT', 'n_tokens_mistral': 4069, 'n_tokens_neox': 3800, 'n_words': 836, 'path': 'Assignments/Assignment_5_Neural_Network.ipynb', 'repo_name': 'bharadwajkastur1/RIT-DSCI-633-FDS', 'size': '10957'}