<jupyter_start><jupyter_text>Report: Ex0 - MNIST in the cloud Tore AadlandFirst lets load required modules and seed the random number generators to make results reproducible:<jupyter_code>import datetime
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, Reshape, Dropout, AveragePooling2D, InputLayer
from keras.datasets import mnist
from keras.utils import to_categorical, plot_model
from keras.utils.vis_utils import model_to_dot
from keras.regularizers import l2
from keras.callbacks import TensorBoard
from IPython.display import SVG
import numpy as np
import os
import random as rn
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
import time

rn.seed(42)
np.random.seed(42)<jupyter_output>C:\Users\Tore\Anaconda3\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.<jupyter_text>Load training data:<jupyter_code>(train_images, train_labels), (test_images, test_labels) = mnist.load_data()<jupyter_output><empty_output><jupyter_text>Seems ok.Scale and cast data:<jupyter_code># Cast:
train_images = train_images.astype(np.float32)
test_images = test_images.astype(np.float32)

# Scale:
scaler = StandardScaler(with_mean=True, with_std=True)
train_images = scaler.fit_transform(train_images.reshape((60000, 784))).reshape((60000, 28, 28))
test_images = scaler.transform(test_images.reshape((10000, 784))).reshape((10000, 28, 28))<jupyter_output><empty_output><jupyter_text>Get some validation data:<jupyter_code>from sklearn.model_selection import train_test_split
validation_images, test_images, validation_labels, test_labels = train_test_split(test_images, test_labels, test_size=0.5, random_state=42)<jupyter_output><empty_output><jupyter_text>Checking that training, validation and test dataset is shufled.<jupyter_code>print(train_labels[0:20])
print(validation_labels[0:20])
print(test_labels[0:20])<jupyter_output>[5 0 4 1 9 2 1 3 1 4 3 5 3 6 1 7 2 8 6 9]
[8 4 3 1 2 9 0 5 8 7 7 7 5 7 6 1 1 3 2 7]
[6 2 3 7 2 2 3 4 7 6 6 9 2 0 9 6 8 0 6 5]<jupyter_text>Plotting a few of the training images:<jupyter_code>plt.figure(figsize=(15,10))
for j, i in enumerate(np.random.choice(len(train_images), 6)):
    plt.subplot(2, 3, j+1)
    plt.imshow(train_images[i], cmap="Greys", vmin=0, vmax=2)
plt.show()<jupyter_output><empty_output><jupyter_text>Lets define a helper function that fits, visualizes and stores trained models and which generates the data requested in the exercise text:<jupyter_code>from collections import namedtuple
ModelPerformance = namedtuple('ModelPerformance', 
        ['name', 'loss', 'val_loss', 'acc', 'val_acc', 'final_loss', 'final_accuracy', 'time'])

d_mp = {}
def fit_and_save_model(name, model, X_train, y_train, X_validation, y_validation,
                       epochs = 10, batch_size=64, validation_split = 1.0/12, eval_batch_size=128):
    required_folders = './output/{}/logs/'.format(name)
    if not os.path.exists(required_folders):
        os.makedirs(required_folders)

    # Validation data is not shuffled! This is probably a problem!
    tensorboard = TensorBoard('./output/{}/logs/{}'.format(name, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
    
    start_time = time.time()
    history = model.fit(X_train, to_categorical(y_train), 
                        validation_data=(validation_images, to_categorical(validation_labels)), 
                        epochs=epochs,
                        batch_size=batch_size, 
                        verbose=0, 
                        callbacks=[tensorboard])
    end_time = time.time()
    score = model.evaluate(X_validation, to_categorical(y_validation), verbose=0)
    

        
    model.save('./output/{}/{}.h5'.format(name, name))
    with open("./output/{}/{}.json".format(name, name), 'w+') as f:
        f.write(model.to_json())
    
    print("I train a model with the following shape:")
    plot_model(model, 
           to_file='./output/{}/model_visualization_{}.png'.format(name,name),
           show_shapes=True,
           show_layer_names=True)
    display(SVG(model_to_dot(model, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg')))
    
    print("It has the following training history:")
    plt.figure(figsize=(10,5))
    plt.subplot(1, 2, 1)
    plt.plot(history.history['loss'], label='Training data loss')
    plt.plot(history.history['val_loss'], label='Validation data loss')
    plt.legend()
    plt.title("Loss")
    
    plt.subplot(1, 2, 2)
    plt.plot(history.history['acc'], label='Training data accuracy')
    plt.plot(history.history['val_acc'], label='Validation data accuracy')
    plt.legend()
    plt.title("Accuracy")
    plt.suptitle("Model performance - {}".format(name))
    
    plt.savefig('./output/{}/performance_{}.png'.format(name,name))
    plt.show()
    
    print("It has the following performance:")
    summary = "Validation loss: {}\nValidation accuracy: {}\nTraining time: {}".format(score[0], score[1], end_time-start_time)
    print(summary)
    d_mp[name] = ModelPerformance(name, 
                                  history.history['loss'], 
                                  history.history['val_loss'],
                                  history.history['acc'], 
                                  history.history['val_acc'],
                                  score[0], 
                                  score[1], 
                                  end_time-start_time)
    
    with open("./output/{}/{}.score".format(name, name), 'w+') as f:
        f.write(summary)
    

    print("It has the following confusion matrix:")
    y_predicted = np.argmax(model.predict(X_validation), axis= 1)
    
    y_true = y_validation.ravel()
    cm = confusion_matrix(y_true, y_predicted)
    for i in range(len(cm)):
        cm[i,i] = 0
    
    plt.figure(figsize=(8,6))
    plt.imshow(cm, cmap="Greys")
    plt.xticks(range(10))
    plt.yticks(range(10))
    plt.savefig('./output/{}/confusion_matrix_{}.png'.format(name,name))
    plt.title("Confusion matrix - {}".format(name))
    plt.xlabel("True label")
    plt.ylabel("Predicted label")
    plt.show()
    
    print("These are examples of misclassified features:")
    plt.figure(figsize=(15,10))
    plt.subplot(2, 2, 1)
    plt.suptitle("Misclassified images - Model {}".format(name))
    poss_indexes = np.argwhere(y_predicted-y_true != 0)
    indexes = poss_indexes[np.random.choice(len(poss_indexes), 6)]
    
    for i, index in enumerate(indexes.ravel()):
        plt.subplot(2, 3, i+1)
        plt.imshow(X_validation[index], cmap="Greys", vmin=0, vmax=2)
        plt.title("A {} misclassified as {}".format(y_true[index], y_predicted[index]))
    plt.savefig('./output/{}/misclassified_features_{}.png'.format(name,name))
    plt.show()<jupyter_output><empty_output><jupyter_text>Now create and train a simple sequential model without any regularization:TODO: Comment on the use of softmax as the final layers activation function.<jupyter_code>simple_sequential = Sequential()
simple_sequential.add(Flatten(input_shape=(28,28), name="Flatten_Layer"))
simple_sequential.add(Dense(1000, activation='sigmoid', name="Hidden_Layer"))
simple_sequential.add(Dense(10, activation='softmax', name="Output_Layer"))
simple_sequential.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("mnist_simple_sequential_batchsize64", simple_sequential, 
                   train_images, train_labels, validation_images, validation_labels,
                   epochs = 10, batch_size=64)<jupyter_output>I train a model with the following shape:<jupyter_text>Now create and train a simple sequential model without any regularization using a batch size of 128.<jupyter_code>simple_sequential = Sequential()
simple_sequential.add(Flatten(input_shape=(28,28), name="Flatten_Layer"))
simple_sequential.add(Dense(1000, activation='sigmoid', name="Hidden_Layer"))
simple_sequential.add(Dense(10, activation='softmax', name="Output_Layer"))
simple_sequential.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("mnist_simple_sequential_batchsize128", simple_sequential, 
                   train_images, train_labels, validation_images, validation_labels,
                   epochs = 10, batch_size=128)<jupyter_output>I train a model with the following shape:<jupyter_text>We see that ... TODO ... Lets try to recreate LeNet-5. I follow the summary of LeNet-5:![](https://engmrk.com/wp-content/uploads/2018/09/LeNEt_Summary_Table.jpg)From: https://engmrk.com/lenet-5-a-classic-cnn-architecture/<jupyter_code>simple_convolutional = Sequential()
simple_convolutional.add(Reshape((28,28, 1), input_shape=(28,28)))
simple_convolutional.add(Conv2D(6, 
                 kernel_size=5,
                 activation="tanh",
                 padding="same"))
simple_convolutional.add(AveragePooling2D(pool_size=(2,2)))
simple_convolutional.add(Conv2D(16, 
                 kernel_size=5,
                 activation="tanh"))
simple_convolutional.add(AveragePooling2D(pool_size=(2,2)))
simple_convolutional.add(Conv2D(120, 
                 kernel_size=5,
                 activation="tanh"))
simple_convolutional.add(Flatten())
simple_convolutional.add(Dense(84, activation='tanh'))
simple_convolutional.add(Dense(10, activation='softmax'))
simple_convolutional.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("LeNet5_batchsize64", simple_convolutional,
                train_images, train_labels, validation_images, validation_labels,
                  epochs = 10, batch_size=64)
simple_convolutional = Sequential()
simple_convolutional.add(Reshape((28,28, 1), input_shape=(28,28)))
simple_convolutional.add(Conv2D(6, 
                 kernel_size=5,
                 activation="tanh",
                 padding="same"))
simple_convolutional.add(AveragePooling2D(pool_size=(2,2)))
simple_convolutional.add(Conv2D(16, 
                 kernel_size=5,
                 activation="tanh"))
simple_convolutional.add(AveragePooling2D(pool_size=(2,2)))
simple_convolutional.add(Conv2D(120, 
                 kernel_size=5,
                 activation="tanh"))
simple_convolutional.add(Flatten())
simple_convolutional.add(Dense(84, activation='tanh'))
simple_convolutional.add(Dense(10, activation='softmax'))
simple_convolutional.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("LeNet5_batchsize128", simple_convolutional,
                train_images, train_labels, validation_images, validation_labels,
                  epochs = 10, batch_size=128)
# Tasks: Experiment with different hyperparameter settings (e.g. learning rate, early stopping, batch sizes, etc)
# and observe their effects on training time, overfitting, and final accuracy.
# See if L2 regularisation has effect on network using relu activation function.
# Compare contrast with relu + dropout.
def compare_performance_of(modelsToBeEvaluated):
    #modelsToBeEvaluated = [n1, n2]
    plt.figure(figsize=(16,8))
    plt.subplot(1, 2, 1)

    for name in modelsToBeEvaluated:
        #plt.plot(d_mp[name].loss, label='{} training data loss'.format(name))
        plt.plot(d_mp[name].val_loss, label='{} validation data loss'.format(name))
    plt.legend()
    plt.title("Loss")

    plt.subplot(1, 2, 2)
    for name in modelsToBeEvaluated:
        #plt.plot(d_mp[name].acc, label='{} training data accuracy'.format(name))
        plt.plot(d_mp[name].val_acc, label='{} validation data accuracy'.format(name))
    plt.legend()
    plt.title("Accuracy")
    plt.suptitle("Performance of the different models on validation data")
    plt.show()
compare_performance_of(["mnist_simple_sequential_batchsize64", "LeNet5_batchsize64", "mnist_simple_sequential_batchsize128", "LeNet5_batchsize128"])<jupyter_output><empty_output><jupyter_text>Exploring the effect of the activation function on a L2 regularizer:<jupyter_code>simple_sequential = Sequential()
simple_sequential.add(Flatten(input_shape=(28,28), name="Flatten_Layer"))
simple_sequential.add(Dense(1000, activation='sigmoid', name="Hidden_Layer", kernel_regularizer=l2(0.01)))
simple_sequential.add(Dense(10, activation='softmax', name="Output_Layer"))
simple_sequential.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("mnist_simple_sequential_batchsize64_L2_sigmoid", simple_sequential, 
                   train_images, train_labels, validation_images, validation_labels,
                   epochs = 10, batch_size=128)
simple_sequential = Sequential()
simple_sequential.add(Flatten(input_shape=(28,28), name="Flatten_Layer"))
simple_sequential.add(Dense(1000, activation='relu', name="Hidden_Layer", kernel_regularizer=l2(0.01)))
simple_sequential.add(Dense(10, activation='softmax', name="Output_Layer"))
simple_sequential.compile(optimizer="SGD", loss="categorical_crossentropy", metrics=["accuracy"])
fit_and_save_model("mnist_simple_sequential_batchsize64_L2_relu", simple_sequential, 
                   train_images, train_labels, validation_images, validation_labels,
                   epochs = 10, batch_size=128)
compare_performance_of(["mnist_simple_sequential_batchsize64_L2_sigmoid", "mnist_simple_sequential_batchsize64_L2_relu"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12456874403582177, 'fraction_numerical': 0.028114218600895545, 'mean_word_length': 4.389240506329114, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '212105', 'license': 'MIT', 'n_tokens_mistral': 4652, 'n_tokens_neox': 4318, 'n_words': 837, 'path': 'notebooks/mnist_classification.ipynb', 'repo_name': 'ToreAad/inf368', 'size': '13623'}