<jupyter_start><jupyter_text>Machine Learning Practical: Coursework 2 Experiment 1/3**Due date: 16:00 Thursday 24th November 2016**  <NAME> - s1687568<jupyter_code>import sys
mlpdir = '/home/student/Dropbox/msc_Artificial_Intelligence/mlp_Machine_Learning_Practical/mlpractical'
sys.path.append(mlpdir)
import matplotlib.pyplot as plt
import numpy as np
import logging

#%matplotlib notebook
%matplotlib inline
from mlp.data_providers import MNISTDataProvider

from mlp.layers import AffineLayer, SoftmaxLayer, SigmoidLayer

from mlp.errors import CrossEntropySoftmaxError

from mlp.models import MultipleLayerModel

from mlp.initialisers import ConstantInit, GlorotUniformInit

from mlp.optimisers import Optimiser

from mlp.learning_rules import GradientDescentLearningRule, MomentumLearningRule, \
    AdaGradLearningRule, RmsPropLearningRule
    
from mlp.schedulers import MomentumCoefficientScheduler, ConstantLearningRateScheduler,\
    ReciprocalLearningRateScheduler, ExponentialLearningRateScheduler
def train_model(model, error, learning_rule, train_data, valid_data, num_epochs, stats_interval, schedulers=[]):
    # As well as monitoring the error over training also monitor classification
    # accuracy i.e. proportion of most-probable predicted classes being equal to targets
    data_monitors={'acc': lambda y, t: (y.argmax(-1) == t.argmax(-1)).mean()}

    # Use the created objects to initialise a new Optimiser instance.
    optimiser = Optimiser(
        model, error, learning_rule, train_data, valid_data, data_monitors, schedulers = schedulers)

    # Run the optimiser for some number of epochs (full passes through the training set)
    # printing statistics every epoch
    stats, keys, runTime = optimiser.train(num_epochs=num_epochs, stats_interval=stats_interval)
    #stats = optimiser.train(num_epochs=num_epochs, stats_interval=stats_interval)

    return stats, keys, runTime
def train_model_and_plot_stats(
        model, error, learning_rule, train_data, valid_data, num_epochs, stats_interval, schedulers=[]):
    
    stats, keys, runTime = train_model(model, error, learning_rule, train_data, valid_data,
                                       num_epochs, stats_interval, schedulers)

    # Plot the change in the validation and training set error over training.
    fig_1 = plt.figure(figsize=(8, 4))
    ax_1 = fig_1.add_subplot(111)
    for k in ['error(train)', 'error(valid)']:
        ax_1.plot(np.arange(1, stats.shape[0]) * stats_interval, 
                  stats[1:, keys[k]], label=k)
    ax_1.legend(loc=0)
    ax_1.set_xlabel('Epoch number')

    # Plot the change in the validation and training set accuracy over training.
    fig_2 = plt.figure(figsize=(8, 4))
    ax_2 = fig_2.add_subplot(111)
    for k in ['acc(train)', 'acc(valid)']:
        ax_2.plot(np.arange(1, stats.shape[0]) * stats_interval, 
                  stats[1:, keys[k]], label=k)
    ax_2.legend(loc=0)
    ax_2.set_xlabel('Epoch number')
    
    return stats, keys, fig_1, ax_1, fig_2, ax_2, runTime
def getFinalValues(statistics):
    lastStats = statistics[-1]
    return {
        "Final Training Error": lastStats[0],
        "Final Training Accuracy": lastStats[1],
        "Final Testing Error": lastStats[2],
        "Final Testing Accuracy": lastStats[3],
    }
# Seed a random number generator
seed = 16011984
rng = np.random.RandomState(seed)<jupyter_output><empty_output><jupyter_text>Note: When in our explanations refer to **accuracy** we mean the final validation accuracy and when we refer to **performance** we mean how fast we reached the optimal accuracy for the current experiment  Grand Experiment 1: Exploring the system architecture in numbers of layers and numbers of nodesWe will explore where the systems breaks/fails for too few layers or too few features and where for too many of themWe are going to a simple system with constant learning rate since here we are not optimizing for accuracy neither optimizing for performance. Rather we want to see how the complexity of the model generally affects the  accuracy and performance on a dataset as complex as MNIST. In other words we care for comparison rather than absolute values.We will use the constant learning rate of 1.1 because it performed relatively well, with good accuracy and performance without making the system too unstableNote: we are using a high learning rate that produced good enough results within 20 epochs in the previous experiment<jupyter_code>weights_init = GlorotUniformInit(rng=rng)
biases_init = ConstantInit(0.)
# Set up a logger object to print info about the training run to stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.handlers = [logging.StreamHandler()]

# Create data provider objects for the MNIST data set
train_data = MNISTDataProvider('train', batch_size=50, rng=rng)
valid_data = MNISTDataProvider('valid', batch_size=50, rng=rng)

error = CrossEntropySoftmaxError() #this does not contain any immutable data that's why it can be set only once
stats_interval = 5<jupyter_output><empty_output><jupyter_text>Constant Learning Rate 1.1  Three Layers and dimensionality of 100 for each hidden layer<jupyter_code>num_epochs = 30

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 3

fig_1.savefig('experiment1_01_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_01_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

print "runtime: " + str(runTime)

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.77e+00, acc(train)=9.94e-02, error(valid)=2.77e+00, acc(valid)=9.90e-02, params_penalty=0.00e+00
Epoch 5: 1.30s to complete
  error(train)=7.49e-02, acc(train)=9.76e-01, error(valid)=1.11e-01, acc(valid)=9.67e-01, params_penalty=0.00e+00
Epoch 10: 1.47s to complete
  error(train)=2.22e-02, acc(train)=9.94e-01, error(valid)=8.48e-02, acc(valid)=9.77e-01, params_penalty=0.00e+00
Epoch 15: 1.20s to complete
  error(train)=6.10e-03, acc(train)=9.99e-01, error(valid)=8.42e-02, acc(valid)=9.80e-01, params_penalty=0.00e+00
Epoch 20: 1.35s to complete
  error(train)=2.13e-03, acc(train)=1.00e+00, error(valid)=8.40e-02, acc(valid)=9.80e-01, params_penalty=0.00e+00
Epoch 25: 1.42s to complete
  error(train)=9.76e-04, acc(train)=1.00e+00, error(valid)=8.75e-02, acc(valid)=9.81e-01, params_penalty=0.00e+00
Epoch 30: 1.18s to complete
  error(train)=7.05e-04, acc(train)=1.00e+00, error(valid)=8.96e-02, acc(valid)=9.81e-01, params_penalty=0.00e+00<jupyter_text>This is our base reference. Within 20 epochs we overfit but as we saw in coursework 1 even for 100 epochs the performance does not get significantly better  Two Layers and dimensionality of 100 for each hidden layer<jupyter_code>num_epochs = 30

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    #AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    #SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 2

fig_1.savefig('experiment1_02_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_02_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.37e+00, acc(train)=9.58e-02, error(valid)=2.37e+00, acc(valid)=9.61e-02, params_penalty=0.00e+00
Epoch 5: 0.91s to complete
  error(train)=5.61e-02, acc(train)=9.85e-01, error(valid)=8.92e-02, acc(valid)=9.75e-01, params_penalty=0.00e+00
Epoch 10: 1.17s to complete
  error(train)=2.12e-02, acc(train)=9.96e-01, error(valid)=7.57e-02, acc(valid)=9.78e-01, params_penalty=0.00e+00
Epoch 15: 0.84s to complete
  error(train)=1.27e-02, acc(train)=9.98e-01, error(valid)=7.75e-02, acc(valid)=9.79e-01, params_penalty=0.00e+00
Epoch 20: 1.01s to complete
  error(train)=5.79e-03, acc(train)=1.00e+00, error(valid)=7.59e-02, acc(valid)=9.79e-01, params_penalty=0.00e+00
Epoch 25: 0.90s to complete
  error(train)=3.77e-03, acc(train)=1.00e+00, error(valid)=7.86e-02, acc(valid)=9.79e-01, params_penalty=0.00e+00
Epoch 30: 0.83s to complete
  error(train)=2.75e-03, acc(train)=1.00e+00, error(valid)=8.02e-02, acc(valid)=9.79e-01, params_penalty=0.00e+00<jupyter_text>For two layers the accuracy and the performance does not seem so bad. The accuracy has slightly dropped from ~98.1% to 97.8%. The best accuracy is achieved within 15 epochs. Meaning that we could consider keeping this much faster implementation.  One Layer without hidden layers<jupyter_code>num_epochs = 200

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, output_dim, weights_init, biases_init)
])

learning_rate = 0.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 1

fig_1.savefig('experiment1_03_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_03_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.42e+00, acc(train)=8.31e-02, error(valid)=2.42e+00, acc(valid)=8.03e-02, params_penalty=0.00e+00
Epoch 5: 0.28s to complete
  error(train)=2.91e-01, acc(train)=9.17e-01, error(valid)=2.79e-01, acc(valid)=9.21e-01, params_penalty=0.00e+00
Epoch 10: 0.28s to complete
  error(train)=2.73e-01, acc(train)=9.23e-01, error(valid)=2.67e-01, acc(valid)=9.26e-01, params_penalty=0.00e+00
Epoch 15: 0.29s to complete
  error(train)=2.65e-01, acc(train)=9.27e-01, error(valid)=2.63e-01, acc(valid)=9.27e-01, params_penalty=0.00e+00
Epoch 20: 0.29s to complete
  error(train)=2.59e-01, acc(train)=9.28e-01, error(valid)=2.61e-01, acc(valid)=9.29e-01, params_penalty=0.00e+00
Epoch 25: 0.28s to complete
  error(train)=2.55e-01, acc(train)=9.29e-01, error(valid)=2.59e-01, acc(valid)=9.28e-01, params_penalty=0.00e+00
Epoch 30: 0.28s to complete
  error(train)=2.52e-01, acc(train)=9.31e-01, error(valid)=2.59e-01, acc(valid)=9.29e-01, params_penalty=0.00e+00
Epoch 35: 0.29s to complet[...]<jupyter_text>The single layer network is not possible to capture the complexity of the model, even after 200 epochs the training accuracy does not seem to be able to grow larger than 93.7% and the validation error is increasing. Moreover it is apparent that the validation accuracy has even start to decrease. The best validation accuracy achieved is ~93% which is much lower than before. We will consider this a failed case. Let's see if adding the layers to go from 3 layers to 4 layers will have a positive impact or not or it will have a neutral effect.  Four Layers and dimensionality of 100 for each hidden layer<jupyter_code>num_epochs = 30

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 4

fig_1.savefig('experiment1_04_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_04_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.58e+00, acc(train)=9.94e-02, error(valid)=2.57e+00, acc(valid)=9.90e-02, params_penalty=0.00e+00
Epoch 5: 2.38s to complete
  error(train)=6.46e-02, acc(train)=9.80e-01, error(valid)=1.07e-01, acc(valid)=9.69e-01, params_penalty=0.00e+00
Epoch 10: 1.53s to complete
  error(train)=3.56e-02, acc(train)=9.88e-01, error(valid)=1.10e-01, acc(valid)=9.69e-01, params_penalty=0.00e+00
Epoch 15: 2.59s to complete
  error(train)=2.53e-02, acc(train)=9.91e-01, error(valid)=1.24e-01, acc(valid)=9.71e-01, params_penalty=0.00e+00
Epoch 20: 1.51s to complete
  error(train)=3.01e-03, acc(train)=9.99e-01, error(valid)=1.07e-01, acc(valid)=9.77e-01, params_penalty=0.00e+00
Epoch 25: 2.12s to complete
  error(train)=6.77e-04, acc(train)=1.00e+00, error(valid)=1.17e-01, acc(valid)=9.76e-01, params_penalty=0.00e+00
Epoch 30: 1.55s to complete
  error(train)=3.95e-04, acc(train)=1.00e+00, error(valid)=1.23e-01, acc(valid)=9.78e-01, params_penalty=0.00e+00<jupyter_text>By adding one more layer we do not seem to have achieved anything significant. We are actually overfitting again since the beginning because the validation error is constantly increasing. We have not done anything better neither in terms of performance or accuracy  Five Layers and dimensionality of 100 for each hidden layer<jupyter_code>num_epochs = 40

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 5

fig_1.savefig('experiment1_05_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_05_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.53e+00, acc(train)=9.90e-02, error(valid)=2.54e+00, acc(valid)=9.67e-02, params_penalty=0.00e+00
Epoch 5: 3.94s to complete
  error(train)=1.21e-01, acc(train)=9.62e-01, error(valid)=1.43e-01, acc(valid)=9.58e-01, params_penalty=0.00e+00
Epoch 10: 1.73s to complete
  error(train)=4.45e-02, acc(train)=9.87e-01, error(valid)=1.09e-01, acc(valid)=9.70e-01, params_penalty=0.00e+00
Epoch 15: 1.77s to complete
  error(train)=2.07e-02, acc(train)=9.94e-01, error(valid)=1.03e-01, acc(valid)=9.73e-01, params_penalty=0.00e+00
Epoch 20: 1.83s to complete
  error(train)=1.53e-02, acc(train)=9.95e-01, error(valid)=1.20e-01, acc(valid)=9.71e-01, params_penalty=0.00e+00
Epoch 25: 1.87s to complete
  error(train)=8.88e-03, acc(train)=9.98e-01, error(valid)=1.27e-01, acc(valid)=9.74e-01, params_penalty=0.00e+00
Epoch 30: 1.73s to complete
  error(train)=5.55e-03, acc(train)=9.99e-01, error(valid)=1.31e-01, acc(valid)=9.73e-01, params_penalty=0.00e+00
Epoch 35: 1.80s to complet[...]<jupyter_text>For five layers again slower speed of algorithm and the accuracy has not improved  Six Layers and dimensionality of 100 for each hidden layer<jupyter_code>num_epochs = 40

input_dim, output_dim, hidden_dim = 784, 10, 100

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), 
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 6

fig_1.savefig('experiment1_06_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_06_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.45e+00, acc(train)=9.72e-02, error(valid)=2.45e+00, acc(valid)=9.83e-02, params_penalty=0.00e+00
Epoch 5: 2.07s to complete
  error(train)=2.31e+00, acc(train)=1.14e-01, error(valid)=2.31e+00, acc(valid)=1.06e-01, params_penalty=0.00e+00
Epoch 10: 2.22s to complete
  error(train)=2.31e+00, acc(train)=9.68e-02, error(valid)=2.31e+00, acc(valid)=1.01e-01, params_penalty=0.00e+00
Epoch 15: 2.07s to complete
  error(train)=4.50e-01, acc(train)=8.85e-01, error(valid)=4.38e-01, acc(valid)=8.91e-01, params_penalty=0.00e+00
Epoch 20: 2.07s to complete
  error(train)=1.35e-01, acc(train)=9.63e-01, error(valid)=1.77e-01, acc(valid)=9.51e-01, params_penalty=0.00e+00
Epoch 25: 2.02s to complete
  error(train)=5.61e-02, acc(train)=9.85e-01, error(valid)=1.31e-01, acc(valid)=9.66e-01, params_penalty=0.00e+00
Epoch 30: 2.44s to complete
  error(train)=3.51e-02, acc(train)=9.90e-01, error(valid)=1.33e-01, acc(valid)=9.67e-01, params_penalty=0.00e+00
Epoch 35: 2.12s to complet[...]<jupyter_text>And finally with six layers we see a different picture. The neural network has a hard time to adjust all the weights to achieve something useful at the first five epochs. The model is slowly learning as the weights are adjusting trying to find a balance. Afterwards we see that the accuracy is at the same terms as before. This means that even though we have all this extra complexity to our model of 6 layers we are not taking advantage any of them. It seems that some features are just being replicated from layer to layer and the model has already found its minimum on the third layer. Let's go back to our experiment with 2 layers that was fast enough and with good enough accuracy and let's try to play with the dimensionality to see what we can achieve with less or more dimensionality on the hidden layer  Two Layers and dimensionality of 2 for the hidden layer<jupyter_code>num_epochs = 100

input_dim, output_dim, hidden_dim = 784, 10, 2

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 2

fig_1.savefig('experiment1_07_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_07_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.33e+00, acc(train)=1.05e-01, error(valid)=2.33e+00, acc(valid)=1.06e-01, params_penalty=0.00e+00
Epoch 5: 0.29s to complete
  error(train)=1.23e+00, acc(train)=4.54e-01, error(valid)=1.23e+00, acc(valid)=4.52e-01, params_penalty=0.00e+00
Epoch 10: 0.30s to complete
  error(train)=1.24e+00, acc(train)=4.65e-01, error(valid)=1.24e+00, acc(valid)=4.58e-01, params_penalty=0.00e+00
Epoch 15: 0.31s to complete
  error(train)=1.24e+00, acc(train)=4.45e-01, error(valid)=1.25e+00, acc(valid)=4.41e-01, params_penalty=0.00e+00
Epoch 20: 0.31s to complete
  error(train)=1.23e+00, acc(train)=4.52e-01, error(valid)=1.24e+00, acc(valid)=4.35e-01, params_penalty=0.00e+00
Epoch 25: 0.30s to complete
  error(train)=1.23e+00, acc(train)=4.64e-01, error(valid)=1.25e+00, acc(valid)=4.53e-01, params_penalty=0.00e+00
Epoch 30: 0.29s to complete
  error(train)=1.22e+00, acc(train)=4.66e-01, error(valid)=1.24e+00, acc(valid)=4.56e-01, params_penalty=0.00e+00
Epoch 35: 0.29s to complet[...]<jupyter_text>Obviously when the features were reduced to only 2 features it would be silly of us to believe that only two features could represent 10 digits from 0 to 9. We executed this experiment to verify our intuition. And the numbers agree with us with a final validation accuracy of ~43%  Two Layers and dimensionality of 1000 for the hidden layer<jupyter_code>num_epochs = 30

input_dim, output_dim, hidden_dim = 784, 10, 1000

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 2

fig_1.savefig('experiment1_08_fig1_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))
fig_2.savefig('experiment1_08_fig2_%d_%d_%d_%d.svg' % (layers, input_dim, output_dim, hidden_dim))

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.65e+00, acc(train)=8.95e-02, error(valid)=2.65e+00, acc(valid)=9.08e-02, params_penalty=0.00e+00
Epoch 5: 9.94s to complete
  error(train)=7.08e-02, acc(train)=9.80e-01, error(valid)=1.05e-01, acc(valid)=9.69e-01, params_penalty=0.00e+00
Epoch 10: 9.91s to complete
  error(train)=3.39e-02, acc(train)=9.91e-01, error(valid)=9.07e-02, acc(valid)=9.74e-01, params_penalty=0.00e+00
Epoch 15: 11.15s to complete
  error(train)=1.96e-02, acc(train)=9.96e-01, error(valid)=9.16e-02, acc(valid)=9.74e-01, params_penalty=0.00e+00
Epoch 20: 9.97s to complete
  error(train)=1.13e-02, acc(train)=9.98e-01, error(valid)=8.94e-02, acc(valid)=9.75e-01, params_penalty=0.00e+00
Epoch 25: 9.80s to complete
  error(train)=5.97e-03, acc(train)=1.00e+00, error(valid)=9.01e-02, acc(valid)=9.76e-01, params_penalty=0.00e+00
Epoch 30: 9.85s to complete
  error(train)=4.02e-03, acc(train)=1.00e+00, error(valid)=9.11e-02, acc(valid)=9.77e-01, params_penalty=0.00e+00<jupyter_text>When we increased the dimensionality to 1000 we got an accuracy of ~97.7% which means that we didn't really do any better than the dimensionality of 100 and once again we overfitted at 15 epochs. The system also became extremely slow because of the number of connections. Therefore once again we achieved nothing special  Two Layers and dimensionality of 10 to 100 for the hidden layer with steps of 5here we are repeating the experiment from 10 to 100 neurons for the hidden layer to see how the final accuracy behaves<jupyter_code>#Note that this is a long running script
num_epochs = 40

input_dim, output_dim = 784, 10

learning_rate = 1.1

def executeExperiment(hidden_dim):
    learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)
    
    model = MultipleLayerModel([
        AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
        SigmoidLayer(),
        AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
    ])

    #fig_1, ax_1, fig_2, ax_2,
    stats, keys, runTime = train_model(
        model, error, 
        learning_rule,
        train_data, valid_data, num_epochs, stats_interval
    )

    return getFinalValues(stats)


hidden_dims = range(10, 105, 5)

results = []

for hidden_dim in hidden_dims:
    results.append(
        executeExperiment(hidden_dim=hidden_dim)
    )

# layers = 2
# fig_1.savefig('experiment1_%d_%d_fig1.svg' % (layers, hidden_dim))
# fig_2.savefig('experiment1_%d_%d_fig2.svg' % (layers, hidden_dim))
def plotting(results):
    trainAccs = [t['Final Training Accuracy'] for t in results]
    validAccs = [t['Final Testing Accuracy'] for t in results]
    trainErrs = [t['Final Training Error'] for t in results]
    validErrs = [t['Final Testing Error'] for t in results]
    
    x = hidden_dims

    fig_1 = plt.figure(figsize=(8, 4))
    ax_1 = fig_1.add_subplot(111)

    ax_1.plot(x, trainErrs, label='Training Errors')
    ax_1.plot(x, validErrs, label='Testing Errors')

    ax_1.legend(loc=0)
    ax_1.set_xlabel('Hidden Layer Dimensionality')

    # Plot the change in the validation and training set accuracy over training.
    fig_2 = plt.figure(figsize=(8, 4))
    ax_2 = fig_2.add_subplot(111)

    ax_2.plot(x, trainAccs, label='Training Accuracies')
    ax_2.plot(x, validAccs, label='Testing Accuracies')

    ax_2.legend(loc=0)
    ax_2.set_xlabel('Hidden Layer Dimensionality')
    
    return fig_1, fig_2
fig_1, fig_2 = plotting(results)

layers = 2
hidden_dim = "10-100"

fig_1.savefig('experiment1_09_fig1_%d_%s.svg' % (layers, hidden_dim))
fig_2.savefig('experiment1_09_fig2_%d_%s.svg' % (layers, hidden_dim))<jupyter_output><empty_output><jupyter_text>From the above two plots we can see how the model becomes more capable of capturing the complexity of the input when the dimensionality of the hidden layer is increasing By following the exact same procedure it worths to try and see if for larger dimensionalities, again for one hidden layer the accuracy is increased  Two Layers and dimensionality from 70 to 200 for the hidden layer with steps of 10<jupyter_code>#Note that this is a long running script
num_epochs = 40

input_dim, output_dim = 784, 10

learning_rate = 1.1

hidden_dims = range(70, 210, 10)

def executeExperiment(hidden_dim):
    learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)
    
    model = MultipleLayerModel([
        AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
        SigmoidLayer(),
        AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
    ])

    stats, keys, runTime = train_model(
        model, error,
        learning_rule,
        train_data, valid_data, num_epochs, stats_interval,
        schedulers = [ConstantLearningRateScheduler(learning_rate)]
    )

    return getFinalValues(stats)

results = []

for hidden_dim in hidden_dims:
    results.append(
        executeExperiment(hidden_dim=hidden_dim)
    )
fig_1, fig_2 = plotting(results)

layers = 2
hidden_dim = "70-200"

fig_1.savefig('experiment1_10_fig1_%d_%s.svg' % (layers, hidden_dim))
fig_2.savefig('experiment1_10_fig2_%d_%s.svg' % (layers, hidden_dim))<jupyter_output><empty_output><jupyter_text>We see that for larger dimensionality we have a better capture of the complexity of the model. We peaked at hidden dimensionality = 190 and 140 has almost equal performance.  We are going to repeat that experiment for this particular dimensionality of 190 to compare its speed with the three layers when the dimensionality was 100 for each layer.  Two Layers and dimensionality of 190 for the hidden layer<jupyter_code>num_epochs = 40

input_dim, output_dim, hidden_dim = 784, 10, 190

model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    SigmoidLayer(),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

learning_rate = 1.1
learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

#stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
    model, error, 
    learning_rule,
    train_data, valid_data, num_epochs, stats_interval,
    schedulers = [ConstantLearningRateScheduler(learning_rate)]
)

layers = 2

fig_1.savefig('experiment1_11_fig1_%d_%d.svg' % (layers, hidden_dim))
fig_2.savefig('experiment1_11_fig2_%d_%d.svg' % (layers, hidden_dim))

print "runtime: " + str(runTime)

getFinalValues(stats)<jupyter_output>Epoch 0:
  error(train)=2.45e+00, acc(train)=9.34e-02, error(valid)=2.44e+00, acc(valid)=9.45e-02, params_penalty=0.00e+00
Epoch 5: 2.32s to complete
  error(train)=5.17e-02, acc(train)=9.85e-01, error(valid)=8.69e-02, acc(valid)=9.76e-01, params_penalty=0.00e+00
Epoch 10: 2.24s to complete
  error(train)=1.94e-02, acc(train)=9.96e-01, error(valid)=7.34e-02, acc(valid)=9.79e-01, params_penalty=0.00e+00
Epoch 15: 2.22s to complete
  error(train)=8.55e-03, acc(train)=9.99e-01, error(valid)=7.19e-02, acc(valid)=9.80e-01, params_penalty=0.00e+00
Epoch 20: 1.63s to complete
  error(train)=4.70e-03, acc(train)=1.00e+00, error(valid)=7.32e-02, acc(valid)=9.80e-01, params_penalty=0.00e+00
Epoch 25: 1.54s to complete
  error(train)=3.00e-03, acc(train)=1.00e+00, error(valid)=7.39e-02, acc(valid)=9.82e-01, params_penalty=0.00e+00
Epoch 30: 1.57s to complete
  error(train)=2.29e-03, acc(train)=1.00e+00, error(valid)=7.54e-02, acc(valid)=9.81e-01, params_penalty=0.00e+00
Epoch 35: 1.67s to complet[...]<jupyter_text>We got an optimal training accuracy of approximately ~98.1% within 25 epochs. However the 30 epochs run took in total 92 seconds which is almost half the speed of the calculation in comparison to the 100 x 100 two hidden layers (three layers in total) for the machine that these tests are currently running at. We will do one final experiment with two hidden layers where the dimensionality of the first hidden layer is going to be 140 which had a good enough accuracy, only slightly smaller than the experiment with 190 neurons.Then for the second hidden layer we are going to do a grid search for the dimesionality within the range of 70 to 190. But this time we are going to keep track both of the accuracy/error and the runtime because apart of the accuracy we also care for the speed on which we can perform the experiments  Three Layers with 140 dimensionality of first hidden layer and a range of 70 to 200 for the second hidden layer<jupyter_code>#Note that this is a long running script
num_epochs = 40

input_dim, output_dim = 784, 10

learning_rate = 1.1

hidden_dims = range(70, 200, 10)

first_hidden_layer_dim = 140

def executeExperiment(hidden_dim):
    learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)
    
    model = MultipleLayerModel([
        AffineLayer(input_dim, first_hidden_layer_dim, weights_init, biases_init),
        SigmoidLayer(),
        AffineLayer(first_hidden_layer_dim, hidden_dim, weights_init, biases_init), 
        SigmoidLayer(),
        AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
    ])

    stats, keys, runTime = train_model(
        model, error,
        learning_rule,
        train_data, valid_data, num_epochs, stats_interval,
        schedulers = [ConstantLearningRateScheduler(learning_rate)]
    )

    return getFinalValues(stats), runTime

results = []

for hidden_dim in hidden_dims:
    results.append(
        executeExperiment(hidden_dim=hidden_dim)
    )
len(results)
stats = [r[0] for r in results]
fig_1, fig_2 = plotting(stats)

layers = 3
hidden_dim = 140
hidden_dim2 = "70-200"

fig_1.savefig('experiment1_12_fig1_%d_%d_%s.svg' % (layers, hidden_dim, hidden_dim2))
fig_2.savefig('experiment1_12_fig2_%d_%d_%s.svg' % (layers, hidden_dim, hidden_dim2))
accuracies = [s['Final Testing Accuracy'] for s in stats]
indMaxAcc = np.argmax(accuracies)
print "We see that we have achieved the best accuracy for %d dimensions in the third layer" % \
    hidden_dims[indMaxAcc]<jupyter_output>We see that we have achieved the best accuracy for 100 dimensions in the third layer<jupyter_text>Let's see how the speed in terms of running time is for each case<jupyter_code>fig_3 = plt.figure()
plt.plot(hidden_dims, [r[1] for r in results], 'b.-')
plt.xlabel('dimensionality of second hidden layer')
plt.ylabel('Run time (seconds)')
plt.show()
fig_3.savefig('experiment1_12_fig3_runtime.svg')
print "metrics for %d dimensionality for second hidden layer" % hidden_dims[indMaxAcc]
results[indMaxAcc][0]
print "running time for %d dimensionality for second hidden layer" % hidden_dims[indMaxAcc]
results[indMaxAcc][1]<jupyter_output>running time for 100 dimensionality for second hidden layer<jupyter_text>We have found out that an architecture of three layers has not provided any significant advantage in comparison with the two layers.Training error for three layers is smaller than training error for two layers which means that the three layer model captures better the complexity of the training data.However the performance of the accuracy has only increased ~0.1% from ~98.18% to ~98.23%By combining two hidden layers we achieved similar results with the two layers model (only one hidden layer).More particularly the final validation accuracy for three layers is 98.12% which is very similar to 98.17% of the two layers with 190 neurons.However the performance of the neural network with three layers is worse than our most-computationally-expesive case of the two layers of 190 neurons. We expect the two layers neural network to be faster for 140 or 150 neurons where the validation accuracy was comparable to the 190 neurons.<jupyter_code># ## Final experiment with 140 neurons for first layer and 120 neurons for second layer
# num_epochs = 40

# input_dim, output_dim, hidden_dim_layer1, hidden_dim_layer2 = 784, 10, 140, 120

# model = MultipleLayerModel([
#     AffineLayer(input_dim, hidden_dim_layer1, weights_init, biases_init),
#     SigmoidLayer(),
#     AffineLayer(hidden_dim_layer1, hidden_dim_layer2, weights_init, biases_init), 
#     SigmoidLayer(),
#     AffineLayer(hidden_dim_layer2, output_dim, weights_init, biases_init)
# ])

# learning_rate = 1.1
# learning_rule = GradientDescentLearningRule(learning_rate=learning_rate)

# #stats, keys, fig_1, ax_1, fig_2, ax_2 = train_model_and_plot_stats(
# stats, keys, fig_1, ax_1, fig_2, ax_2, runTime = train_model_and_plot_stats(
#     model, error, 
#     learning_rule,
#     train_data, valid_data, num_epochs, stats_interval,
#     schedulers = [ConstantLearningRateScheduler(learning_rate)]
# )

# layers = 3

# fig_1.savefig('experiment1_13_fig1_%d_%d_%d_fig1.svg' % (layers, hidden_dim_layer1, hidden_dim_layer2))
# fig_2.savefig('experiment1_13_fig2_%d_%d_%d_fig2.svg' % (layers, hidden_dim_layer1, hidden_dim_layer2))

# print "runtime: " + str(runTime)

# getFinalValues(stats)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1116988866116435, 'fraction_numerical': 0.07242563562842741, 'mean_word_length': 5.360225471199577, 'pattern_counts': {'":': 4, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '483496', 'license': 'MIT', 'n_tokens_mistral': 14471, 'n_tokens_neox': 12945, 'n_words': 3258, 'path': 'notebooks/Coursework_2_experiment1_pligor.ipynb', 'repo_name': 'pligor/msd-music-genre-classification', 'size': '36122'}