<jupyter_start><jupyter_text>Deep Learning=============Assignment 3------------Previously in `2_fullyconnected.ipynb`, you trained a logistic regression and a neural network model.The goal of this assignment is to explore regularization techniques.<jupyter_code># These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle<jupyter_output><empty_output><jupyter_text>First reload the data we generated in _notmist.ipynb_.<jupyter_code>pickle_file = 'notMNIST.pickle'

with open(pickle_file, 'rb') as f:
  save = pickle.load(f)
  train_dataset = save['train_dataset']
  train_labels = save['train_labels']
  valid_dataset = save['valid_dataset']
  valid_labels = save['valid_labels']
  test_dataset = save['test_dataset']
  test_labels = save['test_labels']
  del save  # hint to help gc free up memory
  print('Training set', train_dataset.shape, train_labels.shape)
  print('Validation set', valid_dataset.shape, valid_labels.shape)
  print('Test set', test_dataset.shape, test_labels.shape)<jupyter_output>Training set (200000, 28, 28) (200000,)
Validation set (10000, 28, 28) (10000,)
Test set (10000, 28, 28) (10000,)<jupyter_text>Reformat into a shape that's more adapted to the models we're going to train:- data as a flat matrix,- labels as float 1-hot encodings.<jupyter_code>image_size = 28
num_labels = 10

def reformat(dataset, labels):
  dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
  # Map 2 to [0.0, 1.0, 0.0 ...], 3 to [0.0, 0.0, 1.0 ...]
  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
  return dataset, labels
train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
def accuracy(predictions, labels):
  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
          / predictions.shape[0])<jupyter_output><empty_output><jupyter_text>---Problem 1---------Introduce and tune L2 regularization for both logistic and neural network models. Remember that L2 amounts to adding a penalty on the norm of the weights to the loss. In TensorFlow, you can compute the L2 loss for a tensor `t` using `nn.l2_loss(t)`. The right amount of regularization should improve your validation / test accuracy.---<jupyter_code># Generic Stochastic Gradient Descent Function
# param: graph
# param: hyperparams (steps, batch_size)

def run_simulation(graph, hyperparams):
  with tf.Session(graph=graph) as session:
    tf.initialize_all_variables().run()
    print('Processing...')
    for step in range(hyperparams['num_steps']):
      # Ordered batches
      offset = (step * hyperparams['batch_size']) % (train_labels.shape[0] - hyperparams['batch_size'])
      # Generate a minibatch.
      batch_data = train_dataset[offset:(offset + hyperparams['batch_size']), :]
      batch_labels = train_labels[offset:(offset + hyperparams['batch_size']), :]
      # Dict preparation and run training
      feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
      _, l, predictions = session.run(
        [optimizer, loss, train_prediction], feed_dict=feed_dict) # first param of run is a Tensor that means
      # that maps variables from our graph to the desired variables
      # In this case: "optimizer" to "_"(dummy); "loss" to "l"; and "train_predictions" to "predictions"
    print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
    tf.train.export_meta_graph()<jupyter_output><empty_output><jupyter_text>Logistic Regression Model<jupyter_code># Defining hyper-parameters
batch_size = 128
num_steps = 3001

# Defining the graph
graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Input init and constant init. According to the theory 
  tf_train_dataset = tf.placeholder(tf.float32,
                                    shape=(batch_size, image_size * image_size)) # batch_size
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels)) # batch_size
  tf_valid_dataset = tf.constant(valid_dataset) 
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  weights = tf.Variable(
    tf.truncated_normal([image_size * image_size, num_labels])) # mean 0, low std dev, random values
  biases = tf.Variable(tf.zeros([num_labels])) # zeros
  
  # Training computation.
  logits = tf.matmul(tf_train_dataset, weights) + biases # logistic regression
  loss = tf.reduce_mean(
    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels)) # loss function based on softmax-cross entropy
  
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss) # GD Optimizer
  
  # Predictions for the training, validation, and test data. (Probabilities)
  train_prediction = tf.nn.softmax(logits) 
  valid_prediction = tf.nn.softmax(
    tf.matmul(tf_valid_dataset, weights) + biases) 
  test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)
  
  
run_simulation(graph, hyperparams)<jupyter_output>Processing...
Test accuracy: 86.5%<jupyter_text>Test Acc -> 86.5% (SIMPLE LRM-SGD)  Logistic Regression Model + L2 Regularization<jupyter_code># Defining hyper-parameters
batch_size = 128
num_steps = 3001
l2_beta = 0.003

# Defining the graph
graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Input init and constant init. According to the theory 
  tf_train_dataset = tf.placeholder(tf.float32,
                                    shape=(batch_size, image_size * image_size)) # batch_size
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels)) # batch_size
  tf_valid_dataset = tf.constant(valid_dataset) 
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  weights = tf.Variable(
    tf.truncated_normal([image_size * image_size, num_labels])) # mean 0, low std dev, random values
  biases = tf.Variable(tf.zeros([num_labels])) # zeros
  
  # Training computation.
  logits = tf.matmul(tf_train_dataset, weights) + biases # logistic regression
  d_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels)) 
  loss = d_loss + 0.5*l2_beta*tf.nn.l2_loss(weights) # L' = L + 1/2*beta*||w||^2
  
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss) # GD Optimizer
  
  # Predictions for the training, validation, and test data. (Probabilities)
  train_prediction = tf.nn.softmax(logits) 
  valid_prediction = tf.nn.softmax(
    tf.matmul(tf_valid_dataset, weights) + biases) 
  test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)
  
  
run_simulation(graph, hyperparams)<jupyter_output>Processing...
Test accuracy: 89.0%<jupyter_text>Test Acc -> 89.0% (SIMPLE LRM-SGD-L2 REG)  2-Layers NN<jupyter_code>batch_size = 128
num_steps = 3001
hidden_nodes = 1024 

graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Inputs
  tf_train_dataset = tf.placeholder(tf.float32,
                                    shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  weights_1 = tf.Variable(
    tf.truncated_normal([image_size * image_size, hidden_nodes])) 
  biases_1 = tf.Variable(tf.zeros([hidden_nodes]))
  weights_2 = tf.Variable(
    tf.truncated_normal([hidden_nodes, num_labels])) 
  biases_2 = tf.Variable(tf.zeros([num_labels]))
  
  # Model
  def two_layer_nn(X):
    return tf.matmul(tf.nn.relu(tf.matmul(X, weights_1) + biases_1), weights_2) + biases_2

  logits = two_layer_nn(tf_train_dataset)

  loss = tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
  
  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(logits)
  valid_prediction = tf.nn.softmax(two_layer_nn(tf_valid_dataset))
  test_prediction = tf.nn.softmax(two_layer_nn(tf_test_dataset))
  
run_simulation(graph, hyperparams)<jupyter_output>Processing...
Test accuracy: 89.8%<jupyter_text>Test Acc -> 89.8% (2NN)  2-Layers NN + L2 Regularization<jupyter_code># Hyperparams
batch_size = 128
num_steps = 3001
hidden_nodes = 1024 
l2_beta = 0.003

graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Inputs
  tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  w1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_nodes])) 
  b1 = tf.Variable(tf.zeros([hidden_nodes]))
  w2 = tf.Variable(tf.truncated_normal([hidden_nodes, num_labels])) 
  b2 = tf.Variable(tf.zeros([num_labels]))
  
  # Model
  def two_NN(X):
    return tf.matmul(tf.nn.relu(tf.matmul(X, w1) + b1), w2) + b2

  # Loss and L2 Regularization
  d_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(two_NN(tf_train_dataset), tf_train_labels))
  loss = d_loss + 0.5*l2_beta*(tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2)) # L' = L + 1/2*beta*||wi||^2
  
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
  
  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(two_NN(tf_train_dataset))
  valid_prediction = tf.nn.softmax(two_NN(tf_valid_dataset))
  test_prediction = tf.nn.softmax(two_NN(tf_test_dataset))
  
run_simulation(graph, hyperparams)<jupyter_output>Processing...
Test accuracy: 93.2%<jupyter_text>Test Acc -> 93.2% (2NN - L2 REG) ---Problem 2---------Let's demonstrate an extreme case of overfitting. Restrict your training data to just a few batches. What happens?---<jupyter_code># Generic Stochastic Gradient Descent Function
# param: graph
# param: hyperparams (steps, batch_size)
# Modifing OFFSET value. Moving the pointer of the next batch => old_batch/OVERFITTING_FACTOR

def run_overfitted_simulation(graph, hyperparams):
  with tf.Session(graph=graph) as session:
    tf.initialize_all_variables().run()
    print('Processing...')
    for step in range(hyperparams['num_steps']):
      # Ordered Batches
      # offset = (step * hyperparams['batch_size']) % (train_labels.shape[0] - hyperparams['batch_size'])
      # Overfitting on purpose
      OVERFITTING_FACTOR = 1000
      offset = (((step * hyperparams['batch_size']) % (train_labels.shape[0] - hyperparams['batch_size']))
        /OVERFITTING_FACTOR)
      # Generate a minibatch.
      batch_data = train_dataset[offset:(offset + hyperparams['batch_size']), :]
      batch_labels = train_labels[offset:(offset + hyperparams['batch_size']), :]
      # Dict preparation and run training
      feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
      _, l, predictions = session.run(
        [optimizer, loss, train_prediction], feed_dict=feed_dict) # first param of run is a Tensor that means
      # that maps variables from our graph to the desired variables
      # In this case: "optimizer" to "_"(dummy); "loss" to "l"; and "train_predictions" to "predictions"
    print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
# Hyperparams
batch_size = 128
num_steps = 3001
hidden_nodes = 1024 
l2_beta = 0.003

graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Inputs
  tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  w1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_nodes])) 
  b1 = tf.Variable(tf.zeros([hidden_nodes]))
  w2 = tf.Variable(tf.truncated_normal([hidden_nodes, num_labels])) 
  b2 = tf.Variable(tf.zeros([num_labels]))
  
  # Model
  def two_NN(X):
    return tf.matmul(tf.nn.relu(tf.matmul(X, w1) + b1), w2) + b2

  # Loss and L2 Regularization
  d_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(two_NN(tf_train_dataset), tf_train_labels))
  loss = d_loss + 0.5*l2_beta*(tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2)) # L' = L + 1/2*beta*||wi||^2
  
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
  
  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(two_NN(tf_train_dataset))
  valid_prediction = tf.nn.softmax(two_NN(tf_valid_dataset))
  test_prediction = tf.nn.softmax(two_NN(tf_test_dataset))
  
run_overfitted_simulation(graph, hyperparams)<jupyter_output>Processing...
Test accuracy: 82.9%<jupyter_text>---Problem 3---------Introduce Dropout on the hidden layer of the neural network. Remember: Dropout should only be introduced during training, not evaluation, otherwise your evaluation results would be stochastic as well. TensorFlow provides `nn.dropout()` for that, but you have to make sure it's only inserted during training.What happens to our extreme overfitting case?---  2NN + L2 + Dropout Technique<jupyter_code># Hyperparams
batch_size = 192
num_steps = 5001
hidden_nodes = 1024 
l2_beta = 0.003
keep_prob = 0.7

graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Inputs
  tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  w1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_nodes])) 
  b1 = tf.Variable(tf.zeros([hidden_nodes]))
  w2 = tf.Variable(tf.truncated_normal([hidden_nodes, num_labels])) 
  b2 = tf.Variable(tf.zeros([num_labels]))
  
  # Training Model (DROPOUT)
  def two_NN_dropout(X):
    h1 = tf.matmul(X, w1) + b1
    h2 = tf.nn.relu(h1)
    h3 = tf.nn.dropout(h2, keep_prob)
    y = tf.matmul(h3, w2) + b2
    return y

  # Model
  def two_NN(X):
    return tf.matmul(tf.nn.relu(tf.matmul(X, w1) + b1), w2) + b2

  # Loss and L2 Regularization
  d_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(two_NN_dropout(tf_train_dataset), tf_train_labels))
  regularizers = 0.5*l2_beta*(tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2)) # L' = L + 1/2*beta*||wi||^2
  loss = d_loss + regularizers
  
  # Optimizer.
  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
  
  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(two_NN(tf_train_dataset))
  valid_prediction = tf.nn.softmax(two_NN(tf_valid_dataset))
  test_prediction = tf.nn.softmax(two_NN(tf_test_dataset))

print('Standard Simulation')  
run_simulation(graph, hyperparams)  
#print('Overfitted Simulation')
#run_overfitted_simulation(graph, hyperparams)<jupyter_output>Standard Simulation
Processing...
Test accuracy: 93.9%<jupyter_text>---Problem 4---------Try to get the best performance you can using a multi-layer model! The best reported test accuracy using a deep network is [97.1%](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html?showComment=1391023266211c8758720086795711595).One avenue you can explore is to add multiple layers.Another one is to use learning rate decay:    global_step = tf.Variable(0)   count the number of steps taken.    learning_rate = tf.train.exponential_decay(0.5, global_step, ...)    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)  ---  2-Layers NN + L2 + Dropout + Decay Learning Rate<jupyter_code># Hyperparams
batch_size = 192
num_steps = 5001
hidden_nodes = 1024 
l2_beta = 0.003
keep_prob = 0.7
starter_learning_rate = 0.5
decay_steps = 1500
decay_factor = 0.96

graph = tf.Graph()
hyperparams = {'batch_size' : batch_size, 'num_steps' : num_steps}
with graph.as_default():

  # Inputs
  tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))
  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
  tf_valid_dataset = tf.constant(valid_dataset)
  tf_test_dataset = tf.constant(test_dataset)
  
  # Variables.
  global_step = tf.Variable(0) # Decay Learning Rate
  w1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_nodes])) 
  b1 = tf.Variable(tf.zeros([hidden_nodes]))
  w2 = tf.Variable(tf.truncated_normal([hidden_nodes, num_labels])) 
  b2 = tf.Variable(tf.zeros([num_labels]))
  
  # Training Model (DROPOUT)
  def two_NN_dropout(X):
    h1 = tf.matmul(X, w1) + b1
    h2 = tf.nn.relu(h1)
    h3 = tf.nn.dropout(h2, keep_prob)
    y = tf.matmul(h3, w2) + b2
    return y

  # Model
  def two_NN(X):
    return tf.matmul(tf.nn.relu(tf.matmul(X, w1) + b1), w2) + b2

  # Loss and L2 Regularization
  d_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(two_NN_dropout(tf_train_dataset), tf_train_labels))
  regularizers = l2_beta*(tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2)) # L' = L + 1/2*beta*||wi||^2
  loss = d_loss + regularizers
  
  # Optimizer (Decay Learning Rate)
  learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, decay_steps,
                                             decay_factor, staircase=True)
  optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
  
  # Predictions for the training, validation, and test data.
  train_prediction = tf.nn.softmax(two_NN(tf_train_dataset))
  valid_prediction = tf.nn.softmax(two_NN(tf_valid_dataset))
  test_prediction = tf.nn.softmax(two_NN(tf_test_dataset))
  
  

print('Standard Simulation')  
run_simulation(graph, hyperparams)  
#print('Overfitted Simulation')
#run_overfitted_simulation(graph, hyperparams)<jupyter_output>Standard Simulation
Processing...
Test accuracy: 93.2%
{'fraction_non_alphanumeric': 0.11957398741326448, 'fraction_numerical': 0.02377494486579528, 'mean_word_length': 4.715339686443283, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '626980', 'license': 'Apache-2.0', 'n_tokens_mistral': 6779, 'n_tokens_neox': 6395, 'n_words': 1371, 'path': 'previous_training/udacity/3_regularization.ipynb', 'repo_name': 'archelogos/smart-live-camera', 'size': '18591'}