<jupyter_start><jupyter_text>Test dataset<jupyter_code>for j in range(iterations):
    error, correct_cnt = (0.0, 0)
    if(j % 10 == 0 or j == iterations-1):
      error, correct_cnt = (0.0, 0)

      for i in range(len(test_images)):

            layer_0 = test_images[i:i+1]
            layer_1 = relu(np.dot(layer_0,weights_0_1))
            layer_2 = np.dot(layer_1,weights_1_2)

            error += np.sum((test_labels[i:i+1] - layer_2) ** 2)
            correct_cnt += int(np.argmax(layer_2) == \
                                            np.argmax(test_labels[i:i+1]))
      sys.stdout.write(" Test-Err:" + str(error/float(len(test_images)))[0:5] +\
                 " Test-Acc:" + str(correct_cnt/float(len(test_images))))
      print()<jupyter_output>Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Err:0.738 Test-Acc:0.686
 Test-Er[...]<jupyter_text>Dropout 50% of the weights<jupyter_code>dropout_mask = np.random.randint(2,size=layer_1.shape)
dropout_mask<jupyter_output><empty_output><jupyter_text>You multiply layer_1 by 2. Why do you do this? Remember that layer_2 will perform a weighted sum of layer_1. Even though it’s weighted, it’s still a sum over the values of layer_1. If you turn off half the nodes in layer_1, that sum will be cut in half.<jupyter_code>import numpy, sys
np.random.seed(1)
def relu(x):
   return (x >= 0) * x

def relu2deriv(output):
   return output >= 0

alpha, iterations, hidden_size = (0.005, 300, 100)
pixels_per_image, num_labels = (784, 10)

weights_0_1 = 0.2*np.random.random((pixels_per_image,hidden_size)) - 0.1
weights_1_2 = 0.2*np.random.random((hidden_size,num_labels)) - 0.1

for j in range(iterations):
   error, correct_cnt = (0.0,0)
   for i in range(len(images)):
      layer_0 = images[i:i+1]
      layer_1 = relu(np.dot(layer_0,weights_0_1))
      dropout_mask = np.random.randint(2, size=layer_1.shape)
      layer_1 *= dropout_mask * 2
      layer_2 = np.dot(layer_1,weights_1_2)

      error += np.sum((labels[i:i+1] - layer_2) ** 2)
      correct_cnt += int(np.argmax(layer_2) == \
                                      np.argmax(labels[i:i+1]))
      layer_2_delta = (labels[i:i+1] - layer_2)
      layer_1_delta = layer_2_delta.dot(weights_1_2.T) * relu2deriv(layer_1)
      layer_1_delta *= dropout_mask

      weights_1_2 += alpha * layer_1.T.dot(layer_2_delta)
      weights_0_1 += alpha * layer_0.T.dot(layer_1_delta)

   if(j%10 == 0):
      test_error = 0.0
      test_correct_cnt = 0

      for i in range(len(test_images)):
           layer_0 = test_images[i:i+1]
           layer_1 = relu(np.dot(layer_0,weights_0_1))
           layer_2 = np.dot(layer_1, weights_1_2)

           test_error += np.sum((test_labels[i:i+1] - layer_2) ** 2)
           test_correct_cnt += int(np.argmax(layer_2) == \
                                     np.argmax(test_labels[i:i+1]))

      sys.stdout.write("\n" + \
           "I:" + str(j) + \
           " Test-Err:" + str(test_error/ float(len(test_images)))[0:5] +\
           " Test-Acc:" + str(test_correct_cnt/ float(len(test_images)))+\
           " Train-Err:" + str(error/ float(len(images)))[0:5] +\
           " Train-Acc:" + str(correct_cnt/ float(len(images))))<jupyter_output>I:0 Test-Err:0.632 Test-Acc:0.6388571428571429 Train-Err:0.896 Train-Acc:0.393
I:10 Test-Err:0.433 Test-Acc:0.786 Train-Err:0.479 Train-Acc:0.746
I:20 Test-Err:0.400 Test-Acc:0.8215 Train-Err:0.451 Train-Acc:0.783
I:30 Test-Err:0.395 Test-Acc:0.8272142857142857 Train-Err:0.424 Train-Acc:0.81
I:40 Test-Err:0.385 Test-Acc:0.8395714285714285 Train-Err:0.419 Train-Acc:0.826
I:50 Test-Err:0.394 Test-Acc:0.8359285714285715 Train-Err:0.433 Train-Acc:0.81
I:60 Test-Err:0.400 Test-Acc:0.8310714285714286 Train-Err:0.427 Train-Acc:0.803
I:70 Test-Err:0.383 Test-Acc:0.8357857142857142 Train-Err:0.410 Train-Acc:0.818
I:80 Test-Err:0.387 Test-Acc:0.8302142857142857 Train-Err:0.393 Train-Acc:0.845
I:90 Test-Err:0.378 Test-Acc:0.8209285714285715 Train-Err:0.390 Train-Acc:0.851
I:100 Test-Err:0.376 Test-Acc:0.8196428571428571 Train-Err:0.389 Train-Acc:0.838
I:110 Test-Err:0.370 Test-Acc:0.8242857142857143 Train-Err:0.379 Train-Acc:0.862
I:120 Test-Err:0.386 Test-Acc:0.8200714285714286 Train-Err:0.381 [...]<jupyter_text>Mini-batch gradient descent<jupyter_code>import numpy as np
np.random.seed(1)

def relu(x):
    return (x >= 0) * x

def relu2deriv(output):
    return output >= 0

batch_size = 100
alpha, iterations = (0.001, 300)
pixels_per_image, num_labels, hidden_size = (784, 10, 100)

weights_0_1 = 0.2*np.random.random((pixels_per_image,hidden_size)) - 0.1
weights_1_2 = 0.2*np.random.random((hidden_size,num_labels)) - 0.1

for j in range(iterations):
    error, correct_cnt = (0.0, 0)
    for i in range(int(len(images) / batch_size)):
        batch_start, batch_end = ((i * batch_size),((i+1)*batch_size))

        layer_0 = images[batch_start:batch_end]
        layer_1 = relu(np.dot(layer_0,weights_0_1))
        dropout_mask = np.random.randint(2,size=layer_1.shape)
        layer_1 *= dropout_mask * 2
        layer_2 = np.dot(layer_1,weights_1_2)

        error += np.sum((labels[batch_start:batch_end] - layer_2) ** 2)
        for k in range(batch_size):
            correct_cnt += int(np.argmax(layer_2[k:k+1]) == \
                    np.argmax(labels[batch_start+k:batch_start+k+1]))

            layer_2_delta = (labels[batch_start:batch_end]-layer_2) \
                                                            /batch_size
            layer_1_delta = layer_2_delta.dot(weights_1_2.T)* \
                                                     relu2deriv(layer_1)
            layer_1_delta *= dropout_mask

            weights_1_2 += alpha * layer_1.T.dot(layer_2_delta)
            weights_0_1 += alpha * layer_0.T.dot(layer_1_delta)

    if(j%10 == 0):
        test_error = 0.0
        test_correct_cnt = 0

        for i in range(len(test_images)):
            layer_0 = test_images[i:i+1]
            layer_1 = relu(np.dot(layer_0,weights_0_1))
            layer_2 = np.dot(layer_1, weights_1_2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15002868617326448, 'fraction_numerical': 0.12220309810671257, 'mean_word_length': 2.9417750141322783, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '164477', 'license': 'MIT', 'n_tokens_mistral': 3198, 'n_tokens_neox': 2829, 'n_words': 442, 'path': 'modules/module_14_5/nn_regularization_batching.ipynb', 'repo_name': 'ryanjmccall/sb_ml_eng_capstone', 'size': '6972'}