<jupyter_start><jupyter_text>Parameters<jupyter_code>learning_rate = 1e-3
epoch         = 5
batch_size    = 200
save_path     = './model_ckpt/model.ckpt'
filename      = './60000'<jupyter_output><empty_output><jupyter_text>Prepare MNIST data<jupyter_code>def readImages_hdf5(filename):
    '''Reads hdf5 file.
       Parameter
       ---------
       filename : the name of the hdf5 file
    '''
    file = h5py.File( filename + '.h5', "r+") #open the hdf5 file.
    
    hdf5_images = np.array(file["/images"]).astype("uint8") #read the images as np array
    hdf5_labels = np.array(file["/meta"]).astype("uint8")
    
    return hdf5_images, hdf5_labels
def one_hot_encoder(label_arr):
    '''Returns the given MNIST labels from np arrays of integers to np array of one hot labels.
       Parameter
       ---------
       label_arr : np array of MNIST integer labels
    '''
    total_labels  = label_arr.shape[0] #get the total number of labels
    one_hot_label = np.zeros([total_labels, 10]) #10 for num of classes in MNIST
    
    for i in range(label_arr.shape[0]): #loop through all the labels
        
        one_hot_label[i][int(label_arr[i])] = 1.0 #the label value will be marked as 1.0 at that specific index
        
    return one_hot_label #returns the np one-hot label 
images, labels = readImages_hdf5(filename)
labels = one_hot_encoder(labels)
images = images.reshape(images.shape[0],28, 28, 1) #reshape into a tensor of rank 4 for CNN filtering<jupyter_output><empty_output><jupyter_text>Graph<jupyter_code>tf.reset_default_graph()
sess = tf.InteractiveSession()

class Model():
           
    
    def __init__(self):
    
        self.x = tf.placeholder(tf.float32, [None, 28,28,1], name='inputs')
        self.y = tf.placeholder(tf.float32, [None, 10], name='targets')
        self.is_train = tf.placeholder(tf.bool, name='is_train')

        conv1 = tf.contrib.layers.conv2d(self.x, num_outputs=64, kernel_size=3, stride=1,padding='SAME', activation_fn=None)
    
        #batch normalization before the activation function but after the linear transformation.
        conv1_norm = tf.layers.batch_normalization(conv1, training=self.is_train)
        conv1_actv = tf.nn.relu(conv1_norm)

        conv2 = tf.contrib.layers.conv2d(conv1_actv, num_outputs=64, kernel_size=3, stride=2,padding='SAME', activation_fn=None)
        
        #batch normalization before the activation function but after the linear transformation.
        conv2_norm = tf.layers.batch_normalization(conv2, training=self.is_train)
        
        conv2_actv = tf.nn.relu(conv2_norm)
        

        output_size = 14*14*64
        output_layer = tf.reshape(conv2_actv, (-1, output_size))


        W2 = tf.Variable(tf.truncated_normal([output_size, 100], stddev=0.1))
        B2 = tf.Variable(tf.ones([100]))

        fc2 =  tf.add(tf.matmul(output_layer, W2), B2)
        
        #batch normalization before the activation function but after the linear transformation.
        fc2_norm = tf.layers.batch_normalization(fc2, training=self.is_train)
        fc2_actv = tf.nn.relu(fc2_norm)

        W3 = tf.Variable(tf.truncated_normal([100, 10], stddev=0.1))
        B3 = tf.Variable(tf.ones([10]))

        self.logits = tf.add(tf.matmul(fc2_actv, W3), B3)

        Y_pred = tf.nn.softmax(self.logits)
    
        self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self.y, logits=self.logits))
        
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) #IMPORTANT
        with tf.control_dependencies(update_ops): #IMPORTANT
            self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss)
        correct_prediction = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))<jupyter_output><empty_output><jupyter_text>Building the graph for training<jupyter_code>model_cnn = Model()

sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())

try:

    saver.restore(sess, save_path)
    print("Model has been loaded !")

except:

    print("Model is not loaded !")<jupyter_output>WARNING:tensorflow:From <ipython-input-6-841710c649c7>:47: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See @{tf.nn.softmax_cross_entropy_with_logits_v2}.

INFO:tensorflow:Restoring parameters from ./model_ckpt/model.ckpt
Model is not loaded !<jupyter_text>Training<jupyter_code>for epoch_idx in range(epoch):
    
    loss_total = 0
    accuracy_total = 0
    counter = 0
    
    for firstidx in range(0, images.shape[0], batch_size):
        
        finalidx = firstidx + batch_size
        if finalidx >= images.shape[0] : finalidx = images.shape[0] #to prevent the index to be out of range.
        
        _, loss_val, accuracy_val = sess.run([model_cnn.optimizer, model_cnn.loss, model_cnn.accuracy], 
                                              feed_dict={
                                                        model_cnn.x : images[firstidx:finalidx],
                                                        model_cnn.y : labels[firstidx:finalidx],
                                                        model_cnn.is_train : True
                                                        })
        loss_total += loss_val
        accuracy_total += accuracy_val
        counter += 1
        
    print("Epoch %d, loss %g, accuracy %g"%(epoch_idx, loss_total, accuracy_total/counter))
        
    saver.save(sess, save_path)

sess.close()<jupyter_output>Epoch 0, loss 64.8019, accuracy 0.948533
Epoch 1, loss 14.0743, accuracy 0.9878
Epoch 2, loss 7.84788, accuracy 0.994167
Epoch 3, loss 4.45371, accuracy 0.997233
Epoch 4, loss 2.38387, accuracy 0.999117<jupyter_text>Testing session (Testing with the same trained data because the purpose is just to check if the estimated population mean and population variance works or not.)<jupyter_code>tf.reset_default_graph()
sess = tf.InteractiveSession()

model_cnn = Model()

sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())

try:

    saver.restore(sess, save_path)
    print("Model has been loaded !")

except:

    print("Model is not loaded !")
counter = 0

for firstidx in range(0, images.shape[0], batch_size):

    finalidx = firstidx + batch_size
    if finalidx >= images.shape[0] : finalidx = images.shape[0] #to prevent the index to be out of range.

    loss_val, accuracy_val = sess.run([model_cnn.loss, model_cnn.accuracy], 
                                          feed_dict={
                                                    model_cnn.x : images[firstidx:finalidx],
                                                    model_cnn.y : labels[firstidx:finalidx],
                                                    model_cnn.is_train : False
                                                    })
    
    counter += 1

print("Loss %g, accuracy %g"%(loss_total, accuracy_total/counter))

sess.close()<jupyter_output>Loss 2.38387, accuracy 0.999117
{'fraction_non_alphanumeric': 0.1046463532331449, 'fraction_numerical': 0.030745898249000415, 'mean_word_length': 2.672911392405063, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '155923', 'license': 'MIT', 'n_tokens_mistral': 2430, 'n_tokens_neox': 2279, 'n_words': 514, 'path': 'Tensorflow built-in Batch Normalization.ipynb', 'repo_name': 'Ugenteraan/Batch-Normalization-DIY', 'size': '7253'}