<jupyter_start><jupyter_text>Imports<jupyter_code>from numpy.random import seed
seed(888)
from tensorflow import set_random_seed
set_random_seed(404)
import os
import numpy as np
import tensorflow as tf

from time import strftime
from PIL import Image<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>X_TRAIN_PATH = 'MNIST/digit_xtrain.csv'
X_TEST_PATH = 'MNIST/digit_xtest.csv'
Y_TRAIN_PATH = 'MNIST/digit_ytrain.csv'
Y_TEST_PATH = 'MNIST/digit_ytest.csv'

LOGGING_PATH = 'tensorboard_mnist_digit_logs/'

NR_CLASSES = 10
VALIDATION_SIZE = 10000
IMAGE_WIDTH = 28
IMAGE_HEIGHT = 28
CHANNELS = 1
TOTAL_INPUTS = IMAGE_WIDTH*IMAGE_HEIGHT*CHANNELS<jupyter_output><empty_output><jupyter_text>Get the Data<jupyter_code>%%time

y_train_all = np.loadtxt(Y_TRAIN_PATH, delimiter=',', dtype=int)
y_train_all.shape
y_test = np.loadtxt(Y_TEST_PATH, delimiter=',', dtype=int)
%%time

x_train_all = np.loadtxt(X_TRAIN_PATH, delimiter=',', dtype=int)
%%time

x_test = np.loadtxt(X_TEST_PATH, delimiter=',', dtype=int)<jupyter_output>CPU times: user 9.59 s, sys: 313 ms, total: 9.9 s
Wall time: 10.6 s<jupyter_text>Explore<jupyter_code>x_train_all.shape
x_train_all[0]
y_train_all.shape
x_test.shape
y_train_all[:5]<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code># Re-scale
x_train_all, x_test = x_train_all / 255.0, x_test / 255.0<jupyter_output><empty_output><jupyter_text>Convert target values to one-hot encoding<jupyter_code>values = y_train_all[:5]
np.eye(10)[values]
np.eye(10)
np.eye(10)[2]
values
values[4]
y_train_all = np.eye(NR_CLASSES)[y_train_all]
y_train_all.shape
y_test = np.eye(NR_CLASSES)[y_test]
y_test.shape<jupyter_output><empty_output><jupyter_text>Create validation dataset from training data **Challenge:** Split the training dataset into a smaller training dataset and a validation dataset for the features and the labels. Create four arrays: ```x_val```, ```y_val```, ```x_train```, and ```y_train``` from ```x_train_all``` and ```y_train_all```. Use the validation size of 10,000.<jupyter_code>x_val = x_train_all[:VALIDATION_SIZE]
y_val = y_train_all[:VALIDATION_SIZE]
x_train = x_train_all[VALIDATION_SIZE:]
y_train = y_train_all[VALIDATION_SIZE:]
x_train.shape
x_val.shape<jupyter_output><empty_output><jupyter_text>Setup Tensorflow Graph<jupyter_code>X = tf.placeholder(tf.float32, shape=[None, TOTAL_INPUTS], name='X')
Y = tf.placeholder(tf.float32, shape=[None, NR_CLASSES], name='labels')<jupyter_output><empty_output><jupyter_text>Neural Network Architecture Hyperparameters<jupyter_code>nr_epochs = 50
learning_rate = 1e-3

n_hidden1 = 512
n_hidden2 = 64
def setup_layer(input, weight_dim, bias_dim, name):
    
    with tf.name_scope(name):
        initial_w = tf.truncated_normal(shape=weight_dim, stddev=0.1, seed=42)
        w = tf.Variable(initial_value=initial_w, name='W')

        initial_b = tf.constant(value=0.0, shape=bias_dim)
        b = tf.Variable(initial_value=initial_b, name='B')

        layer_in = tf.matmul(input, w) + b
        
        if name=='out':
            layer_out = tf.nn.softmax(layer_in)
        else:
            layer_out = tf.nn.relu(layer_in)
        
        tf.summary.histogram('weights', w)
        tf.summary.histogram('biases', b)
        
        return layer_out
# Model without dropout
# layer_1 = setup_layer(X, weight_dim=[TOTAL_INPUTS, n_hidden1], 
#                       bias_dim=[n_hidden1], name='layer_1')

# layer_2 = setup_layer(layer_1, weight_dim=[n_hidden1, n_hidden2], 
#                       bias_dim=[n_hidden2], name='layer_2')

# output = setup_layer(layer_2, weight_dim=[n_hidden2, NR_CLASSES], 
#                       bias_dim=[NR_CLASSES], name='out')

# model_name = f'{n_hidden1}-{n_hidden2} LR{learning_rate} E{nr_epochs}'
layer_1 = setup_layer(X, weight_dim=[TOTAL_INPUTS, n_hidden1], 
                      bias_dim=[n_hidden1], name='layer_1')

layer_drop = tf.nn.dropout(layer_1, keep_prob=0.8, name='dropout_layer')

layer_2 = setup_layer(layer_drop, weight_dim=[n_hidden1, n_hidden2], 
                      bias_dim=[n_hidden2], name='layer_2')

output = setup_layer(layer_2, weight_dim=[n_hidden2, NR_CLASSES], 
                      bias_dim=[NR_CLASSES], name='out')

model_name = f'{n_hidden1}-DO-{n_hidden2} LR{learning_rate} E{nr_epochs}'<jupyter_output><empty_output><jupyter_text>Tensorboard Setup<jupyter_code># Folder for Tensorboard

folder_name = f'{model_name} at {strftime("%H:%M")}'
directory = os.path.join(LOGGING_PATH, folder_name)

try:
    os.makedirs(directory)
except OSError as exception:
    print(exception.strerror)
else:
    print('Successfully created directories!')<jupyter_output>Successfully created directories!<jupyter_text>Loss, Optimisation & Metrics  Defining Loss Function<jupyter_code>with tf.name_scope('loss_calc'):
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=Y, logits=output))<jupyter_output><empty_output><jupyter_text>Defining Optimizer<jupyter_code>with tf.name_scope('optimizer'):
    optimizer = tf.train.AdamOptimizer(learning_rate)
    train_step = optimizer.minimize(loss)<jupyter_output><empty_output><jupyter_text>Accuracy Metric<jupyter_code>with tf.name_scope('accuracy_calc'):
    correct_pred = tf.equal(tf.argmax(output, axis=1), tf.argmax(Y, axis=1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
with tf.name_scope('performance'):
    tf.summary.scalar('accuracy', accuracy)
    tf.summary.scalar('cost', loss)<jupyter_output><empty_output><jupyter_text>Check Input Images in Tensorboard<jupyter_code>with tf.name_scope('show_image'):
    x_image = tf.reshape(X, [-1, 28, 28, 1])
    tf.summary.image('image_input', x_image, max_outputs=4)<jupyter_output><empty_output><jupyter_text>Run Session<jupyter_code>sess = tf.Session()<jupyter_output><empty_output><jupyter_text>Setup Filewriter and Merge Summaries<jupyter_code>merged_summary = tf.summary.merge_all()

train_writer = tf.summary.FileWriter(directory + '/train')
train_writer.add_graph(sess.graph)

validation_writer = tf.summary.FileWriter(directory + '/validation')<jupyter_output><empty_output><jupyter_text>Initialise all the variables<jupyter_code>init = tf.global_variables_initializer()
sess.run(init)<jupyter_output><empty_output><jupyter_text>Batching the Data<jupyter_code>size_of_batch = 1000
num_examples = y_train.shape[0]
nr_iterations = int(num_examples/size_of_batch)

index_in_epoch = 0
def next_batch(batch_size, data, labels):
    
    global num_examples
    global index_in_epoch
    
    start = index_in_epoch
    index_in_epoch += batch_size
    
    if index_in_epoch > num_examples:
        start = 0
        index_in_epoch = batch_size
    
    end = index_in_epoch
    
    return data[start:end], labels[start:end]<jupyter_output><empty_output><jupyter_text>Training Loop<jupyter_code>for epoch in range(nr_epochs):
    
    # ============= Training Dataset =========
    for i in range(nr_iterations):
        
        batch_x, batch_y = next_batch(batch_size=size_of_batch, data=x_train, labels=y_train)
        
        feed_dictionary = {X:batch_x, Y:batch_y}
        
        sess.run(train_step, feed_dict=feed_dictionary)
        
    
    s, batch_accuracy = sess.run(fetches=[merged_summary, accuracy], feed_dict=feed_dictionary)
        
    train_writer.add_summary(s, epoch)
    
    print(f'Epoch {epoch} \t| Training Accuracy = {batch_accuracy}')
    
    # ================== Validation ======================
    
    summary = sess.run(fetches=merged_summary, feed_dict={X:x_val, Y:y_val})
    validation_writer.add_summary(summary, epoch)

print('Done training!')<jupyter_output>Epoch 0 	| Training Accuracy = 0.8450000286102295
Epoch 1 	| Training Accuracy = 0.8569999933242798
Epoch 2 	| Training Accuracy = 0.8679999709129333
Epoch 3 	| Training Accuracy = 0.8650000095367432
Epoch 4 	| Training Accuracy = 0.8700000047683716
Epoch 5 	| Training Accuracy = 0.9649999737739563
Epoch 6 	| Training Accuracy = 0.9729999899864197
Epoch 7 	| Training Accuracy = 0.9800000190734863
Epoch 8 	| Training Accuracy = 0.9829999804496765
Epoch 9 	| Training Accuracy = 0.9810000061988831
Epoch 10 	| Training Accuracy = 0.9850000143051147
Epoch 11 	| Training Accuracy = 0.9860000014305115
Epoch 12 	| Training Accuracy = 0.9869999885559082
Epoch 13 	| Training Accuracy = 0.9879999756813049
Epoch 14 	| Training Accuracy = 0.9860000014305115
Epoch 15 	| Training Accuracy = 0.9860000014305115
Epoch 16 	| Training Accuracy = 0.9900000095367432
Epoch 17 	| Training Accuracy = 0.9890000224113464
Epoch 18 	| Training Accuracy = 0.9879999756813049
Epoch 19 	| Training Accuracy = 0.9890000[...]<jupyter_text>Make a Prediction<jupyter_code>img = Image.open('MNIST/test_img.png')
img
bw = img.convert('L')
img_array = np.invert(bw)
img_array.shape
test_img = img_array.ravel()
test_img.shape
prediction = sess.run(fetches=tf.argmax(output, axis=1), feed_dict={X:[test_img]})
print(f'Prediction for test image is {prediction}')<jupyter_output>Prediction for test image is [2]<jupyter_text>Testing and Evaluation **Challenge:** Calculate the accuracy over the test dataset (```x_test``` and ```y_test```). Use your knowledge of running a session to get the accuracy. Display the accuracy as a percentage rounded to two decimal numbers.<jupyter_code>test_accuracy = sess.run(fetches=accuracy, feed_dict={X:x_test, Y:y_test})
print(f'Accuracy on test set is {test_accuracy:0.2%}')<jupyter_output>Accuracy on test set is 97.67%<jupyter_text>Reset for the Next Run<jupyter_code>train_writer.close()
validation_writer.close()
sess.close()
tf.reset_default_graph()<jupyter_output><empty_output><jupyter_text>Code for 1st Part of Module<jupyter_code># with tf.name_scope('hidden_1'):

#     initial_w1 = tf.truncated_normal(shape=[TOTAL_INPUTS, n_hidden1], stddev=0.1, seed=42)
#     w1 = tf.Variable(initial_value=initial_w1, name='w1')

#     initial_b1 = tf.constant(value=0.0, shape=[n_hidden1])
#     b1 = tf.Variable(initial_value=initial_b1, name='b1')

#     layer1_in = tf.matmul(X, w1) + b1

#     layer1_out = tf.nn.relu(layer1_in)<jupyter_output><empty_output><jupyter_text>**Challenge:** Set up the second hindden layer. This layer has 64 neurons and needs to work off the output of the first hidden layer (see above). Then setup the output layer. Remember, the output layer will use the softmax activation function.<jupyter_code># with tf.name_scope('hidden_2'):

#     initial_w2 = tf.truncated_normal(shape=[n_hidden1, n_hidden2], stddev=0.1, seed=42)
#     w2 = tf.Variable(initial_value=initial_w2, name='w2')

#     initial_b2 = tf.constant(value=0.0, shape=[n_hidden2])
#     b2 = tf.Variable(initial_value=initial_b2, name='b2')

#     layer2_in = tf.matmul(layer1_out, w2) + b2
#     layer2_out = tf.nn.relu(layer2_in)
# with tf.name_scope('output_layer'):

#     initial_w3 = tf.truncated_normal(shape=[n_hidden2, NR_CLASSES], stddev=0.1, seed=42)
#     w3 = tf.Variable(initial_value=initial_w3, name='w3')

#     initial_b3 = tf.constant(value=0.0, shape=[NR_CLASSES])
#     b3 = tf.Variable(initial_value=initial_b3, name='b3')

#     layer3_in = tf.matmul(layer2_out, w3) + b3
#     output = tf.nn.softmax(layer3_in)
# b3.eval(sess)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12359750667853962, 'fraction_numerical': 0.05004452359750668, 'mean_word_length': 5.414049114791547, 'pattern_counts': {'":': 0, '<': 92, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '457980', 'license': 'MIT', 'n_tokens_mistral': 4598, 'n_tokens_neox': 4163, 'n_words': 731, 'path': 'src/11_Section_11_Use_Tensorflow_to_Classify_Handwritten_Digits/02_Download_the_Complete_Notebook_Here/11 Neural Networks - TF Handwriting Recognition.ipynb', 'repo_name': 'MilovanTomasevic/Complete-2020-Data-Science-and-Machine-Learning-Bootcamp', 'size': '11230'}