<jupyter_start><jupyter_text>Assignment 4 Part I  Import Libraries and Data<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
type(mnist)
mnist.train.num_examples
mnist.test.num_examples<jupyter_output><empty_output><jupyter_text>Define Supporting Functions  Intialize Weights in the Filter<jupyter_code>"""
Returns a tf.Variable used to store weights in a filter
This variable is initialized with values that can be used to initialize the weights
They are random numbers
"""
def initialize_weights(filter_shape):
    init_random_dist = tf.truncated_normal(filter_shape, stddev=0.1)
    return (tf.Variable(init_random_dist))<jupyter_output><empty_output><jupyter_text>Initialize Bias<jupyter_code>"""
Returns a tf.Variable used to store bias
This variable is initialized with values that can be used to initialize the bias
They are random numbers
"""
def initialize_bias(bias_shape):
    initalize_bias_vals = tf.constant(0.1, shape=bias_shape)
    return (tf.Variable(initalize_bias_vals))<jupyter_output><empty_output><jupyter_text>Setup Conv Layer and Perform Conv Computation: Dot Product (x * W)<jupyter_code>"""
Define a function to set up a conv layer (conv2d)
Paramaters: 
    inputs: [batch, H, W, Channels]
    filter_shape:
    [filter H, filter W, in_channels(in_depth = in_num_filters), out_channels(out_depth = out_num_filters)]
    Return: Outputs of the Layer: the dot product: inputs * weights): x * W
"""

def create_convolutional_layer_and_compute_dot_product(inputs, filter_shape):
    # Initialize the weights in the filter
    filter_initialized_with_weights = initialize_weights(filter_shape)
    
    # Create Conv layer
    conv_layer_outputs = tf.nn.conv2d(inputs, filter_initialized_with_weights, strides=[1, 1, 1, 1], padding='SAME')
    
    # Return the Conv Layer Outputs
    return (conv_layer_outputs)<jupyter_output><empty_output><jupyter_text>Setup a ReLU Layer and perform Computation: Dot Product + Bias (x.W + b)<jupyter_code>"""
Define the function
Setup ReLU layer
Perform Computation
"""

def create_relu_layer_and_compute_dotproduct_plus_b(inputs, filter_shape):
    # Intialize biad for each input channel
    b = initialize_bias([filter_shape[3]])
    
    # Perform the computation by first adding the inputs
    # Then create a ReLU layer associated with this Conv Layer
    relu_layer_outputs = tf.nn.relu(inputs + b)
    
    # Return the outputs of the ReLU Layer
    return (relu_layer_outputs)<jupyter_output><empty_output><jupyter_text>Setup a Pooling Layer and Reduce Spatial Size<jupyter_code>def create_maxpool2by4_and_reduce_spacial_size(inputs):
    # Create a pooling Layer
    pooling_layer_outputs = tf.nn.max_pool(inputs, ksize=[1, 2, 2, 1], strides=[1, 2, 4, 1], padding='SAME')
    
    # Return the pooling Layer
    return(pooling_layer_outputs)<jupyter_output><empty_output><jupyter_text>Setup a Fully Connected Layer and Perform Computaion: (Inputs * Weights) + Bias<jupyter_code>def create_fully_connected_layer_and_compute_dotproduct_plus_bias(inputs, output_size):
    # Get the number of input channels from the input
    # Inputs: the oputputs from the preceding layer or the previous operation like reshaping
    input_size = int(inputs.get_shape()[1])
    
    # Intialize the weights of the filtyer of the FC layer
    # Filter shape [in_channels, out_channels]
    # Each weight for one filter cell
    W = initialize_weights([input_size, output_size])
    
    # Intialize the bias:: each bias output channel
    b = initialize_bias([output_size])
    
    # First: Perform the computation for the FC layer: dot product: inputs * W
    # Then: Add bias to get the results: Outputs of the FC layer
    fc_xW_plus_bias_outputs = tf.matmul(inputs, W) + b
    
    # Return the results: Outputs
    return (fc_xW_plus_bias_outputs)<jupyter_output><empty_output><jupyter_text>Build the Convolutional Neural Network  Create Placeholders for Inputs and Labels: x & y_true<jupyter_code># Placeholder

"""
Create a placeholder for the inputs data: x
x: a 2D array
x: a placeholder that can hold any number of rows/records
    Each row/record is a vector (1D Array) to hold data for one image
    Each row/record has 784 values/elements: 1 pixel = 1 value
"""

x = tf.placeholder(tf.float32, shape=[None, 784])
# Placeholder

"""
Create a placeholder for the labels of the input data: y_true
y_true: a 2D array
y_true: can hold any number of rows/records
    Each row is a vector (1D Array) of 10 values that indicated a digit between 0-9
    Each row/record: the label is stored in the one-hot format
"""

y_true = tf.placeholder(tf.float32, shape=[None,10])<jupyter_output><empty_output><jupyter_text>Reshape the Input Placeholder x<jupyter_code>"""
Prepare to feed inputs into the 1st conv layer
Reshape the input placeholder x
From 1D arrray (vector) original input shape to a 4D-input: [batch, H, W, depth_channels]
Depth = color channels: Grayscale = 1
Reshaped inputs: x_image: [1, 28, 28, 1]
"""

x_image = tf.reshape(x, [-1, 28, 28, 1])<jupyter_output><empty_output><jupyter_text>Create the Convolutional Layer, ReLU Layer, and Perform Computation: x*W + B<jupyter_code>"""
Inputs: x_image with Reshaped inputs with shape [1, 28, 28, 1]
filter_shape: [5, 5, 1, 32]
    Filter: 5x5 
    Input Channels = 1
    Output Channels = 32
"""

# Create the 1st Conv Layer 
# Then learn/extract the features, get the results (outputs): Dot product of inputs * weights
# Return the outputs of the layer
conv_layer_1_outputs = create_convolutional_layer_and_compute_dot_product(x_image, filter_shape=[5, 5, 1, 32])

# Create the ReLU layer for the 1st Conv Layer
# Accepts the outputs from the 1st conv layer as the inputs
# Perform the computation at the layer: add inputs + bias
# Return the outputs of the layer
conv_relu_layer_1_outputs = create_relu_layer_and_compute_dotproduct_plus_b(conv_layer_1_outputs, filter_shape=[5, 5, 1, 32])<jupyter_output><empty_output><jupyter_text>Create the Pooling Layer and Reduce Spacial Size<jupyter_code>"""
Create the 1st Pooling Layer 
Then reduce the spacial size of the input data
Return: Outputs of the layer
"""

pooling_layer_1_outputs = create_maxpool2by4_and_reduce_spacial_size(conv_relu_layer_1_outputs)<jupyter_output><empty_output><jupyter_text>Reshape/Flatten Data to Make it Ready to be Fed into 1st FC Layer<jupyter_code># Reshape and flatten the output of the second pooling Layer
# Prepare to feed the output data into the 1st fully connected layer

pooling_layer_1_outputs_flat = tf.reshape(pooling_layer_1_outputs, [-1, 7 * 7 * 64])<jupyter_output><empty_output><jupyter_text>Create 1st FC Layer, ReLU Layer, and Output Data to Dropout Layer<jupyter_code>"""
Create the FC Layer
Feed the flattened output of the second pooling layer as the inputs into this layer
Then perform the computation: dot product + bias: (x * W) + b

Parameters:
    pooling_layer_2_outputs_flat
    output_size: 1024 out channels

Return: Outputs of the computation: (x * W) + b
"""

fc_layer_1_outputs = create_fully_connected_layer_and_compute_dotproduct_plus_bias(pooling_layer_1_outputs_flat, output_size=1024)

# Create the ReLU layer of the 1st FC Layer
# IMPORTANT: The activation function of this layer is also ReLU

# Return: Outputs of the layer

fc_relu_layer_1_outputs = tf.nn.relu(fc_layer_1_outputs)<jupyter_output><empty_output><jupyter_text>Create Dropout Layer and Dropout a Fraction of Outputs Randomly<jupyter_code>"""
Declare a placeholder to hold the values of probabilitiy to keep:
    The % of total output channels that will be kept
    keep_prob = hold_prob = 50% -> keep 50%, drop 50%
"""

hold_prob = tf.placeholder(tf.float32)

# Dropout:
# Set the outputs to 0 so that they will be ignored in the next layer

fc_dropout_outputs = tf.nn.dropout(fc_relu_layer_1_outputs, keep_prob=hold_prob)<jupyter_output>WARNING:tensorflow:From <ipython-input-19-7dac78920ceb>:12: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.<jupyter_text>Create Final FC Layer, Compute (x.W + b), and Produce Final Outputs<jupyter_code>"""
Create the final FC Layer
Then compute x*W + b
Parameters:
    fc_dropout_outputs: Outputs form the dropout layer
    
Return: y_pred: final predicted outputs, i.e. final classification outputs
"""

y_pred = create_fully_connected_layer_and_compute_dotproduct_plus_bias(fc_dropout_outputs, output_size=10)<jupyter_output><empty_output><jupyter_text>Define Loss Function and Calculate Softmax Cross Entropy Loss<jupyter_code>"""
Define loss function: cross-entropy with logits, i.e. with the final outputs
Calculate the softmax cross-entropy loss
"""

softmax_cross_entropy_loss = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)

# Compute the mean of losses
cross_entropy_mean = tf.reduce_mean(softmax_cross_entropy_loss)<jupyter_output>WARNING:tensorflow:From <ipython-input-21-090b5c94eb32>:6: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See `tf.nn.softmax_cross_entropy_with_logits_v2`.<jupyter_text>Create an Optimizer to Optimize CNN Model and Set Learning Rate<jupyter_code># Get an ADAM Optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)<jupyter_output><empty_output><jupyter_text>Create a Trainer to Train CNN Model<jupyter_code># Create a CNN model trainer
# And optimize the model by minimizing the softmax cross-entropy loss

cnn_trainer = optimizer.minimize(cross_entropy_mean)<jupyter_output><empty_output><jupyter_text>Train and Test CNN Deep Learnign Model on MNIST Dataset  Create a Variable Initializer to Initialize All Variables<jupyter_code># Get a variable initializer

vars_initializer = tf.global_variables_initializer()<jupyter_output><empty_output><jupyter_text>Set the Steps Count to 3000<jupyter_code>"""
x: mnist.train= 50000 images
Each time of training (run the whole process) = 1 step
Each step (i.e. each round of training): Use one batch of inputs
Batch size = 50 images -> total batches = 50000/50 = 1000 batches

Steps = 3000: finish the training after 5000 runs
"""

steps = 3000<jupyter_output><empty_output><jupyter_text>Run tf.Session() to Train and Test Deep Learning CNN Model<jupyter_code>with tf.Session() as sess:
    #First run vars_initializer to initalize all variables
    sess.run(vars_initializer)
    
    for i in range(steps):
        # each batch: 50 images
        batch_x, batch_y = mnist.train.next_batch(50)
        
        # Train the model
        # Dropout keep_prob (% to keep): 0.5 i.e. drop 50%
        sess.run(cnn_trainer, feed_dict={x: batch_x, y_true: batch_y, hold_prob: 0.5})
        
        # Test the model after each 100 steps
        # Run this block of code after each 100 times of training
        if i % 100 == 0:
            print('ON STEP: {}'.format(i))
            print('ACCURACY: ')
            
            # Compare to find matches of y_pred and y_true
            matches = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))
            
            # Cast the matches from integers to tf.float32
            # Calculate the accuracy using the mean of matches
            acc = tf.reduce_mean(tf.cast(matches, tf.float32))
            
            # Test the model at each 100th step using test dataset
            #Dropout: NONE because of test not training
            test_accuracy = sess.run(acc, feed_dict = {x: mnist.test.images, \
                                                      y_true: mnist.test.labels, \
                                                      hold_prob: 1.0})
            
            print(test_accuracy)
            print('\n')<jupyter_output>ON STEP: 0
ACCURACY: 
0.2602


ON STEP: 100
ACCURACY: 
0.9084


ON STEP: 200
ACCURACY: 
0.9416


ON STEP: 300
ACCURACY: 
0.9548


ON STEP: 400
ACCURACY: 
0.9585


ON STEP: 500
ACCURACY: 
0.9604


ON STEP: 600
ACCURACY: 
0.9729


ON STEP: 700
ACCURACY: 
0.9759


ON STEP: 800
ACCURACY: 
0.9749


ON STEP: 900
ACCURACY: 
0.9791


ON STEP: 1000
ACCURACY: 
0.9802


ON STEP: 1100
ACCURACY: 
0.979


ON STEP: 1200
ACCURACY: 
0.982


ON STEP: 1300
ACCURACY: 
0.9816


ON STEP: 1400
ACCURACY: 
0.979


ON STEP: 1500
ACCURACY: 
0.9825


ON STEP: 1600
ACCURACY: 
0.9834


ON STEP: 1700
ACCURACY: 
0.9821


ON STEP: 1800
ACCURACY: 
0.9839


ON STEP: 1900
ACCURACY: 
0.9849


ON STEP: 2000
ACCURACY: 
0.9829


ON STEP: 2100
ACCURACY: 
0.9863


ON STEP: 2200
ACCURACY: 
0.9859


ON STEP: 2300
ACCURACY: 
0.9869


ON STEP: 2400
ACCURACY: 
0.985


ON STEP: 2500
ACCURACY: 
0.9861


ON STEP: 2600
ACCURACY: 
0.9844


ON STEP: 2700
ACCURACY: 
0.9844


ON STEP: 2800
ACCURACY: 
0.9869


ON STEP: 2900
ACCURACY: 
0.987[...]
{'fraction_non_alphanumeric': 0.0780335840741585, 'fraction_numerical': 0.03647139275131069, 'mean_word_length': 4.330903199675983, 'pattern_counts': {'":': 0, '<': 84, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '389768', 'license': 'Unlicense', 'n_tokens_mistral': 4853, 'n_tokens_neox': 4400, 'n_words': 1390, 'path': 'pdf/MNIST_CNN_Classifier.ipynb', 'repo_name': 'ColemanBarkley/ColemanBarkley.github.io', 'size': '13161'}