<jupyter_start><jupyter_text>Script to train an AutoEncoder for the CIFAR-10 dataset. ------------------------------------------------------------------------------------------------------------------- Technology used: Tensorflow-core<jupyter_code># packages used for machine learning
import tensorflow as tf

# packages used for processing: 
from six.moves import cPickle as pickle # for reading the data
import matplotlib.pyplot as plt # for visualization
import numpy as np
from sklearn.preprocessing import OneHotEncoder # for encoding the labels in one hot form

# for operating system related stuff
import os
import sys # for memory usage of objects
from subprocess import check_output

# to plot the images inline
%matplotlib inline
# Input data files are available in the "../Data/" directory.

def exec_command(cmd):
    '''
        function to execute a shell command and see it's 
        output in the python console
        @params
        cmd = the command to be executed along with the arguments
              ex: ['ls', '../input']
    '''
    print(check_output(cmd).decode("utf8"))
# check the structure of the project directory
exec_command(['ls', '..'])
''' Set the constants for the script '''

# various paths of the files
data_path = "../Data/cifar-10-batches-py" # the data path
train_meta = os.path.join(data_path, "batches.meta")
base_model_path = '../Models'

# constant values:
size = 32 # the images of size 32 x 32
channels = 3 # RGB channels
highest_pixel_value = 255.0 # 8 bits for every channel. So, max value is 255
no_of_epochs = 100 # No. of epochs to run
no_of_batches = 5 # There are 5 batches in the dataset
checkpoint_factor = 5 # save the model after every 5 steps (epochs)
# check the contents inside the data folder
exec_command(['ls', data_path])
# function to unPickle a file: 
def unpickle(file):
    '''
        This function takes the file path and unPickles the file acquired from it
        @Param file: the string path of the file
        @return: The dict object unPickled from the file
    '''
    import cPickle
    with open(file, 'rb') as fo:
        dict = cPickle.load(fo)
    return dict<jupyter_output><empty_output><jupyter_text>Let's check the contents of the batches.meta file<jupyter_code>meta_data = unpickle(train_meta)

# check it's contents
meta_data<jupyter_output><empty_output><jupyter_text>Let's read and display some of the images from the dataset along with their labels<jupyter_code>train_batch_preliminary = unpickle(os.path.join(data_path, "data_batch_3"))

# check it's contents
train_batch_preliminary.keys()
# Extract the first 3 images from the dataset
preliminary_data = train_batch_preliminary['data'].reshape((len(train_batch_preliminary['data']), 32, 32, 3), 
                                                           order='F')
preliminary_labels = train_batch_preliminary['labels']

# view some of the data:
preliminary_data[33, :10, :10, 2] #(10 x 10) data of blue channel of 33rd image
# check a few values of the labels of the dataset
preliminary_labels[:10] 
for _ in range(3):
    random_index = np.random.randint(preliminary_data.shape[0])
    
    plt.figure().suptitle("Random Image from the dataset: %s" %(meta_data['label_names'][preliminary_labels[random_index]]))
    plt.imshow(preliminary_data[random_index], interpolation='none')<jupyter_output><empty_output><jupyter_text>The images look blurred out because they are very low resolution images (32 x 32) pixels only. It can be seen that the images in the original dataset are skewed. So, we will have to rotate them by 90 degrees clockwise<jupyter_code># let's try using the numpy.rot90 method for this:
random_index = np.random.randint(preliminary_data.shape[0])
    
plt.figure().suptitle("Random Image from the dataset: %s" %(meta_data['label_names'][preliminary_labels[random_index]]))
plt.imshow(np.rot90(preliminary_data[random_index], axes=(1, 0)), interpolation='none'); # suppress the unnecessary
# output<jupyter_output><empty_output><jupyter_text>This works. So, now we can create a function to put all this together. This function would take the batch pickle file and create the data suitable for feeding it off to a convolutional neural network.<jupyter_code># The batch generator function:
def generateBatch(batchFile):
    '''
        The function to generate a batch of data suitable for performing the convNet operations on it
        @param batchFile -> the path of the input batchfile
        @return batch: (data, labels) -> the processed data.
    '''
    # unpickle the batch file:
    data_dict = unpickle(batchFile)
    
    # extract the data and labels from this dictionary
    unprocessed_data = data_dict['data']
    integer_labels = np.array(data_dict['labels']) # labels in integer form
    
    # reshape and rotate the data
    data = unprocessed_data.reshape((len(unprocessed_data), size, size, channels), order='F')
    processed_data = np.array(map(lambda x: np.rot90(x, axes=(1, 0)), data))
    
    # normalize the images by dividing all the pixels by 255
    processed_data = processed_data.astype(np.float32) / 255
    
    # encode the labels in one-hot encoded form
    # we use the sklearn.preprocessing package for doing this
    encoder = OneHotEncoder(sparse=False)
    encoded_labels = np.array(encoder.fit_transform(integer_labels.reshape(len(integer_labels), 1))).astype(np.int)
    
    # return the processed data and the encoded_labels:
    return (processed_data, encoded_labels)<jupyter_output><empty_output><jupyter_text>Time to test this function<jupyter_code># load the batch no. 1 and check if it works correctly.
batch_data, batch_labels = generateBatch(os.path.join(data_path, "data_batch_1"))
print (batch_data.shape, batch_labels.shape)

# batch_data[0, :12, :12, 2]
# extract one image from the data and display it
randomIndex = np.random.randint(batch_data.shape[0])
randomImage = batch_data[randomIndex]
print "Random image shape: " + str(randomImage.shape)

print "Random image dataType" + str(randomImage.dtype)

print "\n\ncheck if the data has been properly normalized"
print randomImage[:3, :3, 0]

# Visualize the random image from the dataset
plt.figure()
plt.imshow(randomImage, interpolation='none'); # suppress the unnecessary<jupyter_output>Random image shape: (32, 32, 3)
Random image dataTypefloat32


check if the data has been properly normalized
[[ 0.40784314  0.40392157  0.40784314]
 [ 0.39607844  0.39607844  0.40000001]
 [ 0.39215687  0.39607844  0.40000001]]<jupyter_text>Alright! So, the data extraction module is setup. Let's move on to the actual model building and training.  The computation graph (tensorflow graph and not the weights) for this model is inside the computation_graph package. It can be imported from there<jupyter_code># import the graph from the Graph1 module 
import computation_graph.Graph1 

computation_graph = computation_graph.Graph1.graph

# obtain a handle on the encoded_representation tensor of the dataflow computation graph
encoded_representation = computation_graph.get_tensor_by_name("encoded_representation:0")
encoded_representation.shape # The output shape of the encoded representation. It is 32 x 2 x 2 i.e 128 
# Thus the latent representation is 128 dimensional
model_path = os.path.join(base_model_path, "Model1")<jupyter_output><empty_output><jupyter_text>Now, let's write the session code to run this computation graph and perform the training<jupyter_code>''' 
    WARNING WARNING WARNING!!! This is the main training cell. Since, the data used for this task is CIFAR-10, 
    This cell will take a really really long time on low-end machines. It will however not crash your pc, since 
    I have bootstrapped the training in such a way that it loads a small chunk of data at a time to train.
    
    It took me around 5hrs to execute this cell entirely.
'''

with tf.Session(graph=computation_graph) as sess:
    
    if(os.path.isfile(os.path.join(model_path, "checkpoint"))):
         # load the weights from the model1
        saver = tf.train.Saver(max_to_keep=2)

        # instead of global variable initializer, restore the graph:
        saver.restore(sess, tf.train.latest_checkpoint(model_path))
    
    else:
        # create a new saver
        saver = tf.train.Saver(max_to_keep=2)
        
        # initialize all the variables
        sess.run(tf.global_variables_initializer())
    
    for ep in range(50, no_of_epochs):  # epochs loop
        
        print "epoch: " + str(ep + 1)
        print "================================================================================================="
        print "================================================================================================="
        
        for batch_n in range(no_of_batches):  # batches loop
            
            # retrieve the operations from the graph to be evaluated
            loss = sess.graph.get_tensor_by_name("loss:0")
            train_op = sess.graph.get_operation_by_name("train_op")
            inputs = sess.graph.get_tensor_by_name("inputs:0")
            
            # generate the batch images and labels
            batch_images, batch_labels = generateBatch(os.path.join(data_path, "data_batch_" + str(batch_n + 1)))
            
            min_batch_size = 2000 # we look at only 500 images at a time since the machine is small
            
            print "current_batch: " + str(batch_n + 1)
            
            for index in range(len(batch_images) / min_batch_size):
                start = index * min_batch_size
                end = start + min_batch_size
                _, cost = sess.run([train_op, loss], feed_dict={inputs: batch_images[start: end]})
                print('range:{} loss= {}'.format((start, end), cost))
            
            print "\n=========================================================================================\n"
        
        if((ep + 1) % checkpoint_factor == 0):
            # save the model trained so far:
            saver.save(sess, os.path.join(model_path, "model1"), global_step = (ep + 1))
        
    print "================================================================================================="
    print "================================================================================================="<jupyter_output><empty_output><jupyter_text>Let's visualize the representation of a random image and it's reconstructed form<jupyter_code>with tf.Session(graph = computation_graph) as sess:
    # load the weights from the model1
    saver = tf.train.Saver()
    
    # instead of global variable initializer, restore the graph:
    saver.restore(sess, tf.train.latest_checkpoint(model_path))
    
    prediction = sess.graph.get_tensor_by_name("prediction:0")
    inputs = sess.graph.get_tensor_by_name("inputs:0")
    
    random_image = batch_data[np.random.randint(len(batch_data))]
    reconstructed_image = sess.run(prediction, feed_dict={inputs: np.array([random_image])})[0]
    
    # plot the two images with their titles:
    plt.figure().suptitle("Original Image")
    plt.imshow(random_image, interpolation='none')
    
    plt.figure().suptitle("Reconstructed Image")
    plt.imshow(reconstructed_image, interpolation='none')<jupyter_output>INFO:tensorflow:Restoring parameters from ../Models/Model1/model1-100<jupyter_text>Use the 2nd model that has pooling layers to check the results:<jupyter_code># import the graph from the Graph1 module 
import computation_graph.Graph2

computation_graph = computation_graph.Graph2.graph

# obtain a handle on the encoded_representation tensor of the dataflow computation graph
encoded_representation = computation_graph.get_tensor_by_name("encoded_representation:0")
encoded_representation.shape # The output shape of the encoded representation. It is 32 x 2 x 2 i.e 128 
# Thus the latent representation is 128 dimensional
new_model_path = os.path.join(base_model_path, "Model2")
''' 
    WARNING WARNING WARNING!!! This is the main training cell. Since, the data used for this task is CIFAR-10, 
    This cell will take a really really long time on low-end machines. It will however not crash your pc, since 
    I have bootstrapped the training in such a way that it loads a small chunk of data at a time to train.
    
    It took me around 5hrs to execute this cell entirely.
'''

with tf.Session(graph=computation_graph) as sess:
    
    if(os.path.isfile(os.path.join(new_model_path, "checkpoint"))):
         # load the weights from the model2
        saver = tf.train.Saver(max_to_keep=2)

        # instead of global variable initializer, restore the graph:
        saver.restore(sess, tf.train.latest_checkpoint(new_model_path))
    
    else:
        # create a new saver
        saver = tf.train.Saver(max_to_keep=2)
        
        # initialize all the variables
        sess.run(tf.global_variables_initializer())
    
    for ep in range(465, 5 * no_of_epochs):  # start the loop from 10 now
        
        print "epoch: " + str(ep + 1)
        print "================================================================================================="
        print "================================================================================================="
        
        for batch_n in range(no_of_batches):  # batches loop
            
            # retrieve the operations from the graph to be evaluated
            loss = sess.graph.get_tensor_by_name("loss:0")
            train_op = sess.graph.get_operation_by_name("train_op")
            inputs = sess.graph.get_tensor_by_name("inputs:0")
            
            # generate the batch images and labels
            batch_images, batch_labels = generateBatch(os.path.join(data_path, "data_batch_" + str(batch_n + 1)))
            
            min_batch_size = 5000 # we look at only 5000 images at a time since the machine is small
            
            print "current_batch: " + str(batch_n + 1)
            
            for index in range(len(batch_images) / min_batch_size):
                start = index * min_batch_size
                end = start + min_batch_size
                _, cost = sess.run([train_op, loss], feed_dict={inputs: batch_images[start: end]})
                print('range:{} loss= {}'.format((start, end), cost))
            
            print "\n=========================================================================================\n"
        
        if((ep + 1) % checkpoint_factor == 0):
            # save the model trained so far:
            saver.save(sess, os.path.join(new_model_path, "model2"), global_step = (ep + 1))
        
    print "================================================================================================="
    print "================================================================================================="
with tf.Session(graph = computation_graph) as sess:
    # load the weights from the model1
    saver = tf.train.Saver()
    
    # instead of global variable initializer, restore the graph:
    saver.restore(sess, tf.train.latest_checkpoint(new_model_path))
    
    prediction = sess.graph.get_tensor_by_name("prediction:0")
    inputs = sess.graph.get_tensor_by_name("inputs:0")
    
    random_image = batch_data[np.random.randint(len(batch_data))]
    reconstructed_image = sess.run(prediction, feed_dict={inputs: np.array([random_image])})[0]
    
    # plot the two images with their titles:
    plt.figure().suptitle("Original Image")
    plt.imshow(random_image, interpolation='none')
    
    plt.figure().suptitle("Reconstructed Image")
    plt.imshow(reconstructed_image, interpolation='none')<jupyter_output>INFO:tensorflow:Restoring parameters from ../Models/Model2/model2-500
{'fraction_non_alphanumeric': 0.15164038657171922, 'fraction_numerical': 0.01773906408952187, 'mean_word_length': 3.5591304347826087, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '318259', 'license': 'MIT', 'n_tokens_mistral': 4631, 'n_tokens_neox': 4310, 'n_words': 1387, 'path': 'Scripts/Autoencoder.ipynb', 'repo_name': 'akanimax/autoencoder-cifar-10', 'size': '15728'}