<jupyter_start><jupyter_text>Table of Contents[I.  Benchmark with MNIST dataset](Test01)[II. Benchmark with Zalando MNIST dataset](Test02)<jupyter_code>##
# Import required libraries:
#
import tensorflow
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import pandas as pd
import time

##
# Import functions:
#
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import mixed_precision
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split
from utils import process_mnist, models, profiler
##
# Recheck to see if GPU will be available:
#
device_name = tensorflow.test.gpu_device_name()
if not device_name:
  raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))
##
# Set up the precision target:
#
policy = tensorflow.keras.mixed_precision.experimental.Policy('float32')
tensorflow.keras.mixed_precision.experimental.set_policy(policy)

##
# Test for precision target:
#
if not policy:
    raise SystemError('Error when executing this cell block.')<jupyter_output><empty_output><jupyter_text>I. Benchmark with MNIST dataset<jupyter_code>##
# Load the MNIST dataset:
#

data_location = int(input('Select the option to load data (0 = from the server; 1 = manually from the directory): '))

if data_location == 0:
    X, y = fetch_openml('mnist_784', version = 1, return_X_y = True, as_frame = False)
    
    ##
    # Convert the input data into RGB image type and resize the original resolution to 28x28:
    #
    X = process_mnist.resize_mnist(X, 28, 28, 32, 32)
    X = np.stack((X,) * 3, axis = -1)
    
    ##
    # One-hot encoding the output labels:
    #
    y = to_categorical(y, num_classes = 10)
    
elif data_location == 1:
    X_train, y_train = process_mnist.load_mnist('data/mnist', kind = 'train')
    X_test, y_test = process_mnist.load_mnist('data/mnist', kind = 't10k')
    
    ##
    # Convert the input data into RGB image type and resize the resolution to 28x28:
    #
    X_train = process_mnist.resize_mnist(X_train, 28, 28, 32, 32)
    X_train = np.stack((X_train,) * 3, axis = -1)

    X_test = process_mnist.resize_mnist(X_test, 28, 28, 32, 32)
    X_test = np.stack((X_test,) * 3, axis = -1)

    ##
    # One-hot encoding the output labels:
    #
    y_train = to_categorical(y_train, num_classes = 10)
    y_test = to_categorical(y_test, num_classes = 10)
else:
    print('Invalid selection!')
    
##
# Test for data loading:
#
if (not np.any(X_train)) or (not np.any(X_test)) or (not np.any(y_train)) or (not np.any(y_test)):
    raise SystemError('Error when executing this cell block.')
##
# Sanity check for input and output dimensions:
#
if data_location == 0:
    assert X.shape == (70000, 32, 32, 3), "X should have a dimension of (70000, 32, 32, 3)!"
    assert y.shape == (70000,10), "y should have a dimension of (70000,10)!"
elif data_location == 1:
    assert X_train.shape == (60000, 32, 32, 3), "X should have a dimension of (60000, 32, 32, 3)"
    assert y_train.shape == (60000,10), "y should have a dimension of (60000,10)"
    assert X_test.shape == (10000, 32, 32, 3), "X should have a dimension of (60000, 32, 32, 3)"
    assert y_test.shape == (10000,10), "y should have a dimension of (60000,10)"
else:
    print('Invalid selection!')
##
# Prepare the train and test subsets if loaded from the server:
#
if data_location == 0:
    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = None, test_size = 10000)
else:
    None
##
# Compute the flops of this model:
#
session = tensorflow.compat.v1.Session()
graph = tensorflow.compat.v1.get_default_graph()

with graph.as_default():
    with session.as_default():
        models.model_concatenate(32,32,3,'concatenated')
        run_meta = tensorflow.compat.v1.RunMetadata()
        opts = tensorflow.compat.v1.profiler.ProfileOptionBuilder.float_operation()
        flops = tensorflow.compat.v1.profiler.profile(graph=graph,
                                                      run_meta=run_meta, cmd = 'op', options=opts)

tensorflow.compat.v1.reset_default_graph()

##
# Compute the memory usage of this model:
#
model, callbacks_list = models.model_concatenate(X_train.shape[1],X_train.shape[2],X_train.shape[3],'MNIST')
memory_usage = profiler.memory_usage(model,64)

##
# Compute the required memory to store the model's weights
#
memory_weights = profiler.memory_weights(model)

##
# Test for model profiler:
#
if (not flops.total_float_ops) or (not memory_usage) or (not memory_weights):
    raise SystemError('Error when executing this cell block.')
##
# Train the model and store the execution time for evaluation:
#
start = time.time()
history = model.fit(X_train, y_train, epochs = 100, verbose = 1, batch_size = 64, callbacks = callbacks_list,
                         shuffle = True, validation_data = (X_test, y_test))
end = time.time()
duration_mnist = end - start
tensorflow.keras.backend.clear_session()

##
# Test for concatenated model:
#
if not duration_mnist:
    raise SystemError('Error when executing this cell block.')
##
# Visualize the train/validation accuracy and loss after the training duration:
#
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title("MNIST Accuracy")
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc = 'upper left')
plt.grid(True)
plt.show()
###
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title("MNIST Loss")
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc = 'upper left')
plt.grid(True)
plt.show()<jupyter_output><empty_output><jupyter_text>II. Benchmark with Zalando MNIST dataset<jupyter_code>##
# Load the Zalando MNIST dataset:
#
X_train, y_train = process_mnist.load_mnist('data/fashion', kind = 'train')
X_test, y_test = process_mnist.load_mnist('data/fashion', kind = 't10k')


##
# Convert the input data into RGB image type and resize the resolution to 32x32:
#
X_train = process_mnist.resize_mnist(X_train, 28, 28, 32, 32)
X_train = np.stack((X_train,) * 3, axis = -1)
X_test = process_mnist.resize_mnist(X_test, 28, 28, 32, 32)
X_test = np.stack((X_test,) * 3, axis = -1)

##
# One-hot encoding the output labels:
#
y_train = to_categorical(y_train, num_classes = 10)
y_test = to_categorical(y_test, num_classes = 10)

##
# Test for data loading:
#
if (not np.any(X_train)) or (not np.any(X_test)) or (not np.any(y_train)) or (not np.any(y_test)):
    raise SystemError('Error when executing this cell block.')
##
# Sanity check for input and output dimensions:
#
assert X_train.shape == (60000,32,32,3), "X_train should have a dimension of (60000,32,32,3)!"
assert X_test.shape == (10000,32,32,3), "X_test should have a dimension of (10000,32,32,3)!"
assert y_train.shape == (60000,10), "y_train should have a dimension of (60000,10)"
assert y_test.shape == (10000,10), "y_test should have a dimension of (10000,10)"
###
# Train the model and store the execution time for evaluation:
#
start = time.time()
model, callbacks_list = models.model_concatenate(X_train.shape[1],X_train.shape[2],X_train.shape[3],'fashion')
history = model.fit(X_train, y_train, epochs = 100, verbose = 1, batch_size = 64, callbacks = callbacks_list, shuffle = True, validation_data = (X_test, y_test))
end = time.time()
duration_fashion = end - start
tensorflow.keras.backend.clear_session()

##
# Test for concatenated model:
#
if not duration_fashion:
    raise SystemError('Error when executing this cell block.')
##
# Visualize the train/validation accuracy and loss after the training duration:
#
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title("Zalando Accuracy")
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc = 'upper left')
plt.grid(True)
plt.show()
###
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title("Zalando Loss")
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc = 'upper left')
plt.grid(True)
plt.show()
##
# Visualize the execution time on each dataset, respectively: 
#
values = [round(duration_mnist,0) , round(duration_fashion,0)]
names = ['MNIST', 'Zalando']

plt.bar(names, values)
plt.ylabel('Time')
plt.grid(True)

for index, data in enumerate(values):
    plt.text(x = index , y = data + 1 , s = f"{data}" , fontdict = dict(fontsize = 12), ha = 'center')
    
plt.show()
##
# Save the duration results into a .csv file:
#
results = {'Test': ['MNIST','Fashion'],
        'Duration (s)': values,
        'FLOPS': flops.total_float_ops,
        'Memory Usage for Model (GBytes)': memory_usage,
        'Memory Usage for Weights (MBytes)': memory_weights
          }
df = pd.DataFrame(results, columns= ['Test', 'Duration (s)', 'FLOPS', 
                                     'Memory Usage for Model (GBytes)', 'Memory Usage for Weights (MBytes)'])
df.to_csv('results/concatenate.csv', index = False)

##
# Test saving results:
#
if (not np.any(df)):
    raise SystemError('Error when executing this cell block.')
##
# (Optional): Automated debugging:
#
print('1. Total Execution Time for MNIST dataset:')
if (duration_mnist):
    print('Build:passing\n')
    
print('2. Total Execution Time for Zalando MNIST dataset:')    
if (duration_fashion):
    print('Build:passing\n')
    
print('3. The number of FLOPS in the model:')        
if (flops.total_float_ops):
    print('Build:passing\n')

print('4. Memory usage of the model:')    
if (memory_usage):
    print('Build:passing\n')

print('5. Memory required to store the model weights:')
if (memory_weights):
    print('Build:passing\n')

print('6. Save the results:')
if (np.any(df)):
    print('Build:passing\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1378602933634219, 'fraction_numerical': 0.0317981331418607, 'mean_word_length': 4.437813720022309, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '525820', 'license': 'Apache-2.0', 'n_tokens_mistral': 3633, 'n_tokens_neox': 3293, 'n_words': 918, 'path': 'Benchmark Type B.ipynb', 'repo_name': 'vohuynhquangnguyen/Deep-Learning-Hardware-Benchmark', 'size': '9749'}