<jupyter_start><jupyter_text>Jetsoncar Rosey V2Tensorflow 2.0, all in notebook, optimized with RT<jupyter_code>import tensorflow as tf
print(tf.__version__)
tf.config.experimental.list_physical_devices('GPU') # If device does not show and using conda env with tensorflow-gpu then try restarting computer
# verify the image data directory
import os
data_directory = "/media/michael/BigMemory/datasets/jetsoncar/training_data/data/dataset"
os.listdir(data_directory)[:10]

import matplotlib.pyplot as plt
img = plt.imread(os.path.join(data_directory + "/color_images", os.listdir(data_directory + "/color_images")[0]))
print(img.shape)
plt.imshow(img)<jupyter_output>(480, 848, 3)<jupyter_text>Create the datagenerator and augmentation framework<jupyter_code># Include the custom utils.py and perform tests
import importlib
utils = importlib.import_module('utils')
import numpy as np

print(utils.INPUT_SHAPE)

img = utils.load_image(os.path.join(data_directory, 'color_images'),os.listdir(data_directory + "/color_images")[0])
print(img.shape)

fig = plt.figure(figsize=(20,20))
fig.add_subplot(1, 3, 1)
plt.imshow(img)

img, _ = utils.preprocess_data(last_color_image=img)
print(img.shape)

fig.add_subplot(1, 3, 2)
plt.imshow(np.squeeze(img))

plt.show()
# Load the steering angles and image paths from labels.csv
import csv, random
import seaborn as sns

# these will be 2D arrays where each row represents a dataset
x = [] # images
y = [] # steering
z = [] # speed
with open(os.path.join(data_directory, "tags.csv")) as csvfile:
    reader = csv.DictReader(csvfile)
    for row in reader:
        # print(row['Time_stamp'] + ".jpg", row['Steering_angle'])
        if not float(row['raw_speed']) == 0:
            x.append(row['time_stamp'] + ".jpg",)  # get image path
            y.append(float(row['raw_steering']),) # get steering value
            z.append(float(row['raw_speed']))

print("Number of data samples is " + str(len(y)))

data = list(zip(x,y))
random.shuffle(data)
x,y = zip(*data)

# plot of steering angle distribution without correction
sns.distplot(y)
# plot of speed distribution
sns.distplot(z)
# Split the training data
validation_split = 0.2
train_x = x[0:int(len(x)*(1.0-validation_split))]
train_y = y[0:int(len(y)*(1.0-validation_split))]
print("Training data shape: " + str(len(train_x)))
test_x = x[int(len(x)*(1.0-validation_split)):]
test_y = y[int(len(y)*(1.0-validation_split)):]
print("Validation data shape: " + str(len(test_x)) + "\n")
# Define and test batch generator
def batch_generator(data_dir, image_paths, steering_angles, batch_size, is_training):
    """
    Generate training image give image paths and associated steering angles
    """
    images = np.empty([batch_size, utils.IMAGE_HEIGHT, utils.IMAGE_WIDTH, utils.IMAGE_CHANNELS], dtype=np.float32)
    steers = np.empty(batch_size)
    while True:
        i = 0
        for index in np.random.permutation(len(image_paths)):
            img = image_paths[index]
            steering_angle = steering_angles[index]
            # argumentation
            if is_training and np.random.rand() < 0.8:
                image, steering_angle = utils.augument(data_dir, os.path.join("color_images",img), steering_angle)
            else:
                image, _ = utils.preprocess_data(utils.load_image(data_dir, os.path.join("color_images",img)))
            # add the image and steering angle to the batch
            images[i] = image
            steers[i] = steering_angle
            i += 1
            if i == batch_size:
                break
        yield images, steers
        
train_generator = batch_generator(data_directory, train_x, train_y, 32, True)
validation_generator = batch_generator(data_directory, test_x, test_y, 32, False)

train_image = next(train_generator) # returns tuple with steering and throttle
print(train_image[0].shape)
print(train_image[1][0])
plt.imshow(train_image[0][0])<jupyter_output>(32, 240, 640, 3)
0.2661118747113352<jupyter_text>Define the model and start training<jupyter_code>model = tf.keras.models.Sequential([
    tf.keras.Input((utils.IMAGE_HEIGHT, utils.IMAGE_WIDTH, utils.IMAGE_CHANNELS)),
    tf.keras.layers.Conv2D(32, (11,11), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,2)),
    
    tf.keras.layers.Conv2D(32, (7,7), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,2)),
    
    tf.keras.layers.Conv2D(64, (5,5), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,2)),
    
    tf.keras.layers.Conv2D(64, (3,3), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,2)),
    
    tf.keras.layers.Conv2D(32, (3,3), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,3)),
    
    tf.keras.layers.Conv2D(16, (3,3), padding='same', kernel_initializer='lecun_uniform'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.ELU(),
    tf.keras.layers.MaxPool2D((2,3)),
    
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(10, activation='elu'),
    tf.keras.layers.Dense(1, activation='linear')
])
model.summary()
model.compile(loss='mean_squared_error', optimizer='adam')
import datetime
log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
print("To view tensorboard please run `tensorboard --logdir logs/fit` in the code directory from the terminal with deeplearning env active")

checkpoint = tf.keras.callbacks.ModelCheckpoint('rosey_v2.{epoch:03d}-{val_loss:.2f}.h5', # filepath = working directory/
    monitor='val_loss',
    verbose=0,
    save_best_only=True,
    mode='auto')

model.fit_generator(train_generator,
                    steps_per_epoch=100, 
                    epochs=20,
                    validation_data=validation_generator,
                    validation_steps=1,
                   callbacks=[tensorboard_callback, checkpoint])
# Test the model
image, steering = next(train_generator)
print(steering)
print(model.predict(image))
print("")

image, steering = next(validation_generator)
print(steering)
print(model.predict(image))<jupyter_output>[ 0.38866069 -0.01653751  0.15497394  0.18995237 -0.24068273 -0.29661691
 -0.2366807  -0.23459999 -0.09765687 -0.35514895 -0.28255172  0.15013804
 -0.15041455 -0.14102635 -0.04422271 -0.2006     -0.2032     -0.10150293
 -0.18395218  0.34925292 -0.22099999  0.250549    0.28476982  0.40067244
  0.0918623   0.20941769  0.01414458  0.29502599 -0.27545909  0.27571231
 -0.06456293 -0.38859999]
[[-0.01409152]
 [ 0.08915222]
 [-0.0566576 ]
 [-0.03755104]
 [ 0.08308045]
 [-0.15913627]
 [-0.16558295]
 [-0.12865719]
 [ 0.04993952]
 [-0.01573419]
 [-0.02034076]
 [-0.15650505]
 [ 0.06376354]
 [ 0.01313779]
 [-0.12256726]
 [-0.1740151 ]
 [-0.19206235]
 [ 0.08967156]
 [-0.09032293]
 [ 0.2788198 ]
 [-0.13434035]
 [ 0.00317884]
 [ 0.15187286]
 [ 0.15058331]
 [ 0.01717911]
 [ 0.00433371]
 [-0.14564866]
 [ 0.00086454]
 [-0.08476945]
 [-0.05350491]
 [-0.0531204 ]
 [-0.1822269 ]]

[-0.26859999 -0.20259999 -0.31899998 -0.25319999 -0.1032     -0.17560001
 -0.21020001 -0.146      -0.2956     -0.1936     -0.15[...]<jupyter_text>Save the model as tensor RT and export to Jetson format<jupyter_code># Load the model that you would like converted to RT
model_path = 'model.h5'
export_path = "/home/michael/Desktop/model"

import shutil
if not os.path.isdir(export_path):
    os.mkdir(export_path)
else:
    response = input("Do you want to delete existing export_path directory? y/n")
    if response == 'y':
        shutil.rmtree(export_path)
        os.mkdir(export_path)

loaded_model = tf.keras.models.load_model(model_path)

shutil.copy("./utils.py", os.path.join(export_path, "utils.py"))
shutil.copy("./__init__.py", os.path.join(export_path, "__init__.py"))
shutil.copy("./notes.txt", os.path.join(export_path, "notes.txt"))
shutil.copy("./config.yaml", os.path.join(export_path, "config.yaml"))
# Save as tf saved_model (faster than h5)
tf.saved_model.save(loaded_model, export_path)
from tensorflow.python.compiler.tensorrt import trt_convert as trt

conversion_params = trt.DEFAULT_TRT_CONVERSION_PARAMS
conversion_params = conversion_params._replace(max_workspace_size_bytes=(1 << 32))
conversion_params = conversion_params._replace(precision_mode="INT8")
conversion_params = conversion_params._replace(maximum_cached_engines=100)
conversion_params = conversion_params._replace(use_calibration=True)

def my_calibration_input_fn():
    for i in range(20):
        image, _ = utils.preprocess_data(utils.load_image(data_directory, os.path.join("color_images",x[i])))
        yield image.astype(np.float32),

converter = tf.experimental.tensorrt.Converter(input_saved_model_dir=export_path,conversion_params=conversion_params)

gen = my_calibration_input_fn()

converter.convert(calibration_input_fn=my_calibration_input_fn)
converter.build(my_calibration_input_fn)

if not os.path.isdir(os.path.join(export_path, "rt")):
    os.mkdir(os.path.join(export_path, "rt"))
    
converter.save(os.path.join(export_path, "rt"))
# Test normal saved model
saved_model = tf.saved_model.load(export_path) # normal saved model

image, _ = next(validation_generator)

import time
output = saved_model(image.astype(np.float32)) # load once to get more accurate representation of speed
start = time.time()
output = saved_model(image.astype(np.float32))
stop = time.time()
print("inference time: " + str(stop - start))
print("Output: %.20f"%output[8,0])
# Test TRT optimized saved model
saved_model = tf.saved_model.load(os.path.join(export_path, "rt")) # normal saved model


image, _ = next(validation_generator)

import time
output = saved_model(image) # load once to get more accurate representation of speed
start = time.time()
output = saved_model(image)
stop = time.time()
print("inference time: " + str(stop - start))
print("Output: %.20f"%output[8,0])
# Run many samples through and save distribution 
validation_generator = batch_generator(data_directory, test_x, test_y, 32, False)
test = []
for i in range(50):
    img, _ = next(validation_generator)
    test.append(saved_model(img.astype(np.float32))[0][0])
    print(str(i), end="\r")
sns.distplot(test)<jupyter_output>49
{'fraction_non_alphanumeric': 0.14053240522998778, 'fraction_numerical': 0.07656852600884206, 'mean_word_length': 4.331995987963892, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '8725', 'license': 'MIT', 'n_tokens_mistral': 4254, 'n_tokens_neox': 3786, 'n_words': 704, 'path': 'basic_supervised_net/rosey/rosey_v2.ipynb', 'repo_name': 'ARCC-RACE/ai_training_notebooks', 'size': '10631'}