<jupyter_start><jupyter_text>Step 0 - Library import:---<jupyter_code>import numpy as np
import pandas as pd
import cv2

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Step 1 - Data loading and visualization:---  CSV database reading:<jupyter_code># define datapath -where our collected samples are stored-
data_path = "./data/"
# use panda to read csv file that contains images pathes and their corresponding steering values
database = pd.read_csv(data_path+'driving_log.csv', skipinitialspace=True)

database
# load csv file in list, images_list for images pathes
# labels_list for steering values
images_list = []
labels_list = []

# define correction factor for calculation of proper
# steering values from left and right sidesveiew points
correction = 0.2

# load csv file to lists
for i in range(len(database)):
    images_list.append(database.center[i])
    labels_list.append(database.steering[i])
    
    images_list.append(database.left[i])
    labels_list.append(database.steering[i] + correction)
    
    images_list.append(database.right[i])
    labels_list.append(database.steering[i] - correction)

print('check performed correction factor on left and right image:')
print('----------------------------------------------------------')
print(images_list[0], labels_list[0])
print(images_list[1], labels_list[1])
print(images_list[2], labels_list[2])
print('----------------------------------------------------------')
print()
print('Number of all images in dataset:   ', len(images_list))
print('Number of all labels in dataset:   ', len(images_list))<jupyter_output>check performed correction factor on left and right image:
----------------------------------------------------------
IMG/center_2016_12_01_13_30_48_287.jpg 0.0
IMG/left_2016_12_01_13_30_48_287.jpg 0.2
IMG/right_2016_12_01_13_30_48_287.jpg -0.2
----------------------------------------------------------

Number of all images in dataset:    24108
Number of all labels in dataset:    24108<jupyter_text>Batch generator:create a generator to load data batches.<jupyter_code>from sklearn.utils import shuffle

def generator(images_list, labels_list, data_path, batch_size=32, shuffle_on=True, flip_on=True):
    ### generator function, generates data batches of defined baches_size (default 32)
    # inputs:
    #            images_list: list of strings, images pathes in dataset directory.
    #            labels_list: list of steering values corresponded to images in images_list
    #            data_path:   string, of dataset path, where data (csv and images) are stored
    #            batch_size:  integer, size of ouput batches (first dimension of output)
    #            shuffle_on:  bool, turning on shuffle when True
    #            flip_on:     bool, flip images as data augmentation process
    # outputs:
    #            images:      images batch, four dimensional numpy array
    #            steerings:   steerings angels of images in images batch

    
    if flip_on:
        n_img = 2
    else:
        n_img = 1
        
    # in case of flip_on batches size will be devided on 2
    # however as each image produces a flip/miror image the output has batches size as defined
    # in case of odd number of batches_size output batches have 1 image fewer
    batch_size = int(batch_size / n_img)
    
    while 1:
        # shuffle data if shuffle is on
        if shuffle_on:
                images_list, labels_list = shuffle(images_list, labels_list)
                
        # create batches
        for offset in range(0, len(images_list)-batch_size, batch_size):
            images = []
            steerings = []
            # load images and streen values in batch
            for i in range(offset,offset + batch_size):
                
                image = plt.imread(data_path + images_list[i])
                steering = labels_list[i]
                
                images.append(image)
                steerings.append(steering)

            
            # convert list to numpy array as network works only on numpy arrays
            images = np.array(images)
            steerings = np.array(steerings)
            
            # flip loaded images and add it to batch, (douplicate the size of loaded batch)
            if flip_on:
                images = np.append(images, np.copy(images)[:,:,::-1,:], axis=0)
                steerings = np.append(steerings, np.copy(steerings)*-1, axis=0)
                
            # shuffle batch
            if shuffle_on:
                images, steerings = shuffle(images, steerings)
                
            # using yield to make a generator
            yield images, steerings<jupyter_output><empty_output><jupyter_text>Data visualization and test data generator function:<jupyter_code>from tqdm import tqdm
#create generator from all data samples with no shuffle
frame_generator = generator(images_list, labels_list, data_path, batch_size=3, shuffle_on=False, flip_on=False)

# Define the codec and create VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*'MP42')

# VideoWriter objects for center left and right camera-views
center_video = cv2.VideoWriter('./training_cener_video.mp4',fourcc, 20.0, (320,160))
left_video = cv2.VideoWriter('./training_left_video.mp4',fourcc, 20.0, (320,160))
right_video = cv2.VideoWriter('./training_right_video.mp4',fourcc, 20.0, (320,160))

# generate videos from sample driving 
for i in tqdm(range(len(database))):
    frame, _ = next(frame_generator)
    # write the flipped frame
    center_frame = cv2.cvtColor(frame[0],cv2.COLOR_BGR2RGB)
    center_video.write(center_frame)
    
    
    left_frame = cv2.cvtColor(frame[1],cv2.COLOR_BGR2RGB)
    left_video.write(left_frame)
    
    
    right_frame = cv2.cvtColor(frame[2],cv2.COLOR_BGR2RGB)
    right_video.write(right_frame)
    
# Release everything if job is finished
center_video.release()
left_video.release()
right_video.release()
from IPython.display import HTML
# display from samples created videos!
HTML("""
<table style="width:100%">
  <tr><th>Left View</th><th>Center View</th><th>Right View</th></tr>
  <tr><td><video width="320" height="160" controls><source src="{0}"></video></td>
      <td><video width="320" height="160" controls><source src="{1}"></video></td>
      <td><video width="320" height="160" controls><source src="{2}"></video></td>
  </tr>
</table>
""".format('./training_left_video.mp4', './training_cener_video.mp4', './training_right_video.mp4'))<jupyter_output><empty_output><jupyter_text>Test flip functionality (data augmentation):<jupyter_code>frame_generator = generator(images_list, labels_list, data_path, batch_size=6, shuffle_on=False, flip_on=True)
frames, steering = next(frame_generator)
f, axs = plt.subplots(2,3,figsize=(15,6))
for i in range(3):
    axs[0,i].imshow(frames[i])
    axs[0,i].axis('off')
    axs[0,i].set_title('orginal, steering:  {}'.format(steering[i]))
    axs[1,i].imshow(frames[i+3])
    axs[1,i].set_title('fliped, steering:  {}'.format(steering[i+3]))
    axs[1,i].axis('off')
### see how steering value has been generated in data augmentation
data_generator = generator(images_list, labels_list, data_path, batch_size=600, shuffle_on=False, flip_on=True)
_, y = next(data_generator)
#csv_file.center
x = range(600)
plt.plot(x, y)<jupyter_output><empty_output><jupyter_text>Cropping image illustration:<jupyter_code># illustration cropped image view
frames, steering = next(frame_generator)
f, axs = plt.subplots(1,3,figsize=(15,6))
for i in range(3):
    axs[i].imshow(frames[i,55:135,:,:])
    axs[i].axis('off')
    axs[i].set_title('orginal, steering:  {}'.format(steering[i]))<jupyter_output><empty_output><jupyter_text>Step 2 - Training and validation data generation:---<jupyter_code>from sklearn.model_selection import train_test_split
### 
# separate train and validation samples of dataset, 80% training data and 20% validation data
train_images, validation_images, train_labels, validation_labels = train_test_split(images_list, labels_list,\
                                                                                    train_size=0.8)<jupyter_output><empty_output><jupyter_text>Step 3 - Nvidia self driving car network implementation:---<jupyter_code>### Create nerual network model using Keras
from keras.models import Model
from keras.layers import Dense, Flatten, Lambda, Conv2D, Cropping2D, Dropout, Input

# Nvidia self driving car model with dropout layer to generalization model more
# input
inputs = Input(shape=(160,320,3))
# preprocessing, cropping and normalization of images
cropping_1 = Cropping2D(cropping=((55,25),(0,0)))(inputs)
lambda_1 = Lambda(lambda x: x / 256.0 - 0.5)(cropping_1)
# ConvNet
conv2d_1 = Conv2D(24, kernel_size=(5,5), strides=(2,2), activation='relu')(lambda_1)
conv2d_2 = Conv2D(36, kernel_size=(5,5), strides=(2,2), activation='relu')(conv2d_1)
conv2d_3 = Conv2D(48, kernel_size=(5,5), strides=(2,2), activation='relu')(conv2d_2)
conv2d_4 = Conv2D(64, kernel_size=(3,3), activation='relu')(conv2d_3)
conv2d_5 = Conv2D(64, kernel_size=(3,3), activation='relu')(conv2d_4)
# Dropout
dropout_1 = Dropout(0.5)(conv2d_5)
# flatten output of CNN after dropout
flatten_1 = Flatten()(dropout_1)
# MLN
dense_1 = Dense(100)(flatten_1)
dense_2 = Dense(50)(dense_1)
dense_3 = Dense(10)(dense_2)
output  = Dense(1)(dense_3)

# create model, define inputs and outputs of network
model = Model(inputs=inputs, outputs=output)

# print network architecture
print(model.summary())<jupyter_output>Using TensorFlow backend.<jupyter_text>Step 4 - Network training:<jupyter_code># define train and validation data generators
batch_size = 120

train_generator = generator(train_images, train_labels, data_path,\
                            batch_size=batch_size, flip_on=True)

validation_generator = generator(validation_images, validation_labels,\
                                 data_path, batch_size=batch_size, flip_on=True)

# compile model using 'adam' algorithem -which controls learning process
model.compile(optimizer='adam', loss='mse')

# calculate number of steps per epochs, training data
n_train_batches = int(2 * len(train_images) / batch_size) - 1
# calculate number of steps per epochs, validation data
n_valid_batches = int(2 * len(train_images) / batch_size) - 1

# run model and start training process
model.fit_generator(train_generator, steps_per_epoch=n_train_batches, epochs=7,\
validation_data=validation_generator, validation_steps=n_valid_batches, verbose=1)

# save model and exit
model.save('model.h5')<jupyter_output>Epoch 1/7
320/320 [==============================] - 294s 917ms/step - loss: 0.0194 - val_loss: 0.0226
Epoch 2/7
320/320 [==============================] - 299s 935ms/step - loss: 0.0159 - val_loss: 0.0118
Epoch 3/7
320/320 [==============================] - 310s 969ms/step - loss: 0.0147 - val_loss: 0.0170
Epoch 4/7
320/320 [==============================] - 338s 1s/step - loss: 0.0136 - val_loss: 0.0149
Epoch 5/7
320/320 [==============================] - 712s 2s/step - loss: 0.0131 - val_loss: 0.0154
Epoch 6/7
320/320 [==============================] - 302s 945ms/step - loss: 0.0122 - val_loss: 0.0212
Epoch 7/7
320/320 [==============================] - 334s 1s/step - loss: 0.0115 - val_loss: 0.0139
{'fraction_non_alphanumeric': 0.14631336405529954, 'fraction_numerical': 0.042360864941510104, 'mean_word_length': 3.566976932416026, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '107163', 'license': 'MIT', 'n_tokens_mistral': 3860, 'n_tokens_neox': 3521, 'n_words': 898, 'path': 'model.ipynb', 'repo_name': 'nasser-azarbakhsh-hashjin/CarND-Behavioral-Cloning-P3', 'size': '11284'}