<jupyter_start><jupyter_text>Train on Depthreference from https://github.com/asap-report/carla/blob/racetrack/PythonClient/racetrack/train_on_depth.pyhttps://github.com/ManajitPal/DeepLearningForSelfDrivingCars/blob/master/self_driving_car.py  Imports<jupyter_code>%matplotlib inline 

import torch.nn as nn
from pathlib import Path
import numpy as np 
import cv2
import os
import pandas as pd
import matplotlib.pyplot as plt
import PIL.Image as Image<jupyter_output><empty_output><jupyter_text>Check if data exist<jupyter_code>data_dir = Path("/home/michael/Desktop/projects/ROAR/data/custom")
left_depth_dir = data_dir / "left"
center_depth_dir = data_dir / "center"
right_depth_dir = data_dir / "right"
veh_state_dir = data_dir / "state"

assert data_dir.exists(), "data dir does not exist"
assert left_depth_dir.exists(), "left_depth_dir does not exist"
assert center_depth_dir.exists(), "left_depth_dir does not exist"
assert right_depth_dir.exists(), "left_depth_dir does not exist"
assert veh_state_dir.exists(), "veh_state_dir does not exist"

left_depth_paths = [p for p in sorted(left_depth_dir.glob("*.npy", ), key=os.path.getmtime)]
center_depth_paths = [p for p in sorted(center_depth_dir.glob("*.npy", ), key=os.path.getmtime)]
right_depth_paths = [p for p in sorted(right_depth_dir.glob("*.npy", ), key=os.path.getmtime)]
veh_state_paths = [p for p in sorted(veh_state_dir.glob("*.npy", ), key=os.path.getmtime)]

assert len(left_depth_paths) == len(center_depth_paths)== len(right_depth_paths)==len(veh_state_paths), "not the same size of X and y"
print(f"Found { len(left_depth_paths)} data points")
print(left_depth_paths[0])
print(center_depth_paths[0])
print(right_depth_paths[0])
print(veh_state_paths[0])
throttles, steerings = [], []
for st_path in veh_state_paths:
    array = np.load(st_path)
    throttles.append(array[-2])
    steerings.append(array[-1])
df = pd.DataFrame(
    data={"throttle":throttles, "steering":steerings}
)
df.hist()<jupyter_output>/home/michael/anaconda3/envs/ROAR/lib/python3.7/site-packages/pandas/plotting/_matplotlib/tools.py:331: MatplotlibDeprecationWarning: 
The is_first_col function was deprecated in Matplotlib 3.4 and will be removed two minor releases later. Use ax.get_subplotspec().is_first_col() instead.
  if ax.is_first_col():<jupyter_text>TODO: Do data augmentationLet's only train on steering  Define model<jupyter_code>import torch.optim as optim
from torch.utils import data
from torch.utils.data import DataLoader
import torch
class Dataset(data.Dataset):

    def __init__(self, samples, left_depth_dir, center_depth_dir, right_depth_dir, state_dir, transform=None):

        self.samples = samples
        self.transform = transform
        self.left_depth_dir: Path = left_depth_dir
        self.right_depth_dir: Path = right_depth_dir
        self.center_depth_dir: Path = center_depth_dir
        self.state_dir: Path = state_dir

    def __getitem__(self, index):
      
        sample_path = self.samples[index]
        left_depth_path = self.left_depth_dir / sample_path
        center_depth_path = self.center_depth_dir / sample_path
        right_depth_path = self.right_depth_dir / sample_path
        veh_state_path = self.state_dir / sample_path
        
        left_depth = np.load(left_depth_path)
        right_depth = np.load(left_depth_path)
        center_depth = np.load(right_depth_path)
        veh_state = np.load(veh_state_path)
        
        steering_angle_center = veh_state[-1]
        steering_angle_left = steering_angle_center + 0.2
        steering_angle_right = steering_angle_center - 0.2
        return (center_depth, steering_angle_center), \
               (left_depth, steering_angle_left), \
               (right_depth, steering_angle_right)
      
    def __len__(self):
        return len(self.samples)

params = {'batch_size': 1,
          'shuffle': True,
          'num_workers': 4}
samples = [Path(p).name for p in veh_state_paths]
samples = samples[:len(samples) - len(samples)%params['batch_size'] ]
dataset = Dataset(samples=samples, 
                  left_depth_dir = left_depth_dir, 
                  center_depth_dir=center_depth_dir, 
                  right_depth_dir=right_depth_dir,
                  state_dir = veh_state_dir)
data_generator = data.DataLoader(dataset, **params)

# Step6: Check the device and define function to move tensors to that device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 
print('device is: ', device)

def toDevice(datas, device):
    imgs, angles = datas
    return imgs.float().to(device), angles.float().to(device)
def to_tensor(datas, device):
    imgs, angles = datas
    return imgs.float().to(device), angles.float().to(device)

class NvidiaModel(nn.Module):
    def __init__(self):
        super(NvidiaModel, self).__init__()
        self.conv_layers = nn.Sequential(
            nn.Conv2d(1, 24, 5, stride=2),
            nn.ELU(),
            nn.Conv2d(24, 36, 5, stride=2),
            nn.ELU(),
            nn.Conv2d(36, 48, 5, stride=2),
            nn.ELU(),
            nn.Conv2d(48, 64, 3),
            nn.ELU(),
            nn.Conv2d(64, 64, 3),
            nn.Dropout(0.25),
        )
        self.linear_layers = nn.Sequential(
            nn.Linear(in_features=404736, out_features=50),
            nn.ELU(),
            nn.Linear(in_features=50, out_features=10),
            nn.Linear(in_features=10, out_features=1)
        )
        
    def forward(self, input):  
        input = input.view(input.size(0), 1, 600, 800)
        output = self.conv_layers(input)
        output = torch.flatten(output)
        output = self.linear_layers(output)
        return output
model = NvidiaModel()
optimizer = optim.Adam(model.parameters(), lr=0.0001)
criterion = nn.MSELoss()
for epoch in range(10):
    print(f"Staring Epoch {epoch}")
    model.to(device)
    train_loss = 0
    model.train()
    counter = 0
    for i, (centers, lefts, rights) in enumerate(data_generator):
        centers, lefts, rights = toDevice(centers, device), toDevice(lefts, device), toDevice(rights, device)
        optimizer.zero_grad()
        datas = [centers, lefts, rights]
        for data in datas:
            imgs, angles = data
            outputs = model(imgs)
            loss = criterion(outputs, angles.unsqueeze(1))
            loss.backward()
            optimizer.step()
            
            train_loss += loss.data.item()
            
        if i % 100 == 0:
            print(f"    At iteration {i} -> loss: {train_loss / (i+1)}")
    
model.eval()
valid_loss = 0
with torch.set_grad_enabled(False):
    for local_batch, (centers, lefts, rights) in enumerate(data_generator):
        # Transfer to GPU
        centers, lefts, rights = toDevice(centers, device), toDevice(lefts, device), toDevice(rights, device)

        # Model computations
        optimizer.zero_grad()
        datas = [centers, lefts, rights]        
        for data in datas:
            imgs, angles = data
            outputs = model(imgs)
            loss = criterion(outputs, angles.unsqueeze(1))

            valid_loss += loss.data.item()

        if local_batch % 10 == 0:
            print('Valid Loss: %.3f '
                 % (valid_loss/(local_batch+1)))
# Step8: Define state and save the model wrt to state
# state = {
#         'model': model.module if device == 'cuda' else model,
#         }

# torch.save(state, 'model.h5')
torch.save(model, 'model.h5')
# torch.save(model.state_dict(), 'state_dict.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11417950079660116, 'fraction_numerical': 0.014206054168879448, 'mean_word_length': 3.145844799119428, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '357165', 'license': 'Apache-2.0', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2453, 'n_words': 527, 'path': 'misc/train_on_depth.py.ipynb', 'repo_name': 'o0WeiyuFeng0o/ROAR', 'size': '7532'}