<jupyter_start><jupyter_text>Use monte carlo method to understand our distribution<jupyter_code>def describe_loader(loader, N=100):
    i = 0
    loader_generator = iter(loader)
    steerings = []
    while i < N:
        img, steering = next(loader_generator)
        angles = steering.numpy()
        for angle in angles:
            steerings.append(angle)
        i += 1
    df = pd.DataFrame(
        data={"steering":steerings}
    )
    df.hist("steering"), df.describe()<jupyter_output><empty_output><jupyter_text>Split data into Train and valid sets<jupyter_code>X_train, X_valid, y_train, y_valid = train_test_split(center_depth_paths, 
                                                      veh_state_paths, 
                                                      test_size=0.1, 
                                                      shuffle=True)

batch_size = 1
car_train_loader = DataLoader(CarDataset(X=X_train, y=y_train), 
                              batch_size=batch_size, shuffle=True)
car_valid_loader = DataLoader(CarDataset(X=X_valid, y=y_valid), 
                              batch_size=batch_size, shuffle=True)

describe_loader(car_train_loader, N=500)
imgs, angles = next(iter(car_train_loader))
imgs.shape, angles.shape, len(car_train_loader)
from model import CarModel

def toTensor(data, use_cuda, device=None):
    img, target = data
#     img, target = torch.from_numpy(img).float(), torch.from_numpy(np.array([target])).float()
    img = img.float()
    target = target.float()
    if use_cuda:
        img, target = img.to(device), target.to(device)
    return img, target
best_loss = 1000000

def train(epoch, net, train_loader, optimizer, criterion, use_cuda, 
          device=None, save_dir=Path("."),MAX_BATCH=1000):
    net.train()
    train_loss = 0
    for batch_idx, data in enumerate(train_loader):
        optimizer.zero_grad()
        img, steering_angle = toTensor(data, use_cuda, device)
        output = net(img) # Tensor([ITEM])
        loss = criterion(output, steering_angle)
        loss.backward()
        optimizer.step()
        train_loss += loss.data.item()
        if batch_idx % 100 == 0:
            print(f'    Batch: {batch_idx} --> Loss: {train_loss / (batch_idx+1)}') 
        
        if batch_idx >= MAX_BATCH:
            break
    

def valid(epoch, net, validloader, criterion, use_cuda, device=None, save_dir=Path("."),MAX_BATCH=1000):
    global best_loss
    net.eval()
    valid_loss = 0 
    for batch_idx, data in enumerate(validloader):
        img, steering_angle = toTensor(data, use_cuda, device)
        outputs = net(img)
        loss = criterion(outputs, steering_angle)
        valid_loss += loss.data.item()
        
        avg_valid_loss = valid_loss / (batch_idx + 1)
        if batch_idx % 100 == 0:
            print(f"    Valid Loss: {avg_valid_loss}" )
        
        if avg_valid_loss <= best_loss:
            best_loss = avg_valid_loss
            print(f"         Saving.... Best epoch: {epoch} -> {avg_valid_loss}")
            torch.save(net.state_dict(),save_dir / "best_model_state_dict.h5" )
#             torch.save(net, save_dir / "best_model.h5")
        
        if batch_idx >= MAX_BATCH:
            break
    
    torch.save(net.state_dict(),save_dir / "model_state_dict.h5")

#load the previous net or use new net
# net = torch.load("/home/michael/Desktop/projects/ROAR/misc/data/best_model.h5")
# net.training = True

net = CarModel(batch_size=batch_size, 
              image_width=800,
              image_height=600)

optimizer = optim.Adam(net.parameters(), lr=1e-5)
use_cuda = torch.cuda.is_available()
device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
criterion = nn.MSELoss()
save_dir = Path("./data")
save_dir.mkdir(exist_ok=True, parents=True)
if use_cuda:
    net.to(device)
for epoch in range(0, 10):
    print(f"Epoch {epoch}")
    train(epoch, net, car_train_loader, optimizer, criterion, use_cuda, device=device, save_dir=save_dir, 
          MAX_BATCH=len(car_train_loader)//4)
    valid(epoch, net, car_valid_loader, criterion, use_cuda, device=device, save_dir=save_dir, MAX_BATCH=1000)
valid_iter = iter(car_valid_loader)
net.eval()
for i in range(100):
    img, angle = next(valid_iter)
    angle = angle.numpy()[0]
    predicted = net(img.float().to(device)).cpu().detach().numpy()[0]
    print(predicted, angle)
torch.save(net.state_dict(),save_dir / "model_state_dict.h5")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11983842010771993, 'fraction_numerical': 0.014587073608617595, 'mean_word_length': 2.7328308207705194, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '648375', 'license': 'Apache-2.0', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1467, 'n_words': 286, 'path': 'misc/train_depth_to_steering_2.ipynb', 'repo_name': 'pranavpomalapally/ROAR-Semi-Autonomous', 'size': '4456'}