<jupyter_start><jupyter_text>Config  Datasets<jupyter_code>root_path = Path("/media/romet/data2/datasets/rally-estonia/dataset")
trainset = NvidiaTrainDataset(root_path)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True,
                                         num_workers=60, pin_memory=True, persistent_workers=True)
print(len(trainset.frames))
image_path = trainset.frames.iloc[0].image_path
image = torchvision.io.read_image(image_path)
plt.imshow(image.permute(1, 2, 0))
def crop(image):
    xmin = 300
    xmax = 1620

    ymin = 520
    ymax = 864

    scale = 0.2

    height = ymax - ymin
    width = xmax - xmin
    cropped_image = transforms.functional.resized_crop(image, ymin, xmin, height, width,
                                                     (int(scale*height), int(scale*width)))
    return cropped_image
cropped_image = crop(image)
plt.imshow(cropped_image.permute(1, 2, 0))
import cv2

xmin = 300
xmax = 1620
ymin = 520
ymax = 864
scale = 0.2

height = ymax - ymin
width = xmax - xmin
scaled_width = int(scale * width)
scaled_height = int(scale * height)

def resize(img):
    return cv2.resize(img, dsize=(scaled_width, scaled_height), interpolation=cv2.INTER_LINEAR)
images = trainset.frames.image_path.to_numpy()
for image_path in tqdm(images):
    path = Path(image_path)
    image_name = path.name
    root_folder = path.parent.parent
    crop_folder = root_folder / "front_wide_cropped"
    crop_folder.mkdir(exist_ok=True)
    
    #image = torchvision.io.read_image(image_path)
    #cropped_image = crop(image)
    #torchvision.io.write_jpeg(cropped_image, str(crop_folder / image_name), quality=100)
    
    image = cv2.imread(image_path)
    cropped_image = resize(image)
    cv2.imwrite(str(crop_folder / image_name), cropped_image, [cv2.IMWRITE_JPEG_QUALITY, 100])


validset = NvidiaValidationDataset(root_path)
validloader = torch.utils.data.DataLoader(validset, batch_size=64, shuffle=False,
                                         num_workers=32, pin_memory=True, persistent_workers=True)
print(len(validset.frames))<jupyter_output>/media/romet/data2/datasets/rally-estonia/dataset/2021-05-28-15-19-48_e2e_sulaoja_20_30: 10708
/media/romet/data2/datasets/rally-estonia/dataset/2021-06-07-14-20-07_e2e_rec_ss6: 25836
/media/romet/data2/datasets/rally-estonia/dataset/2021-06-07-14-06-31_e2e_rec_ss6: 3003
/media/romet/data2/datasets/rally-estonia/dataset/2021-06-07-14-09-18_e2e_rec_ss6: 4551
/media/romet/data2/datasets/rally-estonia/dataset/2021-06-07-14-36-16_e2e_rec_ss6: 25368
/media/romet/data2/datasets/rally-estonia/dataset/2021-09-24-14-03-45_e2e_rec_ss11_backwards: 25172
/media/romet/data2/datasets/rally-estonia/dataset/2021-10-26-10-49-06_e2e_rec_ss20_elva: 33045
/media/romet/data2/datasets/rally-estonia/dataset/2021-10-26-11-08-59_e2e_rec_ss20_elva_back: 33281
/media/romet/data2/datasets/rally-estonia/dataset/2021-10-20-15-11-29_e2e_rec_vastse_ss13_17_back: 26763
/media/romet/data2/datasets/rally-estonia/dataset/2021-10-11-14-50-59_e2e_rec_vahi: 21211
/media/romet/data2/datasets/rally-estonia/dataset/2021-10-14-[...]
{'fraction_non_alphanumeric': 0.12311153969784636, 'fraction_numerical': 0.09418193506910962, 'mean_word_length': 3.995184590690209, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '364218', 'license': 'Apache-2.0', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1224, 'n_words': 147, 'path': 'notebooks/save_crop.ipynb', 'repo_name': 'UT-ADL/lidar-as-camera', 'size': '3111'}