<jupyter_start><jupyter_text>For LPD (cropping the vehicle patches)<jupyter_code># Generating images of cropped vehicle patches and saving them alongside each image annotation file for all datasets.
save_dir = "vehicles_cropped"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_lp_det_data(i, save_dir)
    print(i, dataset, "Done")
# Checking if the generation was successful.
dataset_i = 0  # Index of which dataset to check, chosen from DATASETS.
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/lp_class_name.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
vehicles_cropped/all_train.txt
vehicles_cropped/all_val.txt
vehicles_cropped/all_test.txt<jupyter_text>For LPR (cropping the LP patches)<jupyter_code># Generating images of cropped vehicle patches and saving them alongside each image annotation file for all datasets.
save_dir = "lps_cropped"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_lp_rec_data(i, save_dir)
    print(i, dataset, "Done")
# Checking if the generation was successful.
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped/all_train.txt
lps_cropped/all_val.txt
lps_cropped/all_test.txt<jupyter_text>For LPR (The LP patches with negative and processed)<jupyter_code># Generating images of cropped vehicle patches and saving them alongside each image annotation file for all datasets.
save_dir = "lps_cropped_neg_grey"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_lp_rec_data(i, save_dir, negative_imgs=True, process=True)
    print(i, dataset, "Done")
# Checking if the generation was successful.
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_neg_grey/all_train.txt
lps_cropped_neg_grey/all_val.txt
lps_cropped_neg_grey/all_test.txt<jupyter_text>For LPR (with only negative images)<jupyter_code># Doubling the # of samples by using the negative of each LP patch.
save_dir = "lps_cropped_neg"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_lp_rec_data(i, save_dir, negative_imgs=True)
    print(i, dataset, "Done")
# Checking if the generation was successful.
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_neg/all_train.txt
lps_cropped_neg/all_val.txt
lps_cropped_neg/all_test.txt<jupyter_text>For LPR (negative images and same aspect ratio)<jupyter_code>save_dir = "lps_cropped_neg_ratio"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_lp_rec_data(i, save_dir, negative_imgs=True, same_ratio=True)
    print(i, dataset, "Done")
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_neg_ratio/all_train.txt
lps_cropped_neg_ratio/all_val.txt
lps_cropped_neg_ratio/all_test.txt<jupyter_text>For LPR (negative images with permutated samples)<jupyter_code># To generate data, please refer to the data_generation.ipynb notebook.
save_dir = "lps_cropped_neg_perm"
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_neg_perm/all_train.txt
lps_cropped_neg_perm/all_val.txt
lps_cropped_neg_perm/all_test.txt<jupyter_text>For LPR (negative images and same ratio with permutated samples)<jupyter_code># To generate data, please refer to the data_generation.ipynb notebook.
save_dir = "lps_cropped_n_r_perm"
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_n_r_perm/all_train.txt
lps_cropped_n_r_perm/all_val.txt
lps_cropped_n_r_perm/all_test.txt<jupyter_text>For LPR (generated images using augmentation)<jupyter_code># To generate data, please refer to the data_generation.ipynb notebook.
save_dir = "lps_cropped_gen"
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_gen/all_train.txt
lps_cropped_gen/all_val.txt
lps_cropped_gen/all_test.txt<jupyter_text>For LPR (negative images and generated images using permutations)<jupyter_code># To generate data, please refer to the data_generation.ipynb notebook.
save_dir = "lps_cropped_perm"
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/char_class_names.txt")
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
lps_cropped_perm/all_train.txt
lps_cropped_perm/all_val.txt
lps_cropped_perm/all_test.txt<jupyter_text>For vehicle detection (ALPR datasets)<jupyter_code># Generating vehicle detection annotation files alongside each sample image for all datasets.
save_dir = "vehicle_det"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_v_det_data(i, save_dir)
    print(i, dataset, "Done")
# Checking if the generation was successful.
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/v_det_class_name.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
vehicle_det/all_train.txt
vehicle_det/all_val.txt
vehicle_det/all_test.txt<jupyter_text>For vehicle detection (COCO dataset)<jupyter_code># Downloading the COCO images (training set)
anno_file = "coco/instances_train2017.json"  # Get this from https://cocodataset.org/#download
class_names = ["car", "motorcycle", "bus", "truck"]
save_dir = "coco"
dir_name = "coco_train"
data_utils.download_coco(anno_file, class_names, dir_name, num_samples=-1)  # Use num_samples=-1 to download all samples.
# Checking if the generation/download was successful.
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{dir_name}", "class_names/v_det_class_name.txt")
# Splitting dataset into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

data_utils.gen_dataset_split_files(dir_name, save_dir, imgs_prefix, train=100, val=0)
# Downloading the COCO images (validation set)
anno_file = "coco/instances_val2017.json"  # Get this from https://cocodataset.org/#download
class_names = ["car", "motorcycle", "bus", "truck"]
save_dir = "coco"
dir_name = "coco_val"
data_utils.download_coco(anno_file, class_names, dir_name, num_samples=-1)  # Use num_samples=-1 to download all samples.
# Checking if the generation/download was successful.
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{dir_name}", "class_names/v_det_class_name.txt")
# Splitting dataset into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

data_utils.gen_dataset_split_files(dir_name, save_dir, imgs_prefix, train=0, val=100)<jupyter_output>coco_val dataset done, files saved:
coco/train_coco_val.txt
coco/val_coco_val.txt
coco/test_coco_val.txt<jupyter_text>For vehicle detection (ALPR dataset, blurred BGs)<jupyter_code># Since the ALPR datasets contain samples where not all vehicles in the image is labelled, they cannot be used to train
# a vehicle detection model because of the false positives. So here, for each sample, the BG is blurred and only the vehicle
# patches are kept unchanged.
save_dir = "v_det_blurred"
for i, dataset in enumerate(data_utils.datasets):
    data_utils.generate_v_det_data(i, save_dir, blur_bg=True)
    print(i, dataset, "Done")
dataset_i = 0
data_utils.check_darknet_yolov4_annos(f"{save_dir}/{DATASETS[dataset_i]}", "class_names/v_det_class_name.txt")
# Splitting datasets into train, val, and test sets.
imgs_prefix = f"../../../../../datasets/{save_dir}"

for i, dataset in enumerate(data_utils.datasets):
    data_utils.gen_dataset_split_files(i, save_dir, imgs_prefix)
# Combining all train, val, test sets of all datasets into three big subsets.
data_utils.combine_subsets(save_dir)<jupyter_output>All subsets combined successfully, files_saved:
v_det_blurred/all_train.txt
v_det_blurred/all_val.txt
v_det_blurred/all_test.txt<jupyter_text>Save all annotations in a JSON file<jupyter_code>data_utils.save_all_annos_as_dict()<jupyter_output>All annotations saved in all_annos.json.
{'fraction_non_alphanumeric': 0.1075230972283326, 'fraction_numerical': 0.00387153541575011, 'mean_word_length': 6.607764390896921, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '385967', 'license': 'MIT', 'n_tokens_mistral': 4098, 'n_tokens_neox': 3970, 'n_words': 861, 'path': 'datasets/darknet_dataset_preperations.ipynb', 'repo_name': 'RedaAlb/IntelligentTransportationInfrastructure_ITI', 'size': '11365'}