<jupyter_start><jupyter_text>Download data and set paths<jupyter_code>dataroot = "/home/ubuntu/Shlok/gtsdb/data"
filename = "FullIJCNN2013.zip"
urlroot = "https://sid.erda.dk/public/archives/ff17dc924eba88d5d01a807357d6614c"
! wget {urlroot}/{filename} -O {dataroot}/{filename}
! unzip {dataroot}/{filename}<jupyter_output><empty_output><jupyter_text>Prepare labels Full<jupyter_code>gtpath = os.path.join(dataroot, "FullIJCNN2013", "gt.txt")
gtlist = open(gtpath).read().split('\n')
gtlist[:10]
gtdict = {}
for line in gtlist:
    if line=="":
        break
    cols = line.split(';')
    fname = cols[0]
    cls_id = int(cols[-1])
    bbox = list(map(lambda x: float(x), cols[1:5]))
    bbox = xyxy2xywhn(bbox)
    if fname in gtdict:
        gtdict[fname].append((cls_id, *bbox))
    else:
        gtdict[fname] = [(cls_id, *bbox)]


split = "full"
labelroot = os.path.join(dataroot, split, "labels")
make_dirs(labelroot)

for key in tqdm(gtdict):
    path = os.path.join(labelroot, str(Path(key).stem)+".txt")
    save_label(path, gtdict[key])<jupyter_output>100%|██████████| 741/741 [00:00<00:00, 8773.48it/s]<jupyter_text>Copy images full<jupyter_code>images_path = os.path.join(dataroot, "FullIJCNN2013")
imglist = glob(images_path+"/*.ppm")

split = "full"
imgroot = os.path.join(dataroot, split, "images")
make_dirs(imgroot)

for img in tqdm(imglist):
    shutil.copy(img, imgroot)<jupyter_output>100%|██████████| 900/900 [00:21<00:00, 42.13it/s]<jupyter_text>Prepare train-valid split Take first 600 images for train and last 300 images for valid.  Random train-valid split is not a good idea here since consecutive images might be similar<jupyter_code>full_imglist = glob(os.path.join(dataroot, "full", "images")+"/*.ppm")
full_imglist.sort()
print(len(full_imglist)); full_imglist[:5]<jupyter_output>900<jupyter_text>Prepare train split<jupyter_code>split = "train"
imglist = full_imglist[:600]

imgroot = os.path.join(dataroot, split, "images")
labelroot = os.path.join(dataroot, split, "labels")
make_dirs(imgroot)
make_dirs(labelroot)
for img in tqdm(imglist):
    p = Path(img)
    src_label = str(p.parent.parent/"labels"/p.stem)+".txt"
    shutil.copy(img, imgroot)
    try:
        shutil.copy(src_label, labelroot)
    except Exception as err:
        pass
split = "valid"
imglist = full_imglist[600:]

imgroot = os.path.join(dataroot, split, "images")
labelroot = os.path.join(dataroot, split, "labels")
make_dirs(imgroot)
make_dirs(labelroot)
for img in tqdm(imglist):
    p = Path(img)
    src_label = str(p.parent.parent/"labels"/p.stem)+".txt"
    shutil.copy(img, imgroot)
    try:
        shutil.copy(src_label, labelroot)
    except Exception as err:
        pass<jupyter_output>100%|██████████| 300/300 [00:00<00:00, 305.00it/s]<jupyter_text>Prepare classes name<jupyter_code>classnames = open(os.path.join(dataroot, "classes.txt")).read().split("\n")[:-1]
classnames[:4]
classnames = list(map(lambda x: x.split('=')[1].split('(')[0].strip(), classnames))
print(classnames)<jupyter_output>['speed limit 20', 'speed limit 30', 'speed limit 50', 'speed limit 60', 'speed limit 70', 'speed limit 80', 'restriction ends 80', 'speed limit 100', 'speed limit 120', 'no overtaking', 'no overtaking', 'priority at next intersection', 'priority road', 'give way', 'stop', 'no traffic both ways', 'no trucks', 'no entry', 'danger', 'bend left', 'bend right', 'bend', 'uneven road', 'slippery road', 'road narrows', 'construction', 'traffic signal', 'pedestrian crossing', 'school crossing', 'cycles crossing', 'snow', 'animals', 'restriction ends', 'go right', 'go left', 'go straight', 'go right or straight', 'go left or straight', 'keep right', 'keep left', 'roundabout', 'restriction ends', 'restriction ends']<jupyter_text>Plot class distribution<jupyter_code>full_labelist = glob(os.path.join(dataroot, "full", "labels")+"/*.txt")
full_labelist[:5]
names = np.array(['speed limit 20', 'speed limit 30', 'speed limit 50', 'speed limit 60', 'speed limit 70', 'speed limit 80', 
          'restriction ends 80', 'speed limit 100', 'speed limit 120', 'no overtaking', 'no overtaking', 
          'priority at next intersection', 'priority road', 'give way', 'stop', 'no traffic both ways', 'no trucks', 
          'no entry', 'danger', 'bend left', 'bend right', 'bend', 'uneven road', 'slippery road', 'road narrows', 
          'construction', 'traffic signal', 'pedestrian crossing', 'school crossing', 'cycles crossing', 'snow', 
          'animals', 'restriction ends', 'go right', 'go left', 'go straight', 'go right or straight', 'go left or straight', 
          'keep right', 'keep left', 'roundabout', 'restriction ends', 'restriction ends'])

def plot_distribution(labelist, split):
    label_to_files = {}
    for lblfile in labelist:
        lines = open(lblfile).read().split('\n')[:-1]
        labels = list(map(lambda x: x.split()[0], lines))
        for label in labels:
            label = int(label)
            if label not in label_to_files:
                label_to_files[label] = [lblfile]
            else:
                label_to_files[label].append(lblfile)

    label_to_no = {_k:len(label_to_files[_k]) for _k in label_to_files}
    # pp(label_to_no)
    _ltn = sorted(list(label_to_no.items()), key=lambda x: x[0])
    _ltnc = [x[0] for x in _ltn]
    _ltnn = [x[1] for x in _ltn]

    label_to_no_set = {_k:len(set(label_to_files[_k])) for _k in label_to_files}
    _ltns = sorted(list(label_to_no_set.items()), key=lambda x: x[0])
    _ltncs = [x[0] for x in _ltns]
    _ltnns = [x[1] for x in _ltns]

    plt.figure(figsize=(16,4))
    plt.bar(names[_ltnc], _ltnn)
    plt.xticks(rotation=90)
    plt.bar(names[_ltncs], _ltnns)
    plt.xticks(rotation=90)
    plt.title("Class distribution ({})".format(split))
    plt.show()

plot_distribution(full_labelist[:600], "train")
plot_distribution(full_labelist[600:], "test")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1669771380186283, 'fraction_numerical': 0.031668077900084676, 'mean_word_length': 4.635496183206107, 'pattern_counts': {'":': 1, '<': 27, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '575453', 'license': 'MIT', 'n_tokens_mistral': 2341, 'n_tokens_neox': 2164, 'n_words': 500, 'path': 'notebooks/prepare_data.ipynb', 'repo_name': 'shloked/yolov5-on-gtsdb', 'size': '5905'}