<jupyter_start><jupyter_text>Merge labels<jupyter_code>set_labels = set()
for filename in os.listdir(path + 'labels/'):
    if filename.endswith(".txt"):
        set_labels.add(filename)

print(len(set_labels))
        
def append_labels(path, labels, destiny):
    for filename in os.listdir(path):
        if filename in labels:
            with open(destiny + '/' + filename, 'a') as outfile:
                with open(path + filename) as infile:
                    for line in infile:
                        outfile.write(line)
        else:
            shutil.copy(path + filename, destiny + filename)
            labels.add(filename)
    return labels
set_labels = append_labels(path + 'Human_arm/labels/', set_labels, path + 'labels/')
print(len(set_labels))
set_labels = append_labels(path + 'Human_body/labels/', set_labels, path + 'labels/')
print(len(set_labels))
set_labels = append_labels(path + 'Human_foot/labels/', set_labels, path + 'labels/')
print(len(set_labels))
set_labels = append_labels(path + 'Human_hand/labels/', set_labels, path + 'labels/')
print(len(set_labels))
set_labels = append_labels(path + 'Human_head/labels/', set_labels, path + 'labels/')
print(len(set_labels))
set_labels = append_labels(path + 'Human_leg/labels/', set_labels, path + 'labels/')
print(len(set_labels))
images = set()
def copy_images(origin, destiny, images):
    for filename in os.listdir(origin):
        if filename.endswith('.jpg'):
            if filename in images:
                continue
            else:
                shutil.copy(origin + filename, destiny + filename)
                images.add(filename)
            
    return images
for key in class_labels_dict_index:
    images = copy_images(path + key + '/', path + 'images/', images)
    print(len(images))
for label in set_labels:
    not_found = True
    for image in images:
        if label[:-3] == image[:-3]:
            not_found = False
            break
    if not_found:
        print(label)<jupyter_output>49262b8f828e9d43.txt
b325b67fb67718de.txt
ba1ea6102b729c92.txt
{'fraction_non_alphanumeric': 0.12135922330097088, 'fraction_numerical': 0.01650485436893204, 'mean_word_length': 2.9108159392789372, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '613412', 'license': 'MIT', 'n_tokens_mistral': 680, 'n_tokens_neox': 645, 'n_words': 147, 'path': 'notebooks/Yolov3(Labels and boxes).ipynb', 'repo_name': 'sysaster/dataset', 'size': '2060'}