<jupyter_start><jupyter_text>Duplicate Images & Class Collisions**Authors:**- [<NAME>](https://github.com/AngusTheMack) ([1106817](mailto:<EMAIL>))- [<NAME>](https://github.com/nmichlo) ([1386161](mailto:<EMAIL>))**Achievement** Detecting which images are duplicates of each other and of those duplicates, which are labeled with differnt classes.----------------------<jupyter_code># Utilities
import sys
import os
from tqdm.notebook import tqdm
from pprint import pprint

# Add root of project to PYTHON_PATH so we can import correctly
if os.path.abspath('../') not in {os.path.abspath(path) for path in sys.path}:
    sys.path.insert(0, os.path.abspath('../'))
    
# Import SSIC common stuffs
from ssic.ssic import SSIC
from ssic.util import set_random_seed, cache_data

# if you dont have a .env file set it here
os.environ.setdefault('DATASET_DIR', '~/downloads/datasets/ssic')

# Initialise SSIC paths, data and other stuffs, searches for a .env file in the project with these variables specified, also checkpoints os.environ and sys.path
SSIC.init()

def get_info_by_class():
    """ Group image info by classes - returns a dict of lists """
    # TODO: this should be in SSIC
    images_by_class = defaultdict(list)
    for name, info in SSIC.get_train_image_info().items():
        images_by_class[info['class_id']].append(info)
    return dict(images_by_class)<jupyter_output>[[93mRESTORED[0m]: os.environ
[[92mLOADED[0m]: 
[[93mRESTORED[0m]: sys.path
[[95mSTORAGE_DIR[0m]: [90m/home/nmichlo/workspace/snake-id/notebooks/out[0m
[[95mDATASET_DIR[0m]: [90m/home/nmichlo/downloads/datasets/ssic[0m
[[95mDATASET_CLASS_CSV[0m]: [90m/home/nmichlo/downloads/datasets/ssic/class_idx_mapping.csv[0m
[[95mDATASET_TRAIN_DIR[0m]: [90m/home/nmichlo/downloads/datasets/ssic/train[0m
[[95mDATASET_TEST_DIR[0m]: [90m/home/nmichlo/downloads/datasets/ssic/round1[0m<jupyter_text>MD5 Collisions<jupyter_code># TODO: this should be moved into class SSIC

def hash_collisions():
    from collections import defaultdict
    hashes = defaultdict(list)
    for info in tqdm(SSIC.get_train_image_info().values()):
        md5 = os.popen(f"md5sum {info['path']}").read().split(' ')[0]
        hashes[md5].append(info['name'])
    return dict(hashes)

hash_collisions = cache_data(path=os.path.join(SSIC.STORAGE_DIR, 'duplicates.json'), generator=hash_collisions)
matching_hashes = {k: v for k, v in hash_collisions.items() if len(v) > 1}

img_info = SSIC.get_train_image_info()
valid_matching_hashes = {k: [name for name in v if img_info[name]['valid']] for k, v in matching_hashes.items()}
valid_matching_hashes = {k: v for k, v in valid_matching_hashes.items() if v}
print('COLLISIONS: ', sum(len(v) for v in matching_hashes.values()))
print('\033[92mVALID COLLISIONS\033[0m: ', sum(len(v) for v in valid_matching_hashes.values()))


conflicting_matching_hashes = {}
for key, collisions in valid_matching_hashes.items():
    start_id = img_info[collisions[0]]['class_id']
    if all(start_id == img_info[c]['class_id'] for c in collisions):
        continue
    conflicting_matching_hashes[key] = list(collisions)

print('\033[91mCONFLICTING COLLISIONS\033[0m:', sum(len(v) for v in conflicting_matching_hashes.values()))<jupyter_output>COLLISIONS:  1220
[92mVALID COLLISIONS[0m:  1036
[91mCONFLICTING COLLISIONS[0m: 48<jupyter_text>Conflicting Classes<jupyter_code># THESE MD5 COLLISIONS HAVE CONFLICTING CLASSES
for collisions in conflicting_matching_hashes.values():
    print(collisions, '->', [img_info[name]['class_id'] for name in collisions])<jupyter_output>['84a61a4c635527c06438b75cfa413c9f.jpg', 'a2bd3bc0d0da494b90b080215603373d.jpg'] -> [460, 448]
['c2d95843d73fdc7aa4a027d94b5403d6.jpg', 'dc4bb8ad3cbc5ec7cefa49c5f64bb36a.jpg'] -> [639, 635]
['dc85f0b6d1b2d87f55687337a9c0c3de.jpg', '254bcaec0e6ce0faa29a19f65fcab453.jpg'] -> [639, 654]
['2c73a88777fbff01c2662c3a23fdc573.jpg', '41bf434b469af0fadb39c407b2e1f364.jpg'] -> [639, 654]
['a15b8c9fe546a91bf47ce5df88b22ca7.jpg', '7334e90eaa98f966455223a81caddf49.jpg'] -> [639, 697]
['74bf306b43c717d51cc234de32dd772c.jpg', '27ea8fbe301561320593737a6cd3af4a.jpg'] -> [394, 872]
['e0fc1a5b5866441ba537dd5990c55048.jpg', 'f834f5e0c839ff10f808e796228bd5e1.jpg'] -> [394, 4]
['693b99e3c8c9f00a109eac6bce2e00b4.jpg', 'c8bf322e35c32924ddf73fec7f4ff4d8.jpg'] -> [362, 204]
['e39e6c80cdc1002d5a4830c92d237d0c.jpg', 'e9580d63e47988d33524129c53d896a4.jpg'] -> [362, 543]
['2d13e228b8b52e51553fca64fb7f9986.jpg', 'e8903fdb6110daabcbe1270202a951da.jpg'] -> [362, 543]
['6154d719cce4d2ad6cb2dd3f7dbd82b9.jpg', '20dafa2cdd[...]
{'fraction_non_alphanumeric': 0.1458469587965991, 'fraction_numerical': 0.12273817309788533, 'mean_word_length': 6.364365971107544, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 26, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '215315', 'license': 'MIT', 'n_tokens_mistral': 2228, 'n_tokens_neox': 1895, 'n_words': 325, 'path': 'notebooks/duplicates.ipynb', 'repo_name': 'AngusTheMack/snake-id', 'size': '4643'}