<jupyter_start><jupyter_text><jupyter_code>from google.colab import auth, drive
auth.authenticate_user()
drive.mount('/content/gdrive')

COLAB_ROOT='/content/gdrive/My Drive/Colab Notebooks/'
DESSERT_ROOT = COLAB_ROOT + 'Dessert/'
DATA_ROOT = DESSERT_ROOT + 'data/'
import os
import hashlib
import subprocess
SOURCE_DIR = DATA_ROOT + 'crop/'
RESULT_DIR = DATA_ROOT + 'crop_dedup/'
if not os.path.isdir(RESULT_DIR):
    os.makedirs(RESULT_DIR)
for source_zip in os.listdir(SOURCE_DIR):
    print(source_zip)
    if os.path.isdir('src/'):
        os.system('rm -rf target.zip dst src; mkdir src; mkdir dst')
    os.system("cp '{}/{}' target.zip".format(SOURCE_DIR, source_zip))
    subprocess.call(['unzip', '-q', 'target.zip', '-d', 'src/'])
    
    name, ext = os.path.splitext(source_zip)

    db = set()
    img_list = os.listdir('src/')
    n_imgs = len(os.listdir('src/'))
    for i, img_file in enumerate(img_list):
        if i%500 == 0:
            print('{} / {}'.format(i, n_imgs))
        with open('src/{}'.format(img_file), 'rb') as f:
            h = hashlib.md5(f.read()).hexdigest()
            if h in db:
                continue
            db.add(h)
            subprocess.call(['cp', 'src/{}'.format(img_file), 'dst/'])
    os.system("cd dst; zip -q '{}_dedup.zip' ./*; mv '{}_dedup.zip' '{}'".format(name, name, RESULT_DIR))
    print('Result: {}'.format(len(db)))<jupyter_output>d9816eadc28a3d8477105088f280f1bc
{'fraction_non_alphanumeric': 0.17782426778242677, 'fraction_numerical': 0.01813110181311018, 'mean_word_length': 3.283582089552239, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '327415', 'license': 'MIT', 'n_tokens_mistral': 561, 'n_tokens_neox': 541, 'n_words': 104, 'path': 'crop_deduplication.ipynb', 'repo_name': 'inhyek/newsest_insta', 'size': '1434'}