<jupyter_start><jupyter_text>guatemala-volcano_00000000_post_disaster.pngguatemala-volcano_00000000_post_disaster.json<jupyter_code>fns=glob(f"{DATADIR}/train/images/*.png")
roots=list(sorted(set(['_'.join(fn.split('/')[-1].split('_')[0:2]) for fn in fns])))
from shutil import copyfile
from tqdm import tqdm
fns[0]
for i, root in tqdm(enumerate(roots)):
    for mode in ('pre', 'post'):
        for dr,ext,ndr in [('images','png', 'images'), ('labels', 'json', 'gt_labels')]:
            src = f"{DATADIR}/train/{dr}/{root}_{mode}_disaster.{ext}"
            dst = f"{DEBUGDIR}/{ndr}/test_{mode}_{i:05d}.{ext}"
            copyfile(src,dst)<jupyter_output>2799it [00:47, 58.54it/s]
{'fraction_non_alphanumeric': 0.20527859237536658, 'fraction_numerical': 0.04985337243401759, 'mean_word_length': 5.098214285714286, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '65033', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 299, 'n_tokens_neox': 272, 'n_words': 40, 'path': 'notebooks/Set up debug run.ipynb', 'repo_name': 'catskillsresearch/xview2-catskills', 'size': '682'}