<jupyter_start><jupyter_text>Detectron sometimes has trouble loading images, and instead of outputting empty json it outputs nothing. Here we make up for that by finding images without any Detectron json, and generating black contour images.This code ingests:- The detectron `.json` files.- The photos from `photos/png1600`.And outputs:- Empty (black) images to `detectron/png1600` for any missing `.json` files.<jupyter_code>from utils.list_all_files import *
from utils.imutil import *
from utils.progress import progress
import numpy as np
import os
detectron_filenames = list(list_all_files('../data/detectron/json/'))
photos_filenames = list(list_all_files('../data/photos/png1600/'))
output_dir = '../data/detectron/png1600/'

def get_id(fn):
    return os.path.splitext(os.path.basename(fn))[0]

detectron_filenames = set([get_id(e) for e in detectron_filenames])
for fn in progress(photos_filenames):
    cur_id = get_id(fn)
    if cur_id not in detectron_filenames:
        img = imread(fn)
        h, w = img.shape
        result = np.zeros((h, w, 3), dtype=np.uint8)
        imwrite(os.path.join(output_dir, cur_id + '.png'), result)<jupyter_output>59278 0:00:51 1164.44/s
{'fraction_non_alphanumeric': 0.1132713440405748, 'fraction_numerical': 0.029585798816568046, 'mean_word_length': 4.5327102803738315, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '84934', 'license': 'MIT', 'n_tokens_mistral': 420, 'n_tokens_neox': 391, 'n_words': 112, 'path': 'notebooks/Create Missing Detectron Contour Images.ipynb', 'repo_name': 'CreativeInquiry/TeenieHarrisProject', 'size': '1183'}