<jupyter_start><jupyter_text>---<jupyter_code>import tqdm

for mask in tqdm.tqdm(mask_list):
    _data = pdfd.utils.load_info_from_mask(MASK_DIR, mask, classes=['face'], values=[128])    
    json_dir = os.path.join(TRAIN_DIR, 'json')
    pdfd.utils.extend_info_into_json(_data, os.path.join(json_dir, mask.replace('bmp', 'json')))
import json

val_json_dir = os.path.join(VAL_DIR, 'json')

json_list = os.listdir(json_dir)
json_path = os.path.join(V_ORIGINAL_DIR, 'region_data.json')

pdfd.utils.extend_info_into_json({}, json_path)
with open(json_path, 'r', encoding="utf-8") as f:
    _data = json.load(f)
    
for data in tqdm.tqdm(json_list):
    _json_path = os.path.join(json_dir, data)
    with open(_json_path, 'r', encoding="utf-8") as f:
        _data.update(json.load(f))
        
pdfd.utils.extend_info_into_json(_data, json_path)<jupyter_output>100%|██████████| 3220/3220 [00:00<00:00, 12905.48it/s]
{'fraction_non_alphanumeric': 0.1686746987951807, 'fraction_numerical': 0.033953997809419496, 'mean_word_length': 5.0131578947368425, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '190738', 'license': 'MIT', 'n_tokens_mistral': 410, 'n_tokens_neox': 385, 'n_words': 57, 'path': 'samples/celebA/.ipynb_checkpoints/manage_celebA_data-checkpoint.ipynb', 'repo_name': 'JhoLee/PDF_FaceDeidentifier', 'size': '913'}