<jupyter_start><jupyter_text>Compress data<jupyter_code>test_img_path = os.path.join(os.getcwd(), 'dataset', 'preprocessed', '1024_cropped', 'density', 'e867fd2ce1495951719e7735577a19b9__256_0__r_180.tif')
test_img = Image.open(test_img_path)
input_dim_1024 = (1024, 1024)
test_img_np = np.array(test_img)
test_img_reconstructed = Image.fromarray(np.uint8(test_img_np.reshape((*input_dim_1024)) * 255), 'L')
test_img_reconstructed
input_dim_512 = (512, 512)
test_img_compressed = test_img.resize(input_dim_512)
test_img_comp_np = np.array(test_img_compressed)
test_img_comp_reconstructed = Image.fromarray(np.uint8(test_img_comp_np.reshape((*input_dim_512)) * 255), 'L')
test_img_comp_reconstructed
test_img_np.sum() / 1000
test_img_np.max()
(test_img_comp_np.sum() * 4) / 1000
test_img_comp_np.max()
input_dim_800 = (800, 800)
test_img_compressed_2 = test_img.resize(input_dim_800)
test_img_compressed_2_np = np.array(test_img_compressed_2)
(test_img_compressed_2_np.sum() * ((1024.0/800.0)**2)) / 1000
test_img_3_path = os.path.join(os.getcwd(), 'dataset', 'preprocessed', '1024_cropped', 'bw_dots', 'sized_Mexico-Monarch-Astrid-Frisch.png')
test_img_3 = Image.open(test_img_3_path)
test_img_3_np = np.array(test_img_3)
print(test_img_3_np.sum() / 255)<jupyter_output>155.0<jupyter_text>Scratchpad<jupyter_code>INPUT_IMAGES_PATH = os.path.join(os.getcwd(), 'dataset', 'preprocessed', '1024_cropped', 'cropped_raw')
images = [f for f in os.listdir(INPUT_IMAGES_PATH) if os.path.isfile(f)]
images<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14089121887287026, 'fraction_numerical': 0.08781127129750983, 'mean_word_length': 9.45890410958904, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '172709', 'license': 'MIT', 'n_tokens_mistral': 741, 'n_tokens_neox': 648, 'n_words': 76, 'path': '_scratch_.ipynb', 'repo_name': 'monarch-counter/monarch-counter', 'size': '1526'}