<jupyter_start><jupyter_text>create random palettes in rgb space<jupyter_code>rgb_palettes = np.random.rand(10_000, 5, 3) * 255
def rgb_to_lab(rgb_palette):
    return rgb2lab(rgb_palette.reshape(-1, 1, 3) / 255).squeeze()


lab_palette = rgb_to_lab(rgb_palettes[1])
lab_palettes = np.stack([rgb_to_lab(palette) for palette in tqdm(rgb_palettes)])<jupyter_output><empty_output><jupyter_text>create permutations<jupyter_code>palette_permutations = np.stack(
    [list(itertools.permutations(palette, 5)) for palette in tqdm(lab_palettes)]
)
palette_permutations.shape<jupyter_output><empty_output><jupyter_text>calculate true distances<jupyter_code>distance_matrix = np.stack(
    [
        (np.linalg.norm(palette_permutations - palette, axis=3).sum(axis=2).min(axis=1))
        for palette in tqdm(lab_palettes)
    ]
)
distance_matrix.shape<jupyter_output><empty_output><jupyter_text>save them<jupyter_code>np.save("/storage/data/palette/lab_palettes", lab_palettes)
np.save("/storage/data/palette/palette_permutations", palette_permutations)
np.save("/storage/data/palette/distance_matrix", distance_matrix)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12543859649122807, 'fraction_numerical': 0.01929824561403509, 'mean_word_length': 7.844961240310077, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '407513', 'license': 'MIT', 'n_tokens_mistral': 461, 'n_tokens_neox': 434, 'n_words': 51, 'path': 'notebooks/palette/notebooks/10 - dataset for palette embedding network.ipynb', 'repo_name': 'wellcomecollection/data-science', 'size': '1140'}