<jupyter_start><jupyter_text>Geometric median pixel composite mosaics<jupyter_code>%cd ~/hdstats
import matplotlib.pyplot as plt
import numpy as np
import hdstats
import joblib
ds = joblib.load('data/6mp63x30.pkl.xz')
times, data = ds
data.shape
gm = hdstats.nangeomedian_pcm(data, num_threads=28)
gm.shape
img = gm[:,:,[2,1,0]].copy()
for i in range(img.shape[-1]):
    dd=img[:,:,i]
    mu = np.mean(dd)
    sd = np.std(dd)
    pl, pu = mu - 3*sd, mu + 3*sd
    dd[dd<pl], dd[dd>pu] = pl, pu
    dd -= pl
    dd /= (pu - pl)
plt.figure(figsize=(5,5))
plt.imshow(img)<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
{'fraction_non_alphanumeric': 0.16017316017316016, 'fraction_numerical': 0.030303030303030304, 'mean_word_length': 3.9571428571428573, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '98004', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 308, 'n_tokens_neox': 287, 'n_words': 74, 'path': 'docs/geomedian.ipynb', 'repo_name': 'daleroberts/hdstats', 'size': '693'}