<jupyter_start><jupyter_text>Create point clouds of "5"s and flipped "5"s<jupyter_code>def get_dataset(angle):
    fives = []

    for i, elem in enumerate(y):
        if elem == '5':
            A = np.zeros((40, 40))
            A[6:34, 6:34] = X[i].reshape((28, 28))
            img = Image.fromarray(np.uint8(A), 'L')
            
            if angle > 0:
                img = img.transpose(Image.FLIP_TOP_BOTTOM)
            
            fives.append(np.asarray(img).flatten())
    
    return np.array(fives)
clouds = []

for angle in [0, 1]:
    clouds.append(get_dataset(angle))
for cloud in clouds:
    print(cloud.shape)<jupyter_output><empty_output><jupyter_text>Compute barcodes<jupyter_code>import mtd
res1 = []
trials = 20

for i in range(1, len(clouds)):
    np.random.seed(7)
    barcs = [mtd.calc_cross_barcodes(clouds[i], clouds[0], batch_size1 = 100, batch_size2 = 1000) for _ in range(trials)]
    res1.append(barcs)
def get_scores(res, args_dict, trials = 10):

    scores = []

    for i in range(len(res)): 
        asum = []
        
        for exp_id, elem in enumerate(res[i]):
            asum.append(mtd.get_score(elem, **args_dict))

        scores.append(sum(asum) / len(res[i]))

    return scores
scores = get_scores(res1, {'h_idx' : 1, 'kind' : 'sum_length'})
for s in scores:
    print(s)
# Geometry Score

import gs

def get_rlts(X):
    N = 2500
    gamma = (1/128)/(N/5000)
    rlts = gs.rlts(X, gamma=gamma, n=N, n_threads = 40)
    
    return rlts
rlts = [None] * len(clouds)
%time
for i in range(len(clouds)):
    rlts[i] = get_rlts(clouds[i])
for i in range(1, len(clouds)):
    print(1e3 * gs.geom_score(rlts[0], rlts[i]))
# additional experiment with IMD
from msid import msid_score

res1 = []
for i in range(len(clouds)):
    v = msid_score(clouds[0], clouds[i])
    res1.append(v)

    print(i, trial, v)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14869888475836432, 'fraction_numerical': 0.032926181625066386, 'mean_word_length': 2.8845360824742268, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '164196', 'license': 'MIT', 'n_tokens_mistral': 783, 'n_tokens_neox': 727, 'n_words': 159, 'path': 'experiments/MNIST.ipynb', 'repo_name': 'IlyaTrofimov/MTopDiv', 'size': '1883'}