<jupyter_start><jupyter_text>`<<<<<<< HEAD`<jupyter_code>from multiinstance.data.realData import buildDataset as buildReal
from multiinstance.utils import *
from multiinstance.ward_clustering import WardClustering
import matplotlib.pyplot as plt
from glob import glob
from tqdm.notebook import tqdm
import numpy as np<jupyter_output><empty_output><jupyter_text>`=======`<jupyter_code>from multiinstance.data.realData import buildDataset as buildReal
from multiinstance.utils import *

from multiinstance.ward_clustering import WardClustering

import matplotlib.pyplot as plt

from glob import glob
from tqdm.notebook import tqdm

import numpy as np<jupyter_output><empty_output><jupyter_text>`>>>>>>> 76a8683cdc145054e4f8b6a1f4525a5b89f04434`<jupyter_code>for f in tqdm(glob("/ssdata/ClassPriorEstimationPrivate/data/rawDatasets/*.mat")):
    ds = buildReal(f,16,nPDistr=lambda: 1 + np.random.poisson(25),
                   nUDistr=lambda: 1 + np.random.poisson(75))

    ds = addTransformScores(ds)


    ds.alphaHats,ds.curves = getBagAlphaHats(ds,numbootstraps=100)

    ward = WardClustering(ds,randomPairing=True,numbootstraps=ds.alphaHats.shape[1])
    ward.cluster()

    meanvars = []
    for level in range(ward.alphaHatMat.shape[0]):
        c_labels = np.unique(ward.clusterAssignment[level]).astype(int)
        meanvars.append(ward.alphaHatMat[level,c_labels].var(1).mean())

    plt.plot(meanvars)
    plt.show()
ds= buildReal("/data/dzeiberg/ClassPriorEstimation/rawDatasets/abalone.mat", 4)
ds = addTransformScores(ds)
ds.alphaHats, ds.curves = getBagAlphaHats(ds,numbootstraps=10)
ward = WardClustering(ds,randomPairing=True,numbootstraps=ds.alphaHats.shape[1])
ward.cluster()
for level in range(ward.clusterAssignment.shape[0]):
    cs = np.unique(ward.clusterAssignment[level]).astype(int)
    aes = []
    for c in cs:
        bags = np.where(ward.clusterAssignment[level] == c)[0]
        alpha = np.dot(ds.trueAlphas.flatten()[bags],ds.numU[bags]) / ds.numU[bags].sum()
        alphaHat= ward.alphaHatMat[level][c]
        ae = np.abs(alpha - alphaHat)
        aes.append(ae)
    print("level: {}    MAE: {:.3f}".format(level, np.mean(aes)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13126709206927986, 'fraction_numerical': 0.022333637192342753, 'mean_word_length': 5.131284916201118, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '576368', 'license': 'Apache-2.0', 'n_tokens_mistral': 828, 'n_tokens_neox': 790, 'n_words': 124, 'path': '28_Tree_Level_Variance.ipynb', 'repo_name': 'Dzeiberg/multiinstance', 'size': '2194'}