<jupyter_start><jupyter_text>Approximation<jupyter_code>approx = defaultdict(int)
mafs = []
for fname in os.listdir('.'):
    if not fname.startswith('tsi-') or not fname.endswith('.frq'):
        continue
    f = open(fname)
    f.readline()
    for cnt, l in enumerate(f):
        toks = [tok for tok in l.rstrip().split(' ') if tok != '']
        maf = float(toks[-2])
        mafs.append(maf)
        approx[maf] += 1
    f.close()
print(len(mafs), type(mafs))
print(len(approx))
%time print(np.median(mafs))
fig, axs = plt.subplots(2, 2, sharex=True, figsize=(16, 9))

xs, ys = zip(*approx.items())
axs[0, 0].plot(xs, ys, '.')

xs = list(approx.keys())
xs.sort()
ys = [approx[x] for x in xs]
axs[0, 1].bar(xs, ys, 0.005)

def get_bins(my_dict, nbins):
    acumulator = [0] * nbins
    xmin, xmax = xs[0], xs[-1]
    interval = (xmax - xmin) / nbins
    bin_xs = [xmin + i * interval + interval / 2 for i in range(nbins)]
    #the starting point, midpoint woudl take more interval / 2
    curr_bin = 0
    for x in xs:
        y_cnt = approx[x]
        while curr_bin + 1 != nbins and abs(x - bin_xs[curr_bin]) > abs(x - bin_xs[curr_bin + 1]):
            curr_bin += 1
        acumulator[curr_bin] += y_cnt
    return bin_xs, acumulator, interval

bin_xs, acumulator, interval = get_bins(approx, 10)
axs[1, 0].bar(np.array(bin_xs) - interval / 2, acumulator, 0.5 / 10)

bin_xs, acumulator, interval = get_bins(approx, 20)
axs[1, 1].bar(np.array(bin_xs) - interval / 2, acumulator, 0.5 / 20)
axs[1, 1].set_xlim(0, 0.5)

fig.tight_layout()
def compute_median_from_dictionary(my_dict):
    xs = list(my_dict.keys())
    xs.sort()
    x_cnt = [my_dict[x] for x in xs]
    start = 0
    end = len(xs) - 1
    while start != end:
        if start == end - 1 and x_cnt[start] == x_cnt[end]:
            return (xs[start] + xs[end]) / 2
        if x_cnt[start] > x_cnt[end]:
            x_cnt[start] -= x_cnt[end]
            end -= 1
        elif x_cnt[start] < x_cnt[end]:
            x_cnt[end] -= x_cnt[start]
            start += 1
        else:
            start += 1
            end -= 1
    return xs[start]
print(compute_median_from_dictionary(approx))
#do not put this, but discuss
for my_round in [1, 2, 3]:
    my_approx = defaultdict(int)
    for val, cnt in approx.items():
        my_approx[round(val, my_round)] += cnt
    print(len(my_approx), compute_median_from_dictionary(my_approx))<jupyter_output>6 0.2
50 0.22
326 0.222<jupyter_text>Subsampling<jupyter_code>import random
import pandas as pd
arr = np.ndarray(shape=(5, 6), dtype=float)
samp_sizes = [1, 10, 100, 1000, 10000]
for rep in range(3):
    for si, samp_size in enumerate([1, 10, 100, 1000, 10000]):
        my_vals = random.sample(mafs, samp_size)
        arr[si, rep] = np.median(my_vals)
        arr[si, rep + 3] = max(my_vals)
df = pd.DataFrame(arr, index=samp_sizes,
                  columns=['Mean #1', 'Mean #2', 'Mean #3', 'Max #1', 'Max #2', 'Max #3'])
df
#bill gates effect<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15603073838957568, 'fraction_numerical': 0.03775476110925493, 'mean_word_length': 2.8933680104031207, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '122917', 'license': 'Apache-2.0', 'n_tokens_mistral': 1282, 'n_tokens_neox': 1170, 'n_words': 292, 'path': 'notebooks/08_Advanced/Median.ipynb', 'repo_name': 'kevintheduu/bioinf-python', 'size': '2993'}