<jupyter_start><jupyter_text>Try bootstrap at each `training_set_size`<jupyter_code>from collections import defaultdict
from math import ceil

import itertools
accuracies = {tss:[] for tss in N}
min_samples = 3
convergence_ratio = 1e-4

def converged(accuracy_list):
    std0 = np.std(accuracy_list[0:2])
    std1 = np.std(accuracy_list[0:3])
    stdn1 = np.std(accuracy_list[0:-1])
    stdn = np.std(accuracy_list)
    d0 = abs(std0 - std1)
    dn = abs(stdn - stdn1)
#     print(f'd0={d0} dn={dn}')
    return dn/d0 < convergence_ratio

for tss in accuracies.keys():
    while len(accuracies[tss]) < min_samples or not converged(accuracies[tss]):
        accuracies[tss].append(noisy_model(tss))
        
print([(k, len(v)) for k,v in accuracies.items()])
p = 0.99
n_tss = len(accuracies.keys())
p_intermediate = p ** (1/n_tss)
print('p_intermediate = {:.3e}'.format(p_intermediate))

configure_plotting_for_publication()
plt.figure(figsize=(4,3))


_ = plt.violinplot(
    accuracies.values(),
    positions=list(accuracies.keys()),
    widths=100,
    showextrema=False
)
bootstraps = {tss: EmpiricalBootstrap(SampleGenerator(data=acc)).\
              bootstrap_confidence_bounds(p_intermediate, n_samples=1000)
             for (tss, acc) in accuracies.items()}
accuracy_combinations = list(itertools.product(*[
    [bs.lb_two_sided, bs.ub_two_sided]
    for tss, bs in bootstraps.items()
]))

Ks = transform_n_to_k_for_knn(N, 'gyorfi', d=d)
m_max = -float('inf')
m_min = float('inf')
i_min = None
i_max = None
for (i, accuracy_combination) in enumerate(accuracy_combinations):
    m_hat, C_hat = asymptotic_curve(Ks.reshape(Ks.size), accuracy_combination)
    if m_hat < m_min:
        m_min = m_hat
        i_min = i
    if m_hat > m_max:
        m_max = m_hat
        i_max = i

for x, y in accuracies.items():
    y= np.array(y)
    plt.scatter(np.ones_like(y)*x, y, marker='.', color='k')

plt.xlabel("training set size")
plt.ylabel("est. accuracy")


plt.scatter(N, accuracy_combinations[i_min], color='b')
plt.scatter(N, accuracy_combinations[i_max], color='r')

plt.axhline(m_max, linestyle='--', color='r')
plt.axhline(m_min, linestyle='--', color='b')
plt.axhline(m, linestyle='-', color='g')<jupyter_output>p_intermediate = 9.950e-01
{'fraction_non_alphanumeric': 0.1497777777777778, 'fraction_numerical': 0.017333333333333333, 'mean_word_length': 4.346793349168646, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '213249', 'license': 'MIT', 'n_tokens_mistral': 919, 'n_tokens_neox': 876, 'n_words': 164, 'path': 'Notebooks/Bootstrap validation with known model.ipynb', 'repo_name': 'maksimt/empirical_privacy', 'size': '2250'}