<jupyter_start><jupyter_text>KNN-based sd estimatation<jupyter_code>import copy
import itertools
import time
import os

import luigi
import dill

from luigi_utils.plots import plot_CCCs
from empirical_privacy.row_distributed_svd import gen_SVD_CCCs_for_multiple_docs, \
svd_dataset_settings
from empirical_privacy.config import LUIGI_COMPLETED_TARGETS_DIR
import dataset_utils.common as cm

CCCs = []

for dataset in ['20NG', 'ml-1m']:
    for trials in range(5,11):
        for part_fraction in [0.01, 0.1, 0.3]:
            for n_max in [2**8, 2**9, 2**10, 2**11]:
                start = time.time()
                print(f'dataset={dataset} trials={trials} pf={part_fraction} n_max={n_max} ')
                ds = svd_dataset_settings(dataset_name=dataset, part_fraction=part_fraction)
                _CCCs = gen_SVD_CCCs_for_multiple_docs(n_max=n_max,
                                                      validation_set_size=512,
                                                      n_docs=5,
                                                      confidence_interval_width=trials,
                                                      dataset_settings=ds)
                luigi.build(_CCCs, workers=3, log_level='ERROR')
                print(f'\tCompleted {time.time()-start}s')
                CCCs += _CCCs
%%time
luigi.build(CCCs, workers=3, log_level='ERROR')<jupyter_output><empty_output><jupyter_text>Plots  ml-1m<jupyter_code>plot_CCCs(CCCs)<jupyter_output><empty_output><jupyter_text>Danger zone (delete completed runs in order to re-run)<jupyter_code>datasets_to_delete = ['20NG']
y = input('Enter "DELETE" to proceed deleting {}'.format(datasets_to_delete))
assert y=="DELETE"

n_del = 0

for trials in range(5,31):
    for n_docs in range(2,5):
        for part_fraction in [0.01, 0.1, 0.3]:
            for n_max in [2**8, 2**9, 2**10, 2**11, 2**12]:
                for dataset in datasets_to_delete :
                    start = time.time()
                    print(f'n_docs={n_docs} trials={trials} pf={part_fraction} n_max={n_max} ds={dataset}')
                    ds = svd_dataset_settings(dataset_name=dataset, part_fraction=part_fraction)
                    _CCCs = gen_SVD_CCCs_for_multiple_docs(n_max=n_max,
                                                          validation_set_size=512,
                                                          n_docs=n_docs,
                                                          confidence_interval_width=trials,
                                                          dataset_settings=ds)
                    for CC in _CCCs:
                        n_del += CC.delete_deps()
print(f'{n_del} items deleted')
print(n_del)<jupyter_output>4197
{'fraction_non_alphanumeric': 0.09452004413387274, 'fraction_numerical': 0.023905847738139023, 'mean_word_length': 1.869198312236287, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '408655', 'license': 'MIT', 'n_tokens_mistral': 875, 'n_tokens_neox': 802, 'n_words': 153, 'path': 'Notebooks/development/Row Distributed SVD.ipynb', 'repo_name': 'maksimt/empirical_privacy', 'size': '2719'}