<jupyter_start><jupyter_text>Load template dataset<jupyter_code>background_dataset = pd.read_csv('../.data/background_template_dataset.csv')
source_dataset = pd.read_csv('../data/shielded_templates_200kev_dataset.csv')

source_dataset, spectra_dataset, all_keys = load_easy(source_dataset, background_dataset)<jupyter_output>indicies dropped: []<jupyter_text>Create Dataset<jupyter_code>total_spectra = 1e3
dataset = {'sources': [],
           'backgrounds': [],
           'keys': []}

# simulate sources
isotopes_processed = 0
for isotope in set(source_dataset['isotope'].values):
    for _ in np.arange(total_spectra):

        background_cps = np.random.poisson(200)
        integration_time = 10 ** np.random.uniform(np.log10(60),
                                                   np.log10(600))
        signal_to_background = np.random.uniform(0.5, 2)
        calibration = [np.random.uniform(0, 10),
                       np.random.uniform(0.9, 1.1),
                       0]

        source_spectrum, background_spectrum = make_random_spectrum(
            source_dataset,
            background_dataset,
            background_cps=background_cps,
            integration_time=integration_time,
            signal_to_background=signal_to_background,
            calibration=calibration,
            isotope=isotope,
            fwhm=7.5,)
        dataset['sources'].append(source_spectrum)
        dataset['backgrounds'].append(background_spectrum)
        dataset['keys'].append(isotope)
    isotopes_processed += 1
    print(isotopes_processed)

np.save('spectra_dataset_simple_'+str(int(total_spectra)),dataset)<jupyter_output>1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
{'fraction_non_alphanumeric': 0.10400929692039512, 'fraction_numerical': 0.04938988959907031, 'mean_word_length': 2.9047619047619047, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '282009', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 606, 'n_tokens_neox': 539, 'n_words': 92, 'path': 'examples/source-interdiction/dataset-generation/Dataset-Generation-Simple.ipynb', 'repo_name': 'arfc/annsa', 'size': '1721'}