<jupyter_start><jupyter_text>Load template dataset<jupyter_code>background_dataset = pd.read_csv('../.data/background_template_dataset.csv')
source_dataset = pd.read_csv('../data/shielded_templates_200kev_dataset.csv')

source_dataset, spectra_dataset, all_keys = load_full(source_dataset, background_dataset)<jupyter_output>indicies dropped: [552 553 554 555 556 557 564 565 566 582 583 584 585 586 587 594 595 596
 612 613 614 615 616 617 624 625 626 642 643 644 645 646 647 654 655 656
 672 673 674 675 676 677 684 685 686 702 703 704 705 706 707 714 715 716]<jupyter_text>Create Dataset<jupyter_code>total_spectra = 1e1
dataset = {'sources': [],
           'backgrounds': [],
           'keys': []}

# simulate sources
isotopes_processed = 0
for isotope in set(source_dataset['isotope'].values):
    for _ in np.arange(total_spectra):

        background_cps = np.random.poisson(200)
        integration_time = 10 ** np.random.uniform(np.log10(10), np.log10(3600))
        signal_to_background = np.random.uniform(0.1, 3)
        calibration = [np.random.uniform(0, 10),
                       np.random.uniform(0.8, 1.2),
                       0]
        fwhm = choice([7.0, 7.5, 8.0])

        source_spectrum, background_spectrum = make_random_spectrum(source_dataset,
                                                                    background_dataset,
                                                                    background_cps=background_cps,
                                                                    integration_time=integration_time,
                                                                    signal_to_background=signal_to_background,
                                                                    calibration=calibration,
                                                                    isotope=isotope,
                                                                    fwhm=fwhm,)
        dataset['sources'].append(source_spectrum)
        dataset['backgrounds'].append(background_spectrum)
        dataset['keys'].append(isotope)
    isotopes_processed+=1
    print(isotopes_processed)

np.save('spectra_dataset_complete_'+str(int(total_spectra)),dataset)<jupyter_output>1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
{'fraction_non_alphanumeric': 0.08149111400086692, 'fraction_numerical': 0.10923276983094929, 'mean_word_length': 1.7574671445639187, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '505279', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 834, 'n_tokens_neox': 630, 'n_words': 147, 'path': 'examples/source-interdiction/dataset-generation/Dataset-Generation-Complete.ipynb', 'repo_name': 'arfc/annsa', 'size': '2307'}