<jupyter_start><jupyter_text>Import model, training function<jupyter_code>from annsa.model_classes import (cnn1d_model_features,
                                 CNN1D,
                                 generate_random_cnn1d_architecture,
                                 save_model,
                                 train_earlystop)<jupyter_output><empty_output><jupyter_text>Dataset Construction  Load dataset<jupyter_code>background_dataset = pd.read_csv('../../source-interdiction/training_testing_data/background_template_dataset.csv')
source_dataset = pd.read_csv('../../source-interdiction/training_testing_data/shielded_templates_200kev_dataset.csv')<jupyter_output><empty_output><jupyter_text>Dataset details<jupyter_code>print('sourcedist: ' + str(sorted(set(source_dataset['sourcedist']))))
print('sourceheight: ' + str(sorted(set(source_dataset['sourceheight']))))
print('alum shieldingdensity: ' + str(sorted(set(source_dataset[source_dataset['shielding']=='alum']['shieldingdensity']))))
print('iron shieldingdensity: ' + str(sorted(set(source_dataset[source_dataset['shielding']=='iron']['shieldingdensity']))))
print('lead shieldingdensity: ' + str(sorted(set(source_dataset[source_dataset['shielding']=='lead']['shieldingdensity']))))
print('fwhm: ' + str(sorted(set(source_dataset['fwhm']))))<jupyter_output>sourcedist: [50.0, 112.5, 175.0, 237.5, 300.0]
sourceheight: [50.0, 75.0, 100.0, 125.0, 150.0]
alum shieldingdensity: [1.82, 4.18, 7.49, 13.16]
iron shieldingdensity: [1.53, 3.5, 6.28, 11.02]
lead shieldingdensity: [0.22, 0.51, 0.92, 1.61]
fwhm: [6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0]<jupyter_text>Customize dataset<jupyter_code>source_dataset = source_dataset[(source_dataset['fwhm']==7.0) | 
                                (source_dataset['fwhm']==7.5) |
                                (source_dataset['fwhm']==8.0)]

source_dataset = source_dataset[(source_dataset['sourcedist']==50.5) | 
                                (source_dataset['sourcedist']==175.0) | 
                                (source_dataset['sourcedist']==300.0)]

source_dataset = source_dataset[(source_dataset['sourceheight']==50.0) |
                                (source_dataset['sourceheight']==100.0) |
                                (source_dataset['sourceheight']==150.0)]

# remove 80% shielding
source_dataset = source_dataset[source_dataset['shieldingdensity']!=13.16]
source_dataset = source_dataset[source_dataset['shieldingdensity']!=11.02]
source_dataset = source_dataset[source_dataset['shieldingdensity']!=1.61]<jupyter_output><empty_output><jupyter_text>Remove empty spectra<jupyter_code>zero_count_indicies = np.argwhere(np.sum(source_dataset.values[:,6:],axis=1) == 0).flatten()

print('indicies dropped: ' +str(zero_count_indicies))

source_dataset.drop(source_dataset.index[zero_count_indicies], inplace=True)<jupyter_output>indicies dropped: [552 553 554 555 556 557 564 565 566 582 583 584 585 586 587 594 595 596
 612 613 614 615 616 617 624 625 626 642 643 644 645 646 647 654 655 656
 672 673 674 675 676 677 684 685 686 702 703 704 705 706 707 714 715 716]<jupyter_text>Add empty spectra for background<jupyter_code>blank_spectra = []
for fwhm in set(source_dataset['fwhm']):
    num_examples = source_dataset[(source_dataset['fwhm']==fwhm) &
                                  (source_dataset['isotope']==source_dataset['isotope'].iloc()[0])].shape[0]
    for k in range(num_examples):
        blank_spectra_tmp = [0]*1200
        blank_spectra_tmp[5] = fwhm
        blank_spectra_tmp[0] = 'background'
        blank_spectra_tmp[3] = 'background'
        blank_spectra.append(blank_spectra_tmp)

source_dataset = source_dataset.append(pd.DataFrame(blank_spectra,
                                                    columns=source_dataset.columns))<jupyter_output><empty_output><jupyter_text>Create dataset from spectra<jupyter_code>spectra_dataset = source_dataset.values[:,5:].astype('float64')
all_keys = source_dataset['isotope'].values<jupyter_output><empty_output><jupyter_text>Model Construction<jupyter_code>def make_model():
    '''
    Makes a random model given some parameters.

    '''
    cnn_filters_choices = (
                       (8, 16),
                       (8, 16),
                      )

    cnn_kernel_choices = ((8,),)
    pool_size_choices = ((8,),)

    
    model_features = generate_random_cnn1d_architecture(
        cnn_filters_choices= cnn_filters_choices,
        cnn_kernel_choices=cnn_kernel_choices,
        pool_size_choices=pool_size_choices,
    )
    model_features.trainable = True
    model_features.learining_rate = 10**np.random.uniform(-4,-1)
    model_features.batch_size = 2**np.random.randint(4,6)
    model_features.output_size = all_keys_binarized.shape[1]
    model_features.scaler = choice([make_pipeline(FunctionTransformer(np.log1p, validate=True)),
                                    make_pipeline(FunctionTransformer(np.sqrt, validate=True))])
    model_features.activation_function = tf.nn.tanh
    model_features.output_function = None
    model_features.Pooling = tf.layers.MaxPooling1D
    model_features.l2_regularization_scale = 0.
    model_features.dropout_probability = 0.
    model_features.pool_strides = ((2,2,2))
    number_layers = choice([2])
    dense_nodes = [128,128]
    dense_nodes = np.sort(dense_nodes)
    dense_nodes = np.flipud(dense_nodes)
    
    model_features.dense_nodes = dense_nodes

    model = CNN1D(model_features)

    return model, model_features 

model_class = CNN1D<jupyter_output><empty_output><jupyter_text>Define Training Parameters  Define online data augmentation<jupyter_code>def integration_time():
    return np.random.uniform(np.log10(10),np.log10(3600))

def background_cps():
    return np.random.poisson(200)

def signal_to_background():
    return np.random.uniform(0.1,2)

def calibration():
    return [np.random.uniform(0,10),
            np.random.uniform(2500/3000,3500/3000),
            0]

online_data_augmentation = online_data_augmentation_vanilla(background_dataset,
                                background_cps,
                                integration_time,
                                signal_to_background,
                                calibration,)<jupyter_output><empty_output><jupyter_text>Create temporary testing dataset based on training dataset<jupyter_code>examples_per_isotope=10

testing_spectra = []
testing_keys = []

for key in set(all_keys):
    for _ in range(examples_per_isotope):
        testing_spectra_tmp = source_dataset[source_dataset['isotope']==key].sample().values[:,5:].astype('float64')
        testing_spectra_tmp = online_data_augmentation(testing_spectra_tmp).numpy()[0]
        testing_keys.append(key)
        testing_spectra.append(testing_spectra_tmp)

testing_spectra = np.array(testing_spectra)<jupyter_output><empty_output><jupyter_text>Train network<jupyter_code>model_id='CNN-kfoldseasy-1_8_dae_features'
with open('../hyperparameter_search/hyperparameter-search-results/'+model_id,"rb" ) as f:
        model_features=pickle.load(f)
        
model_class = CNN1D
model_id='CNN_onlinedataaugfull_log10integrationtime'
mlb=LabelBinarizer()

training_errors = []
total_networks = 10
for network_id in range(total_networks):
    # reset model on each iteration

    all_keys_binarized = mlb.fit_transform(all_keys.reshape([all_keys.shape[0],1]))
    testing_keys_binarized = mlb.transform(testing_keys)
    # model, model_features = make_model()
    model = model_class(model_features)
    optimizer = tf.train.AdamOptimizer(model_features.learining_rate)
    
    _, f1_error = model.fit_batch(
        (spectra_dataset, all_keys_binarized),
        (testing_spectra, testing_keys_binarized),
        optimizer=optimizer,
        num_epochs=1500,
        verbose=1,
        obj_cost=model.cross_entropy,
        earlystop_cost_fn=model.f1_error,
        earlystop_patience=1000,
        data_augmentation=online_data_augmentation,
        augment_testing_data=False,
        print_errors=True,
        record_train_errors=False)

    training_errors.append(f1_error['test'])
    np.save('./final-models/final_test_errors_'+model_id, training_errors)
    model.save_weights('./final-models/'+model_id+'_checkpoint_'+str(network_id))<jupyter_output>Epoch 1: CostFunc loss: 0.00 1.11, EarlyStop loss: 0.00 0.33
Epoch 2: CostFunc loss: 0.00 1.16, EarlyStop loss: 0.00 0.36
Epoch 3: CostFunc loss: 0.00 1.27, EarlyStop loss: 0.00 0.41
Epoch 4: CostFunc loss: 0.00 1.09, EarlyStop loss: 0.00 0.33
Epoch 5: CostFunc loss: 0.00 1.16, EarlyStop loss: 0.00 0.37
Epoch 6: CostFunc loss: 0.00 1.16, EarlyStop loss: 0.00 0.39
Epoch 7: CostFunc loss: 0.00 1.20, EarlyStop loss: 0.00 0.35
Epoch 8: CostFunc loss: 0.00 1.13, EarlyStop loss: 0.00 0.34
Epoch 9: CostFunc loss: 0.00 1.21, EarlyStop loss: 0.00 0.38
Epoch 10: CostFunc loss: 0.00 1.23, EarlyStop loss: 0.00 0.39
Epoch 11: CostFunc loss: 0.00 1.14, EarlyStop loss: 0.00 0.36
Epoch 12: CostFunc loss: 0.00 1.16, EarlyStop loss: 0.00 0.38
Epoch 13: CostFunc loss: 0.00 1.14, EarlyStop loss: 0.00 0.35
Epoch 14: CostFunc loss: 0.00 1.12, EarlyStop loss: 0.00 0.35
Epoch 15: CostFunc loss: 0.00 1.21, EarlyStop loss: 0.00 0.39
Epoch 16: CostFunc loss: 0.00 1.18, EarlyStop loss: 0.00 0.37
Epoch 17: CostFun[...]
{'fraction_non_alphanumeric': 0.11926109970832884, 'fraction_numerical': 0.06773252673652372, 'mean_word_length': 3.7894464562855665, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '80360', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3591, 'n_tokens_neox': 3108, 'n_words': 521, 'path': 'examples/source-interdiction/training-notebooks/CNN Training - Online Data Augmentation - Full.ipynb', 'repo_name': 'samgdotson/annsa', 'size': '9257'}