<jupyter_start><jupyter_text>Model<jupyter_code>def SixtyFourNet(input_shape, drop_rate=0., weight_dcay=0.):
    model = Sequential()
    model.add(Conv3D(64, kernel_size=3, 
                     input_shape=(input_shape[1], input_shape[2], input_shape[3], 1), 
                     activation='elu', padding='valid', name='Conv_1'))
    model.add(MaxPooling3D(pool_size=3, name='Pool_1'))
    model.add(Dropout(drop_rate))
    model.add(Conv3D(64, kernel_size=3, padding='valid', activation='elu', name='Conv_2'))

    model.add(MaxPooling3D(pool_size=3, name='Pool_2'))
    model.add(Dropout(drop_rate))

    model.add(Conv3D(64, kernel_size=3, padding='valid', activation='elu', name='Conv_3',
                     kernel_regularizer=l2(weight_dcay)))
    model.add(Conv3D(64, kernel_size=3, padding='valid', activation='elu', name='Conv_4',
                     kernel_regularizer=l2(weight_dcay)))
    model.add(MaxPooling3D(pool_size=3, name='Pool_4'))
    model.add(Dropout(drop_rate))

    model.add(Flatten())
    model.add(Dense(units=1, activation='sigmoid'))
    return model<jupyter_output><empty_output><jupyter_text>Normalization<jupyter_code>files_train = np.array(files_train)
import re
def return_subject_ID(string):
    """ Match subject IDs with the pattern: [3 digits]_S_[4 digits]"""
    subject_ID = re.search('(\d{3}_S_\d{4})', string)
    return subject_ID.group(0)
names_train = [return_subject_ID(item.decode('utf-8')) for item in files_train]<jupyter_output><empty_output><jupyter_text>Normalize min-max<jupyter_code>for i in range(len(X_train)):
    X_train[i] -= np.min(X_train[i])
    X_train[i] /= np.max(X_train[i])
    
for i in range(len(X_holdout)):
    X_holdout[i] -= np.min(X_holdout[i])
    X_holdout[i] /= np.max(X_holdout[i])<jupyter_output><empty_output><jupyter_text>Normalize by voxel and split data<jupyter_code>GSS = GroupShuffleSplit(n_splits=1, test_size=0.15, random_state=22)
indices = list(GSS.split(names_train, groups=names_train))[0]
train_idx, val_idx = indices[0], indices[1]

mean = np.mean(X_train[train_idx], axis=0)
std = np.std(X_train[train_idx], axis=0)
X_train = (X_train - mean) / (std)
# shuffle indices test
np.random.shuffle(train_idx)
np.random.shuffle(val_idx)
plt.imshow(X_train[4][:,:,46,0], cmap='gray')
plt.show()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># training args
lr = 0.001
lr_decay = 0.

num_trials = 10
patience = 10
store_models = True

best_epoch_acc = []
final_epoch_sens = []
final_epoch_spec = []

for i in range(num_trials):
    # init model
    model = SixtyFourNet(drop_rate=0.3, weight_dcay=0.01, input_shape=X_train.shape)
    opti = Adam(lr=lr, decay=lr_decay)
    model.compile(loss='binary_crossentropy', optimizer=opti, metrics=['accuracy'])
    
    #callbacks
    earlystopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=0, mode='auto')
    model_checkpoint = ModelCheckpoint(os.path.join(result_dir, "model_trial_%i.h5" %i), monitor='val_acc', verbose=0, save_best_only=True, mode='max')

    if store_models:
        callbacks = [earlystopping, model_checkpoint]
    else:
        callbacks = [earlystopping]
    
    # Start training
    history = model.fit(
        X_train[train_idx],
        y_train[train_idx],
        epochs=num_epochs,
        batch_size=b,
        validation_data=(X_train[val_idx],y_train[val_idx]),
        callbacks=callbacks,
        shuffle=True
    )
    training_time = time.time() - start_time
    # Plot training curves
    plt.figure(figsize=(11, 4))
    plt.subplot(1, 2, 1)
    plt.plot(history.history["loss"])
    plt.plot(history.history["val_loss"])
    plt.legend(["Train", "Val"])
    plt.title("Loss")
    plt.subplot(1, 2, 2)
    plt.plot(history.history["acc"])
    plt.plot(history.history["val_acc"])
    plt.legend(["Train", "Val"])
    plt.title("Accuracy")
    plt.show()    
    # Evaluation
    y_pred = model.predict(X_train[val_idx], batch_size=1).flatten() # predict and turn into 1-dimensional array
    sens = sensitivity(y_train[val_idx], y_pred>0.5)
    spec = specificity(y_train[val_idx], y_pred>0.5)
    # Store results
    best_epoch = np.argmax(history.history["val_acc"])
    best_epoch_acc.append(history.history["val_acc"][best_epoch])
    final_epoch_sens.append(sens)
    final_epoch_spec.append(spec)
    # Print results
    print("Final epoch results:")
    print("Balanced Accuracy: {:.2f} %".format(((sens + spec) * 100) / 2))
    print("Sensitivity: {:.2f} %".format(sens * 100))
    print("Specificity: {:.2f} %".format(spec * 100))
print("All best val accuracies: {}".format(best_epoch_acc))
print("Average best validation accuracy over {} trials: {:.5f} (std: {:.5f})".format(num_trials, np.mean(best_epoch_acc), np.std(best_epoch_acc)))
print("Final epochs sensitivity: {}".format(final_epoch_sens))
print("Final epochs specificity: {}".format(final_epoch_spec))
print("Training Time: {}h:{}m:{}s".format(
            training_time//3600, (training_time//60)%60, training_time%60))
import numpy as np
sens = [0.835820895522388, 0.835820895522388, 0.8656716417910447, 0.8507462686567164, 0.8805970149253731, 0.746268656716418, 0.8507462686567164, 0.7164179104477612, 0.7910447761194029, 0.7611940298507462]
specs = [0.6981132075471698, 0.6981132075471698, 0.7924528301886793, 0.7547169811320755, 0.660377358490566, 0.9056603773584906, 0.7358490566037735, 0.9433962264150944, 0.6792452830188679, 0.6792452830188679]
final_balanced_accs = []
for sen, spec in zip(sens, specs):
    final_balanced_accs.append((sen + spec)/2)
print(final_balanced_accs)
print(np.mean(final_balanced_accs))<jupyter_output>[0.7669670515347788, 0.7669670515347788, 0.8290622359898621, 0.802731624894396, 0.7704871867079696, 0.8259645170374543, 0.793297662630245, 0.8299070684314278, 0.7351450295691354, 0.7202196564348071]
0.7840749084764855
{'fraction_non_alphanumeric': 0.15046968403074296, 'fraction_numerical': 0.10913748932536294, 'mean_word_length': 5.030895983522142, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '460307', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2238, 'n_words': 330, 'path': 'Pretraining on ADNI/Training on ADNI.ipynb', 'repo_name': 'derEitel/explainableMS', 'size': '5855'}