<jupyter_start><jupyter_text>The val that varies between experiments<jupyter_code>ds = 4<jupyter_output><empty_output><jupyter_text>Prepare to load in some input data and masks and remove outliers<jupyter_code>patients = hu.get_each_normalized_hgg_folder()
patients = hu.remove_outliers(patients)

masks = hu.get_each_hgg_folder()
masks = hu.remove_outliers(masks)

patients, masks = shuffle(patients, masks, random_state=1)<jupyter_output><empty_output><jupyter_text>Train, Test indices<jupyter_code>train_data_ratio = 0.8
train_start = 0
train_stop = int(np.round(train_data_ratio * len(patients)))
print(train_start)
train_stop
test_start = train_stop
test_stop = len(patients)
print(test_start)
test_stop<jupyter_output>194<jupyter_text>Save paths for train<jupyter_code>train_data = patients[:test_start]
train_masks = masks[:test_start]

fname_train_data = "ds_"+str(ds)+"_train_data.pkl"
fname_train_masks = "ds_"+str(ds)+"_train_masks.pkl"

with open(fname_train_data, 'wb') as file_pi:
    pickle.dump(train_data, file_pi)
    
with open(fname_train_masks, 'wb') as file_pi:
    pickle.dump(train_masks, file_pi)<jupyter_output><empty_output><jupyter_text>Save the paths for testing<jupyter_code>test_data = patients[test_start:]
test_masks = masks[test_start:]

fname_test_data = "ds_"+str(ds)+"_test_data.pkl"
fname_test_masks = "ds_"+str(ds)+"_test_masks.pkl"

with open(fname_test_data, 'wb') as file_pi:
    pickle.dump(test_data, file_pi)
    
with open(fname_test_masks, 'wb') as file_pi:
    pickle.dump(test_masks, file_pi)<jupyter_output><empty_output><jupyter_text>Preallocate arrays to hold data & masks<jupyter_code>some_data = np.ones([ num_to_load*155, 240, 240, 4])
some_masks = np.ones([ num_to_load*155, 240, 240, 1])
def load_n_brains(data, start, stop, paths, end):

    data_idx = 0
    num_slices = 155
    brains_seen = 0
    
    #for multimodal_tensor in tqdm(range(start, stop)):
    for multimodal_tensor in range(start, stop):

        if multimodal_tensor != end:
            four_channel_scan = hu.reshape_tensor_with_slices_first(
                                    hu.get_a_multimodal_tensor( 
                                                paths[multimodal_tensor] 
                                    )[data_idx]
            )
            #print(paths[multimodal_tensor])


            for slic in range(num_slices):
                data[slic+(num_slices*brains_seen),:,:,:] = four_channel_scan[slic,:,:,:]

            brains_seen += 1
        else:
            break
        
    #print(multimodal_tensor)
    return data
def load_n_masks(data, start, stop, paths, end):

    data_idx = 0
    num_slices = 155
    brains_seen = 0

    for mask_idx in range(start, stop):
        if mask_idx != end:

            mask =  hu.reshape_tensor_with_slices_first(
                                    hu.convert_mask_to_binary_mask(
                                         hu.get_a_mask_tensor( paths[mask_idx] )

                                   )
            )
            #print(paths[mask_idx])


            for slic in range(num_slices):
                data[slic+(num_slices*brains_seen),:,:,:] = mask[slic,:,:,:]

            brains_seen += 1
        else:
            break
            
    return data<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>chunks = 39

beg = 0
end = num_to_load*155
truncated = 155*(train_stop - ( (chunks-1) * num_to_load  ) )

#truncated
#155*(train_stop - ( (chunks-1) * num_to_load  ) )

my_opt = tf.keras.optimizers.Adam(learning_rate=1e-5)

start_time = time.time()

for run in tqdm(range(5)):
    
    print("********************")
    print("Run:", run)
    
    model = unet( input_size=(240,240,4), ds=ds )
    # Save architecture
    model_json_name = "unet_ds_{}.json".format(ds)
    with open(model_json_name, "w") as json_file:
        json_file.write(model.to_json())
        
    model.compile(optimizer=my_opt, loss=dice, metrics=[dice_coef])
    
    run_history = []

    for epoch in tqdm(range(20)):
        
        epoch_history = []
        
        for i in range(chunks):
            #print("Loading chunk of data...")
            some_data = load_n_brains(some_data,  (num_to_load*i), (num_to_load*i)+num_to_load, patients, train_stop).astype(np.float32)
            some_masks = load_n_masks(some_masks, (num_to_load*i), (num_to_load*i)+num_to_load, masks, train_stop).astype(np.float32)

            some_data, some_masks = shuffle(some_data, some_masks, random_state=1)

            if num_to_load*i+num_to_load <= train_stop:
                history = model.fit(some_data[beg:end,...], some_masks[beg:end,...], validation_split=0.2, epochs=1, batch_size=16, verbose=0)

            else:
                history = model.fit(some_data[beg:truncated,...], some_masks[beg:truncated,...], validation_split=0.2, epochs=1, batch_size=16, verbose=0)


            epoch_history.append(history.history)
            gc.collect()
        print("Epoch", epoch, "completed")
        print("Elapsed time:", (time.time() - start_time)/60.0, "minutes" )
        run_history.append(epoch_history)
    
    print()
    print("Saving run", run, "loss etc.")
    history_name = "ds_"+str(ds)+"_run_" + str(run) +"_histories.pkl"

    with open(history_name, 'wb') as file_pi:
        pickle.dump(run_history, file_pi)
    
    model_weights_name = "ds_"+str(ds)+"_run_" + str(run) +"_model_weights.h5"
    
    print("Saving run", run, "model weights as", model_weights_name)
    model.save_weights(model_weights_name)
    
    del model
    gc.collect()
    
print("Total time:", (time.time() - start_time)/60.0, "minutes"  )
#print( run_history[0].history )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12609764664559184, 'fraction_numerical': 0.01598173515981735, 'mean_word_length': 3.094176851186197, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '177519', 'license': 'MIT', 'n_tokens_mistral': 2173, 'n_tokens_neox': 2008, 'n_words': 334, 'path': 'experiment_ds_4.ipynb', 'repo_name': 'CSCI4850/S20-team0-project', 'size': '5694'}