<jupyter_start><jupyter_text>IMPORT PARAMETERS FROM CONFIG FILE<jupyter_code># config['data_dict'] = pickle file containing a dictionary with at least the following keys: subject and contrast_foldname
# This dict is load as a panda dataframe and used by the function utils.fetch_data_files
# IMPORTANT NOTE: the testing dataset is not included in this dataframe
DATA_PD = pd.read_pickle(config['data_dict'])
# print(DATA_PD)
DATA_FOLD = config["data_dir"]  # where the preprocess data are stored
MODEL_FOLD = config["path2save"]  # where to store the trained models

MEAN_TRAIN_T2, STD_TRAIN_T2 = 871.309, 557.916  # Mean and SD of the training dataset of the original paper<jupyter_output><empty_output><jupyter_text>CONVERT INPUT IMAGES INTO AN HDF5 FILE<jupyter_code>len_train = int(0.8 * len(DATA_PD.index)) # 80% of the dataset is used for the training 
idx_train = random.sample(range(len(DATA_PD.index)), len_train)
idx_valid = [ii for ii in range(len(DATA_PD.index)) if ii not in idx_train] # the remaining images are used for the validation
training_files = fetch_data_files(data_frame=DATA_PD[DATA_PD.index.isin(idx_train)],
                                  data_fold=DATA_FOLD,
                                  im_suffixe='_norm',
                                  target_suffixe='_crop_MASK')
validation_files = fetch_data_files(data_frame=DATA_PD[DATA_PD.index.isin(idx_valid)],
                                  data_fold=DATA_FOLD,
                                  im_suffixe='_norm',
                                  target_suffixe='_crop_MASK')

print(training_files)<jupyter_output><empty_output><jupyter_text>EXTRACT 3D PATCHES<jupyter_code># The extracted patches are stored as pickle files (one for training, one for validation).
# If these files already exist, we load them directly (i.e. do not re run the patch extraction).
pkl_train_fname = DATA_FOLD + 'lesion_train_data_t2.pkl'
print(pkl_train_fname)
if not os.path.isfile(pkl_train_fname):
    X_train, y_train = load_3Dpatches(fname_lst=training_files,patch_shape=config["patch_size"],overlap=config["patch_overlap"]) 
    X_train = normalize_data(X_train, MEAN_TRAIN_T2, STD_TRAIN_T2)
    X_train, y_train = shuffle(X_train, y_train, random_state=2611)
    print(X_train.shape)
    with open(pkl_train_fname, 'wb') as fp:
        pickle.dump(np.array([X_train, y_train]), fp)
else:
    with open (pkl_train_fname, 'rb') as fp:
        X_train, y_train = pickle.load(fp)

pkl_valid_fname = DATA_FOLD + 'lesion_valid_data_t2.pkl'
print(pkl_valid_fname)

if not os.path.isfile(pkl_valid_fname):
    X_valid, y_valid = load_3Dpatches(fname_lst=validation_files,
                                        patch_shape=config["patch_size"],
                                        overlap=0)
    
    X_valid = normalize_data(X_valid, MEAN_TRAIN_T2, STD_TRAIN_T2)
    
    with open(pkl_valid_fname, 'wb') as fp:
        pickle.dump(np.array([X_valid, y_valid]), fp)
else:
    with open (pkl_valid_fname, 'rb') as fp:
        X_valid, y_valid = pickle.load(fp)
print 'Number of Training patches:\n\t' + str(X_train.shape[0])
print 'Number of Validation patches:\n\t' + str(X_valid.shape[0])<jupyter_output><empty_output><jupyter_text>LOAD TRAINED MODEL<jupyter_code>model_fname = os.path.join(commands.getoutput('$SCT_DIR').split(': ')[2], 'data', 'deepseg_lesion_models', 't2_lesion.h5')
model = load_trained_model(model_fname)<jupyter_output><empty_output><jupyter_text>GET TRAINING AND VALIDATION GENERATORS<jupyter_code>train_generator, nb_train_steps = get_training_and_validation_generators(
                                                    [X_train, y_train],
                                                    batch_size=config["batch_size"],
                                                    augment=True,
                                                    augment_flip=True)

print(train_generator,nb_train_steps)
validation_generator, nb_valid_steps = get_training_and_validation_generators(
                                                    [X_valid, y_valid],
                                                    batch_size=1,
                                                    augment=False,
                                                    augment_flip=False)
print(validation_generator,nb_valid_steps)
for g, g_name in zip([train_generator, validation_generator], ['train_visu', 'valid_visu']):
    print '\n' + g_name
    X_visu_, y_visu_ = g.next()
    idx_random = random.randint(0, X_visu_.shape[-1])
    visualize_data(X=X_visu_[0,0,:,:,idx_random], Y=y_visu_[0,0,:,:,idx_random])<jupyter_output><empty_output><jupyter_text>RUN NET ---> Cell to change --> Change it to fine-tuning / transfer learning...etc.<jupyter_code>train_model(model=model,
            path2save=config["path2save"],
            model_name=config["model_name"],
            training_generator=train_generator,
            validation_generator=validation_generator,
            steps_per_epoch=nb_train_steps,
            validation_steps=nb_valid_steps,
            n_epochs=config["n_epochs"],
            learning_rate_drop=config["learning_rate_drop"],
            learning_rate_patience=config["learning_rate_patience"]
           )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09550133434998094, 'fraction_numerical': 0.009149828440716736, 'mean_word_length': 2.9098360655737703, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '443750', 'license': 'Unlicense', 'n_tokens_mistral': 1699, 'n_tokens_neox': 1659, 'n_words': 315, 'path': 'Scripts/Main_file.ipynb', 'repo_name': 'sct-pipeline/deepseg-training', 'size': '5246'}