<jupyter_start><jupyter_text>1) Create a csv of labels and show how it can be returned by the CSV Reader<jupyter_code>from niftynet.utilities.download import download
download('mr_ct_regression_model_zoo_data')
labels_location = 'ct.csv'
files = [file for file in os.listdir('/home/tom/niftynet/data/mr_ct_regression/CT_zero_mean') if file.endswith('.nii.gz')]
pd.DataFrame(data=[(file.replace('.nii.gz', ''), file.replace('.nii.gz', '')) for file in files]).to_csv('label.csv', index=None, header=['subject_id', 'label'])
pd.read_csv('label.csv')
#### Testing the CSV Reader on labels
# Make sure we accept 'Label', 'label', 'LABEL'
task_param = TaskParam(['image'])
image_data_param = {'CT': {'path_to_search': '~/niftynet/data/mr_ct_regression/CT_zero_mean', 'filename_contains': 'nii'}}
#csv_data_file is a csv with data
csv_data_param = {'label': {'csv_data_file': 'label.csv', 'to_ohe': True}}
grouping_param = {'image': (['CT'])}

image_sets_partitioner = ImageSetsPartitioner().initialise(image_data_param)
image_reader = ImageReader().initialise(image_data_param, grouping_param, file_list=image_sets_partitioner.all_files)
preprocessing = Preprocessing(net_param, action_param, task_param)
normalisation_layers = preprocessing.prepare_normalisation_layers()
augmentation_layers = preprocessing.prepare_augmentation_layers()
image_reader.add_preprocessing_layers(normalisation_layers + augmentation_layers)
csv_reader = CSVReader(('label',)).initialise(csv_data_param, {'label': (['label'])}, file_list=image_sets_partitioner.all_files)
print('One sample from the csv_reader:', np.squeeze(csv_reader(idx=13)[1]['label']))
window_sizes = {'image': (100, 100, 1), 'label': (1, 1, 1)}
sampler = ResizeSampler(reader=image_reader,
                        csv_reader=csv_reader,
                        window_sizes=window_sizes,
                        num_threads=2,
                        smaller_final_batch_mode='drop',
                        batch_size=2,
                        queue_length=2)
sample = next(sampler())
print(sample['image'].shape)
print(sample['label'].shape)<jupyter_output>[1mINFO:niftynet:[0m 

Number of subjects 15, input section names: ['subject_id', 'CT']
-- using all subjects (without data partitioning).

[1mINFO:niftynet:[0m Image reader: loading 15 subjects from sections ['CT'] as input [image]
[1mWARNING:niftynet:[0m This method will read your entire csv into memory
One sample from the csv_reader: [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[1mINFO:niftynet:[0m reading size of preprocessed images
[1mWARNING:niftynet:[0m sampler queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (10).
(1, 100, 100, 1, 1, 1)
(1, 15, 1, 1, 1, 1)<jupyter_text>2) Create a csv of features and show how it can be returned by the CSV Reader<jupyter_code>from niftynet.utilities.download import download
download('mr_ct_regression_model_zoo_data')
labels_location = 'ct.csv'
files = [file.replace('.nii.gz', '') for file in os.listdir('/home/tom/niftynet/data/mr_ct_regression/CT_zero_mean') if file.endswith('.nii.gz')]

pd.DataFrame(data=[tuple([file] + list(np.random.randn(10))) for file in files]).to_csv('features.csv', index=None, header=['subject_id'] + [str(x) for x in range(10)])
pd.read_csv('features.csv')
task_param = TaskParam(['image'])
image_data_param = {'CT': {'path_to_search': '~/niftynet/data/mr_ct_regression/CT_zero_mean', 'filename_contains': 'nii'}}
csv_data_param = {'features': {'csv_data_file': 'features.csv', 'to_ohe': False}}
grouping_param = {'image': (['CT'])}
image_sets_partitioner = ImageSetsPartitioner().initialise(image_data_param)
image_reader = ImageReader().initialise(image_data_param, grouping_param, file_list=image_sets_partitioner.all_files)
preprocessing = Preprocessing(net_param, action_param, task_param)
normalisation_layers = preprocessing.prepare_normalisation_layers()
augmentation_layers = preprocessing.prepare_augmentation_layers()
image_reader.add_preprocessing_layers(normalisation_layers + augmentation_layers)
csv_reader = CSVReader(('features',)).initialise(csv_data_param, {'features': ['features']}, file_list=image_sets_partitioner.all_files)
print('One sample from the csv_reader:', np.squeeze(csv_reader(idx=13)[1]['features']))
window_sizes = {'image': (100, 100, 1), 'features': (1, 1, 1)}
sampler = ResizeSampler(reader=image_reader,
                        csv_reader=csv_reader,
                        window_sizes=window_sizes,
                        num_threads=2,
                        smaller_final_batch_mode='drop',
                        batch_size=2,
                        queue_length=2)
sample = next(sampler())
print(sample['image'].shape)
print(sample['features'].shape)
print(sample.keys())<jupyter_output>[1mINFO:niftynet:[0m 

Number of subjects 15, input section names: ['subject_id', 'CT']
-- using all subjects (without data partitioning).

[1mINFO:niftynet:[0m Image reader: loading 15 subjects from sections ['CT'] as input [image]
[1mWARNING:niftynet:[0m This method will read your entire csv into memory
One sample from the csv_reader: [ 0.23841972 -0.96603888 -0.07318273  1.59560139 -0.09326917 -1.04853203
 -0.65709902 -1.17890471  0.7956195  -0.97414747]
[1mINFO:niftynet:[0m reading size of preprocessed images
[1mWARNING:niftynet:[0m sampler queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (10).
(1, 100, 100, 1, 1, 1)
(1, 10, 1, 1, 1, 1)
dict_keys(['image_location', 'image', 'features', 'features_location'])<jupyter_text>Testing the CSV Reader on labels AND Features<jupyter_code># Make sure we accept 'Label', 'label', 'LABEL'
task_param = TaskParam(['image'])
image_data_param = {'CT': {'path_to_search': '~/niftynet/data/mr_ct_regression/CT_zero_mean', 'filename_contains': 'nii'}}
csv_data_param = {'label': {'csv_data_file': 'label.csv', 'to_ohe': True},
                  'features': {'csv_data_file': 'features.csv', 'to_ohe': False}}
grouping_param = {'image': (['CT'])}

image_sets_partitioner = ImageSetsPartitioner().initialise(image_data_param)
image_reader = ImageReader().initialise(image_data_param, grouping_param, file_list=image_sets_partitioner.all_files)
preprocessing = Preprocessing(net_param, action_param, task_param)
normalisation_layers = preprocessing.prepare_normalisation_layers()
augmentation_layers = preprocessing.prepare_augmentation_layers()
image_reader.add_preprocessing_layers(normalisation_layers + augmentation_layers)

csv_reader = CSVReader(('label', 'features')).initialise(csv_data_param,
                                                         {'label': (['label']), 'features': (['features'])},
                                                         file_list=image_sets_partitioner.all_files)


print('One sample from the csv_reader:', np.squeeze(csv_reader(idx=13)[1]['label']))
window_sizes = {'image': (100, 100, 1), 'label': (1, 1, 1)}
sampler = ResizeSampler(reader=image_reader,
                        csv_reader=csv_reader,
                        window_sizes=window_sizes,
                        num_threads=2,
                        smaller_final_batch_mode='drop',
                        batch_size=2,
                        queue_length=2)
sample = next(sampler())
print(sample['image'].shape)
print(sample['label'].shape)
print(sample['features'].shape)

modalities = ['t1ce.', 't1.', 'flair.', 't2.']
def get_modality(string):
    return modalities[[True if mod in string else False for mod in modalities].index(True)][:-1]
    
files = [(file.replace('.nii.gz', ''), get_modality(file)) \
         for file in os.listdir('/home/tom/data/BRATS_18_SPLITS/train') if 'seg' not in file]
pd.DataFrame(data=files, columns=['subject_id', 'label']).to_csv('/home/tom/phd/NiftyNet-Generator-PR/NiftyNet/modality_labels.csv', index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14263862332695984, 'fraction_numerical': 0.02982791586998088, 'mean_word_length': 4.76487876561352, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '57562', 'license': 'Apache-2.0', 'n_tokens_mistral': 2759, 'n_tokens_neox': 2539, 'n_words': 533, 'path': 'demos/module_examples/FullCSVReaderDemo.ipynb', 'repo_name': 'tdml13/NiftyNet', 'size': '7845'}