<jupyter_start><jupyter_text>Data loading pipeline examplesThe purpose of this notebook is to illustrate reading Nifti files and test speed of different methods.<jupyter_code>%matplotlib inline

import os
import sys
from glob import glob
import tempfile

import numpy as np
import nibabel as nib


import torch
from torch.utils.data import DataLoader
from torch.multiprocessing import Pool, Process, set_start_method
try:
     set_start_method('spawn')
except RuntimeError:
    pass

import monai
from monai.transforms.compose import Compose
from monai.data.nifti_reader import NiftiDataset
from monai.transforms import (AddChannel, Rescale, ToTensor, 
                              UniformRandomPatch, Rotate, RandAffine)

monai.config.print_config()<jupyter_output>MONAI version: 0.0.1
Python version: 3.5.6 |Anaconda, Inc.| (default, Aug 26 2018, 16:30:03)  [GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]
Numpy version: 1.18.1
Pytorch version: 1.4.0
Ignite version: 0.3.0<jupyter_text>0. Preparing input data (nifti images)Create a number of test Nifti files, 3d single channel images with spatial size (256, 256, 256) voxels.<jupyter_code>tempdir = tempfile.mkdtemp()

for i in range(5):
    im, seg = monai.data.synthetic.create_test_image_3d(256,256,256)
    
    n = nib.Nifti1Image(im, np.eye(4))
    nib.save(n, os.path.join(tempdir, 'im%i.nii.gz'%i))
    
    n = nib.Nifti1Image(seg, np.eye(4))
    nib.save(n, os.path.join(tempdir, 'seg%i.nii.gz'%i))
# prepare list of image names and segmentation names
images = sorted(glob(os.path.join(tempdir,'im*.nii.gz')))
segs = sorted(glob(os.path.join(tempdir,'seg*.nii.gz')))<jupyter_output><empty_output><jupyter_text>1. Test image loading with minimal preprocessing<jupyter_code>imtrans = Compose([
    AddChannel(),
    ToTensor()
])    

segtrans = Compose([
    AddChannel(),
    ToTensor()
])    
    
ds = NiftiDataset(images, segs, transform=imtrans, seg_transform=segtrans)
loader = DataLoader(ds, batch_size=3, num_workers=8)

im, seg = monai.utils.misc.first(loader)
print(im.shape, seg.shape)
%timeit data = next(iter(loader))<jupyter_output>5.11 s ± 207 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>2. Test image-patch loading with CPU multi-processing:- rotate (256, 256, 256)-voxel in the plane axes=(1, 2)- extract random (64, 64, 64) patches- implemented in MONAI using ` scipy.ndimage.rotate`<jupyter_code>images = sorted(glob(os.path.join(tempdir,'im*.nii.gz')))
segs = sorted(glob(os.path.join(tempdir,'seg*.nii.gz')))

imtrans = Compose([
    Rescale(),
    AddChannel(),
    Rotate(angle=45.),
    UniformRandomPatch((64, 64, 64)),
    ToTensor()
])    

segtrans = Compose([
    AddChannel(),
    Rotate(angle=45.),
    UniformRandomPatch((64, 64, 64)),
    ToTensor()
])    
    
ds = NiftiDataset(images, segs, transform=imtrans, seg_transform=segtrans)
loader = DataLoader(ds, batch_size=3, num_workers=8, pin_memory=torch.cuda.is_available())

im, seg = monai.utils.misc.first(loader)
print(im.shape, seg.shape)
%timeit -n 3 data = next(iter(loader))<jupyter_output>10.3 s ± 175 ms per loop (mean ± std. dev. of 7 runs, 3 loops each)<jupyter_text>(the above results were based on a 2.9 GHz 6-Core Intel Core i9)  3. Test image-patch loading with preprocessing on GPU:- random rotate (256, 256, 256)-voxel in the plane axes=(1, 2)- extract random (64, 64, 64) patches- implemented in MONAI using native pytorch resampling<jupyter_code>images = sorted(glob(os.path.join(tempdir,'im*.nii.gz')))
segs = sorted(glob(os.path.join(tempdir,'seg*.nii.gz')))

# same parameter with different interpolation mode for image and segmentation
rand_affine_img = RandAffine(prob=1.0, rotate_range=np.pi/4, translate_range=(96, 96, 96),
                             spatial_size=(64, 64, 64), mode='bilinear',
                             as_tensor_output=True, device=torch.device('cuda:0'))
rand_affine_seg = RandAffine(prob=1.0, rotate_range=np.pi/4, translate_range=(96, 96, 96),
                             spatial_size=(64, 64, 64), mode='nearest',
                             as_tensor_output=True, device=torch.device('cuda:0'))
    
imtrans = Compose([
    Rescale(),
    AddChannel(),
    rand_affine_img,
])    

segtrans = Compose([
    AddChannel(),
    rand_affine_seg,
])    
    
ds = NiftiDataset(images, segs, transform=imtrans, seg_transform=segtrans)
loader = DataLoader(ds, batch_size=3, num_workers=0)

im, seg = monai.utils.misc.first(loader)

print(im.shape, seg.shape)
%timeit -n 3 data = next(iter(loader))
print(torch.cuda.get_device_name(0))
print(torch.cuda.memory_summary(0, abbreviated=True))
!rm -rf {tempdir}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1305650684931507, 'fraction_numerical': 0.03809931506849315, 'mean_word_length': 4.262387387387387, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '77788', 'license': 'Apache-2.0', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1646, 'n_words': 399, 'path': 'examples/transform_speed.ipynb', 'repo_name': 'mhubii/MONAI', 'size': '4672'}