<jupyter_start><jupyter_text>Preparing the dataset for hippocampus segmentation<jupyter_code># Import necessary libraries: nibabel, matplotlib, numpy
import nibabel as nib
import numpy as np
import os, glob, shutil
import matplotlib.pyplot as plt
image_path = "/data/TrainingSet/images/"
label_path = "/data/TrainingSet/labels/"

images = sorted(glob.glob(os.path.join(image_path,'*.nii.gz')))
labels = sorted(glob.glob(os.path.join(label_path,'*.nii.gz')))
len(images), len(labels)
image_names = [x.split('/')[-1] for x in images]
label_names = [x.split('/')[-1] for x in labels]
mismatch = list(set(image_names) - set(label_names))
print(mismatch)
for file in mismatch:
    image_names.remove(file)
len(image_names), len(label_names)
def shape(path, names, threshold):
    outlier = []
    for name in names:  
        x, y, z = nib.load(os.path.join(path, name)).get_fdata().shape
        if (x > threshold) | (y > threshold) | (z > threshold):
            outlier.append(name)
            print(name)
            print(x, y, z)
            print()
    return outlier

image_outlier = shape(image_path, image_names, 60)
label_outlier = shape(label_path, label_names, 60)

print(list(set(image_outlier).union(set(label_outlier))))

# remove the outliers
for name in list(set(image_outlier).union(set(label_outlier))):
    image_names.remove(name)
    label_names.remove(name)

assert image_names == label_names, 'FileMismatchError.'

print(len(image_names), len(label_names))
# Load an image and a segmentation mask into variables called image and label

img_n = 65
image = nib.load(os.path.join(image_path, image_names[img_n])).get_fdata()
label = nib.load(os.path.join(label_path, label_names[img_n])).get_fdata()
print(image.dtype)
print(label.dtype)
print(image.shape)
print(label.shape)
def display_volume_slices(img, w, h):
    plot_w = w
    plot_h = h

    # You can play with figsize parameter to adjust how large the images are
    fig, ax = plt.subplots(plot_h, plot_w, figsize=[35,25])

    for i in range(plot_w*plot_h):
        plt_x = i % plot_w
        plt_y = i // plot_w
        
        if (i < len(img)):
            ax[plt_y, plt_x].set_title(f"slice {i}")
            ax[plt_y, plt_x].imshow(np.fliplr(np.rot90(img[i])), cmap='gray')
        ax[plt_y, plt_x].axis("off")

    plt.show()
display_volume_slices(np.transpose(image, (2, 0, 1)), 7, 5)
display_volume_slices(np.transpose(label, (2, 0, 1)), 7, 5)<jupyter_output><empty_output><jupyter_text>Looking at single image dataIn this section we will look closer at the NIFTI representation of our volumes. In order to measure the physical volume of hippocampi, we need to understand the relationship between the sizes of our voxels and the physical world.<jupyter_code># format of images?
img = nib.load(os.path.join(image_path, image_names[img_n]))
print(img.header_class)
print(img.header)<jupyter_output><class 'nibabel.nifti1.Nifti1Header'> object, endian='<'
sizeof_hdr      : 348
data_type       : b''
db_name         : b''
extents         : 0
session_error   : 0
regular         : b'r'
dim_info        : 0
dim             : [ 3 34 52 35  1  1  1  1]
intent_p1       : 0.0
intent_p2       : 0.0
intent_p3       : 0.0
intent_code     : none
datatype        : float32
bitpix          : 32
slice_start     : 0
pixdim          : [1. 1. 1. 1. 1. 0. 0. 0.]
vox_offset      : 0.0
scl_slope       : nan
scl_inter       : nan
slice_end       : 0
slice_code      : unknown
xyzt_units      : 10
cal_max         : 0.0
cal_min         : 0.0
slice_duration  : 0.0
toffset         : 0.0
glmax           : 0
glmin           : 0
descrip         : b'5.0.10'
aux_file        : b'none'
qform_code      : scanner
sform_code      : scanner
quatern_b       : 0.0
quatern_c       : 0.0
quatern_d       : 0.0
qoffset_x       : 1.0
qoffset_y       : 1.0
qoffset_z       : 1.0
srow_x          : [1. 0. 0. 1.]
srow_y          : [[...]<jupyter_text>Further down we will be inspecting .header attribute that provides access to NIFTI metadata. You can use this resource as a reference for various fields: https://brainder.org/2012/09/23/the-nifti-file-format/<jupyter_code># bits per pixel?
print('Bits per pixels: ', img.header['bitpix'])
# units of measurement?
unit_b = int(str(img.header['xyzt_units'].item()), base=2)
unit_d = img.header['xyzt_units']

print(f'Units of measurement: {unit_d} in decimal , {unit_b} in binary.')
spatial_unit, temporal_unit = img.header.get_xyzt_units()
print(f"Spatial unit: {spatial_unit}, Temporal unit: {temporal_unit}")
# regular grid? grid spacings?
print('Grid Spacing: ', img.header['pixdim'])<jupyter_output>Grid Spacing:  [1. 1. 1. 1. 1. 0. 0. 0.]<jupyter_text>According to the grid spacing, we have a regular grid with a voxel size of 1x1x1 mm3<jupyter_code># What dimensions represent axial, sagittal, and coronal slices? How do you know?
print(nib.aff2axcodes(img.affine))
print("Orientation is R A S, i.e., left to Right (sagittal or X direction), \
posterior to Anterior (coronal or Y direction), and inferior to Superior \
(axial or Z direction), respectively.")
# Compute the volume (in mm³) of a hippocampus using one of the labels loaded. 
label = nib.load(os.path.join(label_path, label_names[img_n]))
label_np = label.get_fdata()
np.sum(label_np != 0)<jupyter_output><empty_output><jupyter_text>Plotting some charts<jupyter_code># Plot a histogram of all volumes that we have in our dataset and see how 
# our dataset measures against a slice of a normal population represented by the chart below.<jupyter_output><empty_output><jupyter_text><jupyter_code>volumes = []

for label_name in label_names:
    label_file = os.path.join(label_path, label_name)
    label = nib.load(label_file)          # Load the NIFTI file
    label_slices = label.get_fdata()      # Load the array 
    volume = (label_slices > 0).sum()     # Calculate the volume
    
    # Check for outliers
    if volume > 5000 or volume < 2000:
        file_name = label_file.split('/')[-1]
        outliers.add(file_name)
        print("Volume of file {} ({}) exceeds expected value (~4500).".format(file_name, volume))
    
    volumes.append(volume)

plt.plot(volumes)<jupyter_output><empty_output><jupyter_text>In the real world we would have precise information about the ages and conditions of our patients, and understanding how our dataset measures against population norm would be the integral part of clinical validation that we talked about in last lesson. Unfortunately, we do not have this information about this dataset, so we can only guess why it measures the way it is. The mask seems to have two classes, labeled with values `1` and `2` respectively. Class 1 is the anterior segment of the hippocampus and class 2 is the posterior one. For the purpose of volume calculation we do not care about the distinction, however we will still train our network to differentiate between these two classes and the background.<jupyter_code>plt.hist(volumes, bins=50)
# Copy the clean dataset to the output folder inside section1/out

out_images_path = 'out/images/'
out_labels_path = 'out/labels/'

os.makedirs(os.path.dirname(out_images_path), exist_ok=True)
os.makedirs(os.path.dirname(out_labels_path), exist_ok=True)

for image_name in image_names:
    image_file = os.path.join(image_path, image_name)
    shutil.copy(image_file, out_images_path)
    label_file = os.path.join(label_path, image_name)
    shutil.copy(label_file, out_labels_path)
        
print("Number of images in clean dataset: ", len(os.listdir(out_images_path)))
print("Number of labels in clean dataset: ", len(os.listdir(out_labels_path)))<jupyter_output>Number of images in clean dataset:  260
Number of labels in clean dataset:  260
{'fraction_non_alphanumeric': 0.1090601845833875, 'fraction_numerical': 0.019498245157935787, 'mean_word_length': 3.720245398773006, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '574533', 'license': 'MIT', 'n_tokens_mistral': 2738, 'n_tokens_neox': 2561, 'n_words': 761, 'path': 'Section-1/Final Project EDA.ipynb', 'repo_name': 'sams-chw/Quantifying-Hippocampus-Volume-for-Alzheimers-Progression', 'size': '7693'}