<jupyter_start><jupyter_text>Setup<jupyter_code>import os
import PIL
import cv2
import tarfile
from tqdm import tqdm
import numpy as np
import pydicom
import pandas as pd
from glob import glob
import nibabel as nib
from PIL import Image
import matplotlib.pyplot as plt
from scipy import ndimage as ndi<jupyter_output><empty_output><jupyter_text>Extracting Data<jupyter_code># Extract Main Training Data
print("... Extracting Main Task1 Training Files (3-5 Minutes) ...\n")
tar = tarfile.open("../../input/zips/BraTS2021_Training_Data.tar")
tar.extractall("../../input/zips/task_1_zips/")
tar.close()
# !del C:\Users\alcka\F_drive\RSNA-MICCAI-brain-tumor\input\zips\task_1_zips\.DS_STORE
all_dirs = []
types = ["flair", "t1", "t1ce", "t2"]
root = r"C:\Users\alcka\F_drive\RSNA-MICCAI-brain-tumor\input\zips\task_1_zips"

for patient_id in os.listdir(root):
    dir_dict = {}
    patient_dir = os.path.join(root, patient_id)
    for t in types:
        patient_modality_dir = os.path.join(patient_dir, patient_id + f"_{t}.nii.gz")
        dir_dict[t] = patient_modality_dir
    dir_dict["segm"] = os.path.join(patient_dir, patient_id + f"_seg.nii.gz")
    all_dirs.append(dir_dict)<jupyter_output><empty_output><jupyter_text>Saving to npy<jupyter_code>save_root = r"C:\Users\alcka\F_drive\RSNA-MICCAI-brain-tumor\input\task_1_data"
os.makedirs(save_root, exist_ok=True)
for patient_dict in tqdm(all_dirs):
    flair_path = patient_dict["flair"]
    t1_path = patient_dict["t1"]
    t1ce_path = patient_dict["t1ce"]
    t2_path = patient_dict["t2"]
    segm_path = patient_dict["segm"]
    
    patientid = flair_path.split("\\")[-2]
    
    flair_image = nib.load(flair_path).get_fdata().transpose([2, 0, 1])
    t1_image = nib.load(t1_path).get_fdata().transpose([2, 0, 1])
    t1ce_image = nib.load(t1ce_path).get_fdata().transpose([2, 0, 1])
    t2_image = nib.load(t2_path).get_fdata().transpose([2, 0, 1])
    segm_image = nib.load(segm_path).get_fdata().transpose([2, 0, 1])
    
    np.save(os.path.join(save_root, patientid + "_flair"), flair_image)
    np.save(os.path.join(save_root, patientid + "_t1"), t1_image)
    np.save(os.path.join(save_root, patientid + "_t1ce"), t1ce_image)
    np.save(os.path.join(save_root, patientid + "_t2"), t2_image)
    np.save(os.path.join(save_root, patientid + "_segm"), segm_image)<jupyter_output>100%|██████████████████████████████████████████████████████████████████████████████| 1251/1251 [56:53<00:00,  2.73s/it]
{'fraction_non_alphanumeric': 0.17986195696305318, 'fraction_numerical': 0.02882663418595209, 'mean_word_length': 4.894736842105263, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '376923', 'license': 'MIT', 'n_tokens_mistral': 1111, 'n_tokens_neox': 1020, 'n_words': 159, 'path': 'src/preprocessing/rsna-miccai-convert-task-1-to-npy.ipynb', 'repo_name': 'alckasoc/RSNA-MICCAI-brain-tumor', 'size': '2463'}