<jupyter_start><jupyter_text>This notebook contains some code for transfering the atlas data:1. Gets the histology data (labels, cell density maps)2. Gets the MRI data3. Gets the contours (prostate, PZ, urethra) and performs morphological interpolation<jupyter_code>%load_ext autoreload
%autoreload 2
import shutil
import pathlib

from birt_utils import morphological_interpolate
case_id_list = [
    "005","006","007","008","009","010",
    "011","012","013","014","015","017",
    "018","019","020","021","022","023",
    "024","025","026","027","028","030",
    "031","032","033","034","035","036",
    "037","038","039","040","041","042",
    "043","044","045","046","047","048",
    "049","050","051","052","053","054",
    "055","056","057","058","059","060",
    "061","062","063","064","065","066",
    "068","069","070",
]

# Not included:
# 001
# 002
# 003
# 004
# 016
# 029
# 067
    
print(len(case_id_list))
for case_id in case_id_list:
    
    print('Processing ', case_id)
    
    # Set output directory
    output_dir = pathlib.Path(f"../../1_data/atlas_data/MRHIST{case_id}")

    # Make subdirectories
    (output_dir / "IMAGES").mkdir(exist_ok=True, parents=True)
    (output_dir / "LABELS").mkdir(exist_ok=True, parents=True)
    
    """
    Copy relevant histology-based data
    """

    # Set input directory
    input_dir_regresults = pathlib.Path(f"../../1_data/RegistrationOutput/output/mrhist{case_id}")

    # Cell density maps
    file_in_cd = input_dir_regresults / "cell_density_maps.nii.gz"
    file_out_cd = output_dir / "IMAGES" / f"MRHIST{case_id}_CELL_DENSITY_MAP.nii.gz"

    # Annotation masks
    file_in_am = input_dir_regresults / "annotation_mask.nii.gz"
    file_out_am = output_dir / "LABELS" / f"MRHIST{case_id}_LABEL_HISTOLOGY.nii.gz"
        
    # Histology
    file_in_hist = input_dir_regresults / "histology.nii.gz"
    file_out_hist = output_dir / "IMAGES" / f"MRHIST{case_id}_HISTOLOGY.nii.gz"

    # Copy files
    shutil.copy(file_in_cd, file_out_cd)
    shutil.copy(file_in_hist, file_out_hist)
    
    """
    Fix annotation masks
    """
    
    img = sitk.ReadImage(str(file_in_am))
    arr = sitk.GetArrayFromImage(img)

    # Get unique values
    vals = np.unique(arr)

    # Check to see if there are errors
    okay_vals = [0, 64, 96, 128, 160, 192, 224, 234, 244, 255, 100, 200]

    total_bad_vals = 0

    if np.any(np.in1d(vals, okay_vals)):
        # We ned to correct
        bad_vals = vals[~np.in1d(vals, okay_vals)]

        for v in bad_vals:
            n_bad = np.count_nonzero(arr == v)
            total_bad_vals += n_bad
            print(f"{v}: {n_bad}")

            # Find location of bad values
            locz, locy, locx = np.where(arr == v)
            print(list(zip(locz, locy, locx)))

            # Get neighbourhood values in slice
            for i in range(n_bad):
                neighbours = arr[
                    locz[i], locy[i] - 2 : locy[i] + 3, locx[i] - 2 : locx[i] + 3
                ]

                print(neighbours)

                replace_val_list, counts = np.unique(neighbours, return_counts=True)

                counts = counts[replace_val_list != 0]
                replace_val_list = replace_val_list[replace_val_list != 0]

                replace_val = replace_val_list[np.argmax(counts)]
                print(f"Replace value: {replace_val}")

                arr[locz[i], locy[i], locx[i]] = replace_val

    img_fix = sitk.GetImageFromArray(arr)
    img_fix.CopyInformation(img)
    
    sitk.WriteImage(img_fix, str(file_out_am))
    
    print('  Copied histology.')
    
    """
    Copy relevant MRI-based data
    """

    # Set input directory
    #if int(case_id) > 45:
    input_dir_mri = pathlib.Path(f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/PatientData/mrhist{case_id}/ex_vivo_mri")

    # 3D MRI
    file_in_3d_list = [i.as_posix() for i in list(input_dir_mri.glob("*3D*.nii.gz"))]
    print("Number of 3D MRI matches: ", len(file_in_3d_list))
    file_in_3d = file_in_3d_list[0]

    # 2D MRI
    file_in_2d_list = [i.as_posix() for i in list(input_dir_mri.glob("*TSE_TRA*.nii.gz"))]
    
    if len(file_in_2d_list)==1:
        file_in_2d = file_in_2d_list[0]
        print(f'Success: {file_in_2d}')
    
    
    elif len(file_in_2d_list)==0:
        print('Using back-up naming for 2D MRI.')
        file_in_2d_list = [i.as_posix() for i in list(input_dir_mri.glob("*2D*.nii.gz"))]
        
        if len(file_in_2d_list)==1:
            file_in_2d = file_in_2d_list[0]
            print(f'Success: {file_in_2d}')
            
        elif len(file_in_2d_list)==0:
            file_in_2d_list = [i.as_posix() for i in list(input_dir_mri.glob("*TSE_AX*.nii.gz"))]
            if len(file_in_2d_list)==1:
                file_in_2d = file_in_2d_list[0]
                print(f'Success: {file_in_2d}')
            
            else:
                raise ValueError("Too many 2D MRI files found!", file_in_2d_list)

        else:
            raise ValueError("Too many 2D MRI files found!", file_in_2d_list)
            
    else:
        raise ValueError("Too many 2D MRI files found!", file_in_2d_list)

    #else:
    #file_in_2d = f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/RegistrationResults/mrhist{case_id}/registration/invivo_to_exvivo/ex_2d.nii"
    #file_in_3d = f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/RegistrationResults/mrhist{case_id}/registration/invivo_to_exvivo/ex_3d.nii"


    file_out_3d = output_dir / "IMAGES" / f"MRHIST{case_id}_MRI_T2W_3D.nii.gz"
    file_out_2d = output_dir / "IMAGES" / f"MRHIST{case_id}_MRI_T2W_2D.nii.gz"


    # Copy files
    shutil.copy(file_in_3d, file_out_3d)
    shutil.copy(file_in_2d, file_out_2d)
    
    print('  Copied MRI.')
    
    """
    MRI contours - whole gland
    """
    
    file_out_wg = output_dir / "LABELS" / f"MRHIST{case_id}_CONTOUR_PROSTATE.nii.gz"
    
    # Set input directory - whole gland contours defined on 2D T2w MRI
    if int(case_id) > 45:
        file_in_wg = f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/ExVivoAutoContouring/edit_contours/mrhist{case_id}_ex_2d_edit.nii.gz"
        
        # Copy files
        shutil.copy(file_in_wg, file_out_wg)
    
    else:
        #file_in_wg = f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/RegistrationResults/mrhist{case_id}/registration/invivo_to_exvivo/ex_2d_contour.nii"
        input_dir_wg = pathlib.Path(f"/media/robbie/My Passport/Work/3_ResearchProjects/ProstateAtlas/1_data/ExVivoAutoContouring/MRI_data_Gonzalo_invivo_exvivo/MRI_data_Gonzalo_invivo_exvivo/mrhist{case_id}")
        
        file_in_wg = sorted([i.as_posix() for i in list(input_dir_wg.glob("ex_2d_contour*"))], key=lambda x:len(x))[-1]
        
        print("Prostate contour file: ", file_in_wg)
        
        im_in_wg = sitk.ReadImage(file_in_wg)
        im_in_wg = sitk.Cast(im_in_wg>0, sitk.sitkUInt8)
        
        sitk.WriteImage(im_in_wg, file_out_wg.as_posix())
        
    # # Optional - copy 3D imaging: wait until finished?
    #     if int(case_id) <= 45:
    #         file_in_wg = f"/home/robbie/Work/3_ResearchProjects/BiRT/ProstateAtlas/1_data/RegistrationResults/mrhist{case_id}/registration/invivo_to_exvivo/ex_3d_contour.nii"
    #         file_out_wg = output_dir / "LABELS" / f"mrhist{case_id}_CONTOUR_T2W_3D_PROSTATE.nii.gz"

    #         # Copy files
    #         shutil.copy(file_in_wg, file_out_wg)
    
    """
    MRI contours - peripheral zone and urethra
    This step requires some processing
    """

    # Set input directory - contours defined on 2D T2w MRI using histology
    file_in_pz = f"/home/robbie/Work/3_ResearchProjects/BiRT/ProstateAtlas/1_data/PZ_UR_contours/mrhist{case_id}_pz_ur_contours.nii.gz"

    image_pz_ur = sitk.ReadImage(file_in_pz)
    image_pz_interp = morphological_interpolate(image_pz_ur==1)
    # image_pz_interp = sitk.Mask(im_in_wg)

    file_out_pz = output_dir / "LABELS" / f"MRHIST{case_id}_CONTOUR_PZ_INTERP.nii.gz"
    sitk.WriteImage(image_pz_interp, file_out_pz.as_posix())

    image_ur = (image_pz_ur==2)

    file_out_ur = output_dir / "LABELS" / f"MRHIST{case_id}_CONTOUR_URETHRA.nii.gz"
    sitk.WriteImage(image_ur, file_out_ur.as_posix())
    

    print('  Copied contours.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1279028224365845, 'fraction_numerical': 0.04180064308681672, 'mean_word_length': 2.991444866920152, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '270992', 'license': 'Apache-2.0', 'n_tokens_mistral': 3538, 'n_tokens_neox': 3214, 'n_words': 492, 'path': '1A_transfer_atlas_data.ipynb', 'repo_name': 'rnfinnegan/prostate-biological-model', 'size': '8397'}