<jupyter_start><jupyter_text>Connect to the drive to get the data<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output><empty_output><jupyter_text>Sometimes it's needed to hard reset the machine using command below.<jupyter_code>!kill -9 -1<jupyter_output><empty_output><jupyter_text>Check how much RAM on GPU is available<jupyter_code>!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi
!pip install gputil
!pip install psutil
!pip install humanize
import psutil
import humanize
import os
import GPUtil as GPU
GPUs = GPU.getGPUs()

# XXX: only one GPU on Colab and isn’t guaranteed
gpu = GPUs[0]

def printm():
    process = psutil.Process(os.getpid())
    print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
    print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))

    printm()<jupyter_output><empty_output><jupyter_text>Install deps<jupyter_code>!curl https://course-v3.fast.ai/setup/colab | bash
!pip install pydicom<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import os
import re

import numpy as np
import pydicom
import matplotlib.pyplot as plt

from fastai import *
from fastai.vision import *
import cv2

from fastai.layers import FlattenedLoss<jupyter_output><empty_output><jupyter_text>Copy data from drive to the filesystem - this might make things faster.Note: from time to time you need to update compressed file.<jupyter_code>drive_data_path = './drive/My Drive/Data/'
data_path = './data'
!cp ./drive/My\ Drive/data.tar.gz .
!tar xzf data.tar.gz
scans = []
for root, dirs, files in os.walk(data_path):
    if 'CT' in root:
        # remove wrongly labeled data
        if not 'P1B1' in root:
            for _file in files:
                scans.append(root + '/' + _file)
np.random.seed(42)
np.random.shuffle(scans)<jupyter_output><empty_output><jupyter_text>Overwrite fastai loading images methods<jupyter_code>def open_dcm_image(fn, *args, **kwargs)->Image:
#     window_min = -100
#     window_max = 400
    window_min = -100
    window_max = 100

    array = pydicom.dcmread(fn).pixel_array
    array = np.clip(array, a_min=window_min, a_max=window_max)
            
    array = (((array - array.min()) / (array.max() - array.min())) * (255 - 0) + 0).astype(np.uint8)
    array = cv2.equalizeHist(array.astype(np.uint8))

    array = np.repeat(array[:, :, None], 3, axis=2)
    
    # we can store images in this format :top: to make stuff faster...
    return Image(pil2tensor(array, np.float32).div_(255))

def open_dcm_mask(fn, *args, **kwargs)->Image:
    x = pydicom.dcmread(fn).pixel_array
    x = pil2tensor(x, np.float32)
    return ImageSegment(x)


def annotate_metadata(fn, ax):
    subdirs = fn.split('/')
    patient_id = subdirs[-3]
    slice_number = re.findall(r'\d+', subdirs[-1])[0]
    ax.annotate(
        '{} [{}]'.format(patient_id, slice_number),
        xy=(.25, .25),
        xycoords='data', 
        xytext=(30, 10),
        fontsize=20,
        textcoords='offset points',
    )

                                  
# monkey patch
fastai.vision.image.open_image = open_dcm_image
fastai.vision.image.open_mask = open_dcm_mask
fastai.vision.data.open_image = open_dcm_image
fastai.vision.data.open_mask = open_dcm_mask
open_image = open_dcm_image
open_mask = open_dcm_mask<jupyter_output><empty_output><jupyter_text>Look at the data<jupyter_code>open_image(scans[1003])
get_y_fn = lambda path: str('.' / Path(path).parent / '../label' / Path(path).name)

open_mask(get_y_fn(scans[1003]))
codes = ['void', 'water']
src = (
    SegmentationItemList.from_df(pd.DataFrame(scans, columns=['files']), '.')
    .split_by_valid_func(lambda img_src: 'P7' in str(img_src) or 'P6' in str(img_src))
    .label_from_func(get_y_fn, classes=codes)
)
src
img = open_image(scans[600]).data
src_size = np.array(img.shape[1:])
size = src_size // 4
size
bs = 80
data = (
    # note wrap might deform images. For now I've set up 0, maybe we can use it.
    src.transform(get_transforms(max_rotate=5., max_lighting=0, p_lighting=0, max_warp=0), size=size, tfm_y=True)
    .databunch(bs=bs)
    .normalize(imagenet_stats)
)
data.show_batch(2, figsize=(10,7))
data.show_batch(2, figsize=(10,7), ds_type=DatasetType.Valid)<jupyter_output><empty_output><jupyter_text>Choose metrics to evaluate<jupyter_code>from fastai.metrics import accuracy, dice
def acc(input, target):
    target = target.squeeze(1)
    return (input.argmax(dim=1)==target).float().mean()
metrics=[acc, dice]<jupyter_output><empty_output><jupyter_text>Implement new loss functions<jupyter_code>from torch.nn.modules.loss import _Loss
class DiceLoss(_Loss):
    def __init__(self, **kwargs):
        super(DiceLoss, self).__init__(**kwargs)
        self.softmax = nn.Softmax(1)

    def forward(self, input, target):
        input = self.softmax(input)[:, 1]
        target = target.float()
        smooth = 1.
        intersection = (input * target).sum()

        return 1 - ((2. * intersection + smooth) /
                  (input.sum() + target.sum() + smooth))


class GeneralizedDiceLoss(_Loss):
    # reference: https://niftynet.readthedocs.io/en/dev/_modules/niftynet/layer/loss_segmentation.html#generalised_dice_loss
    def __init__(self, **kwargs):
        super(GeneralizedDiceLoss, self).__init__(**kwargs)
        self.softmax = nn.Softmax(1)

    def forward(self, input, target):
        prediction = self.softmax(input)
        one_hot = (
            torch.sparse.torch.eye(2).cuda()
            .index_select(0, target.long())
        )
          
        ref_vol = torch.sum(one_hot, 0)
        
        seg_vol = torch.sum(prediction, 0)
        intersect = torch.sum(one_hot * prediction, 0)
        
        weights = torch.reciprocal(ref_vol ** 2)
        weights[weights == float("Inf")] = 0

        generalised_dice_numerator = 2 * torch.sum(weights * intersect)
        generalised_dice_denominator = torch.sum(
            weights * torch.max(seg_vol + ref_vol, torch.ones_like(weights))
        )
        generalised_dice_score = \
            generalised_dice_numerator / generalised_dice_denominator
        
        generalised_dice_score[torch.isnan(generalised_dice_score)] =  1.
        return 1 - generalised_dice_score
            
dice_loss = FlattenedLoss(DiceLoss, axis=1)
generalized_dice_loss = FlattenedLoss(GeneralizedDiceLoss, axis=1)
dice_loss(torch.Tensor([[10, 1], [10, 0]]), torch.Tensor([[1], [1]]))
generalized_dice_loss(torch.Tensor([[10, 1], [10, 0]]).cuda(), torch.Tensor([[1], [1]]).cuda())<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>learn = unet_learner(
    data, models.resnet34, metrics=metrics, 
    self_attention=False,
    loss_func=generalized_dice_loss,
)
lr_find(learn)
learn.recorder.plot()
lr=3e-5
learn.fit_one_cycle(10, slice(lr), pct_start=0.9)
learn.save('3_1')
learn.load('3_1');
!cp ./models/3_1.pth ./drive/My\ Drive/
learn.show_results(rows=20)
learn.unfreeze()
lr_find(learn)
learn.recorder.plot()
lrs = slice(1e-6, 8e-5)
learn.fit_one_cycle(12, lrs, pct_start=0.8)
learn.recorder.plot_losses()
learn.recorder.plot_lr()
learn.save('3_2');
!mkdir -p ./drive/My\ Drive/Code/
!cp ./models/3_2.pth ./drive/My\ Drive/
learn = learn.load('3_2')
learn.show_results(rows=24)<jupyter_output><empty_output><jupyter_text>Go big - full size of an image<jupyter_code>!mkdir -p models
!cp ./drive/My\ Drive/Code/Mateusz/stage-1.pth ./models/stage-1.pth 
size = src_size
bs = 5
data = (
    src.transform(get_transforms(max_rotate=5., max_lighting=0, p_lighting=0), size=size, tfm_y=True)
    .databunch(bs=bs)
    .normalize(imagenet_stats)
)
learn = unet_learner(
    data, models.resnet34, metrics=metrics, self_attention=True,
)
learn.load('stage-1');
lr_find(learn)
learn.recorder.plot()
lr=1e-5
learn.fit_one_cycle(3, slice(lr))
learn.save('stage-1-big')
learn.show_results()
!cp ./models/stage-1-big.pth ./drive/My\ Drive/
learn.load('stage-1-big');
learn.unfreeze()
lrs = slice(1e-6,1e-4)
learn.fit_one_cycle(10, lrs, wd=1e-3)
learn.save('stage-2-big')
learn.load('stage-2-big')
learn.show_results()
!cp ./models/stage-2-big.pth ./drive/My\ Drive/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14338897562147232, 'fraction_numerical': 0.02269724990993155, 'mean_word_length': 3.9571428571428573, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '199017', 'license': 'MIT', 'n_tokens_mistral': 3214, 'n_tokens_neox': 3090, 'n_words': 574, 'path': 'notebooks/collab/U-Net-fastai.ipynb', 'repo_name': 'hydrocephalus-poznan/exploration', 'size': '8327'}