<jupyter_start><jupyter_text>Brain Tumor Detection<jupyter_code>!pip install opendatasets
! pip install pydot
!pip install graphviz
import opendatasets as od
od.download("https://www.kaggle.com/mateuszbuda/lgg-mri-segmentation")
# downloading helper_functions.py
! pip install wget
import wget
!python -m wget https://raw.githubusercontent.com/Hrushi11/Dogs_VS_Cats/main/helper_functions.py
!pip install tensorflow-gpu<jupyter_output>Collecting tensorflow-gpu
  Downloading tensorflow_gpu-2.7.0-cp37-cp37m-win_amd64.whl (430.8 MB)
Requirement already satisfied: absl-py>=0.4.0 in f:\ml_projects\assignments\env\lib\site-packages (from tensorflow-gpu) (0.12.0)
Requirement already satisfied: grpcio<2.0,>=1.24.3 in f:\ml_projects\assignments\env\lib\site-packages (from tensorflow-gpu) (1.36.1)
Collecting flatbuffers<3.0,>=1.12
  Downloading flatbuffers-2.0-py2.py3-none-any.whl (26 kB)
Collecting typing-extensions>=3.6.6
  Downloading typing_extensions-4.0.1-py3-none-any.whl (22 kB)
Collecting libclang>=9.0.1
  Using cached libclang-12.0.0-py2.py3-none-win_amd64.whl (13.1 MB)
Requirement already satisfied: opt-einsum>=2.3.2 in f:\ml_projects\assignments\env\lib\site-packages (from tensorflow-gpu) (3.1.0)
Requirement already satisfied: h5py>=2.9.0 in f:\ml_projects\assignments\env\lib\site-packages (from tensorflow-gpu) (2.10.0)
Requirement already satisfied: keras-preprocessing>=1.1.1 in f:\ml_projects\assignments\env\lib\[...]<jupyter_text>Importing dependancies<jupyter_code># Importing dependancies
import os
import cv2
import random
import numpy as np
import pandas as pd
from skimage import io
import tensorflow as tf
import matplotlib.pyplot as plt

from tensorflow.keras import layers
from tensorflow.keras import models
from tensorflow.keras import backend as K
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from helper_functions import unzip_data, walk_through_dir, plot_loss_curves
# lookthroug our data
walk_through_dir(".\lgg-mri-segmentation\kaggle_3m")
# getting the directories
dir = []
for dir_ in os.listdir("./lgg-mri-segmentation/kaggle_3m/"):
  if dir_.split("_")[0] == "TCGA":
    dir.append("./lgg-mri-segmentation/kaggle_3m/" + dir_)
len(dir)
dir[:5]
# Getting the filepaths of all images
file_paths = []
for dir_ in dir:
  for file_path in os.listdir(dir_):
    if len(file_path.split("_")) == 5:
      file_paths.append(dir_ + "/" + file_path)
len(file_paths)
file_paths[:5]
# Getting file paths of masked images
file_paths_masked = []
for elem in file_paths:
  file_paths_masked.append("." + elem.split(".")[1] + "_mask.tif")
len(file_paths_masked)
file_paths_masked[:5]
file_paths[0].split("/")[3]
# Plotting original image with its mask
def plot_orig_mask():
  plt.figure(figsize=(10, 7))
  # Getting the image
  rand_ = np.random.randint(len(file_paths))
  img = plt.imread(file_paths[rand_])
  img_mask = plt.imread(file_paths_masked[rand_])

  # Plotting the image
  plt.subplot(1, 2, 1)
  plt.imshow(img/255)
  plt.title("Original", color="green")
  plt.axis(False)
  
  plt.subplot(1, 2, 2)
  plt.imshow(img_mask/255)
  plt.title("Masked", color="blue")
  plt.axis(False);
# Plotting the image
plot_orig_mask()
# Creating train, val and test splits
df = pd.DataFrame({"filename":file_paths,
                   "mask":file_paths_masked})

df_train, df_test = train_test_split(df,test_size=0.1)
df_train, df_val = train_test_split(df_train,test_size=0.2)

df_train.shape, df_val.shape, df_test.shape
def train_generator(data_frame, batch_size, aug_dict, image_color_mode="rgb", 
                    mask_color_mode="grayscale", image_save_prefix="image", mask_save_prefix="mask", 
                    save_to_dir=None, target_size=(256,256), seed=1):
    
    image_datagen = ImageDataGenerator(**aug_dict)
    mask_datagen = ImageDataGenerator(**aug_dict)
    
    image_generator = image_datagen.flow_from_dataframe(
        data_frame,
        x_col = "filename",
        class_mode = None,
        color_mode = image_color_mode,
        target_size = target_size,
        batch_size = batch_size,
        save_to_dir = save_to_dir,
        save_prefix  = image_save_prefix,
        seed = seed)

    mask_generator = mask_datagen.flow_from_dataframe(
        data_frame,
        x_col = "mask",
        class_mode = None,
        color_mode = mask_color_mode,
        target_size = target_size,
        batch_size = batch_size,
        save_to_dir = save_to_dir,
        save_prefix  = mask_save_prefix,
        seed = seed)
    
    train_gen = zip(image_generator,mask_generator)
    
    for (img, mask) in train_gen:
      img , mask = adjust_data(img,mask)
      yield (img,mask)
        
def adjust_data(img,mask):
  img=img/255
  mask=mask/255
  mask[mask>0.5]=1
  mask[mask<=0.5]=0
        
  return (img,mask)
def dice_coef(y_true,y_pred):
    y_truef = K.flatten(y_true)
    y_predf = K.flatten(y_pred)
    cal=K.sum(y_truef*y_predf)
    
    return ((2*cal+100)/(K.sum(y_truef)+K.sum(y_predf)+100))

def dice_coef_loss(y_true,y_pred):
    return -dice_coef(y_true,y_pred)

def iou(y_true,y_pred):
    intersection = K.sum(y_true*y_pred)
    sum_ = K.sum(y_true+y_pred)
    jac = (intersection+100)/(sum_-intersection+100)
    return jac

def jac_distance(y_true,y_pred):
    y_truef = K.flatten(y_true)
    y_predf = K.flatten(y_pred)
    
    return -iou(y_true,y_pred)
def unet(input_size=(256,256,3)):
    inputs = layers.Input(input_size)
    
    conv1 = layers.Conv2D(64, (3, 3), padding='same')(inputs)
    bn1 = layers.Activation('relu')(conv1)
    conv1 = layers.Conv2D(64, (3, 3), padding='same')(bn1)
    bn1 = layers.BatchNormalization(axis=3)(conv1)
    bn1 = layers.Activation('relu')(bn1)
    pool1 = layers.MaxPooling2D(pool_size=(2, 2))(bn1)

    conv2 = layers.Conv2D(128, (3, 3), padding='same')(pool1)
    bn2 = layers.Activation('relu')(conv2)
    conv2 = layers.Conv2D(128, (3, 3), padding='same')(bn2)
    bn2 = layers.BatchNormalization(axis=3)(conv2)
    bn2 = layers.Activation('relu')(bn2)
    pool2 = layers.MaxPooling2D(pool_size=(2, 2))(bn2)

    conv3 = layers.Conv2D(256, (3, 3), padding='same')(pool2)
    bn3 = layers.Activation('relu')(conv3)
    conv3 = layers.Conv2D(256, (3, 3), padding='same')(bn3)
    bn3 = layers.BatchNormalization(axis=3)(conv3)
    bn3 = layers.Activation('relu')(bn3)
    pool3 = layers.MaxPooling2D(pool_size=(2, 2))(bn3)

    conv4 = layers.Conv2D(512, (3, 3), padding='same')(pool3)
    bn4 = layers.Activation('relu')(conv4)
    conv4 = layers.Conv2D(512, (3, 3), padding='same')(bn4)
    bn4 = layers.BatchNormalization(axis=3)(conv4)
    bn4 = layers.Activation('relu')(bn4)
    pool4 = layers.MaxPooling2D(pool_size=(2, 2))(bn4)

    conv5 = layers.Conv2D(1024, (3, 3), padding='same')(pool4)
    bn5 = layers.Activation('relu')(conv5)
    conv5 = layers.Conv2D(1024, (3, 3), padding='same')(bn5)
    bn5 = layers.BatchNormalization(axis=3)(conv5)
    bn5 = layers.Activation('relu')(bn5)

    up6 = layers.concatenate([layers.Conv2DTranspose(512, (2, 2), strides=(2, 2), padding='same')(bn5), conv4], axis=3)
    conv6 = layers.Conv2D(512, (3, 3), padding='same')(up6)
    bn6 = layers.Activation('relu')(conv6)
    conv6 = layers.Conv2D(512, (3, 3), padding='same')(bn6)
    bn6 = layers.BatchNormalization(axis=3)(conv6)
    bn6 = layers.Activation('relu')(bn6)

    up7 = layers.concatenate([layers.Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(bn6), conv3], axis=3)
    conv7 = layers.Conv2D(256, (3, 3), padding='same')(up7)
    bn7 = layers.Activation('relu')(conv7)
    conv7 = layers.Conv2D(256, (3, 3), padding='same')(bn7)
    bn7 = layers.BatchNormalization(axis=3)(conv7)
    bn7 = layers.Activation('relu')(bn7)

    up8 = layers.concatenate([layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(bn7), conv2], axis=3)
    conv8 = layers.Conv2D(128, (3, 3), padding='same')(up8)
    bn8 = layers.Activation('relu')(conv8)
    conv8 = layers.Conv2D(128, (3, 3), padding='same')(bn8)
    bn8 = layers.BatchNormalization(axis=3)(conv8)
    bn8 = layers.Activation('relu')(bn8)

    up9 = layers.concatenate([layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(bn8), conv1], axis=3)
    conv9 = layers.Conv2D(64, (3, 3), padding='same')(up9)
    bn9 = layers.Activation('relu')(conv9)
    conv9 = layers.Conv2D(64, (3, 3), padding='same')(bn9)
    bn9 = layers.BatchNormalization(axis=3)(conv9)
    bn9 = layers.Activation('relu')(bn9)

    conv10 = layers.Conv2D(1, (1, 1), activation='sigmoid')(bn9)

    return models.Model(inputs=[inputs], outputs=[conv10])
model = unet()
model.summary()<jupyter_output>Model: "model"
__________________________________________________________________________________________________
 Layer (type)                   Output Shape         Param #     Connected to                     
==================================================================================================
 input_1 (InputLayer)           [(None, 256, 256, 3  0           []                               
                                )]                                                                
                                                                                                  
 conv2d (Conv2D)                (None, 256, 256, 64  1792        ['input_1[0][0]']                
                                )                                                                 
                                                                                                  
 activation (Activation)        (None, 256, 256, 64  0           ['conv2d[0][0]']             [...]<jupyter_text>The model was trained on Cloud GPU<jupyter_code>EPOCHS = 150
BATCH_SIZE = 2
learning_rate = 1e-4
BATCH_SIZE = 2
decay_rate = learning_rate / EPOCHS
opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=decay_rate, amsgrad=False)
train_generator_args = dict(rotation_range=0.2,
                            width_shift_range=0.05,
                            height_shift_range=0.05,
                            shear_range=0.05,
                            zoom_range=0.05,
                            horizontal_flip=True,
                            fill_mode='nearest')

train_gen = train_generator(df_train, BATCH_SIZE,
                            train_generator_args,
                            target_size=(256, 256))
    
test_gener = train_generator(df_val, BATCH_SIZE,
                             dict(),
                             target_size=(256, 256))
    
model = unet(input_size=(256, 256, 3))

model.compile(optimizer=opt, 
              loss=dice_coef_loss, 
              metrics=["binary_accuracy", iou, dice_coef])

callbacks = [ModelCheckpoint('./Brain_MRI_Model', verbose=1, save_best_only=True)]

history = model.fit(train_gen,
                    steps_per_epoch=len(df_train) / BATCH_SIZE, 
                    epochs=EPOCHS, 
                    callbacks=callbacks,
                    validation_data = test_gener,
                    validation_steps=len(df_val) / BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Predictions with model<jupyter_code># Loading the trained and downloaded model from the cloud
load_model = tf.keras.models.load_model("unet_brain_mri_seg.hdf5", custom_objects={'dice_coef_loss': dice_coef_loss, 
                                                                                   'iou': iou, 
                                                                                   'dice_coef': dice_coef})
load_model.summary()
# Generating test paths
test_filepaths = list(df_test["filename"])
test_mask_filepaths = list(df_test["mask"])
len(test_filepaths), len(test_mask_filepaths)
plt.axis(False);

img = io.imread(file_paths[444])
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

mask = io.imread(file_paths_masked[444])

img[mask == 255] = (255, 0, 0)
plt.imshow(img)
plt.axis(False);
# To plot our predictions
def plot_random_preds():
    plt.figure(figsize=(17, 10))
    rand_arr = random.sample(range(len(test_filepaths)), 3)
    i_ = 0
    for i in range(0, 11, 5):
        # getting the images
        img = io.imread(test_filepaths[rand_arr[i_]])
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        true_mask = io.imread(test_mask_filepaths[rand_arr[i_]])
        
        # making predictions
        pred_mask = load_model.predict(tf.expand_dims(img, axis=0)/255)
        pred_mask = pred_mask[np.newaxis, :, :, :]
        pred_mask = np.squeeze(pred_mask) > .5

        # Plotting the images
        plt.subplot(3, 5, i+1)
        plt.axis(False)
        plt.title("Image", color="blue")
        plt.imshow(img)
        
        # To plot original image
        plt.subplot(3, 5, i+2)
        plt.axis(False)
        plt.title("True Mask", color="green")
        plt.imshow(true_mask)
        
        # To plot pred 
        plt.subplot(3, 5, i+3)
        plt.axis(False)
        plt.title("Pred Mask", color="red")
        plt.imshow(pred_mask)
        
        # To plot orig mask on image
        img[true_mask == 255] = (0, 255, 0)
        plt.subplot(3, 5, i+4)
        plt.axis(False)
        plt.title("True Mask", color="green")
        plt.imshow(img)
        
        # To plot pred mask on image 
        img_ = io.imread(test_filepaths[rand_arr[i_]])
        img_ = cv2.cvtColor(img_, cv2.COLOR_BGR2RGB)
        img_[pred_mask == 1] = (255, 0, 0)
        plt.subplot(3, 5, i+5)
        plt.axis(False)
        plt.title("Pred Mask", color="red")
        i_ += 1
        plt.imshow(img_);
# Plotting random preds from test
plot_random_preds()
# Plotting random preds from test
plot_random_preds()
# Plotting random preds from test
plot_random_preds()
# Plotting random preds from test
plot_random_preds()
img = io.imread("TCGA_CS_4944_20010208_11.tif")
img.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1372661048421654, 'fraction_numerical': 0.043065276389087276, 'mean_word_length': 2.91582774049217, 'pattern_counts': {'":': 3, '<': 18, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '197465', 'license': 'MIT', 'n_tokens_mistral': 5195, 'n_tokens_neox': 4872, 'n_words': 875, 'path': 'Brain_MRI.ipynb', 'repo_name': 'Hrushi11/Brain-Tumor-Detection', 'size': '14002'}