<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive

drive.mount('/content/gdrive/')
# %tensorflow_version 2.x
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow
from tensorflow import keras
from tensorflow.keras import backend
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.preprocessing import label_binarize
from sklearn.metrics import roc_curve, auc, confusion_matrix, accuracy_score
from sklearn.svm import OneClassSVM
from sklearn.neighbors import LocalOutlierFactor
from keras.utils import plot_model
import matplotlib.pyplot as plt
from scipy import interp
import numpy as np
import tqdm
import math
import os
import cv2
! pip install git+https://github.com/divamgupta/image-segmentation-keras.git<jupyter_output><empty_output><jupyter_text>**Prepare data**<jupyter_code># https://lhncbc.nlm.nih.gov/publication/pub9356

# DATASET

left_path = '/content/gdrive/My Drive/ECE1512/MontgomerySet/ManualMask/leftMask'
right_path = '/content/gdrive/My Drive/ECE1512/MontgomerySet/ManualMask/rightMask'
cxr_path = '/content/gdrive/My Drive/ECE1512/MontgomerySet/CXR_png'
mask_path = '/content/gdrive/My Drive/ECE1512/MontgomerySet/Mask'
cxr_dirs = os.listdir(cxr_path)

if not os.path.exists(mask_path):
  for item in cxr_dirs:
    try:
      left = cv2.imread(os.path.join(left_path, item))
      right = cv2.imread(os.path.join(right_path, item))
      mask = cv2.add(left, right)
      cv2.imwrite(mask_path + '/' + item, mask)
    except:
      continue
# PREPARE DATASET FOR keras_segmentation
from math import ceil
from tqdm import tqdm

cxr_dirs = os.listdir('/content/gdrive/My Drive/ECE1512/MontgomerySet/CXR_png')
segmentation_train_size = ceil(len(cxr_dirs) * 0.9)

if cxr_dirs:
  idx = 0
  pbar = tqdm(total=len(cxr_dirs))
  for item in cxr_dirs:
    try:
      if idx < segmentation_train_size:
        image = cv2.resize(cv2.imread(os.path.join(cxr_path, item)), (473, 473))
        cv2.imwrite(os.path.join('/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_images', item), image)
        mask = cv2.resize(cv2.imread(os.path.join(mask_path, item)), (473, 473))
        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
        _, mask = cv2.threshold(mask,100,255,cv2.THRESH_BINARY)
        mask = np.repeat(np.expand_dims(mask/255, axis=-1), repeats=3, axis=2)
        cv2.imwrite(os.path.join('/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_segmentation', item), mask)
      else:
        image = cv2.resize(cv2.imread(os.path.join(cxr_path, item)), (473, 473))
        cv2.imwrite(os.path.join('/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_images', item), image)
        mask = cv2.resize(cv2.imread(os.path.join(mask_path, item)), (473, 473))
        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
        _, mask = cv2.threshold(mask,100,255,cv2.THRESH_BINARY)
        mask = np.repeat(np.expand_dims(mask/255, axis=-1), repeats=3, axis=2)
        cv2.imwrite(os.path.join('/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_segmentation', item), mask)
    except:
      print("failed at " + item)
      continue
    finally:
      idx += 1
      pbar.update(1)
  pbar.close()<jupyter_output><empty_output><jupyter_text>**Define visualization functions**<jupyter_code># DEFINE FUNCTION TO VISUALIZE SEGMENTATION RESULTS on MontgomerySet and cxr
import cv2
import os
import random

def visualize_MS_segmentation(model_to_visualize, dataset="MontgomerySet", subset="val", idx=0):
  dirs = os.listdir(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, 'dataset', (subset + '_images')))
  image_name = dirs[idx]
  x = cv2.resize(cv2.imread(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, 'dataset', (subset + '_images'), image_name)), (473, 473))
  ground_truth = cv2.resize(cv2.imread(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, 'dataset', (subset + '_segmentation'), image_name)), (473, 473))
  ground_truth[ground_truth == 1] = 255
  y = model_to_visualize.predict_segmentation(x)
  y[y == 1] = 255
  # show image and segmentation
  plt.figure(figsize=(10,10))
  plt.subplot(1,3,1)
  plt.title(dataset + ' ' + subset + ' image x')
  plt.imshow(x)
  plt.subplot(1,3,2)
  plt.title(dataset + ' ' + subset + ' segmentation y')
  plt.imshow(y, cmap='gray')
  plt.subplot(1,3,3)
  plt.title(dataset + ' ' + subset + ' ground truth')
  _ = plt.imshow(ground_truth, cmap='gray')

def visualize_cxr_segmentation(model_to_visualize, dataset="chest_xray", subset="train", idx=0):
  plt.figure(figsize=(10,10))
  for i, subfolder in enumerate(['bacteria', 'normal', 'virus']):
    dirs = os.listdir(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, subset, subfolder))
    image_name = dirs[idx]
    x = cv2.resize(cv2.imread(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, subset, subfolder, image_name)), (473, 473))
    # ground_truth = cv2.resize(cv2.imread(os.path.join('/content/gdrive/My Drive/ECE1512', dataset, subset, subfolder, image_name)), (473, 473))
    # ground_truth[ground_truth == 1] = 255
    y = model_to_visualize.predict_segmentation(x)
    y[y == 1] = 255
    # show image and segmentation
    plt.subplot(3,2,i*2+1)
    plt.title(dataset + ' ' + subset + ' ' + subfolder + ' image x')
    plt.imshow(x)
    plt.subplot(3,2,i*2+2)
    plt.title(dataset + ' ' + subset + ' ' + subfolder + ' segmentation y')
    plt.imshow(y, cmap='gray')<jupyter_output><empty_output><jupyter_text>**Define some segmentation models**<jupyter_code># TRY SOME SEGMENTATION MODELS
from keras_segmentation.models.segnet import segnet
from keras_segmentation.models.pspnet import pspnet_50, pspnet_101

lungseg_segnet = segnet(n_classes=2, input_height=473, input_width=473, encoder_level=3)
lungseg_pspnet50 = pspnet_50(n_classes=2, input_height=473, input_width=473)
lungseg_pspnet101 = pspnet_101(n_classes=2, input_height=473, input_width=473)<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.

WARNING:tensorflow:From /usr/lo[...]<jupyter_text>**Lungseg_segnet**<jupyter_code>lungseg_segnet.summary()
# plot_model(lungset_segnet, show_shapes=True)
# TRAIN lungseg_segnet
lungseg_segnet_history = lungseg_segnet.train(
    train_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_images',
    train_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_segmentation',
    checkpoints_path='/content/gdrive/My Drive/ECE1512/MontgomerySet/lungseg_segnet/lungseg_segnet_weights.hdf5',
    epochs=50,
    batch_size=2,
    validate=True,
    val_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_images',
    val_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_segmentation',
    auto_resume_checkpoint=True,
    steps_per_epoch=63)
# VISUALIZEE SEGMENTATION RESULTS ON MontgomerySet and cxr
visualize_MS_segmentation(lungseg_segnet, subset='train', idx=1)
visualize_cxr_segmentation(lungseg_segnet, subset='train', idx=1)<jupyter_output><empty_output><jupyter_text>**Lungseg_pspnet50**<jupyter_code>plot_model(lungseg_pspnet50, show_shapes=True)
# TRAIN lungseg_pspnet50

lungseg_pspnet50_history = lungseg_pspnet50.train(
    train_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_images',
    train_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_segmentation',
    checkpoints_path='/content/gdrive/My Drive/ECE1512/MontgomerySet/lungseg_pspnet50/lungseg_pspnet50_weights.hdf5',
    epochs=50,
    batch_size=2,
    validate=True,
    val_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_images',
    val_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_segmentation',
    auto_resume_checkpoint=False,
    steps_per_epoch=63)
# VISUALIZEE SEGMENTATION RESULTS ON MontgomerySet and cxr
visualize_MS_segmentation(lungseg_pspnet50, subset='train', idx=1)
visualize_cxr_segmentation(lungseg_pspnet50, subset='train', idx=1)<jupyter_output><empty_output><jupyter_text>**Lungseg_pspnet101**<jupyter_code>plot_model(lungseg_pspnet101, show_shapes=True)
# TRAIN lungseg_pspnet101

lungseg_pspnet101_history = lungseg_pspnet101.train(
    train_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_images',
    train_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_segmentation',
    checkpoints_path='/content/gdrive/My Drive/ECE1512/MontgomerySet/lungseg_pspnet101/lungseg_pspnet101_weights.hdf5',
    epochs=10,
    batch_size=2,
    validate=True,
    val_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_images',
    val_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_segmentation',
    auto_resume_checkpoint=False,
    steps_per_epoch=63)
# VISUALIZEE SEGMENTATION RESULTS ON MontgomerySet and cxr
# visualize_MS_segmentation(lungseg_pspnet101, subset='train')
visualize_cxr_segmentation(lungseg_pspnet101, subset='train')<jupyter_output><empty_output><jupyter_text>**lungseg**<jupyter_code># https://divamgupta.com/image-segmentation/2019/06/06/deep-learning-semantic-segmentation-keras.html

# LOAD PRETRAINED SEGMENTATION MODEL pspnest_101_voc12
from keras_segmentation.pretrained import pspnet_101_voc12
from keras.utils import plot_model

pspnet_101 = pspnet_101_voc12() # load the pretrained model trained on Pascal VOC 2012 dataset
# pspnet_101.summary()
# plot_model(pspnet_101, show_shapes=True)
# BUILD NEW SEGMENTATION MODEL lungseg BASED ON pspnet_101
from types import MethodType
from keras import metrics
from keras.models import Model
from keras.optimizers import RMSprop
from keras.layers import Conv2D, Reshape, Dense, Softmax, Activation
from keras_segmentation.models._pspnet_2 import Interp
from keras_segmentation.models.model_utils import get_segmentation_model, resize_image
from keras_segmentation.train import train
from keras_segmentation.predict import predict, predict_multiple, evaluate

# make adjustment to pspnet_101
# add a global spatial average pooling layer
x = pspnet_101.layers[-5]
x = Conv2D(2, (1, 1), strides=(1, 1), name="last_conv")(x.output)
x = Interp([473, 473], name='last_interp')(x)

# ...get_segmentation_model...
o = (Reshape((473*473, -1)))(x)
o = (Activation('softmax'))(o)
lungseg = Model(inputs=pspnet_101.input, outputs=o)
lungseg.n_classes = 2
lungseg.input_height = 473
lungseg.input_width = 473
lungseg.output_height = 473
lungseg.output_width = 473
lungseg.model_name = "pspnet_101"
lungseg.train = MethodType(train, lungseg)
lungseg.predict_segmentation = MethodType(predict, lungseg)
lungseg.predict_multiple = MethodType(predict_multiple, lungseg)
lungseg.evaluate_segmentation = MethodType(evaluate, lungseg)

# freeze pspnet_101 layers before concatenation layer
for layer in pspnet_101.layers[:-8]:
   layer.trainable = False
# train all layers
for layer in lungseg.layers:
   layer.trainable = True
# rmsprop optimizer of lungseg
rmsprop = RMSprop()
# compile the model (should be done *after* setting layers to non-trainable)
lungseg.compile(optimizer=rmsprop, loss='binary_crossentropy',
               metrics=[metrics.binary_accuracy])
# lungseg.summary()
# plot_model(lungseg, show_shapes=True)
# TRAIN lungseg MODEL on MontgomerySet
from keras.callbacks import ModelCheckpoint

lungseg_history = lungseg.train(
    train_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_images',
    train_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/train_segmentation',
    checkpoints_path='/content/gdrive/My Drive/ECE1512/MontgomerySet/lungseg/lungseg_weights.hdf5',
    epochs=1,
    batch_size=2,
    validate=True,
    val_images='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_images',
    val_annotations='/content/gdrive/My Drive/ECE1512/MontgomerySet/dataset/val_segmentation',
    steps_per_epoch=63)
# LOAD lungseg
from keras_segmentation.models.pspnet import pspnet_101

# re-create a fresh new pspnet_101 model and load weights
lungseg = pspnet_101(n_classes=2, input_height=473, input_width=473)
lungseg.load_weights('/content/gdrive/My Drive/ECE1512/MontgomerySet/lungseg/lungseg_weights.hdf5.0')
# VISUALIZEE SEGMENTATION RESULTS ON MontgomerySet and cxr
# visualize_MS_segmentation(lungseg, subset='train')
visualize_cxr_segmentation(lungseg, subset='train')<jupyter_output><empty_output><jupyter_text>**Save ROI images**<jupyter_code>count = 0
for folder in ['train', 'validation', 'test']:
  for subfolder in ['virus', 'bacteria']:
    dirs = os.listdir(os.path.join('/content/gdrive/My Drive/ECE1512/chest_xray/', folder, subfolder))
    print(folder, subfolder, len(dirs))
    count += len(dirs)
print(count)

count = 0
for folder in ['train', 'validation', 'test']:
  for subfolder in ['virus', 'bacteria']:
    dirs = os.listdir(os.path.join('/content/gdrive/My Drive/ECE1512/chest_xray/dataset/', folder, subfolder))
    print(folder, subfolder, len(dirs))
    count += len(dirs)
print(count)
# SAVE ROI IMAGES
from tqdm import tqdm
import cv2

folders = ['train', 'validation', 'test']
subfolders = ['bacteria', 'normal', 'virus']

# make folders
for folder in folders:
  folder_path = '/content/gdrive/My Drive/ECE1512/chest_xray/dataset/' + folder
  if not os.path.exists(folder_path):
    os.mkdir(folder_path)
  for subfolder in subfolders:
    subfolder_path = '/content/gdrive/My Drive/ECE1512/chest_xray/dataset/' + folder + '/' + subfolder
    if not os.path.exists(subfolder_path):
      os.mkdir(subfolder_path)

# produce masked images
for folder in ['train']:
  for subfolder in ['virus']:
    # print("Working on", folder, subfolder)
    dirs = os.listdir(os.path.join('/content/gdrive/My Drive/ECE1512/chest_xray/', folder, subfolder))
    pbar = tqdm(total=len(dirs))
    for image_name in dirs:
      pbar.update(1)
      x = cv2.resize(cv2.imread(os.path.join('/content/gdrive/My Drive/ECE1512/chest_xray', folder, subfolder, image_name)), (473, 473))
      y = lungseg.predict_segmentation(x)
      masked = np.copy(x)
      masked[y == 0] = 0
      target_subfolder = os.path.join('/content/gdrive/My Drive/ECE1512/chest_xray/dataset/', folder, subfolder)
      cv2.imwrite(os.path.join(target_subfolder, image_name), masked)
    pbar.close()<jupyter_output>100%|██████████| 1196/1196 [16:31<00:00,  1.23it/s]
{'fraction_non_alphanumeric': 0.12259960809928151, 'fraction_numerical': 0.040431090790333114, 'mean_word_length': 5.51531914893617, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '644967', 'license': 'MIT', 'n_tokens_mistral': 5921, 'n_tokens_neox': 5504, 'n_words': 935, 'path': 'ipynb/Segmentation.ipynb', 'repo_name': 'yiyinglai/Pneumonia', 'size': '15310'}