<jupyter_start><jupyter_text>**UNet wall segmentation model using Keras & TesorFlow****Author: <NAME>****ETIS Laboratory UMR 8051, CY Cergy Paris University, ENSEA, CNRS**Version.0.0 - 02/02/2021* input: 512x512x1 (gray scale image)* data augemnted to 50 times* Opetimiser: RMS-prop* Loss function: Dice_coef loss function* learning epoch : 100* early stoping after 50 iteration* initial model performance : 93% of dice score tested on 30% of CVC-FP dataset<jupyter_code>import cv2
import os
import random
import re
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf
#import tensorflow.keras as keras
from tensorflow import keras
import tensorflow.keras as keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.callbacks import EarlyStopping
from keras.optimizers import Adam
import numpy as np
import os
import skimage.io as io
import skimage.transform as trans
import numpy as np
from keras.models import *
from keras.layers import *
from keras.optimizers import *
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.losses import categorical_crossentropy
from keras import backend as keras

from google.colab import drive
drive.mount('/content/gdrive')
# !!!!!!! IMPORTANT !!!!!!!!
# change data directories
train_frame_path = '/content/gdrive/MyDrive/data1/Training/Frames/frame/'
train_mask_path = '/content/gdrive/MyDrive/data1/Training/Masks/mask/'

val_frame_path = '/content/gdrive/MyDrive/data1/Validation/Frames/frame/'
val_mask_path = '/content/gdrive/MyDrive/data1/Validation/Masks/mask/'

weights_path = '/content/gdrive/MyDrive/data1/unet_CVC_modif_BckGrnd.hdf5'<jupyter_output><empty_output><jupyter_text>Learning Parameters<jupyter_code>NO_OF_TRAINING_IMAGES = len(os.listdir('/content/gdrive/MyDrive/data1/Training/Frames/frame/'))
NO_OF_VAL_IMAGES = len(os.listdir('/content/gdrive/MyDrive/data1/Validation/Frames/frame/'))

NO_OF_EPOCHS = 150

BATCH_SIZE = 32

LEARNING_RATE=0.0001

IMG_WIDTH = 512
IMG_HEIGHT = 512
IMG_CHANNELS = 1<jupyter_output><empty_output><jupyter_text>Data laoding and augmentation<jupyter_code>train_datagen = ImageDataGenerator(rescale=1./255,
                                   zoom_range=0.3,
                                   horizontal_flip=True,
                                   vertical_flip=True,
                                   fill_mode="nearest",
                                   width_shift_range=0.1,
                                   height_shift_range=0.1)
        
val_datagen = ImageDataGenerator(rescale=1./255)

# Don't forget to change data directories
seed = 1
train_image_generator = train_datagen.flow_from_directory('/content/gdrive/MyDrive/data1/Training/Frames/', 
                                                          target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                          color_mode="grayscale", 
                                                          classes=None, 
                                                          class_mode=None, 
                                                          batch_size=BATCH_SIZE,
                                                          follow_links=False,
                                                          subset=None,
                                                          interpolation="nearest",
                                                          seed=seed,
                                                          shuffle=True)

train_mask_generator = train_datagen.flow_from_directory('/content/gdrive/MyDrive/data1/Training/Masks/', 
                                                         target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                         color_mode="grayscale", 
                                                         classes=None, 
                                                         class_mode=None, 
                                                         batch_size=BATCH_SIZE,
                                                         follow_links=False,
                                                         subset=None,
                                                         interpolation="nearest",
                                                         seed=seed,
                                                         shuffle=True)

val_image_generator = val_datagen.flow_from_directory('/content/gdrive/MyDrive/data1/Validation/Frames/', 
                                                          target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                          color_mode="grayscale", 
                                                          classes=None, 
                                                          class_mode=None, 
                                                          batch_size=BATCH_SIZE,
                                                          follow_links=False,
                                                          subset=None,
                                                          interpolation="nearest",
                                                          seed=seed)

val_mask_generator = val_datagen.flow_from_directory('/content/gdrive/MyDrive/data1/Validation/Masks/', 
                                                         target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                         color_mode="grayscale", 
                                                         classes=None, 
                                                         class_mode=None, 
                                                         batch_size=BATCH_SIZE,
                                                         follow_links=False,
                                                         subset=None,
                                                         interpolation="nearest", seed=seed)

train_generator = zip(train_image_generator, train_mask_generator)
val_generator = zip(val_image_generator, val_mask_generator)<jupyter_output>Found 300 images belonging to 1 classes.
Found 300 images belonging to 1 classes.
Found 100 images belonging to 1 classes.
Found 100 images belonging to 1 classes.<jupyter_text>Metrics and loss functions for testing<jupyter_code># Define IoU metric
from keras import backend as K
def iou_coeff(y_true, y_pred, smooth=1):
    intersection = K.sum(K.abs(y_true * y_pred), axis=[1,2,3])
    union = K.sum(y_true,[1,2,3])+K.sum(y_pred,[1,2,3])-intersection
    iou = K.mean((intersection + smooth) / (union + smooth), axis=0)
    return iou

def iou_coef(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    union = K.sum(y_true_f) + K.sum(y_pred_f) - intersection
    return intersection/union  
# Define Dice metric
def dice_coeff(y_true, y_pred, smooth=1):
    #print(y_true.shape,'<<<<<<<<<<<<>>>>>>>>>>',y_pred.shape)
    intersection = K.sum(y_true * y_pred, axis=[1,2,3])
    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])
    dice = K.mean((2. * intersection + smooth)/(union + smooth), axis=0)
    return dice



def dice_coef(y_true, y_pred, smooth=1):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)


def dice_coef_loss(y_true, y_pred):
    return 1-dice_coef(y_true, y_pred, smooth=1)
# Define specific loss function : focal loss
#https://arxiv.org/pdf/1708.02002.pdf
#https://github.com/keras-team/keras/issues/6261

def focal_loss(target, output, gamma=2):
    output /= K.sum(output, axis=-1, keepdims=True)
    eps = K.epsilon()
    output = K.clip(output, eps, 1. - eps)
    return -K.sum(K.pow(1. - output, gamma) * target * K.log(output), axis=-1)
# ignore some of the pixels as they are unknown, So all pixels with class 0 will have zero loss.
def ignore_unknown_xentropy(ytrue, ypred):
    #return (1-ytrue[:, :, :, 0])*categorical_crossentropy(ytrue, ypred)
    return (1-ytrue[:, :, 0])*categorical_crossentropy(ytrue, ypred)
def weighted_categorical_crossentropy(weights):
    # weights = [0.9,0.05,0.04,0.01]
    def wcce(y_true, y_pred):
        Kweights = K.constant(weights)
        if not K.is_tensor(y_pred): y_pred = K.constant(y_pred)
        y_true = K.cast(y_true, y_pred.dtype)
        return K.categorical_crossentropy(y_true, y_pred) * K.sum(y_true * Kweights, axis=-1)
    return wcce
def pixelwise_crossentropy(target, output):
    output = tf.clip_by_value(output, 10e-8, 1. - 10e-8)
    return - tf.reduce_sum(target * tf.math.log(output))<jupyter_output><empty_output><jupyter_text>Loading and compile pre-trained unet model on CVC original dataset<jupyter_code>import tensorflow as tf
#Build the model
inputs = tf.keras.layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
#s = tf.keras.layers.Lambda(lambda x: x / 255)(inputs)
s = tf.keras.layers.Lambda(lambda x: x / 1.)(inputs)

#Contraction path
c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(s)
c1 = tf.keras.layers.Dropout(0.1)(c1)
c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)
p1 = tf.keras.layers.MaxPooling2D((2, 2))(c1)

c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)
c2 = tf.keras.layers.Dropout(0.1)(c2)
c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)
p2 = tf.keras.layers.MaxPooling2D((2, 2))(c2)
 
c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)
c3 = tf.keras.layers.Dropout(0.2)(c3)
c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)
p3 = tf.keras.layers.MaxPooling2D((2, 2))(c3)
 
c4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)
c4 = tf.keras.layers.Dropout(0.2)(c4)
c4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)
p4 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(c4)
 
c5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)
c5 = tf.keras.layers.Dropout(0.3)(c5)
c5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)
p5 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(c5)

c6 = tf.keras.layers.Conv2D(512, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p5)
c6 = tf.keras.layers.Dropout(0.3)(c6)
c6 = tf.keras.layers.Conv2D(512, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)
p6 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(c6)

c7 = tf.keras.layers.Conv2D(1024, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p6)
c7 = tf.keras.layers.Dropout(0.3)(c7)
c7 = tf.keras.layers.Conv2D(1024, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)

#Expansive path 
u8 = tf.keras.layers.Conv2DTranspose(512, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = tf.keras.layers.concatenate([u8, c6])
c8 = tf.keras.layers.Conv2D(512, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)
c8 = tf.keras.layers.Dropout(0.3)(c8)
c8 = tf.keras.layers.Conv2D(512, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)

u9 = tf.keras.layers.Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = tf.keras.layers.concatenate([u9, c5])
c9 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)
c9 = tf.keras.layers.Dropout(0.3)(c9)
c9 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)

u10 = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c9)
u10 = tf.keras.layers.concatenate([u10, c4])
c10 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u10)
c10 = tf.keras.layers.Dropout(0.2)(c10)
c10 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c10)
 
u11 = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c10)
u11 = tf.keras.layers.concatenate([u11, c3])
c11 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u11)
c11 = tf.keras.layers.Dropout(0.2)(c11)
c11 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c11)
 
u12 = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c11)
u12 = tf.keras.layers.concatenate([u12, c2])
c12 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u12)
c12 = tf.keras.layers.Dropout(0.1)(c12)
c12 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c12)
 
u13 = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c12)
u13 = tf.keras.layers.concatenate([u13, c1], axis=3)
c13 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u13)
c13 = tf.keras.layers.Dropout(0.1)(c13)
c13 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c13)
 
outputs = tf.keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(c13)
model = tf.keras.Model(inputs=[inputs], outputs=[outputs])
#model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',iou_coef, dice_coef])


##model = tf.keras.Model(inputs=[inputs], outputs=[outputs])
#model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', iou_coef, dice_coef])
#loss = weighted_categorical_crossentropy([0.9,0.05,0.04,0.01])
opt = tf.keras.optimizers.RMSprop(learning_rate=LEARNING_RATE)#'RMSprop'
model.compile(optimizer=opt, loss=dice_coef_loss, metrics=['accuracy', tf.keras.metrics.MeanIoU(num_classes=2), iou_coef, dice_coef])
model.summary()<jupyter_output>Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 512, 512, 1) 0                                            
__________________________________________________________________________________________________
lambda (Lambda)                 (None, 512, 512, 1)  0           input_1[0][0]                    
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 512, 512, 16) 160         lambda[0][0]                     
__________________________________________________________________________________________________
dropout (Dropout)               (None, 512, 512, 16) 0           conv2d[0][0]                 [...]<jupyter_text>to load an existing model and continue the training sequentially<jupyter_code>from tensorflow import keras
model = keras.models.load_model('/content/gdrive/MyDrive/data1/v500.hdf5', compile=False)
opt = tf.keras.optimizers.RMSprop(learning_rate=LEARNING_RATE)#'RMSprop'
model.compile(optimizer=opt, loss=dice_coef_loss, metrics=['accuracy', tf.keras.metrics.MeanIoU(num_classes=2), iou_coef, dice_coef])
model.summary()<jupyter_output>Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 512, 512, 1) 0                                            
__________________________________________________________________________________________________
lambda (Lambda)                 (None, 512, 512, 1)  0           input_1[0][0]                    
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 512, 512, 16) 160         lambda[0][0]                     
__________________________________________________________________________________________________
dropout (Dropout)               (None, 512, 512, 16) 0           conv2d[0][0]                 [...]<jupyter_text>Training Phase<jupyter_code>from PIL import Image
Image.MAX_IMAGE_PIXELS = 1000000000         

checkpoint = ModelCheckpoint('/content/gdrive/MyDrive/data1/v500.hdf5', verbose=1, save_best_only=True)

csv_logger = CSVLogger('/content/gdrive/MyDrive/data1/log_v500.out', append=True, separator=';')

earlystopping = EarlyStopping(monitor = 'loss', verbose = 1, min_delta = 0.0000001, patience = 50, mode = 'max')

callbacks_list = [checkpoint, csv_logger, earlystopping]


results = model.fit(train_generator, epochs=NO_OF_EPOCHS, 
                          steps_per_epoch=(NO_OF_TRAINING_IMAGES//BATCH_SIZE),
                          validation_data=val_generator, 
                          validation_steps=(NO_OF_VAL_IMAGES//BATCH_SIZE), 
                          callbacks=callbacks_list)
model.save('/content/gdrive/MyDrive/data1/Model__v500.h5')<jupyter_output>Epoch 1/150
9/9 [==============================] - 242s 30s/step - loss: 0.1068 - accuracy: 0.9370 - mean_io_u: 0.8216 - iou_coef: 0.8066 - dice_coef: 0.8928 - val_loss: 0.1407 - val_accuracy: 0.9555 - val_mean_io_u: 0.6500 - val_iou_coef: 0.7539 - val_dice_coef: 0.8593

Epoch 00001: val_loss improved from inf to 0.14069, saving model to /content/gdrive/MyDrive/data1/v500.hdf5
Epoch 2/150
9/9 [==============================] - 239s 30s/step - loss: 0.1864 - accuracy: 0.9221 - mean_io_u: 0.7784 - iou_coef: 0.7374 - dice_coef: 0.8209 - val_loss: 0.1703 - val_accuracy: 0.9526 - val_mean_io_u: 0.5520 - val_iou_coef: 0.7101 - val_dice_coef: 0.8297

Epoch 00002: val_loss did not improve from 0.14069
Epoch 3/150
9/9 [==============================] - 239s 30s/step - loss: 0.1966 - accuracy: 0.9213 - mean_io_u: 0.7572 - iou_coef: 0.7258 - dice_coef: 0.8094 - val_loss: 0.1549 - val_accuracy: 0.9547 - val_mean_io_u: 0.5719 - val_iou_coef: 0.7319 - val_dice_coef: 0.8451

Epoch 00003: val_loss did[...]<jupyter_text>analyses<jupyter_code>#plot the training and validation accuracy and loss at each epoch
loss = results.history['loss']
val_loss = results.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'y', label='Training loss')
plt.plot(epochs, val_loss, 'r', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Test Phase<jupyter_code>#test_frame_path = './CVC_walls_dataset_with_VERSPERA_background/test_frames/test/'
#test_mask_path = './CVC_walls_dataset_with_VERSPERA_background/test_masks/test/'
#NO_OF_TEST_IMAGES = len(os.listdir('./CVC_walls_dataset_with_VERSPERA_background/test_frames/'))


test_frame_path = './CVC_walls_dataset_with_VERSPERA_background/test_frames/test/'
test_mask_path = './CVC_walls_dataset_with_VERSPERA_background/test_masks/test/'
NO_OF_TEST_IMAGES = len(os.listdir('./CVC_walls_dataset_with_VERSPERA_background/test_frames/test/'))



test_image_generator = val_datagen.flow_from_directory('./CVC_walls_dataset_with_VERSPERA_background/test_frames/', 
                                                          target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                          color_mode="grayscale", 
                                                          classes=None, 
                                                          class_mode=None, 
                                                          batch_size=BATCH_SIZE,
                                                          follow_links=False,
                                                          subset=None,
                                                          interpolation="nearest",
                                                          seed=seed)

test_mask_generator = val_datagen.flow_from_directory('./CVC_walls_dataset_with_VERSPERA_background/test_masks/', 
                                                          target_size=(IMG_WIDTH,IMG_HEIGHT), 
                                                          color_mode="grayscale", 
                                                          classes=None, 
                                                          class_mode=None, 
                                                          batch_size=BATCH_SIZE,
                                                          follow_links=False,
                                                          subset=None,
                                                          interpolation="nearest",
                                                          seed=seed)

test_generator = zip(test_image_generator, test_mask_generator)

#pred= model.predict(test_gen, NO_OF_TEST_IMAGES//BATCH_SIZE, verbose=1)
score = model.evaluate(test_generator, steps=1, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=1)
BATCH_SIZE
#print(f'Test loss: {score[0]} / Test accuracy: {score[1]}')<jupyter_output><empty_output><jupyter_text>visualisation<jupyter_code>from tensorflow import keras

SIZE_X = IMG_HEIGHT #Resize images (height  = X, width = Y)
SIZE_Y = IMG_WIDTH

model = keras.models.load_model('./CVC_walls_dataset_with_VERSPERA_background/unet_CVC_modif_BckGrnd.hdf5', compile=False)
#Test on a different image
#READ EXTERNAL IMAGE...
test_img = cv2.imread('./CVC_walls_dataset_with_VERSPERA_background/test_frames/test/120.png', cv2.IMREAD_GRAYSCALE)       
test_img = cv2.resize(test_img, (SIZE_Y, SIZE_X))
#test_img = cv2.cvtColor(test_img, cv2.COLOR_RGB2BGR)
test_img = np.expand_dims(test_img, axis=0)

prediction = model.predict(test_img)
#View and Save segmented image
prediction_image = prediction.reshape(512,512)
plt.imshow(prediction_image, cmap='gray')
plt.imsave('./CVC_walls_dataset_with_VERSPERA_background/test0_segmented.jpg', prediction_image, cmap='gray')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13083333333333333, 'fraction_numerical': 0.041973684210526316, 'mean_word_length': 2.590708661417323, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 48, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 120, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '134925', 'license': 'MIT', 'n_tokens_mistral': 7629, 'n_tokens_neox': 6979, 'n_words': 1215, 'path': 'Unet_FP_Wall_Seg_Keras_model.ipynb', 'repo_name': 'swaileh/U-net_wall_Segmentation_FP_Keras_model', 'size': '22808'}