<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load libraries2&nbsp;&nbsp;Define loss functions3&nbsp;&nbsp;Define models4&nbsp;&nbsp;Training4.1&nbsp;&nbsp;Functions, generators and data4.2&nbsp;&nbsp;Training  Load libraries<jupyter_code>import cv2
import numpy as np
import pandas as pd

from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard
from keras.models import Model
from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Activation, UpSampling2D, BatchNormalization
from keras.optimizers import RMSprop
from keras.losses import binary_crossentropy
import keras.backend as K

from sklearn.model_selection import train_test_split
import math
import random
import gzip
import pickle
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Define loss functions<jupyter_code>def dice_coeff(y_true, y_pred):
    smooth = 1.
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
    return score


def dice_loss(y_true, y_pred):
    loss = 1 - dice_coeff(y_true, y_pred)
    return loss


def bce_dice_loss(y_true, y_pred):
    loss = binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
    return loss<jupyter_output><empty_output><jupyter_text>Define models<jupyter_code>def unet_down_one_block(inputs, num_filters):
    x = Conv2D(num_filters, (3, 3), padding='same')(inputs)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(num_filters, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    return x
def unet_max_pool(inputs):
    x = Conv2D(inputs.get_shape().as_list()[3],
               (2,2), strides=(2,2))(inputs)
    return x
def unet_up_one_block(up_input, down_input, num_filters):
    x = UpSampling2D((2,2))(up_input)
    x = concatenate([down_input, x], axis=3)
    x = Conv2D(num_filters, (3,3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(num_filters, (3,3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(num_filters, (3,3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    return x
def get_unet(input_shape = (256, 256, 3),
             num_classes = 1,
             initial_filters = 32,
             central_filters = 1024):
    
    num_filters = initial_filters
    
    out_list    = [Input(shape=input_shape)]
    down_interim_list = []
    
    while num_filters <= central_filters/2:
        x = unet_down_one_block(out_list[-1], num_filters)
        down_interim_list.append(x)
        num_filters = num_filters * 2
        y = unet_max_pool(x)
        out_list.append(y)
    
    x = unet_down_one_block(out_list[-1], num_filters)
    out_list.append(x)
    num_filters = int(num_filters / 2)
    
    while num_filters >= initial_filters:
        x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)
        out_list.append(x)
        num_filters = int(num_filters / 2)
    
    classify = Conv2D(num_classes, (1,1), activation = 'sigmoid')(out_list[-1])
    
    model = Model(inputs=out_list[0], outputs=classify)
    
    return model
model = get_unet(input_shape=(896,1280,3), initial_filters=8)
model.summary()<jupyter_output>____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (None, 896, 1280, 3)  0                                            
____________________________________________________________________________________________________
conv2d_1 (Conv2D)                (None, 896, 1280, 8)  224         input_1[0][0]                    
____________________________________________________________________________________________________
batch_normalization_1 (BatchNorm (None, 896, 1280, 8)  32          conv2d_1[0][0]                   
____________________________________________________________________________________________________
activation_1 (Activation)        (None, 896, 1280, 8)  0           batch_normalization_1[0][...]<jupyter_text>Training  Functions, generators and data<jupyter_code>df_train = pd.read_csv('data/train_masks.csv')
df_train.head()
train_bboxes = pickle.load(open('./train_bboxes_resized_m512_ang_09.pkl', 'rb'))
actual_bboxes = pickle.load(open('./train_bboxes_actuals.pkl', 'rb'))
ids_train = list(train_bboxes.keys())
ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42)
def randomHueSaturationValue(image, hue_shift_limit=(-180, 180),
                             sat_shift_limit=(-255, 255),
                             val_shift_limit=(-255, 255), u=0.5):
    if np.random.random() < u:
        image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(image)
        hue_shift = np.random.uniform(hue_shift_limit[0], hue_shift_limit[1])
        h = cv2.add(h, hue_shift)
        sat_shift = np.random.uniform(sat_shift_limit[0], sat_shift_limit[1])
        s = cv2.add(s, sat_shift)
        val_shift = np.random.uniform(val_shift_limit[0], val_shift_limit[1])
        v = cv2.add(v, val_shift)
        image = cv2.merge((h, s, v))
        image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)

    return image
def randomShiftScaleRotate(image, mask,
                           shift_limit=(-0.0625, 0.0625),
                           scale_limit=(-0.1, 0.1),
                           rotate_limit=(-45, 45), aspect_limit=(0, 0),
                           borderMode=cv2.BORDER_CONSTANT, u=0.5):
    if np.random.random() < u:
        height, width, channel = image.shape

        angle = np.random.uniform(rotate_limit[0], rotate_limit[1])  # degree
        scale = np.random.uniform(1 + scale_limit[0], 1 + scale_limit[1])
        aspect = np.random.uniform(1 + aspect_limit[0], 1 + aspect_limit[1])
        sx = scale * aspect / (aspect ** 0.5)
        sy = scale / (aspect ** 0.5)
        dx = round(np.random.uniform(shift_limit[0], shift_limit[1]) * width)
        dy = round(np.random.uniform(shift_limit[0], shift_limit[1]) * height)

        cc = np.math.cos(angle / 180 * np.math.pi) * sx
        ss = np.math.sin(angle / 180 * np.math.pi) * sy
        rotate_matrix = np.array([[cc, -ss], [ss, cc]])

        box0 = np.array([[0, 0], [width, 0], [width, height], [0, height], ])
        box1 = box0 - np.array([width / 2, height / 2])
        box1 = np.dot(box1, rotate_matrix.T) + np.array([width / 2 + dx, height / 2 + dy])

        box0 = box0.astype(np.float32)
        box1 = box1.astype(np.float32)
        mat = cv2.getPerspectiveTransform(box0, box1)
        image = cv2.warpPerspective(image, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,
                                    borderValue=(
                                        0, 0,
                                        0,))
        mask = cv2.warpPerspective(mask, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,
                                   borderValue=(
                                       0, 0,
                                       0,))

    return image, mask

def randomHorizontalFlip(image, mask, u=0.5):
    if np.random.random() < u:
        image = cv2.flip(image, 1)
        mask = cv2.flip(mask, 1)

    return image, mask

all_imgs  = {}
all_masks = {}
for id in ids_train:
    img  = cv2.imread('data/train/{}.jpg'.format(id))
    mask = cv2.imread('data/train_masks/{}_mask.png'.format(id), cv2.IMREAD_GRAYSCALE)
    all_imgs[id]  = img
    all_masks[id] = mask
def train_generator(train_batch_size):
    while True:
        this_ids_train_split = random.sample(ids_train_split, len(ids_train_split))
        for start in range(0, len(ids_train_split), train_batch_size):
            x_batch = []
            y_batch = []
            end = min(start + train_batch_size, len(ids_train_split))
            ids_train_batch = this_ids_train_split[start:end]
            for id in ids_train_batch:
                img  = all_imgs[id]
                mask = all_masks[id]
                
                this_bbox = train_bboxes[id]
                this_bbox = [math.floor(this_bbox[0]-25), math.ceil(this_bbox[1]+25),
                             math.floor(this_bbox[2]-20), math.ceil(this_bbox[3]+10)]
                
                x_size = this_bbox[1]-this_bbox[0]+1
                y_size = this_bbox[3]-this_bbox[2]+1
                
                x_diff=1280-x_size
                y_diff=896-y_size
                
                x_top=random.randint(0, min(x_diff, this_bbox[0]))
                x_bottom=min(x_diff-x_top, 1917-this_bbox[1])
                if (x_top+x_bottom) < x_diff:
                    if x_top==this_bbox[0]:
                        x_bottom = x_diff-x_top
                    else:
                        x_top = x_diff-x_bottom
                
                
                y_top=random.randint(0, min(y_diff, this_bbox[2]))
                y_bottom=min(y_diff-y_top, 1279-this_bbox[3])
                if (y_top+y_bottom) < y_diff:
                    if y_top==this_bbox[2]:
                        y_bottom = y_diff-y_top
                    else:
                        y_top = y_diff-y_bottom
                
                this_bbox[0] = this_bbox[0]-x_top
                this_bbox[1] = this_bbox[1]+x_bottom
                this_bbox[2] = this_bbox[2]-y_top
                this_bbox[3] = this_bbox[3]+y_bottom
                
                img  = img[this_bbox[2]:(this_bbox[3]+1), this_bbox[0]:(this_bbox[1]+1),:]
                mask = mask[this_bbox[2]:(this_bbox[3]+1), this_bbox[0]:(this_bbox[1]+1)]
                
                img = randomHueSaturationValue(img,
                                               hue_shift_limit=(-50, 50),
                                               sat_shift_limit=(-5, 5),
                                               val_shift_limit=(-15, 15))
                img, mask = randomHorizontalFlip(img, mask)
                
                mask = np.expand_dims(mask, axis=2)
                
                if img.shape[0]!=896 or img.shape[1]!=1280:
                    print(id)
                    print(x_top)
                    print(x_bottom)
                    print(y_top)
                    print(y_bottom)
                    print(x_diff)
                    print(y_diff)
                
                x_batch.append(img)
                y_batch.append(mask)
                
            x_batch = np.array(x_batch, np.float32) / 255
            y_batch = np.array(y_batch, np.float32) / 255
            yield x_batch, y_batch
def valid_generator(val_batch_size):
    while True:
        for start in range(0, len(ids_valid_split), val_batch_size):
            
            x_batch = []
            y_batch = []
            
            end = min(start + val_batch_size, len(ids_valid_split))
            ids_valid_batch = ids_valid_split[start:end]
            for id in ids_valid_batch:
                img  = all_imgs[id]
                mask = all_masks[id]
                
                this_bbox = list(actual_bboxes[id])
                
                x_size = this_bbox[1]-this_bbox[0]+1
                y_size = this_bbox[3]-this_bbox[2]+1
                
                x_diff=1280-x_size
                y_diff=896-y_size
                
                x_top=min(int(x_diff/2), this_bbox[0])
                x_bottom=min(x_diff-x_top, 1917-this_bbox[1])
                if (x_top+x_bottom) < x_diff:
                    if x_top==this_bbox[0]:
                        x_bottom = x_diff-x_top
                    else:
                        x_top = x_diff-x_bottom
                
                y_top=min(int(y_diff/2), this_bbox[2])
                y_bottom=min(y_diff-y_top, 1279-this_bbox[3])
                if (y_top+y_bottom) < y_diff:
                    if y_top==this_bbox[2]:
                        y_bottom = y_diff-y_top
                    else:
                        y_top = y_diff-y_bottom
                
                this_bbox[0] = this_bbox[0]-x_top
                this_bbox[1] = this_bbox[1]+x_bottom
                this_bbox[2] = this_bbox[2]-y_top
                this_bbox[3] = this_bbox[3]+y_bottom
                
                img  = img[this_bbox[2]:(this_bbox[3]+1), this_bbox[0]:(this_bbox[1]+1),:]
                mask = mask[this_bbox[2]:(this_bbox[3]+1), this_bbox[0]:(this_bbox[1]+1)]
                
                mask = np.expand_dims(mask, axis=2)
                
                x_batch.append(img)
                y_batch.append(mask)  
            x_batch = np.array(x_batch, np.float32) / 255
            y_batch = np.array(y_batch, np.float32) / 255
            yield x_batch, y_batch<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>train_batch_size = 6
val_batch_size   = 16
max_epochs = 50
model.compile(optimizer=RMSprop(lr=0.01), loss=bce_dice_loss, metrics=[dice_coeff])
callbacks = [EarlyStopping(monitor='val_loss',
                           patience=8,
                           verbose=1,
                           min_delta=1e-4),
             ReduceLROnPlateau(monitor='val_loss',
                               factor=0.1,
                               patience=4,
                               verbose=1,
                               epsilon=1e-4),
             ModelCheckpoint(monitor='val_loss',
                             filepath='weights/full_res_ang_09_no_pool.hdf5',
                             save_best_only=True,
                             save_weights_only=True),
             TensorBoard(log_dir='logs')]

history = model.fit_generator(generator=train_generator(train_batch_size),
                    steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),
                    epochs=max_epochs,
                    verbose=2,
                    callbacks=callbacks,
                    validation_data=valid_generator(val_batch_size),
                    validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))
callbacks = [EarlyStopping(monitor='val_loss',
                           patience=8,
                           verbose=1,
                           min_delta=1e-4),
             ReduceLROnPlateau(monitor='val_loss',
                               factor=0.1,
                               patience=4,
                               verbose=1,
                               epsilon=1e-4),
             ModelCheckpoint(monitor='val_loss',
                             filepath='weights/full_res_ang_09_no_pool.hdf5',
                             save_best_only=True,
                             save_weights_only=True),
             TensorBoard(log_dir='logs')]

history = model.fit_generator(generator=train_generator(train_batch_size),
                    steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),
                    epochs=max_epochs,
                    verbose=2,
                    callbacks=callbacks,
                    validation_data=valid_generator(val_batch_size),
                    validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))<jupyter_output>Epoch 1/50
159s - loss: 0.0416 - dice_coeff: 0.9828 - val_loss: 0.0742 - val_dice_coeff: 0.9741
Epoch 2/50
160s - loss: 0.0398 - dice_coeff: 0.9835 - val_loss: 0.0740 - val_dice_coeff: 0.9720
Epoch 3/50
160s - loss: 0.0402 - dice_coeff: 0.9834 - val_loss: 0.0995 - val_dice_coeff: 0.9643
Epoch 4/50
160s - loss: 0.0415 - dice_coeff: 0.9829 - val_loss: 0.0497 - val_dice_coeff: 0.9802
Epoch 5/50
160s - loss: 0.0374 - dice_coeff: 0.9843 - val_loss: 0.0525 - val_dice_coeff: 0.9799
Epoch 6/50
160s - loss: 0.0387 - dice_coeff: 0.9839 - val_loss: 0.0481 - val_dice_coeff: 0.9806
Epoch 7/50
160s - loss: 0.0370 - dice_coeff: 0.9845 - val_loss: 0.0579 - val_dice_coeff: 0.9773
Epoch 8/50
160s - loss: 0.0350 - dice_coeff: 0.9853 - val_loss: 0.0590 - val_dice_coeff: 0.9795
Epoch 9/50
160s - loss: 0.0395 - dice_coeff: 0.9840 - val_loss: 0.0896 - val_dice_coeff: 0.9653
Epoch 10/50
160s - loss: 0.0377 - dice_coeff: 0.9846 - val_loss: 0.1005 - val_dice_coeff: 0.9695
Epoch 11/50

Epoch 00010: reducing lear[...]
{'fraction_non_alphanumeric': 0.11581037179641439, 'fraction_numerical': 0.043556732041872215, 'mean_word_length': 1.9364069952305247, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '386597', 'license': 'MIT', 'n_tokens_mistral': 5930, 'n_tokens_neox': 5430, 'n_words': 887, 'path': '18-full-res-model-angle-09-bbox-no-maxpool.ipynb', 'repo_name': 'sainathadapa/kaggle-carvana-image-masking-challenge', 'size': '16622'}