<jupyter_start><jupyter_text>Model.py<jupyter_code>from tensorflow.keras.layers import Conv2D, BatchNormalization, Activation, MaxPool2D, Conv2DTranspose, Concatenate, Input
from tensorflow.keras.models import Model
def conv_block(input, num_filters):
    x = Conv2D(num_filters, 3, padding="same")(input)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)

    x = Conv2D(num_filters, 3, padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)

    return x
def encoder_block(input, num_filters):
    x = conv_block(input, num_filters)
    p = MaxPool2D((2, 2))(x)
    return x, p
def decoder_block(input, skip_features, num_filters):
    x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding="same")(input)
    x = Concatenate()([x, skip_features])
    x = conv_block(x, num_filters)
    return x
def build_unet(input_shape):
    inputs = Input(input_shape)

    s1, p1 = encoder_block(inputs, 64)
    s2, p2 = encoder_block(p1, 128)
    s3, p3 = encoder_block(p2, 256)
    s4, p4 = encoder_block(p3, 512)

    b1 = conv_block(p4, 1024)

    d1 = decoder_block(b1, s4, 512)
    d2 = decoder_block(d1, s3, 256)
    d3 = decoder_block(d2, s2, 128)
    d4 = decoder_block(d3, s1, 64)

    outputs = Conv2D(1, 1, padding="same", activation="sigmoid")(d4)

    model = Model(inputs, outputs, name="U-Net")
    return model
input_shape = (512, 512, 3)
model = build_unet(input_shape)
model.summary()<jupyter_output>Model: "U-Net"
__________________________________________________________________________________________________
 Layer (type)                   Output Shape         Param #     Connected to                     
==================================================================================================
 input_1 (InputLayer)           [(None, 512, 512, 3  0           []                               
                                )]                                                                
                                                                                                  
 conv2d (Conv2D)                (None, 512, 512, 64  1792        ['input_1[0][0]']                
                                )                                                                 
                                                                                                  
 batch_normalization (BatchNorm  (None, 512, 512, 64  256        ['conv2d[0][0]']             [...]<jupyter_text>Metrics.py<jupyter_code>import numpy as np
import tensorflow as tf
from tensorflow.keras import backend as K
def iou(y_true, y_pred):
    def f(y_true, y_pred):
        intersection = (y_true * y_pred).sum()
        union = y_true.sum() + y_pred.sum() - intersection
        x = (intersection + 1e-15) / (union + 1e-15)
        x = x.astype(np.float32)
        return x
    return tf.numpy_function(f, [y_true, y_pred], tf.float32)
smooth = 1e-15
def dice_coef(y_true, y_pred):
    y_true = tf.keras.layers.Flatten()(y_true)
    y_pred = tf.keras.layers.Flatten()(y_pred)
    intersection = tf.reduce_sum(y_true * y_pred)
    return (2. * intersection + smooth) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + smooth)
def dice_loss(y_true, y_pred):
    return 1.0 - dice_coef(y_true, y_pred)<jupyter_output><empty_output><jupyter_text>train.py<jupyter_code>import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import cv2
from glob import glob
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger, ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import Recall, Precision<jupyter_output><empty_output><jupyter_text>Global parameters<jupyter_code>H = 512
W = 512
def load_data(path, split=0.1):
    images = sorted(glob(os.path.join(path, "CXR_png", "*.png")))
    masks1 = sorted(glob(os.path.join(path, "ManualMask", "leftMask", "*.png")))
    masks2 = sorted(glob(os.path.join(path, "ManualMask", "rightMask", "*.png")))

    split_size = int(len(images) * split)

    train_x, valid_x = train_test_split(images, test_size=split_size, random_state=42)
    train_y1, valid_y1 = train_test_split(masks1, test_size=split_size, random_state=42)
    train_y2, valid_y2 = train_test_split(masks2, test_size=split_size, random_state=42)

    train_x, test_x = train_test_split(train_x, test_size=split_size, random_state=42)
    train_y1, test_y1 = train_test_split(train_y1, test_size=split_size, random_state=42)
    train_y2, test_y2 = train_test_split(train_y2, test_size=split_size, random_state=42)

    return (train_x, train_y1, train_y2), (valid_x, valid_y1, valid_y2), (test_x, test_y1, test_y2)
def read_image(path):
    x = cv2.imread(path, cv2.IMREAD_COLOR)
    x = cv2.resize(x, (W, H))
    x = x/255.0
    x = x.astype(np.float32)
    return x
def read_mask(path1, path2):
    x1 = cv2.imread(path1, cv2.IMREAD_GRAYSCALE)
    x2 = cv2.imread(path2, cv2.IMREAD_GRAYSCALE)
    x = x1 + x2
    x = cv2.resize(x, (W, H))
    x = x/np.max(x)
    x = x > 0.5
    x = x.astype(np.float32)
    x = np.expand_dims(x, axis=-1)
    return x
def tf_parse(x, y1, y2):
    def _parse(x, y1, y2):
        x = x.decode()
        y1 = y1.decode()
        y2 = y2.decode()

        x = read_image(x)
        y = read_mask(y1, y2)
        return x, y

    x, y = tf.numpy_function(_parse, [x, y1, y2], [tf.float32, tf.float32])
    x.set_shape([H, W, 3])
    y.set_shape([H, W, 1])
    return x, y
def tf_dataset(X, Y1, Y2, batch=8):
    dataset = tf.data.Dataset.from_tensor_slices((X, Y1, Y2))
    dataset = dataset.shuffle(buffer_size=200)
    dataset = dataset.map(tf_parse)
    dataset = dataset.batch(batch)
    dataset = dataset.prefetch(4)
    return dataset<jupyter_output><empty_output><jupyter_text>Seeding<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)<jupyter_output><empty_output><jupyter_text>Hyperparameters<jupyter_code>batch_size = 2
lr = 1e-5
num_epochs = 10
model_path = os.path.join("/content/drive/MyDrive/files", "model.h5")
csv_path = os.path.join("/content/drive/MyDrive/files", "data.csv")<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>dataset_path = "/content/drive/MyDrive/MontgomerySet"
(train_x, train_y1, train_y2), (valid_x, valid_y1, valid_y2), (test_x, test_y1, test_y2) = load_data(dataset_path)
print(f"Train: {len(train_x)} - {len(train_y1)} - {len(train_y2)}")
print(f"Valid: {len(valid_x)} - {len(valid_y1)} - {len(valid_y2)}")
print(f"Test: {len(test_x)} - {len(test_y1)} - {len(test_y2)}")
train_dataset = tf_dataset(train_x, train_y1, train_y2, batch=batch_size)
valid_dataset = tf_dataset(valid_x, valid_y1, valid_y2, batch=batch_size)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>model = build_unet((H, W, 3))
metrics = [dice_coef, iou, Recall(), Precision()]
model.compile(loss=dice_loss, optimizer=Adam(lr), metrics=metrics)
callbacks = [
        ModelCheckpoint(model_path, verbose=1, save_best_only=True),
        ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, min_lr=1e-7, verbose=1),
        CSVLogger(csv_path)
    ]
model.fit(
        train_dataset,
        epochs=num_epochs,
        validation_data=valid_dataset,
        callbacks=callbacks
    )<jupyter_output>Epoch 1/10
56/56 [==============================] - ETA: 0s - loss: 0.5702 - dice_coef: 0.4298 - iou: 0.2780 - recall: 0.6587 - precision: 0.4490
Epoch 1: val_loss improved from inf to 0.66813, saving model to /content/drive/MyDrive/files/model.h5
56/56 [==============================] - 211s 3s/step - loss: 0.5702 - dice_coef: 0.4298 - iou: 0.2780 - recall: 0.6587 - precision: 0.4490 - val_loss: 0.6681 - val_dice_coef: 0.3258 - val_iou: 0.1959 - val_recall: 0.0683 - val_precision: 0.2616 - lr: 1.0000e-05
Epoch 2/10
56/56 [==============================] - ETA: 0s - loss: 0.3609 - dice_coef: 0.6391 - iou: 0.4737 - recall: 0.9093 - precision: 0.6488
Epoch 2: val_loss improved from 0.66813 to 0.66085, saving model to /content/drive/MyDrive/files/model.h5
56/56 [==============================] - 94s 2s/step - loss: 0.3609 - dice_coef: 0.6391 - iou: 0.4737 - recall: 0.9093 - precision: 0.6488 - val_loss: 0.6608 - val_dice_coef: 0.3346 - val_iou: 0.2023 - val_recall: 0.3499 - val_precision:[...]<jupyter_text>eval.py<jupyter_code>from tqdm import tqdm
from tensorflow.keras.utils import CustomObjectScope<jupyter_output><empty_output><jupyter_text>Seeding<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)<jupyter_output><empty_output><jupyter_text>Loading model<jupyter_code>with CustomObjectScope({'iou': iou, 'dice_coef': dice_coef, 'dice_loss': dice_loss}):
  model = tf.keras.models.load_model("/content/drive/MyDrive/files/model.h5")<jupyter_output><empty_output><jupyter_text>Predicting the mask<jupyter_code>for x, y1, y2 in tqdm(zip(test_x, test_y1, test_y2), total=len(test_x)):
  """ Extracing the image name. """
  image_name = x.split("/")[-1]

  """ Reading the image """
  ori_x = cv2.imread(x, cv2.IMREAD_COLOR)
  ori_x = cv2.resize(ori_x, (W, H))
  x = ori_x/255.0
  x = x.astype(np.float32)
  x = np.expand_dims(x, axis=0)

  """ Reading the mask """
  ori_y1 = cv2.imread(y1, cv2.IMREAD_GRAYSCALE)
  ori_y2 = cv2.imread(y2, cv2.IMREAD_GRAYSCALE)
  ori_y = ori_y1 + ori_y2
  ori_y = cv2.resize(ori_y, (W, H))
  ori_y = np.expand_dims(ori_y, axis=-1)  ## (512, 512, 1)
  ori_y = np.concatenate([ori_y, ori_y, ori_y], axis=-1)  ## (512, 512, 3)

  """ Predicting the mask. """
  y_pred = model.predict(x)[0] > 0.5
  y_pred = y_pred.astype(np.int32)

  """ Saving the predicted mask along with the image and GT """
  save_image_path = f"/content/drive/MyDrive/results/{image_name}"
  y_pred = np.concatenate([y_pred, y_pred, y_pred], axis=-1)

  sep_line = np.ones((H, 10, 3)) * 255

  cat_image = np.concatenate([ori_x, sep_line, ori_y, sep_line, y_pred*255], axis=1)
  cv2.imwrite(save_image_path, cat_image)<jupyter_output>100%|██████████| 13/13 [00:10<00:00,  1.29it/s]
{'fraction_non_alphanumeric': 0.15787358054466888, 'fraction_numerical': 0.0551703346397347, 'mean_word_length': 3.4748201438848922, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '629037', 'license': 'MIT', 'n_tokens_mistral': 4148, 'n_tokens_neox': 3780, 'n_words': 707, 'path': 'U-Net/UNETFinalImplementation.ipynb', 'repo_name': 'writetoharsh/Face-Recognition-Using-Siamese-Network', 'size': '9951'}