<jupyter_start><jupyter_text>Mask R-CNN - Train on Camus DatasetThis notebook shows how to perform transfer learning of Mask R-CNN models using echocardiogram images from the [Camus Project](https://www.creatis.insa-lyon.fr/Challenge/camus/) dataset. Authors: **<NAME>** and **<NAME>**  1. Setup and data preparation1. import the necessary Python modules 2. Download pre-trained model weights3. preparing the data to be used by the Mask R-CNN model<jupyter_code>%load_ext autoreload
%autoreload 2

# Imports and definition of constant paths
import os
import sys
from glob import glob
from tqdm import tqdm

from camus_image_processing import fix_resolution, pre_filter
from camus_image_processing import CamusConfig, CamusDataset, InferenceConfig

# Root directory of the project
ROOT_DIR = os.path.abspath("../../")

# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "logs")

# Local path to trained weights file
COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")

# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library

from mrcnn.config import Config
from mrcnn import utils
import mrcnn.model as modellib
from mrcnn import visualize
from mrcnn.model import log

# Download COCO trained weights from Releases if needed
if not os.path.exists(COCO_MODEL_PATH):
    utils.download_trained_weights(COCO_MODEL_PATH)

import cv2
import skimage
import numpy as np

import warnings
warnings.filterwarnings('ignore')

import matplotlib.pyplot as plt
%matplotlib inline
# The images are in data/ directory
patients_path = glob("../../../../data_chambers/patient*/")
# patients_path = glob("../../../../train/patient*/")

n_images = 0
for patient_path in tqdm(patients_path, ncols=80):
    masks_filenames = glob(patient_path + "*CH_*_gt_chambers.png")
    for mask_filename in masks_filenames:
        raw_filename = mask_filename.replace('_gt_chambers', '')
        
        try:
            image = cv2.imread(raw_filename).max(axis=2).astype(np.uint8)
            mask = cv2.imread(mask_filename).max(axis=2).astype(np.uint8)
            if image.shape != mask.shape:
                raise RuntimeError("Image and mask shapes are not equal")
            
            image_resized = fix_resolution(image).astype(np.uint8)
            mask_resized = fix_resolution(mask)
            mask_ventricule_filtered = pre_filter(mask_resized).astype(np.uint8)
            if len(np.unique(mask_ventricule_filtered)) == 2:
                mask_channels_resized = mask_ventricule_filtered.copy()
            else:
                mask_channels_resized = cv2.cvtColor(mask_ventricule_filtered, cv2.COLOR_GRAY2BGR)
                mask_channels_resized[:,:,0][mask_channels_resized[:,:,0] != 1] = 0
                mask_channels_resized[:,:,1][mask_channels_resized[:,:,1] != 2] = 0
                mask_channels_resized[:,:,2][mask_channels_resized[:,:,2] != 3] = 0

            resized_raw_filename = raw_filename.replace(".png", "_resized.png")
            resized_mask_filename = mask_filename.replace(".png", "_resized.png")
            cv2.imwrite(resized_raw_filename, image_resized)
            cv2.imwrite(resized_mask_filename, mask_channels_resized)
        except:
            continue
        
        n_images += 1
str(n_images) + " processed images"
# display the last processed images
img1, img2 = image_resized, mask_channels_resized
try:
    img3 = cv2.cvtColor(img1, cv2.COLOR_GRAY2RGB)
except:
    img3 = img1.copy()

edged = cv2.Canny(((img2 == 1)*255).astype(np.uint8), 30, 200) 
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) 
cv2.drawContours(img3, contours, -1, (0, 255, 0), 1) 
  
plt.figure(figsize=(15, 4))
plt.subplot(1,3,1)
plt.title("Original image")
plt.imshow(img1, cmap='gray');
plt.subplot(1,3,2)
plt.title("Mask")
plt.imshow(img2 * 85, cmap='gray');
plt.subplot(1,3,3)
plt.title("Masked image")
plt.imshow(img3, cmap='gray');<jupyter_output><empty_output><jupyter_text>2. Creating instances for the classes that handle the CamusDatasetThe classes **CamusConfig** and **CamusDataset** extend mrcnn configuration classes to account for specificities of the Camus data set.<jupyter_code>dataset_train = CamusDataset()
dataset_val = CamusDataset()

config = CamusConfig()
config.display()<jupyter_output>Configurations:
BACKBONE                       resnet101
BACKBONE_STRIDES               [4, 8, 16, 32, 64]
BATCH_SIZE                     4
BBOX_STD_DEV                   [0.1 0.1 0.2 0.2]
COMPUTE_BACKBONE_SHAPE         None
DETECTION_MAX_INSTANCES        100
DETECTION_MIN_CONFIDENCE       0.7
DETECTION_NMS_THRESHOLD        0.3
FPN_CLASSIF_FC_LAYERS_SIZE     1024
GPU_COUNT                      1
GRADIENT_CLIP_NORM             5.0
IMAGES_PER_GPU                 4
IMAGE_CHANNEL_COUNT            3
IMAGE_MAX_DIM                  128
IMAGE_META_SIZE                14
IMAGE_MIN_DIM                  128
IMAGE_MIN_SCALE                0
IMAGE_RESIZE_MODE              square
IMAGE_SHAPE                    [128 128   3]
LEARNING_MOMENTUM              0.9
LEARNING_RATE                  0.001
LOSS_WEIGHTS                   {'rpn_class_loss': 1.0, 'rpn_bbox_loss': 1.0, 'mrcnn_class_loss': 1.0, 'mrcnn_bbox_loss': 1.0, 'mrcnn_mask_loss': 1.0}
MASK_POOL_SIZE                 14
MASK_SHAPE             [...]<jupyter_text>3. Training the model (transfer learning)1. Split patients data set to train and validate the model2. Train model on train set3. Export weights to file<jupyter_code># Patients that have ids with endding in any digit but 1 are used to train the model
dataset_train.load_camus("../../../../data_chambers/patient*[2,3,4,5,6,7,8,9,0]/", config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1])
# dataset_train.load_camus("../../../../train/patient*[2,3,4,5,6,7,8,9,0]/", config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1])
dataset_train.prepare()

# Patients that have ids with endding in 1 are used to validate/test the model
dataset_val.load_camus("../../../../data_chambers/patient*[1]/", config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1])
# dataset_val.load_camus("../../../../train/patient*[1]/", config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1])
dataset_val.prepare()

str(len(dataset_train)) + " images loaded for training", str(len(dataset_val)) + " images loaded for validation"
# Load and display random samples
image_ids = np.random.randint(0, len(dataset_train), 4)
for image_id in image_ids:
    image = dataset_train.load_image(image_id)
    mask, class_ids = dataset_train.load_mask(image_id)
    visualize.display_top_masks(image, mask, class_ids, dataset_train.class_names)
    print(mask.shape)
# Create model in training mode
model = modellib.MaskRCNN(mode="training", config=config,
                          model_dir=MODEL_DIR)
# Load weights trained on MS COCO, but skip layers that
# are different due to the different number of classes
# See README for instructions to download the COCO weights
model.load_weights(COCO_MODEL_PATH, by_name=True,
                   exclude=["mrcnn_class_logits", "mrcnn_bbox_fc", 
                            "mrcnn_bbox", "mrcnn_mask"])
model.train(dataset_train, dataset_val, 
            learning_rate=config.LEARNING_RATE, 
            epochs=12,
            layers='heads')
# Save the trained model weights as a .h5 file
model_path = os.path.join(MODEL_DIR, "mask_rcnn_camus_chambers.h5")
model.keras_model.save_weights(model_path)<jupyter_output><empty_output><jupyter_text>4. Inference - predicting atrium, ventricle and heart muscle in new echocardiogram studies1. Configure the inference task2. Load the weights for the trained model3. Perform inference for new data and visualize results<jupyter_code>inference_config = InferenceConfig()

# Recreate the model in inference mode
model = modellib.MaskRCNN(mode="inference", 
                          config=inference_config,
                          model_dir=MODEL_DIR)

# Get path to saved weights
# Either set a specific path or find last trained weights
# model_path = os.path.join(ROOT_DIR, ".h5 file name here")
model_path = model.find_last()

# Load trained weights
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# Test on a random image from validation set
image_id = np.random.randint(0, len(dataset_val))
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
    modellib.load_image_gt(dataset_val, inference_config, 
                           image_id, use_mini_mask=False)

log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)

visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, 
                            dataset_train.class_names, figsize=(8, 8))
def get_ax(rows=1, cols=1, box_id=1, size=8):
    """Return a Matplotlib Axes array to be used in
    all visualizations in the notebook. Provide a
    central point to control graph sizes.
    
    Change the default size attribute to control the size
    of rendered images
    """
    ax = plt.subplot(rows, cols, box_id)
    return ax

results = model.detect([original_image], verbose=True)

r = results[0]
plt.figure(figsize=(8,8))
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], 
                            dataset_val.class_names, r['scores'], ax=get_ax())
image_ids = np.random.randint(0, len(dataset_val), 12)

plt.figure(figsize=(16,12))
for i, id in enumerate(image_ids):
    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
        modellib.load_image_gt(dataset_val, inference_config, 
                               id, use_mini_mask=False)

    results = model.detect([original_image])
    r = results[0]
    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], 
                                dataset_val.class_names, r['scores'], ax=get_ax(3, 4, i+1))
plt.tight_layout();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1072073880746838, 'fraction_numerical': 0.022485444689821322, 'mean_word_length': 3.4677130044843048, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '123446', 'license': 'MIT', 'n_tokens_mistral': 3406, 'n_tokens_neox': 3264, 'n_words': 768, 'path': 'samples/camus/Build dataset with PNG images and traing Mask-CRNN model.ipynb', 'repo_name': 'Tauffer-Consulting/Mask_RCNN', 'size': '9975'}