<jupyter_start><jupyter_text>Mask R-CNN - Test on Old Shapes DatasetRun the Mask R-CNN net in inference mode, with the additional PCILayer that generates the context-based tensor<jupyter_code>from IPython.core.display import display, HTML
display(HTML("<style>.container { width:95% !important; }</style>"))
%matplotlib inline
%load_ext autoreload
%autoreload 2
import sys
sys.path.append('../')
import tensorflow as tf
import keras.backend as KB
import numpy as np
from mrcnn.datagen     import data_generator, load_image_gt
from mrcnn.callbacks   import get_layer_output_1,get_layer_output_2
from mrcnn.utils       import mask_string
import mrcnn.visualize as visualize
from mrcnn.prep_notebook import prep_oldshapes_test
import matplotlib.pyplot as plt
# model_file  = 'E:\\Models\\mrcnn_logs\\shapes20180509T1928\\mask_rcnn_shapes_2192.h5'
model_file  = 'E:\Models\mrcnn_development_logs\shapes20180513T1946\\mask_rcnn_shapes_4044.h5'
# model_file  = 'E:\\Models\\mrcnn_oldshape_test_logs\\mask_rcnn_shapes_3877.h5'
# model_file   = 'E:\Models\mrcnn_logs\TESTshapes20180511T1742\mask_rcnn_shapes_0023.h5'
model, dataset_test, test_generator, inference_config = prep_oldshapes_test(init_with = model_file, FCN_layers = True, batch_sz = 1)
model.layer_info()
import matplotlib.pyplot as plt
def get_ax(rows=1, cols=1, size=8):
    """Return a Matplotlib Axes array to be used in
    all visualizations in the notebook. Provide a
    central point to control graph sizes.
    
    Change the default size attribute to control the size
    of rendered images
    """
    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))
    return ax<jupyter_output><empty_output><jupyter_text>Detection  Test on a random image<jupyter_code>import random
import pprint
from mrcnn.utils import log
pp = pprint.PrettyPrinter(indent=4, width=100)
image_id = random.choice(dataset_test.image_ids)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
    load_image_gt(dataset_test, inference_config, image_id, use_mini_mask=False)
    

print('Image Id :', image_id)    
shape_list = dataset_test.image_info[image_id]['shapes']
pp.pprint(shape_list)
log("original_image", original_image)
log("image_meta", image_meta)
print(image_meta)
log("gt_class_id", gt_bbox)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)

print(" 1: person   2: car  3: sun  4: building  5: tree  6: cloud ")
visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, 
                            dataset_test.class_names, figsize=(8, 8))
results = model.detect([original_image], verbose=1)<jupyter_output><empty_output><jupyter_text>Display Detection Results<jupyter_code>r = results[0]
print('  rois       : ', r['rois'])

print('  class ids  : ', r['class_ids'])
print('  class names: ', dataset_test.class_names)
print('  scores     : ', r['scores'])
visualize.display_instances_wo_mask(original_image, r['rois'],r['class_ids'], 
                            dataset_test.class_names, r['scores'])<jupyter_output><empty_output><jupyter_text>Run generator to create images<jupyter_code>train_batch_x, train_batch_y = next(test_generator)
imgmeta_idx = model.keras_model.input_names.index('input_image_meta')
img_meta    = train_batch_x[imgmeta_idx]

for img_idx in range(inference_config.BATCH_SIZE):
    image_id = img_meta[img_idx,0]
    image = dataset_test.load_image(image_id)
    mask, class_ids = dataset_test.load_mask(image_id)
    print('Image id: ',image_id)
    print('Image meta', img_meta[img_idx])
    print('Classes (1: circle, 2: square, 3: triangle ): ',class_ids)
    visualize.display_top_masks(image, mask, class_ids, dataset_test.class_names)
train_batch_x, train_batch_y = next(test_generator)

model_output = get_layer_output_2(model.keras_model, train_batch_x, 1)
# model_output = get_layer_output_1(model.keras_model, train_batch_x, [ 26], 1)

print(len(model_output))
 
for i in model_output:
    print( i.shape)
# print('FCN Normalized Loss is :', fcn_normalized_loss)    

detections                = model_output[0]          # layer:  0   shape: (16, 100, 6)
rpn_proposal_rois         = model_output[1]          # layer:  1   shape: (16, 1000, 4)
rpn_class                 = model_output[2]          # layer:  2   shape: (16, 4092, 2)
rpn_bbox                  = model_output[3]          # layer:  3   shape: (16, 4092, 4)
mrcnn_class               = model_output[4]          # layer:  4   shape: (16, 1000, 4)
mrcnn_bbox                = model_output[5]          # layer:  5   shape: (16, 1000, 4, 4)
input_image      =  train_batch_x[0]
input_image_meta =  train_batch_x[1]
input_image      =  train_batch_x[0]
input_image_meta =  train_batch_x[1]
print(rpn_roi_proposals.shape)
print(mrcnn_class.shape)
print(mrcnn_bbox.shape)
print(input_image_meta.shape)
print(input_image.shape , input_image_meta.shape)<jupyter_output><empty_output><jupyter_text>Inference Detection Layer<jupyter_code>from mrcnn.utils import parse_image_meta, apply_box_deltas
a1, a2, windows, a4 =  parse_image_meta(input_image_meta)
print(windows)
# image
# for b in range(self.config.BATCH_SIZE):
#     _, _, window, _ =  parse_image_meta(image_meta)
b=0
print(rpn_proposal_rois[b].shape, mrcnn_class[b].shape, mrcnn_bbox[b].shape)

# detections = refine_detections(rois[b], mrcnn_class[b], mrcnn_bbox[b], window[b], self.config)

rois = rpn_proposal_rois[b]
probs = mrcnn_class[b]
deltas = mrcnn_bbox[b]
window = windows[b]
config = inference_config
##  1. Find Class IDs with higest scores for each per ROI
class_ids       = np.argmax(probs, axis=1)
print(class_ids)
print(probs[0:100])
##  1. Find Class IDs with higest scores for each per ROI
class_ids       = np.argmax(probs, axis=1)
print(class_ids[:200])
##  2. Get Class probability(score) and bbox delta of the top class of each ROI
print(class_ids.shape)
print(deltas.shape)
class_scores    =  probs[np.arange(class_ids.shape[0]), class_ids]
deltas_specific = deltas[np.arange(deltas.shape[0])   , class_ids]
print(class_scores.shape)
print(deltas_specific.shape)

print(class_scores[:200])
# Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates
refined_rois    = apply_box_deltas(rois, deltas_specific * config.BBOX_STD_DEV)

##  4. Convert the refined roi coordiates from normalized to image domain
# TODO: better to keep them normalized until later   
height, width   = config.IMAGE_SHAPE[:2]
refined_rois   *= np.array([height, width, height, width])

##  5.  Clip boxes to image window
refined_rois    = clip_to_window(window, refined_rois)

##  6.  Round and cast to int since we're deadling with pixels now
refined_rois    = np.rint(refined_rois).astype(np.int32)

##  7.  TODO: Filter out boxes with zero area

##  8.  Filter out background boxes
keep = np.where(class_ids > 0)[0]
scores = np.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[0]
# Filter out low confidence boxes
print(class_ids[:40])
print(class_scores[:40])
print(keep[:20])
print(scores[:20])
print(class_ids[keep[:20]])
print(class_scores[scores[:20]])
keep1 = np.intersect1d(keep, scores )
print(keep1.size)
print(keep1[:20])
pre_nms_class_ids = class_ids[keep1]
pre_nms_scores    = class_scores[keep1]
pre_nms_rois      = refined_rois[keep1]
nms_keep          = []
print(pre_nms_class_ids[:20])
print(pre_nms_scores[:20])
print(np.unique(pre_nms_class_ids))
# Pad with zeros if detections < DETECTION_MAX_INSTANCES
gap = self.config.DETECTION_MAX_INSTANCES - detections.shape[0]
assert gap >= 0
if gap > 0:
    detections = np.pad(detections, [(0, gap), (0, 0)], 'constant', constant_values=0)
detections_batch.append(detections)

# Stack detections and cast to float32
# TODO: track where float64 is introduced
detections_batch = np.array(detections_batch).astype(np.float32)
# Reshape output
def refine_detections(rois, probs, deltas, window, config):
    '''
    Refine classified proposals and filter overlaps and return final detections.

    Inputs:
    ------
        
    rois:           rpn_rois    - [N, (y1, x1, y2, x2)] in normalized coordinates
    probs:          mrcnn_class - [N, num_classes]. Class probabilities.
    deltas:         mrcnn_bbox  - [N, num_classes, (dy, dx, log(dh), log(dw))]. 
                                  Class-specific bounding box deltas.
                                  
    window:         (y1, x1, y2, x2) in image coordinates. The part of the image
                    that contains the image excluding the padding.

    Returns:
    --------
    detections      [N, (y1, x1, y2, x2, class_id, score)]
    '''

    
    ##  1. Find Class IDs with higest scores for each per ROI
    class_ids       = np.argmax(probs, axis=1)
    print(class_ids)
    ##  2. Get Class probability(score) and bbox delta of the top class of each ROI
    class_scores    =  probs[np.arange(class_ids.shape[0]), class_ids]
    deltas_specific = deltas[np.arange(deltas.shape[0])   , class_ids]
    
    ##  3. Apply bounding box delta to the corrsponding rpn_proposal
    # Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates
    refined_rois    = apply_box_deltas(rois, deltas_specific * config.BBOX_STD_DEV)
    
    ##  4. Convert the refined roi coordiates from normalized to image domain
    # TODO: better to keep them normalized until later   
    height, width   = config.IMAGE_SHAPE[:2]
    refined_rois   *= np.array([height, width, height, width])
    
    ##  5.  Clip boxes to image window
    refined_rois    = clip_to_window(window, refined_rois)
    
    ##  6.  Round and cast to int since we're deadling with pixels now
    refined_rois    = np.rint(refined_rois).astype(np.int32)

    ##  7.  TODO: Filter out boxes with zero area

    ##  8.  Filter out background boxes
    keep = np.where(class_ids > 0)[0]
    # Filter out low confidence boxes
    if config.DETECTION_MIN_CONFIDENCE:
        keep = np.intersect1d(keep, np.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])

    ##----------------------------------------------------------------------------
    ##  9.  Apply per-class NMS
    ##----------------------------------------------------------------------------
    pre_nms_class_ids = class_ids[keep]
    pre_nms_scores    = class_scores[keep]
    pre_nms_rois      = refined_rois[keep]
    nms_keep          = []
    # print(' apply per class nms')    
    for class_id in np.unique(pre_nms_class_ids):
        # Pick detections of this class
        ixs = np.where(pre_nms_class_ids == class_id)[0]

        # print('class_id : ', class_id)
        # print('pre_nms_rois.shape:', pre_nms_rois[ixs].shape)
        # pp.pprint(pre_nms_rois[ixs])
        # print('pre_nms_scores.shape :', pre_nms_scores[ixs].shape)
        # pp.pprint(pre_nms_scores[ixs])    
        # Apply NMS
        class_keep = non_max_suppression(pre_nms_rois[ixs], 
                                         pre_nms_scores[ixs],
                                         config.DETECTION_NMS_THRESHOLD)
        # Map indicies
        class_keep = keep[ixs[class_keep]]
        nms_keep   = np.union1d(nms_keep, class_keep)
    
    keep = np.intersect1d(keep, nms_keep).astype(np.int32)

    ##----------------------------------------------------------------------------
    ## 10.  Keep top detections
    ##----------------------------------------------------------------------------
    roi_count = config.DETECTION_MAX_INSTANCES
    top_ids   = np.argsort(class_scores[keep])[::-1][:roi_count]
    keep      = keep[top_ids]

    ##----------------------------------------------------------------------------
    ## 11.  Arrange output as [N, (y1, x1, y2, x2, class_id, score)]
    ##      Coordinates are in image domain.
    ##----------------------------------------------------------------------------
    result = np.hstack((refined_rois[keep],
                        class_ids   [keep][..., np.newaxis],
                        class_scores[keep][..., np.newaxis]))

    return result
############################################################
#  Detection Layer
############################################################

def clip_to_window(window, boxes):
    '''
    window: (y1, x1, y2, x2). The window in the image we want to clip to.
    boxes: [N, (y1, x1, y2, x2)]
    '''
    
    boxes[:, 0] = np.maximum(np.minimum(boxes[:, 0], window[2]), window[0])
    boxes[:, 1] = np.maximum(np.minimum(boxes[:, 1], window[3]), window[1])
    boxes[:, 2] = np.maximum(np.minimum(boxes[:, 2], window[2]), window[0])
    boxes[:, 3] = np.maximum(np.minimum(boxes[:, 3], window[3]), window[1])
    return boxes
original_image = train_batch_x
results = model.detect(original_image, verbose=1)
r = results[0]
print('  rois       : ', r['rois'])
print('  masks      : ', r['masks'].shape)
print('  class ids  : ', r['class_ids'])
print('  class names: ', dataset_test.class_names)
print('  scores     : ', r['scores'])
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], 
                            dataset_test.class_names, r['scores'], ax=get_ax())<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
from mrcnn.datagen     import data_generator, load_image_gt
import  mrcnn.utils as utils 

image_ids = np.random.choice(dataset_test.image_ids, 100)
APs = []
for image_id in image_ids:
    # Load image and ground truth data
    image, image_meta, gt_class_id, gt_bbox, gt_mask =\
        load_image_gt(dataset_test, inference_config,
                               image_id, use_mini_mask=False)
    molded_images = np.expand_dims(utils.mold_image(image, inference_config), 0)
    # Run object detection
    results = model.detect([image], verbose=0)
    r = results[0]
    # Compute AP
    AP, precisions, recalls, overlaps =\
        utils.compute_ap(gt_bbox, gt_class_id,
                         r["rois"], r["class_ids"], r["scores"])
    APs.append(AP)
    
print("mAP: ", np.mean(APs))<jupyter_output><empty_output><jupyter_text>Get next shapes from generator and display loaded shapes  Save<jupyter_code># Save weights
# Typically not needed because callbacks save after every epoch
# Uncomment to save manually
model_path = os.path.join(MODEL_DIR, "mask_rcnn_shapes.h5")
model.keras_model.save_weights(model_path)<jupyter_output><empty_output><jupyter_text>Notebook Preferences<jupyter_code>def get_ax(rows=1, cols=1, size=8):
    """Return a Matplotlib Axes array to be used in
    all visualizations in the notebook. Provide a
    central point to control graph sizes.
    
    Change the default size attribute to control the size
    of rendered images
    """
    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))
    return ax<jupyter_output><empty_output><jupyter_text>Simulation of `detect()` routine<jupyter_code># print('>>> model detect()')
print(model.config.BATCH_SIZE)
images = train_batch_x[0]
print(len(input_images))
verbose = 1
# images  = [original_image]
assert model.mode   == "inference", "Create model in inference mode."
assert len(images) == model.config.BATCH_SIZE, "len(images) must be equal to BATCH_SIZE"

if verbose:
    log("Processing {} images".format(len(images)))
    for image in images:
        log("image", image)

# Mold inputs to format expected by the neural network
molded_images, image_metas, windows = model.mold_inputs(images)
if verbose:
    log("molded_images", molded_images)
    log("image_metas"  , image_metas)

## Run object detection pipeline
# print('    call predict()')
detections, rpn_rois, rpn_class, rpn_bbox,\
            mrcnn_class, mrcnn_bbox \
                      =  model.keras_model.predict([molded_images, image_metas], verbose=0)

print('    return from  predict()')
print('    Length of detections : ', len(detections))
print('    Length of rpn_rois   : ', len(rpn_rois   ))
print('    Length of rpn_class  : ', len(rpn_class  ))
print('    Length of rpn_bbox   : ', len(rpn_bbox   ))
print('    Length of mrcnn_class: ', len(mrcnn_class))
print('    Length of mrcnn_bbox : ', len(mrcnn_bbox ))
# print('    Length of mrcnn_mask : ', len(mrcnn_mask ))

####  detection array layout is `[ y1, x1, y2, x2, class, score]`

detections[0].shape
print(detections[0])

## Process detections
results = []
for i, image in enumerate(images):
    final_rois, final_class_ids, final_scores  =\
        model.unmold_detections(detections[i], 
                               image.shape  ,
                               windows[i])
    results.append({
        "rois"     : final_rois,
        "class_ids": final_class_ids,
        "scores"   : final_scores,
        
    })
original_image.shape
for i in range(config.BATCH_SIZE)   :
    r = results[i]
    print('  rois       : ', r['rois'])
    print('  class ids  : ', r['class_ids'])
    print('  class names: ', dataset_test.class_names)
    print('  scores     : ', r['scores'])
    visualize.display_instances_wo_mask(input_image[i], r['rois'], r['class_ids'], dataset_test.class_names, r['scores'], figsize=(8,8))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475958433628838, 'fraction_numerical': 0.01884576997592908, 'mean_word_length': 3.2531835205992508, 'pattern_counts': {'":': 1, '<': 40, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '212775', 'license': 'MIT', 'n_tokens_mistral': 6072, 'n_tokens_neox': 5787, 'n_words': 1297, 'path': 'notebooks/dev - inference process .ipynb', 'repo_name': 'kbardool/Mask_RCNN_2', 'size': '17033'}