<jupyter_start><jupyter_text>Mask R-CNN - Train FCN using MRCNN in Predict Mode<jupyter_code>from IPython.core.display import display, HTML
display(HTML("<style>.container { width:95% !important; }</style>"))
%matplotlib inline
%load_ext autoreload
%autoreload 2
import os, sys, math, io, time, gc, argparse, platform, pprint
import numpy as np
import tensorflow as tf
import keras
import keras.backend as KB
sys.path.append('../')
import mrcnn.model_mrcnn  as mrcnn_modellib
import mrcnn.model_fcn    as fcn_modellib
import mrcnn.visualize    as visualize
import mrcnn.new_shapes   as shapes
from datetime import datetime   
from mrcnn.utils        import command_line_parser, Paths
from mrcnn.config       import Config
from mrcnn.dataset      import Dataset 
from mrcnn.utils        import log, stack_tensors, stack_tensors_3d, write_stdout
from mrcnn.datagen      import data_generator, load_image_gt
from mrcnn.callbacks    import get_layer_output_1,get_layer_output_2
from mrcnn.coco         import CocoDataset, CocoConfig, CocoInferenceConfig, evaluate_coco, build_coco_results
from mrcnn.prep_notebook import prep_coco_dataset

pp = pprint.PrettyPrinter(indent=2, width=100)
np.set_printoptions(linewidth=100,precision=4,threshold=1000, suppress = True)
start_time = datetime.now().strftime("%m-%d-%Y @ %H:%M:%S")
print()
print('--> Execution started at:', start_time)
print("    Tensorflow Version: {}   Keras Version : {} ".format(tf.__version__,keras.__version__))

####  Pass input parameters to argparse

# args = parser.parse_args("--epochs 100 --steps_in_epoch 128  --last_epoch 1264 --batch_size 8  --lr 0.5               --logs_dir train_fcn_adagrad --model /home/kbardool/models/train_mrcnn/shapes20180621T1554/mask_rcnn_shapes_1119.h5 --fcn_model init".split())
# input_parms = "--epochs 100 --steps_in_epoch 100  --last_epoch 1264 --batch_size 25 --lr 0.8 --val_steps 5 --logs_dir train_fcn_adagrad --model /home/kbardool/models/train_mrcnn/shapes20180621T1554/mask_rcnn_shapes_1119.h5 --fcn_model /home/kbardool/models/train_fcn_adagrad/shapes20180709T1732/fcn_shapes_1167.h5"
# input_parms +=" --model     /home/kbardool/models/train_mrcnn/shapes20180621T1554/mask_rcnn_shapes_1119.h5 "
##------------------------------------------------------------------------------------
## Parse command line arguments
##------------------------------------------------------------------------------------
parser = command_line_parser()
input_parms = "--epochs 1 --steps_in_epoch 10  --last_epoch 0 --batch_size 1 --lr 0.001 --val_steps 4 " 
input_parms +="--mrcnn_logs_dir train_mrcnn_coco "
input_parms +="--fcn_logs_dir train_fcn_coco "
input_parms +="--mrcnn_model last "
input_parms +="--fcn_model vgg16 "
input_parms +="--opt adagrad"
# input_parms +="--fcn_model /home/kbardool/models/train_fcn_adagrad/shapes20180709T1732/fcn_shapes_1167.h5"
print(input_parms)

args = parser.parse_args(input_parms.split())
# args = parser.parse_args()

##----------------------------------------------------------------------------------------------
## if debug is true set stdout destination to stringIO
##----------------------------------------------------------------------------------------------            
# debug = False
args.sysout = args.sysout.upper()
if args.sysout == 'FILE':
    sys.stdout = io.StringIO()

# print("    Dataset            : ", args.dataset)
# print("    Logs               : ", args.logs)
# print("    Limit              : ", args.limit)
print("    MRCNN Model        : ", args.mrcnn_model)
print("    FCN Model          : ", args.fcn_model)
print("    MRCNN Log Dir      : ", args.mrcnn_logs_dir)
print("    FCN Log Dir        : ", args.fcn_logs_dir)
print("    Last Epoch         : ", args.last_epoch)
print("    Epochs to run      : ", args.epochs)
print("    Steps in each epoch: ", args.steps_in_epoch)
print("    Validation steps   : ", args.val_steps)
print("    Batch Size         : ", args.batch_size)
print("    Optimizer          : ", args.opt.upper())
# print("    OS Platform        : ", syst)
##------------------------------------------------------------------------------------
## setup project directories
##   ROOT_DIR         : Root directory of the project 
##   MODEL_DIR        : Directory to save logs and trained model
##   COCO_MODEL_PATH  : Path to COCO trained weights
##---------------------------------------------------------------------------------
paths = Paths()
paths.display()

##------------------------------------------------------------------------------------
## Build configuration object 
##------------------------------------------------------------------------------------                          
mrcnn_config                    = CocoConfig()
mrcnn_config.NAME               = 'mrcnn'              
mrcnn_config.TRAINING_PATH      = paths.MRCNN_TRAINING_PATH
mrcnn_config.COCO_DATASET_PATH  = paths.COCO_DATASET_PATH 
mrcnn_config.COCO_MODEL_PATH    = paths.COCO_MODEL_PATH   
mrcnn_config.RESNET_MODEL_PATH  = paths.RESNET_MODEL_PATH 
mrcnn_config.VGG16_MODEL_PATH   = paths.VGG16_MODEL_PATH  

mrcnn_config.COCO_CLASSES       = None 
mrcnn_config.DETECTION_PER_CLASS = 200
mrcnn_config.HEATMAP_SCALE_FACTOR = 4

mrcnn_config.BATCH_SIZE         = int(args.batch_size)                  # Batch size is 2 (# GPUs * images/GPU).
mrcnn_config.IMAGES_PER_GPU     = int(args.batch_size)                  # Must match BATCH_SIZE
mrcnn_config.STEPS_PER_EPOCH    = int(args.steps_in_epoch)
mrcnn_config.LEARNING_RATE      = float(args.lr)
mrcnn_config.EPOCHS_TO_RUN      = int(args.epochs)
mrcnn_config.FCN_INPUT_SHAPE    = mrcnn_config.IMAGE_SHAPE[0:2]
mrcnn_config.LAST_EPOCH_RAN     = int(args.last_epoch)
mrcnn_config.WEIGHT_DECAY       = 2.0e-4
mrcnn_config.VALIDATION_STEPS   = int(args.val_steps)
mrcnn_config.REDUCE_LR_FACTOR   = 0.5
mrcnn_config.REDUCE_LR_COOLDOWN = 30
mrcnn_config.REDUCE_LR_PATIENCE = 40
mrcnn_config.EARLY_STOP_PATIENCE= 80
mrcnn_config.EARLY_STOP_MIN_DELTA = 1.0e-4
mrcnn_config.MIN_LR             = 1.0e-10
mrcnn_config.NEW_LOG_FOLDER     = False
mrcnn_config.OPTIMIZER          = args.opt.upper()
mrcnn_config.display()<jupyter_output>>>> Initialize Paths
 Linx  Linux

Paths:
-------------------------
COCO_DATASET_PATH              /home/kbardool/MLDatasets/coco2014
COCO_MODEL_PATH                /home/kbardool/PretrainedModels/mask_rcnn_coco.h5
DIR_DATASET                    /home/kbardool/MLDatasets
DIR_PRETRAINED                 /home/kbardool/PretrainedModels
DIR_ROOT                       /home/kbardool/git_projs/mrcnn3/notebooks
DIR_TRAINING                   /home/kbardool/models
FCN_TRAINING_PATH              /home/kbardool/models/train_fcn_coco
FCN_VGG16_MODEL_PATH           /home/kbardool/PretrainedModels/fcn_vgg16_weights_tf_dim_ordering_tf_kernels.h5
MRCNN_TRAINING_PATH            /home/kbardool/models/train_mrcnn_coco
RESNET_MODEL_PATH              /home/kbardool/PretrainedModels/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5
VGG16_MODEL_PATH               /home/kbardool/PretrainedModels/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5



Configuration Parameters:
-------------------------
BACKBO[...]<jupyter_text>Heatmap Dataset<jupyter_code>from mrcnn.heatmap import HeatmapDataset
HEATMAP_PATH = os.path.join(paths.DIR_DATASET, 'coco2014_heatmaps')
print(HEATMAP_PATH)
# del hm_dataset
hm_dataset = HeatmapDataset()
hm_dataset.__dict__


hm_dataset.load_heatmap(mrcnn_config.COCO_DATASET_PATH, HEATMAP_PATH, 'minival' )
hm_dataset.image_info
pp = next(os.walk(join(HEATMAP_PATH, 'train_heatmaps')))[2]
for i in range(3):
    number = os.path.splitext(pp[i].lstrip('hm_'))[0]
    print(type(number))
from os.path import join, getsize
print(HEATMAP_PATH)
for root, dirs, files in os.walk(HEATMAP_PATH):
    print('Root       : ', root)
    print('Directories: ', dirs)
    print('Files      : ', len(files))
#     print(sum(getsize(join(root, name)) for name in files),end = " ")
#     print("bytes in", len(files), "non-directory files")
from mrcnn.prep_notebook import coco_dataset
# dataset_train = coco_dataset(["train",  "val35k"], mrcnn_config)
train_generator= data_generator(dataset_train, mrcnn_model.config, shuffle=False,
                                batch_size=2 )

train_batch_x, train_batch_y = next(train_generator)
print(train_batch_x[1])
print(dataset_train.image_info[1]['path'])<jupyter_output><empty_output><jupyter_text>Build MRCNN<jupyter_code>##------------------------------------------------------------------------------------
## Build Mask RCNN Model in TRAINFCN mode
##------------------------------------------------------------------------------------
# Recreate the model in training mode
try :
    del mrcnn_model,model
    print('delete model is successful')
    gc.collect()
except: 
    pass
KB.clear_session()
mrcnn_model = mrcnn_modellib.MaskRCNN(mode='trainfcn', config=mrcnn_config)
##------------------------------------------------------------------------------------
## Load Mask RCNN Model Weight file
##------------------------------------------------------------------------------------
mrcnn_model.load_model_weights(init_with = args.mrcnn_model, exclude = [])   
mrcnn_model.layer_info()
mrcnn_model.config.display()<jupyter_output>Inputs:
 -------
 index:  0    input name : input_image:0                              Type: float32           Shape: (?, 1024, 1024, 3)
 index:  1    input name : input_image_meta:0                         Type: float32           Shape: (?, ?)
 index:  2    input name : input_rpn_match:0                          Type: int32             Shape: (?, ?, 1)
 index:  3    input name : input_rpn_bbox:0                           Type: float32           Shape: (?, ?, 4)
 index:  4    input name : input_gt_class_ids:0                       Type: int32             Shape: (?, ?)
 index:  5    input name : input_gt_boxes:0                           Type: float32           Shape: (?, ?, 4)


 Outputs:
 --------
 layer:  0    output name: cntxt_layer/pred_heatmap_norm:0            Type: float32           Shape: (1, 256, 256, 81)
 layer:  1    output name: cntxt_layer/pred_heatmap_scores:0          Type: float32           Shape: (1, 81, 200, 11)
 layer:  2    output name: cntxt_layer_gt/gt_heatmap[...]<jupyter_text>Define training and validation datasets<jupyter_code>##------------------------------------------------------------------------------------
## Build & Load Training and Validation datasets
##------------------------------------------------------------------------------------
# del dataset_train, dataset_val, train_generator
# dataset_train = prep_coco_dataset(["train",  "val35k"], mrcnn_config)
# del dataset_train
dataset_val   = prep_coco_dataset(["minival"]         , mrcnn_config)
# train_batch_x, train_batch_y = next(tr_generator)
# dataset_val   = prep_coco_dataset(["minival"]         , mrcnn_config, generator = False)
# print(dataset_val.image_info[0])

# val_generator= data_generator(dataset_val, mrcnn_config, 
#                                 shuffle=False,
#                                 batch_size= mrcnn_config.BATCH_SIZE)
x, y = next(val_generator)
print(x[1])
type(dataset_val)
dest_path = os.path.join(paths.DIR_DATASET, 'coco2014_heatmaps','val2014')
print(dest_path)

batch_size = 1
start_from = 100
##--------------------------------------------------------------------------------
## Build Heatmap files 
##--------------------------------------------------------------------------------
build_heatmap_files(mrcnn_model, dataset_val, iterations= 10, start_from = start_from, dest_path = dest_path)
##--------------------------------------------------------------------------------
## Build Heatmap files 
##--------------------------------------------------------------------------------
build_heatmap_files(mrcnn_model, dataset_val, iterations= 10, start_from = start_from, dest_path = dest_path)
##----------------------------------------------------------------------------------------------
## 
##---------------------------------------------------------------------------------------------- 
def build_heatmap_files(
          mrcnn_model,
          dataset, 
          iterations        = 5,
          start_from        = 0, 
          dest_path         = None):
   
    '''
    train_dataset:  Training Dataset objects.

    '''
    assert mrcnn_model.mode == "trainfcn", "Create model in training mode."
    log("Starting for  {} iterations - batch size of each iteration: {}".format(iterations, batch_size))
    log(" Output destination: {}".format(dest_path))
    tr_generator= data_generator(dataset, mrcnn_model.config, 
                                    shuffle=False,
                                    batch_size= mrcnn_model.config.BATCH_SIZE,
                                    image_index = start_from)
    
    ## Start main loop
    epoch_idx = 0
    for  epoch_idx in range(iterations) :
        tm_start = time.time()
        
        train_batch_x, train_batch_y = next(tr_generator)
        print(' ==> mrcnn_model: step {} of {} iterations, image_id: {} '.format(epoch_idx, iterations, train_batch_x[1][:,0]))
        
        # print('   length of train_batch_x:', len(train_batch_x), ' number of things in batch x :', train_batch_x[1].shape)        
        # for i in train_batch_x:
            # print('       ', i.shape)
        # print('length of train_batch_y:', len(train_batch_y))

        # results = get_layer_output_1(mrcnn_model.keras_model, train_batch_x, [0,1,2,3], 1)
        
        results = mrcnn_model.keras_model.predict(train_batch_x) 
        
        # pr_hm_norm, gt_hm_norm, pr_hm_scores, gt_hm_scores = results[:4]              
        
        for i in range(batch_size):
            # print('  pr_hm_norm shape   :', results[0][i].shape)
            # print('  pr_hm_scores shape :', results[1][i].shape)
            # print('  gt_hm_norm shape   :', results[2][i].shape)
            # print('  gt_hm_scores shape :', results[3][i].shape)
            image_id = train_batch_x[1][i,0]
            coco_image_id = dataset.image_info[image_id]['id']
            coco_filename = os.path.basename(dataset.image_info[image_id]['path'])
            filename = 'hm_{:012d}.npz'.format(image_id)
            print('  output: {}  image_id: {}  coco_image_id: {} coco_filename: {} output file: {}'.format(
                            i, image_id, coco_image_id, coco_filename, filename))
#             print('  output file: ',os.path.join(dest_path, filename))
            np.savez_compressed(os.path.join(dest_path, filename), 
                             input_image_meta=train_batch_x[1][i], 
                             pr_hm_norm   = results[0][i],
                             pr_hm_scores = results[1][i],
                             gt_hm_norm   = results[2][i],
                             gt_hm_scores = results[3][i],
                             coco_info    = np.array([coco_image_id, coco_filename])    )        
        tm_stop= time.time()            
        print(' ==> Elapsed time {:.4f}s #        of items in results: {} '.format(tm_stop - tm_start,len(train_batch_x)))

    print('Final : mrcnn_model epoch_idx{}   iterations {}'.format(epoch_idx, iterations))
    return



dest = 'abcd'
filename = dest+'/{:05d}'.format(0)
print(filename)

epochs = fcn_model.epoch + epochs_to_run
log("Starting at epoch {} of {} epochs. LR={}\n".format(fcn_model.epoch, epochs, learning_rate))
log("Steps per epochs {} ".format(steps_per_epoch))
log("    Last epoch completed : {} ".format(fcn_model.epoch))
log("    Starting from epoch  : {} for {} epochs".format(fcn_model.epoch, epochs_to_run))
log("    Learning Rate        : {} ".format(learning_rate))
log("    Steps per epoch      : {} ".format(steps_per_epoch))
log("    Batch Size           : {} ".format(batch_size))
log("    Checkpoint Folder    : {} ".format(fcn_model.checkpoint_path))
results = mrcnn_model.keras_model.predict(train_batch_x)
print('# of items in results:', len(results))

for i in results:
    print( i.shape)<jupyter_output><empty_output><jupyter_text>NPZ Code<jupyter_code>pr_hm_norm, gt_hm_norm, pr_hm_scores, gt_hm_scores = results[11:]                 
print(pr_hm_norm.shape)
print(gt_hm_norm.shape)
print(pr_hm_scores.shape)
print(gt_hm_scores.shape)
# model.keras_model.summary(line_length=132, positions=[0.30,0.75, .83, 1. ])
train_batch_x, train_batch_y = next(train_generator)
imgmeta_idx = model.keras_model.input_names.index('input_image_meta')
img_meta    = train_batch_x[imgmeta_idx]

for img_idx in range(config.BATCH_SIZE):
    image_id = img_meta[img_idx,0]
    image = dataset_train.load_image(image_id)
    mask, class_ids = dataset_train.load_mask(image_id)
    print('Image id: ',image_id)
    print('Image meta', img_meta[img_idx])
    print('Classes (1: circle, 2: square, 3: triangle ): ',class_ids)
    visualize.display_top_masks(image, mask, class_ids, dataset_train.class_names)
model.layer_info()
# train_batch_x, train_batch_y = next(train_generator)

model_output = get_layer_output_2(model.keras_model, train_batch_x, 1)
# model_output = get_layer_output_1(model.keras_model, train_batch_x, [ 26], 1)

print(len(model_output))

# rpn_class_loss            = model_output[0]          # layer: 11   shape: (1, 1)
# rpn_bbox_loss             = model_output[1]          # layer: 12   shape: (1, 1)
# mrcnn_class_loss          = model_output[2]          # layer: 13   shape: (1, 1)
# mrcnn_bbox_loss           = model_output[3]          # layer: 14   shape: (1, 1)
# fcn_normalized_loss       = model_output[0]          # layer: 26   shape: (1, 1)

# print(type(output_rois))
for i in model_output:
    print( i.shape)
# print('FCN Normalized Loss is :', fcn_normalized_loss)    

input_image      =  train_batch_x[0]
input_image_meta =  train_batch_x[1]
input_rpn_match  =  train_batch_x[2]
input_rpn_bbox   =  train_batch_x[3]
input_gt_class_ids = train_batch_x[4]
input_gt_bboxes    = train_batch_x[5]
input_gt_masks     = train_batch_x[6]
print(' Input image shape is :', input_image.shape)
h, w = input_image.shape[1], input_image.shape[2]      #  tf.shape(input_image)[1], tf.shape(input_image)[2]
input_normlzd_gt_bboxes = tf.identity(input_gt_bboxes / [h,w,h,w])

# gt_masks   =  train_batch_x[6]
print(' input_rpn_match    ', input_rpn_match.shape)
print(' input_rpn_bbox     ', input_rpn_bbox.shape)
print(' input_gt_class_ids ', input_gt_class_ids.shape)
print(' input_gt_bboxes    ', input_gt_bboxes.shape)
print(' input_normlzd_gt_bboxes    ', input_normlzd_gt_bboxes.shape)

np.savez_compressed('../hm_'+str(input_image_meta[0,0]), input_image_meta=input_image_meta[0], 
                 pred_heatmap_norm=model_pred_heatmap_norm[0],
                 pred_heatmap_scores = model_pred_heatmap_scores[0],
                 gt_heatmap_norm = model_gt_heatmap_norm[0], 
                 gt_heatmap_scores = model_gt_heatmap_scores[0])<jupyter_output><empty_output><jupyter_text>Load npz file<jupyter_code>image_id = 0
for image_id in range(20):
    filename = 'hm_{:012d}.npz'.format(image_id)
    # print('  output: {} of {} iterations, image_id: {}  coco_image_id: {} coco_filename: {}'.format(
    #                 i, batch_size, image_id, coco_image_id, coco_filename))
    print('  output file: ',filename)


    loaddata = np.load(os.path.join(dest_path, filename))
    print(loaddata['coco_info'])
# input_obj = {}
for i in loaddata.keys():
        print('   ',i, '   ',loaddata[i].shape)
#      input_obj[i]=loaddata[i]
# loaddata.close()    
print(loaddata['coco_info'])
type(data)
data.keys()
print(input_image_meta[0,0])
for i in data.keys():
        print(i, data[i].shape)
print(np.all(data['gt_heatmap_norm'] == model_gt_heatmap_norm[0]))  
print(np.all(data['gt_heatmap_scores'] == model_gt_heatmap_scores[0]))  
print(np.all(data['pred_heatmap_norm'] == model_pred_heatmap_norm[0]))  
print(np.all(data['pred_heatmap_scores'] == model_pred_heatmap_scores[0]))  
print(np.all(data['input_image_meta'] == input_image_meta[0]))   
loaddata = np.load(dest_path+'/hm_00001.npz')
input_image_meta = data['input_image_meta']

res = mrcnn_model.keras_model.predict(train_batch_x)
for i in res:
    print(i.shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1930024249022616, 'fraction_numerical': 0.019894096105310042, 'mean_word_length': 2.3220450435640307, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '181432', 'license': 'MIT', 'n_tokens_mistral': 6927, 'n_tokens_neox': 6594, 'n_words': 1163, 'path': 'notebooks/Utils/Utils - Build npz files using MRCNN in Trainfcn mode.ipynb', 'repo_name': 'kbardool/mrcnn3', 'size': '20207'}