<jupyter_start><jupyter_text>Adding Object Detection Predictions to a Voxel51 DatasetThis notebook will add predictions from an object detection model to the samples in a Voxel51 Dataset.Adapted from: https://voxel51.com/docs/fiftyone/recipes/model_inference.html<jupyter_code>model_path = '/tf/model-export/lb-400images-efficientdet-d0-model/image_tensor_saved_model/saved_model' # The path of the saved Object Detection model
dataset_name = "jsm-test-dataset" # Name of the Voxel51 Dataset to use
field_name = "dolt_bg_predict" # Name of the field to store the predictions in
labelmap_file = '/tf/dataset-export/lb-400images-efficientdet-d0-model/label_map.pbtxt' # the location of the labelmap file to use
min_score = 0.8 # This is the minimum score for adding a prediction. This helps keep out bad predictions but it may need to be adjusted if your model is not that good yet.
# dimensions of images
#config
import fiftyone as fo
import os

dataset = fo.load_dataset(dataset_name)  
import io
import os
import scipy.misc
import numpy as np
import six
import time

from six import BytesIO

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.pyplot import imshow
from PIL import Image, ImageDraw, ImageFont
from object_detection.utils import label_map_util
import tensorflow as tf
from object_detection.utils import visualization_utils as viz_utils
# small function that preprocesses the images so that the model can read them in
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load saved modelLoading a saved objection detection model is a little weird. I found some info on it:https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/inference_from_saved_model_tf2_colab.ipynb<jupyter_code>start_time = time.time()
tf.keras.backend.clear_session()
detect_fn = tf.saved_model.load(model_path)
end_time = time.time()
elapsed_time = end_time - start_time
print('Elapsed time: ' + str(elapsed_time) + 's')<jupyter_output>Importing a function (__inference_bifpn_layer_call_and_return_conditional_losses_119378) with ops with custom gradients. Will likely fail if a gradient is requested.
Importing a function (__inference_bifpn_layer_call_and_return_conditional_losses_60229) with ops with custom gradients. Will likely fail if a gradient is requested.
Importing a function (__inference_bifpn_layer_call_and_return_conditional_losses_55664) with ops with custom gradients. Will likely fail if a gradient is requested.
Importing a function (__inference_bifpn_layer_call_and_return_conditional_losses_120998) with ops with custom gradients. Will likely fail if a gradient is requested.
Importing a function (__inference_EfficientDet-D0_layer_call_and_return_conditional_losses_92047) with ops with custom gradients. Will likely fail if a gradient is requested.
Importing a function (__inference_EfficientDet-D0_layer_call_and_return_conditional_losses_88439) with ops with custom gradients. Will likely fail if a gradient is[...]<jupyter_text>Load the LabelMap file<jupyter_code>label_map = label_map_util.load_labelmap(labelmap_file)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=100)
category_index = label_map_util.create_category_index(categories)

def findClassName(class_id):
    return category_index[class_id]["name"]<jupyter_output><empty_output><jupyter_text>Add predictionsItterate through all the samples, run them through the model and add the predictions to the sample  Predictions with TilingUse tiling to break up large images to sizes closer to the input tensor of the model.<jupyter_code># from: https://github.com/google-coral/pycoral/blob/master/examples/small_object_detection.py
import collections
Object = collections.namedtuple('Object', ['label', 'score', 'bbox'])

def tiles_location_gen(img_size, tile_size, overlap):
  """Generates location of tiles after splitting the given image according the tile_size and overlap.
  Args:
    img_size (int, int): size of original image as width x height.
    tile_size (int, int): size of the returned tiles as width x height.
    overlap (int): The number of pixels to overlap the tiles.
  Yields:
    A list of points representing the coordinates of the tile in xmin, ymin,
    xmax, ymax.
  """

  tile_width, tile_height = tile_size
  img_width, img_height = img_size
  h_stride = tile_height - overlap
  w_stride = tile_width - overlap
  for h in range(0, img_height, h_stride):
    for w in range(0, img_width, w_stride):
      xmin = w
      ymin = h
      xmax = min(img_width, w + tile_width)
      ymax = min(img_height, h + tile_height)
      yield [xmin, ymin, xmax, ymax]
    
def non_max_suppression(objects, threshold):
  """Returns a list of indexes of objects passing the NMS.
  Args:
    objects: result candidates.
    threshold: the threshold of overlapping IoU to merge the boxes.
  Returns:
    A list of indexes containings the objects that pass the NMS.
  """
  if len(objects) == 1:
    return [0]

  boxes = np.array([o.bbox for o in objects])
  xmins = boxes[:, 0]
  ymins = boxes[:, 1]
  xmaxs = boxes[:, 2]
  ymaxs = boxes[:, 3]

  areas = (xmaxs - xmins) * (ymaxs - ymins)
  scores = [o.score for o in objects]
  idxs = np.argsort(scores)

  selected_idxs = []
  while idxs.size != 0:

    selected_idx = idxs[-1]
    selected_idxs.append(selected_idx)

    overlapped_xmins = np.maximum(xmins[selected_idx], xmins[idxs[:-1]])
    overlapped_ymins = np.maximum(ymins[selected_idx], ymins[idxs[:-1]])
    overlapped_xmaxs = np.minimum(xmaxs[selected_idx], xmaxs[idxs[:-1]])
    overlapped_ymaxs = np.minimum(ymaxs[selected_idx], ymaxs[idxs[:-1]])

    w = np.maximum(0, overlapped_xmaxs - overlapped_xmins)
    h = np.maximum(0, overlapped_ymaxs - overlapped_ymins)

    intersections = w * h
    unions = areas[idxs[:-1]] + areas[selected_idx] - intersections
    ious = intersections / unions

    idxs = np.delete(
        idxs, np.concatenate(([len(idxs) - 1], np.where(ious > threshold)[0])))

  return selected_idxs

def reposition_bounding_box(bbox, tile_location):
  """Relocates bbox to the relative location to the original image.
  Args:
    bbox (int, int, int, int): bounding box relative to tile_location as xmin,
      ymin, xmax, ymax.
    tile_location (int, int, int, int): tile_location in the original image as
      xmin, ymin, xmax, ymax.
  Returns:
    A list of points representing the location of the bounding box relative to
    the original image as xmin, ymin, xmax, ymax.
  """
  bbox[0] = bbox[0] + tile_location[0]
  bbox[1] = bbox[1] + tile_location[1]
  bbox[2] = bbox[2] + tile_location[0]
  bbox[3] = bbox[3] + tile_location[1]
  return bbox

def get_resize(input_size, img_size):
  """Copies a resized and properly zero-padded image to a model's input tensor.
  Args:
    interpreter: The ``tf.lite.Interpreter`` to update.
    size (tuple): The original image size as (width, height) tuple.
    resize: A function that takes a (width, height) tuple, and returns an
      image resized to those dimensions.
  Returns:
    The resized tensor with zero-padding as tuple
    (resized_tensor, resize_ratio).
  """
  width, height = input_size
  w, h = img_size
  scale = min(width / w, height / h)
  print(scale)
  w, h = int(w * scale), int(h * scale)
  return w,h
  tensor = input_tensor(interpreter)
  tensor.fill(0)  # padding
  _, _, channel = tensor.shape
  result = resize((w, h))
  tensor[:h, :w] = np.reshape(result, (h, w, channel))
  return result, (scale, scale)
# remove the older labels
dataset.delete_sample_field(field_name)
view = dataset.shuffle() # Adjust the view as needed
tile_string="1920x1080,768x768"
tile_overlap=50
iou_threshold=0


for sample in view.select_fields("filepath"):

    start_time = time.time()
    img = load_img(sample.filepath,)
    img_size = img.size
    img_width, img_height = img_size
    objects_by_label = dict()
    exportDetections = []
    tile_sizes = []
    for tile_size in tile_string.split(','):
        tile_size=tile_size.split('x')
        tile_sizes.append([int(tile_size[0]),int(tile_size[1])])
    #print(tile_sizes)
    #tile_sizes = [map(int, tile_size.split('x')) for tile_size in tile_string.split(',')]
    for tile_size in tile_sizes:
        tile_width, tile_height = tile_size
        for tile_location in tiles_location_gen(img_size, tile_size, tile_overlap):
            
            tile = img.crop(tile_location)
            
            old_size = tile.size  # old_size[0] is in (width, height) format

            ratio = float(512)/max(old_size)
            if (ratio > 1):
                continue
            new_size = tuple([int(x*ratio) for x in old_size])
            
            im = tile.resize(new_size, Image.ANTIALIAS)
            # create a new image and paste the resized on it

            new_im = Image.new("RGB", (512, 512))
            new_im.paste(im, (0,0)) #((512-new_size[0])//2, (512-new_size[1])//2))
            
            
            img_array = img_to_array(new_im,dtype='uint8')
            img_batch = np.array([img_array])

            detections = detect_fn(img_batch)
            for i, detectScore in enumerate(detections['detection_scores'][0]):
                if detectScore > min_score:
                    
                    
                    x1 = detections['detection_boxes'][0][i][1].numpy() * 512 #tile_width
                    y1 = detections['detection_boxes'][0][i][0].numpy() * 512 #tile_height
                    x2 = detections['detection_boxes'][0][i][3].numpy() * 512 #tile_width
                    y2 = detections['detection_boxes'][0][i][2].numpy() * 512 #tile_height
                    
                    
                    
                    bbox = [x1,y1,x2,y2]
                    #draw = ImageDraw.Draw(new_im)
                    #draw.rectangle((x1,y1,x2,y2),outline="red")
                    #new_im.show()
                    
                    scaled_bbox = []                  
                    for number in bbox:
                        scaled_bbox.append(number / ratio)
                    repositioned_bbox = reposition_bounding_box(scaled_bbox, tile_location)   
                    #print("tile size: {} tile_location: {} ratio: {}".format(tile_size, tile_location, ratio))
                    #print("bbox: {} scaled_bbox: {} ".format(bbox, scaled_bbox))
                    
                    #print("repositiond: {}".format(repositioned_bbox))
                    #draw = ImageDraw.Draw(img)
                    #draw.rectangle(repositioned_bbox,outline="red")
                    #img.show()
                    
                    confidence = detections['detection_scores'][0][i]
                    label = findClassName(int(detections['detection_classes'][0][i]))
                    objects_by_label.setdefault(label,[]).append(Object(label, confidence, repositioned_bbox))

    #img = load_img(sample.filepath,)
    #draw = ImageDraw.Draw(img)

   
    for label, objects in objects_by_label.items():
        idxs = non_max_suppression(objects, iou_threshold)
        for idx in idxs:
            #print(objects[idx])
            x1 = objects[idx].bbox[0] / img_width
            y1 = objects[idx].bbox[1] / img_height
            x2 = objects[idx].bbox[2] / img_width
            y2 = objects[idx].bbox[3] / img_height
            #draw.rectangle((objects[idx].bbox[0],objects[idx].bbox[1],objects[idx].bbox[2],objects[idx].bbox[3]),outline="red")
            w = x2 - x1
            h = y2 - y1
            bbox = [x1, y1, w, h]
            exportDetections.append( fo.Detection(label=objects[idx].label, bounding_box=bbox, confidence=objects[idx].score))

    #img.show() 
    # Store detections in a field name of your choice
    sample[field_name] = fo.Detections(detections=exportDetections)
    sample.save()    
    end_time = time.time()
    print("Total time: {}".format(end_time-start_time))<jupyter_output>Total time: 1.0665056705474854
Total time: 1.071852684020996
Total time: 1.0863239765167236
Total time: 1.0705442428588867
Total time: 1.069521427154541
Total time: 1.065279483795166
Total time: 1.0477983951568604
Total time: 1.114544153213501
Total time: 1.0657985210418701
Total time: 1.0854346752166748
Total time: 1.1064269542694092
Total time: 1.0548880100250244
Total time: 1.0774283409118652
Total time: 1.1101186275482178
Total time: 1.0602049827575684
Total time: 1.0558576583862305
Total time: 1.0735247135162354
Total time: 1.0723848342895508
Total time: 1.059493064880371
Total time: 1.053640604019165
Total time: 1.0666377544403076
Total time: 1.0707149505615234
Total time: 1.0645191669464111
Total time: 1.0332753658294678
Total time: 1.1061675548553467
Total time: 1.0988528728485107
Total time: 1.0437288284301758
Total time: 1.0634734630584717
Total time: 1.1143126487731934
Total time: 1.0947096347808838
Total time: 1.0828886032104492
Total time: 1.090792179107666
Total time: 1.0[...]<jupyter_text>Run Prediction against the whole imageThis approach is faster but misses small objects<jupyter_code>view = dataset.shuffle() # Adjust the view as needed


start_time = time.time()
for sample in view.select_fields("filepath"):


    img = load_img(sample.filepath)
    img_array = img_to_array(img)
    input_tensor = np.expand_dims(img_array, 0)
    detections = detect_fn(input_tensor)

    for i, detectScore in enumerate(detections['detection_scores'][0]):
        if detectScore > min_score:
            print("\t- {}: {}".format(findClassName(int(detections['detection_classes'][0][i])), detections['detection_scores'][0][i]))

            label = findClassName(int(detections['detection_classes'][0][i]))
            confidence = detections['detection_scores'][0][i]
            # TF Obj Detect bounding boxes are: [ymin, xmin, ymax, xmax]
            
            # For Voxel 51 - Bounding box coordinates should be relative values
            # in [0, 1] in the following format:
            # [top-left-x, top-left-y, width, height]
            x1 = detections['detection_boxes'][0][i][1]
            y1 = detections['detection_boxes'][0][i][0]
            x2 = detections['detection_boxes'][0][i][3]
            y2 = detections['detection_boxes'][0][i][2]
            w = x2 - x1
            h = y2 - y1
            bbox = [x1, y1, w, h]

            exportDetections.append( fo.Detection(label=label, bounding_box=bbox, confidence=confidence))
    

    # Store detections in a field name of your choice
    sample[field_name] = fo.Detections(detections=exportDetections)
    sample.save()    
end_time = time.time()
print("Total time: {}".format(end_time-start_time))<jupyter_output><empty_output><jupyter_text>Examine the resultsHere is some example code on how you could test how well the predictions match ground truth data.  View the ResultsUse the UI to examine the predictions. You can select poorly performing samples and tag them for relabeling.<jupyter_code>session = fo.launch_app(dataset, auto=False)
print(dataset)
view = dataset.exists("predict_model")#.match({"relabel": {"$exists": False, "$eq": None}})
session = fo.launch_app(view, auto=False)
print(view)
#session.view = view<jupyter_output><empty_output><jupyter_text>Select SamplesSelect poorly performing samples in the UI and then run to code below to tag the selected samples for relabeling.<jupyter_code># Create a view containing only the selected samples
selected_view = dataset.select(session.selected)
print(selected_view)
for sample in selected_view:
    sample.tags.append("relabel")
    sample.save() 
predict_model_view = dataset.exists(field_name)
total=0
top3_total=0
for sample in predict_model_view:
    top_detect = sample["predict_model"].detections[0]
    bb_area = top_detect["bounding_box"][2] * top_detect["bounding_box"][3]
    
    if sample["norm_model"].label==top_detect["label"]:
        match="Match"
        top3_match="Top3 Match"
        total = total+1
        top3_total=top3_total+1
        found=True
        top3_found=True
    else:
        match="!NO Match!"
        top3_match="!NO TOP3 Match!"
        found=False
        top3_found=False
        for i,guess in enumerate(sample["predict_model"].detections):
            if i>3:
                break
            if sample["norm_model"].label==guess["label"]:
                top3_match="Top3 Match"
                top3_found=True
                top3_total=top3_total+1
                break
                
    #print("{}\t{}\t\t{}\t\t{}".format(bb_area,sample["norm_model"].label,match,top3_match))
    print("{}, {}, {}, {}".format(bb_area,sample["norm_model"].label,found,top3_found))
print("{}\n{}\n\n{}\n{}".format(total,100-total,top3_total,100-top3_total))<jupyter_output>0.003363115119441318, A321, False, False
0.004131421679733904, A321, False, True
0.00306788170649952, 757-200, False, False
0.003407978022078595, 757-200, False, False
0.004803034913855697, A330, False, True
0.002368545882305284, ERJ-170, True, True
0.0023542858515170906, ERJ-170, True, True
0.002527645226944486, ERJ-170, False, True
0.0012630497521257666, Learjet 45/60, False, True
0.001340467587462868, CRJ700, False, False
0.0031371196698231074, 737-900, False, False
0.002852709476428572, 737-900, False, False
0.0024098699581678318, 737-900, False, False
0.0013388308064037346, Cessna Jet, True, True
0.002016396768755868, A320, False, True
0.0026610918499443414, 757-200, False, False
0.0029156908491483335, 757-200, False, False
0.002778857595231443, 757-200, False, False
0.0032397564838007042, 757-200, False, True
0.00216837580980922, A321, False, True
0.002636617286100318, A321, False, False
0.004325663871298957, 787-800, False, True
0.0020077339682273987, A320, True, True
0.00236754[...]
{'fraction_non_alphanumeric': 0.10424970527143097, 'fraction_numerical': 0.07488912591927244, 'mean_word_length': 3.491679273827534, 'pattern_counts': {'":': 3, '<': 26, '<?xml version=': 0, '>': 31, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '19073', 'license': 'Apache-2.0', 'n_tokens_mistral': 6693, 'n_tokens_neox': 5794, 'n_words': 1462, 'path': 'ml-model/notebooks/Add Object Detection to Voxel51 using Image Tensor Save Model.ipynb', 'repo_name': 'wiseman/SkyScan', 'size': '17813'}