<jupyter_start><jupyter_text>Settings<jupyter_code>from google.colab import drive
from google.colab import files
import glob
import cv2
from io import BytesIO

import tensorflow as tf
import tensorflow_hub as hub
#get image
import matplotlib.pyplot as plt
import tempfile
from six.moves.urllib.request import urlopen
from six import BytesIO
#draw on the image
import numpy as np
from PIL import Image as IMG
from PIL import ImageColor
from PIL import ImageDraw
from PIL import ImageFont
from PIL import ImageOps
from IPython.display import Image as IMAGE
import os
import shutil

''' 
This file is for constructing a data set consist of "poo, pee, nothing" dog image from video.
First, get sequential frame images from the video and put it to the object detection model one by one.
if the "dog" is detected, crop that part and put it as input to the pose decision model.
according to the prediction result, save the image in the matching folder.
'''<jupyter_output><empty_output><jupyter_text>Import module<jupyter_code># import module and obejct detector
module_handle = "https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1" #@param ["https://tfhub.dev/google/openimages_v4/ssd/mobilenet_v2/1", "https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1"]
detector = hub.load(module_handle).signatures['default']<jupyter_output>INFO:tensorflow:Saver not created because there are no variables in the graph to restore<jupyter_text>Define functions<jupyter_code># function for showing image
def display_image(image):
  fig = plt.figure(figsize=(15, 10))
  plt.grid(False)
  plt.imshow(image)

# function that draws bounding box around the detected object
def draw_bounding_box_on_image(image,
                               ymin,
                               xmin,
                               ymax,
                               xmax,
                               color,
                               font,
                               thickness=4,
                               display_str_list=()):
  """Adds a bounding box to an image."""
  draw = ImageDraw.Draw(image)
  im_width, im_height = image.size
  (left, right, top, bottom) = (xmin * im_width, xmax * im_width,
                                ymin * im_height, ymax * im_height)
  draw.line([(left, top), (left, bottom), (right, bottom), (right, top),
             (left, top)],
            width=thickness,
            fill=color)

  display_str_heights = [font.getsize(ds)[1] for ds in display_str_list]
  # Each display_str has a top and bottom margin of 0.05x.
  total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights)

  if top > total_display_str_height:
    text_bottom = top
  else:
    text_bottom = top + total_display_str_height
  # Reverse list and print from bottom to top.
  for display_str in display_str_list[::-1]:
    text_width, text_height = font.getsize(display_str)
    margin = np.ceil(0.05 * text_height)
    draw.rectangle([(left, text_bottom - text_height - 2 * margin),
                    (left + text_width, text_bottom)],
                   fill=color)
    draw.text((left + margin, text_bottom - text_height - margin),
              display_str,
              fill="black",
              font=font)
    text_bottom -= text_height - 2 * margin

# function that calls "draw_bounding_box_on_image" when the "dog" is detected
def draw_boxes(image, boxes, class_names, scores, max_boxes=10, min_score=0.1):
  """Overlay labeled boxes on an image with formatted scores and label names."""
  colors = list(ImageColor.colormap.values())
  dog_coord = []

  try:
    font = ImageFont.truetype("/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf",
                              25)
  except IOError:
    print("Font not found, using default font.")
    font = ImageFont.load_default()

  for i in range(min(boxes.shape[0], max_boxes)):
    if scores[i] >= min_score:
      ymin, xmin, ymax, xmax = tuple(boxes[i])
      if (class_names[i].decode("ascii") == "Dog") :
        dog_coord.append([xmin, ymin, xmax, ymax])
      display_str = "{}: {}%".format(class_names[i].decode("ascii"),
                                     int(100 * scores[i]))
      color = colors[hash(class_names[i]) % len(colors)]
      image_pil = IMG.fromarray(np.uint8(image)).convert("RGB")
      draw_bounding_box_on_image(
          image_pil,
          ymin,
          xmin,
          ymax,
          xmax,
          color,
          font,
          display_str_list=[display_str])
      np.copyto(image, np.array(image_pil))
  display_image(image)
  
  return dog_coord
def run_detector(detector, img):
  converted_img  = tf.image.convert_image_dtype(img, tf.float32)[tf.newaxis, ...]
  result = detector(converted_img)
  result = {key:value.numpy() for key,value in result.items()}
  coord = draw_boxes(
      img, result["detection_boxes"],
      result["detection_class_entities"], result["detection_scores"])

  return coord
  
# drive.mount('/content/gdrive/')<jupyter_output>Mounted at /content/gdrive/<jupyter_text>Load pose decision model<jupyter_code>from tensorflow import keras
model = keras.models.load_model('/content/gdrive/My Drive/soma/good_model/additional_train_4/')
def createFolder(directory) :
  try :
    if not os.path.exists(directory) :
      os.makedirs(directory)
  except OSError :
      print('Error : Creating dir : ' + directory)

vid_num = 0
image_x = 224
image_y = 224

for fname in glob.glob('/content/gdrive/My Drive/soma/vid/*.mp4') :
  # try :
    print("target : "+ fname)
    realname = fname.replace('/content/gdrive/My Drive/soma/vid/', "")
    realname = realname.replace('.mp4', "")
    fakename = realname[len(realname)-6:len(realname)] # generate folder with the unique video code

    newfolder = '/content/gdrive/My Drive/soma/result/'+fakename
    folder_o = newfolder+'/poo'
    folder_e = newfolder+'/pee'
    folder_n = newfolder+'/nothing'

    dir_list = [newfolder, folder_o, folder_e, folder_n]

    for p in dir_list :
      createFolder(p)

    texpath = os.path.join(newfolder, "predict.txt")
    fid = open(texpath, "a")
    fid.write("Time\tPredicted\tAnswer\n")

    # get frames from the video
    vidcap = cv2.VideoCapture(fname)
    fps = vidcap.get(5)
#     print("fps : ", fps)
    vid_num += 1
    count = 0
    # to get 4 frames for each second
    if (fps < 4) :
      div = 1
    else :
      div = fps // 4
    
    while (vidcap.isOpened()) :
      ret, image = vidcap.read()
      if (ret == False) :
        break
      if (int(vidcap.get(1)) % div == 0) :
        im2 = image.copy()
        im2[:,:,0] = image[:,:,2]
        im2[:,:,2] = image[:,:,0]
        image = im2
        # plt.imshow(image, aspect="auto")
        # plt.show()
        
        '''object detection'''
        coord = run_detector(detector, image)
        img = IMG.fromarray(image, 'RGB') # img = original image
        w,h = img.size
        temps = []

        count += 1

        if (coord == []) :
          continue

        for elem in coord :
          temp = [0] * 4
          temp[0] = elem[0] * w
          temp[1] = elem[1] * h
          temp[2] = elem[2] * w
          temp[3] = elem[3] * h

          y = temp[3] - temp[1]
          x = temp[2] - temp[0]

          if (x >= y) :
            gap = x - y
            gap = gap/2
            temp[1] -= gap
            temp[3] += gap
          else :
            gap = y - x
            gap = gap /2
            temp[0] -= gap
            temp[2] += gap

          temps.append(temp)
        cropped_img = img.crop(temps[0])
        final_img = cropped_img
        # rescale
        cropped_img = np.array(cropped_img, dtype=np.float32)
        cropped_img = np.expand_dims(cropped_img, axis=0)
        cropped_img = cropped_img/255.0

        '''predict'''
        pred_prob = model.predict(cropped_img)
        pred_class = np.argmax(pred_prob)

        classes = ['poo','nothing','pee']

        # save file & write text file
        if (pred_class == 0) : # predicted poo
          newdir = folder_o + "/"+fakename + "_%0.2f.jpg" %(count*0.25)
          final_img.save(newdir)
          line = "%0.2f\tpoo\t\n" %(count*0.25)
          fid.write(line)

        elif (pred_class == 1) : #predicted nothing
          newdir = folder_n + "/"+fakename + "_%0.2f.jpg" %(count*0.25)
          final_img.save(newdir)
          line = "%0.2f\tnothing\t\n" %(count*0.25)
          fid.write(line)

        elif (pred_class == 2) : # predicted pee
          newdir = folder_e + "/"+fakename + "_%0.2f.jpg" %(count*0.25)
          final_img.save(newdir)
          line = "%0.2f\tpee\t\n" %(count*0.25)
          fid.write(line)

        else :
          print("prediction failed")
          line = "%0.2f\tX\t\n" %(count*0.25)

          fid.write(line)
          

        # print("frame #" + str(count) + " : " + classes[pred_class])
    fid.close()
    vidcap.release()<jupyter_output>target : /content/gdrive/My Drive/soma/vid/Dog Does Cartwheel While Peeing-RBtV6S4hkdo.mp4
fps :  30.0
WARNING:tensorflow:Model was constructed with shape (None, 224, 224, 3) for input Tensor("resnet50_input:0", shape=(None, 224, 224, 3), dtype=float32), but it was called on an input with incompatible shape (None, 92, 92, 3).
{'fraction_non_alphanumeric': 0.12224622030237581, 'fraction_numerical': 0.018250539956803456, 'mean_word_length': 2.428730099962977, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '639596', 'license': 'MIT', 'n_tokens_mistral': 3176, 'n_tokens_neox': 3063, 'n_words': 741, 'path': 'deep_learning/pose_decision/2. construct_data_set.ipynb', 'repo_name': 'livlikwav/GoodPooPee_ML_backend', 'size': '9260'}