<jupyter_start><jupyter_text>Yolo V3 Object DetectionNotebook is copied from face_extraction.ipynb.Useful [resource](https://medium.com/@amrokamal_47691/yolo-yolov2-and-yolov3-all-you-want-to-know-7e3e92dc4899) on YOLO<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=True)


!ls drive/'My Drive'/cs231n-project/datasets/emotiw/train | head

!pip3 install dlib
!pip3 install face_recognition
import glob
import json
import math
import os
import pickle 
import sys

import pprint

# enter the foldername in your Drive where you have saved the project contents
TRAINFOLDERNAME = 'datasets/emotiw/train'
MODELFOLDERNAME = 'datasets/emotiw/models'
assert TRAINFOLDERNAME is not None, "[!] Enter the foldername."
assert MODELFOLDERNAME is not None, "[!] Enter the foldername."

PROJECT_PATH = os.path.join('/content', 'drive', 'My Drive', 'cs231n-project')
TRAIN_PATH = os.path.join(PROJECT_PATH, TRAINFOLDERNAME)
MODEL_PATH = os.path.join(PROJECT_PATH, MODELFOLDERNAME)

sys.path.append(TRAIN_PATH)

from concurrent.futures import ThreadPoolExecutor
import cv2
import face_recognition
from google.colab.patches import cv2_imshow
from keras.models import load_model
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
import numpy as np

from yolov3 import yolo3_one_file_to_detect_them_all as yolo3_functions

# Note: No trailing slash
DOWNSAMPLE_RATIO = 1
SAMPLE_EVERY = 10 # eg. if using 30 FPS and SAMPLE_EVERY=10, we would produce at 3 FPS

def create_yolov3_model():
    """
    Create YOLO V3 Model using Pretrained weights

    Mostly following this tutorial: https://machinelearningmastery.com/how-to-perform-object-detection-with-yolov3-in-keras/
    """
    print('Defining the YOLO v3 Model')
    model = yolo3_functions.make_yolov3_model()

    print('Loading the model weights')
    weight_file = os.path.join(TRAIN_PATH, 'yolov3', 'yolov3.weights')
    weight_reader = yolo3_functions.WeightReader(weight_file)

    # Next, call the load_weights() function of the WeightReader instance
    # passing in our defined Keras model to set the weights into the layers.
    print('Setting the model weights')
    weight_reader.load_weights(model)

    # Next, save this model to a Keras compatible .h5 model file ready for later use.
    print('Saving the model')
    model.save(os.path.join(MODEL_PATH, 'yolov3_model.h5'))

def process_video(video_dir, video_name, video_num, total_videos, model):
    """
    Processes a video by extracting the faces from each frame and creating a 
    list of list of faces which is then saved as a pickled object.

    NOTE: Faces are not guaranteed to be the same across frames.
          eg. 'face 1' in frame 1 may not be the same as 'face 1' in frame 2

    Pickle Object Format:
        [
            [
                [frame 1, face 1],
                [frame 1, face 2],
                [frame 1, face 3]
            ],
            [
                [frame 2, face 1],
                [frame 2, face 2]
            ],
            ...
        ]

    """
    print(f"Processing video {video_num}/{total_videos} with name {video_name}\n")
    vidcap = cv2.VideoCapture(os.path.join(video_dir, video_name))

    input_length = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
    frame_width = int(vidcap.get(cv2.CAP_PROP_FRAME_WIDTH))
    frame_height = int(vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    fps = int(vidcap.get(cv2.CAP_PROP_FPS))

    metadata = {}
    metadata[video_name] = {}
    faces_all_frames = []
    success, orig_image = vidcap.read()
    count = 0
    i = 1
    while success:
        if count % SAMPLE_EVERY == 0:
            print(f"   Working on {video_name} frame {count} video num {video_num}")
            # NOTE:
            # cv2_imshow is used by Colab in place of cv2.imshow('image', image) as 
            # the latter command is known to crash Jupyter sessions in Colab

            if video_num == 0 and count == 0:
                print(f"Showing sample frame for {video_name} with shape {orig_image.shape}")
                print("=" * 20)

            image_h, image_w = orig_image.shape[:2]
            print('printing type image')
            print(type(orig_image))

            if video_num == 0 and count == 0:
                print("Original frame: ")
                cv2_imshow(orig_image)
                print("")

            # Convert from BGR color (OpenCV) to RGB color (face_recognition)
            # rgb_image = image[:, :, ::-1]

            # https://machinelearningmastery.com/how-to-perform-object-detection-with-yolov3-in-keras/
            # The first step is to load the Keras model. This might be the slowest part of making a prediction.
            # Resize the image because YOLO expects 416 by 416
            # define the expected input shape for the model
            input_w, input_h = 416, 416
            image = cv2.resize(orig_image, (input_w, input_h), interpolation = cv2.INTER_CUBIC)
            print('Shape of image is {}'.format(image.shape))

            # scale pixel values to [0, 1]
            image = image.astype('float32')
            image /= 255.0

            # add a dimension so that we have one sample
            image = np.expand_dims(image, 0)

            # We can now feed the photo into the Keras model and make a prediction.
            # Make Prediction
            yhat = model.predict(image)

            print('Summarize the shape of the list of arrays')
            print([a.shape for a in yhat])

            # define the anchors
            anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]
            
            # define the probability threshold for detected objects
            class_threshold = 0.6
            boxes = list()
            for i in range(len(yhat)):
                # decode the output of the network
                boxes += yolo3_functions.decode_netout(yhat[i][0], anchors[i], class_threshold, input_h, input_w)
            
            # correct the sizes of the bounding boxes for the shape of the image
            yolo3_functions.correct_yolo_boxes(boxes, image_h=image_h, image_w=image_w, net_h=input_h, net_w=input_w)
            
            # suppress non-maximal boxes
            yolo3_functions.do_nms(boxes, 0.5)
            
            # define the labels
            # Double check all the categories that YoloV3 can predict 
            labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck",
              "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
              "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe",
              "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
              "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
              "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana",
              "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
              "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse",
              "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator",
              "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"]
           
            # get the details of the detected objects
            v_boxes, v_labels, v_scores = yolo3_functions.get_boxes(boxes, labels, class_threshold)
            
            # summarize what we found
            for i in range(len(v_boxes)):
                print(v_labels[i], v_scores[i])
            
            # draw what we found
            yolo3_functions.draw_boxes(orig_image, v_boxes, v_labels, v_scores)

            v_box_locations = [(v_box.xmin, v_box.xmax, v_box.ymin, v_box.ymax) for v_box in v_boxes]

            metadata[video_name][count] = {
                'vboxes': v_box_locations,
                'v_labels': v_labels,
                'v_scores': v_scores,
            }

        success, orig_image = vidcap.read()
        count += 1
    video_num += 1
    vidcap.release()

    return metadata

create_yolov3_model()
# Testing for one video
# Note you might get an error that says UserWarning: No training configuration found in save file
# Seems like it's ok to ignore for now: https://github.com/fizyr/keras-retinanet/issues/207
print('Loading YOLO v3 Keras Model')
model = load_model(os.path.join(MODEL_PATH, 'yolov3_model.h5'))
metadata = process_video(video_dir=TRAIN_PATH, video_name="2_1.mp4", video_num=0, total_videos=10, model=model)
pprint.pprint(metadata)
# Running over all videos now

# Getting a list of all video files in a directory
file_path_names = glob.glob(os.path.join(TRAIN_PATH, '*.mp4'))
filenames = [os.path.basename(file_path_name) for file_path_name in file_path_names]

# # Just testing a few, can comment out later
# filenames = ['344_7.mp4', '260_13.mp4', '220_4.mp4', '245_8.mp4']

print(filenames)

# Process all videos, run YOLO, and store output
metadata_all = {}
for video_filename in filenames:
    print('Processing video: {}'.format(video_filename))
    metadata = process_video(video_dir=TRAIN_PATH, video_name=video_filename, video_num=0, total_videos=10, model=model)
    metadata_all.update(metadata)
with open(os.path.join(TRAIN_PATH, 'yolov3', 'metadata_all.json'), 'w') as fp:
    json.dump(metadata_all, fp, sort_keys=True, indent=4, separators=(',', ': '))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10952925283175725, 'fraction_numerical': 0.020887457133949912, 'mean_word_length': 2.7830188679245285, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '129186', 'license': 'MIT', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3035, 'n_words': 862, 'path': 'notebooks/yolo_obj_detection/yolov3_object_detection.ipynb', 'repo_name': 'kevincong95/cs231n-emotiw', 'size': '9623'}