<jupyter_start><jupyter_text>Event detector

This notebook uses de [movienet models](https://github.com/movienet/movienet-tools) to detect characters and actions in a animated short, using sort for pos-processing, and exports the generated data to JSON format.<jupyter_code>from movienet.tools import ActionExtractor, PersonDetector, PersonExtractor # Models

import cv2 as cv # Image manipulation
import numpy as np 
import json # To export the results<jupyter_output><empty_output><jupyter_text>Step 1: Detection

- Loads the two person detection and action extractor models and action labels
- Run the models in the video, with a utility cell to save the data, in case you need to perform the process in parts or an exception occurs
  - It might take a while to process the entire short<jupyter_code>act = ActionExtractor(require_normalized_bbox=False) # Action detector
psd = PersonDetector() # Person detector

action_raw= { "name": "bend/bow (at the waist)" ,"_id": 1 ,"_type": "PERSON_MOVEMENT" } , { "name": "crawl" ,"_id": 2 ,"_type": "PERSON_MOVEMENT" } , { "name": "crouch/kneel" ,"_id": 3 ,"_type": "PERSON_MOVEMENT" } , { "name": "dance" ,"_id": 4 ,"_type": "PERSON_MOVEMENT" } , { "name": "fall down" ,"_id": 5 ,"_type": "PERSON_MOVEMENT" } , { "name": "get up" ,"_id": 6 ,"_type": "PERSON_MOVEMENT" } , { "name": "jump/leap" ,"_id": 7 ,"_type": "PERSON_MOVEMENT" } , { "name": "lie/sleep" ,"_id": 8 ,"_type": "PERSON_MOVEMENT" } , { "name": "<NAME>" ,"_id": 9 ,"_type": "PERSON_MOVEMENT" } , { "name": "run/jog" ,"_id": 10 ,"_type": "PERSON_MOVEMENT" } , { "name": "sit" ,"_id": 11 ,"_type": "PERSON_MOVEMENT" } , { "name": "stand" ,"_id": 12 ,"_type": "PERSON_MOVEMENT" } , { "name": "swim" ,"_id": 13 ,"_type": "PERSON_MOVEMENT" } , { "name": "walk" ,"_id": 14 ,"_type": "PERSON_MOVEMENT" } , { "name": "<NAME>" ,"_id": 15 ,"_type": "OBJECT_MANIPULATION" } , { "name": "brush teeth" ,"_id": 16 ,"_type": "OBJECT_MANIPULATION" } , { "name": "carry/hold (an object)" ,"_id": 17 ,"_type": "OBJECT_MANIPULATION" } , { "name": "catch (an object)" ,"_id": 18 ,"_type": "OBJECT_MANIPULATION" } , { "name": "chop" ,"_id": 19 ,"_type": "OBJECT_MANIPULATION" } , { "name": "climb (e.g., a mountain)" ,"_id": 20 ,"_type": "OBJECT_MANIPULATION" } , { "name": "clink glass" ,"_id": 21 ,"_type": "OBJECT_MANIPULATION" } , { "name": "close (e.g., a door, a box)" ,"_id": 22 ,"_type": "OBJECT_MANIPULATION" } , { "name": "cook" ,"_id": 23 ,"_type": "OBJECT_MANIPULATION" } , { "name": "cut" ,"_id": 24 ,"_type": "OBJECT_MANIPULATION" } , { "name": "dig" ,"_id": 25 ,"_type": "OBJECT_MANIPULATION" } , { "name": "dress/put on clothing" ,"_id": 26 ,"_type": "OBJECT_MANIPULATION" } , { "name": "drink" ,"_id": 27 ,"_type": "OBJECT_MANIPULATION" } , { "name": "drive (e.g., a car, a truck)" ,"_id": 28 ,"_type": "OBJECT_MANIPULATION" } , { "name": "eat" ,"_id": 29 ,"_type": "OBJECT_MANIPULATION" } , { "name": "enter" ,"_id": 30 ,"_type": "OBJECT_MANIPULATION" } , { "name": "exit" ,"_id": 31 ,"_type": "OBJECT_MANIPULATION" } , { "name": "extract" ,"_id": 32 ,"_type": "OBJECT_MANIPULATION" } , { "name": "fishing" ,"_id": 33 ,"_type": "OBJECT_MANIPULATION" } , { "name": "hit (an object)" ,"_id": 34 ,"_type": "OBJECT_MANIPULATION" } , { "name": "kick (an object)" ,"_id": 35 ,"_type": "OBJECT_MANIPULATION" } , { "name": "lift/pick up" ,"_id": 36 ,"_type": "OBJECT_MANIPULATION" } , { "name": "listen (e.g., to music)" ,"_id": 37 ,"_type": "OBJECT_MANIPULATION" } , { "name": "open (e.g., a window, a car door)" ,"_id": 38 ,"_type": "OBJECT_MANIPULATION" } , { "name": "paint" ,"_id": 39 ,"_type": "OBJECT_MANIPULATION" } , { "name": "play board game" ,"_id": 40 ,"_type": "OBJECT_MANIPULATION" } , { "name": "play musical instrument" ,"_id": 41 ,"_type": "OBJECT_MANIPULATION" } , { "name": "play with pets" ,"_id": 42 ,"_type": "OBJECT_MANIPULATION" } , { "name": "point to (an object)" ,"_id": 43 ,"_type": "OBJECT_MANIPULATION" } , { "name": "press" ,"_id": 44 ,"_type": "OBJECT_MANIPULATION" } , { "name": "pull (an object)" ,"_id": 45 ,"_type": "OBJECT_MANIPULATION" } , { "name": "push (an object)" ,"_id": 46 ,"_type": "OBJECT_MANIPULATION" } , { "name": "put down" ,"_id": 47 ,"_type": "OBJECT_MANIPULATION" } , { "name": "read" ,"_id": 48 ,"_type": "OBJECT_MANIPULATION" } , { "name": "ride (e.g., a bike, a car, a horse)" ,"_id": 49 ,"_type": "OBJECT_MANIPULATION" } , { "name": "row boat" ,"_id": 50 ,"_type": "OBJECT_MANIPULATION" } , { "name": "sail boat" ,"_id": 51 ,"_type": "OBJECT_MANIPULATION" } , { "name": "shoot" ,"_id": 52 ,"_type": "OBJECT_MANIPULATION" } , { "name": "shovel" ,"_id": 53 ,"_type": "OBJECT_MANIPULATION" } , { "name": "smoke" ,"_id": 54 ,"_type": "OBJECT_MANIPULATION" } , { "name": "stir" ,"_id": 55 ,"_type": "OBJECT_MANIPULATION" } , { "name": "take a photo" ,"_id": 56 ,"_type": "OBJECT_MANIPULATION" } , { "name": "text on/look at a cellphone" ,"_id": 57 ,"_type": "OBJECT_MANIPULATION" } , { "name": "throw" ,"_id": 58 ,"_type": "OBJECT_MANIPULATION" } , { "name": "touch (an object)" ,"_id": 59 ,"_type": "OBJECT_MANIPULATION" } , { "name": "turn (e.g., a screwdriver)" ,"_id": 60 ,"_type": "OBJECT_MANIPULATION" } , { "name": "watch (e.g., TV)" ,"_id": 61 ,"_type": "OBJECT_MANIPULATION" } , { "name": "work on a computer" ,"_id": 62 ,"_type": "OBJECT_MANIPULATION" } , { "name": "write" ,"_id": 63 ,"_type": "OBJECT_MANIPULATION" } , { "name": "fight/hit (a person)" ,"_id": 64 ,"_type": "PERSON_INTERACTION" } , { "name": "give/serve (an object) to (a person)" ,"_id": 65 ,"_type": "PERSON_INTERACTION" } , { "name": "grab (a person)" ,"_id": 66 ,"_type": "PERSON_INTERACTION" } , { "name": "hand clap" ,"_id": 67 ,"_type": "PERSON_INTERACTION" } , { "name": "hand shake" ,"_id": 68 ,"_type": "PERSON_INTERACTION" } , { "name": "hand wave" ,"_id": 69 ,"_type": "PERSON_INTERACTION" } , { "name": "hug (a person)" ,"_id": 70 ,"_type": "PERSON_INTERACTION" } , { "name": "kick (a person)" ,"_id": 71 ,"_type": "PERSON_INTERACTION" } , { "name": "kiss (a person)" ,"_id": 72 ,"_type": "PERSON_INTERACTION" } , { "name": "lift (a person)" ,"_id": 73 ,"_type": "PERSON_INTERACTION" } , { "name": "listen to (a person)" ,"_id": 74 ,"_type": "PERSON_INTERACTION" } , { "name": "play with kids" ,"_id": 75 ,"_type": "PERSON_INTERACTION" } , { "name": "push (another person)" ,"_id": 76 ,"_type": "PERSON_INTERACTION" } , { "name": "sing to (e.g., self, a person, a group)" ,"_id": 77 ,"_type": "PERSON_INTERACTION" } , { "name": "take (an object) from (a person)" ,"_id": 78 ,"_type": "PERSON_INTERACTION" } , { "name": "talk to (e.g., self, a person, a group)" ,"_id": 79 ,"_type": "PERSON_INTERACTION" } , { "name": "watch (a person)" ,"_id": 80 ,"_type": "PERSON_INTERACTION" }

action_label = [] # Labels for the action detector results

for action in action_raw:
    action_label.append(action["name"])
# Open the video
cap = cv.VideoCapture("One man band.mp4")
total_frames = cap.get(cv.CAP_PROP_FRAME_COUNT)

# Loads stored data, if you're going to continue where you left off
total_data_result = []
last_frame = -1
try:
    result_file = open("one_man_band_events.json", "r")
    total_data_result = json.load(result_file)
    result_file.close()

    last_frame = total_data_result[-1]["frame"]
except:
    pass

# Process all the frames
total_time = 0
start_tick = cv.getTickCount()

for i in range(last_frame+1, int(total_frames)):
    start_frame_tick = cv.getTickCount()
    
    cap.set(cv.CAP_PROP_POS_FRAMES, i)
    ret, frame = cap.read()

    result = {"frame":i}

    # Detect persons
    person_bbox = psd.detect(frame)
    
    # Extract the action of each person
    if person_bbox.shape[0] > 0:
        result["person"] = person_bbox.tolist()

        action_result = act.extract([frame], person_bbox)

        result["action"] = []

        for j in range(person_bbox.shape[0]):
            action = action_label[np.argmax(action_result[j]["score"])]
            result["action"].append(action)
    
    # Store the frame data
    total_data_result.append(result)

    end_frame_tick = cv.getTickCount()
    frame_time = (end_frame_tick - start_frame_tick) / cv.getTickFrequency()
    total_time += frame_time
    mean_time = total_time / (i - last_frame)

    if i % 100 == 0 or i == last_frame+1:
        print(i, "frames completed,", i/total_frames*100, "%", mean_time, "sec/frame", mean_time*(total_frames-i)/60, "min to go")


end_tick = cv.getTickCount()

print("time =", (end_tick - start_tick) / cv.getTickFrequency())


result_file = open("one_man_band_events.json", "w")
json.dump(total_data_result, result_file)
result_file.close()
# Stores the data
result_file = open("one_man_band_events.json", "r")
total_data_result = json.load(result_file)
result_file.close()
# Show the detected persons and actions

cap = cv.VideoCapture("One man band.mp4")
total_frames = cap.get(cv.CAP_PROP_FRAME_COUNT)

for i in range(int(total_frames)):
    cap.set(cv.CAP_PROP_POS_FRAMES, i)
    ret, frame = cap.read()

    if "person" in total_data_result[i]:
        for j in range(len(total_data_result[i]["person"])):
            bbox = np.array(total_data_result[i]["person"][j], int)
            cv.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
            
            action = total_data_result[i]["action"][j]
            cv.putText(frame, action, (bbox[0],bbox[1]), cv.FONT_HERSHEY_SIMPLEX,0.7,(0,255,0),2,cv.LINE_AA)

    
    
    cv.imshow("frame", frame)
    if cv.waitKey(1) == ord('q'):
        break

cv.destroyAllWindows()<jupyter_output><empty_output><jupyter_text>Step 2: Pos-processing

- Uses sort algorithm to track the persons
  - Use intersection over union to link the detected action to the tracked bounding box
- Use a person extractor to see the corresponding character to each tracked person<jupyter_code>def intersection_over_union(bbox1, bbox2):
    '''
        Computes the distance between two bounding boxes,
        using the intersection over union metric
    '''
    area1 = abs((bbox1[2]-bbox1[0]) * (bbox1[3]-bbox1[1]))
    area2 = abs((bbox2[2]-bbox2[0]) * (bbox2[3]-bbox2[1]))
    
    inter_x = max(bbox1[0], bbox2[0]) - min(bbox1[2], bbox2[2])
    inter_y = max(bbox1[1], bbox2[1]) - min(bbox1[3], bbox2[3])
    inter_area = inter_x*inter_y

    union_area = area1+area2 - inter_area
   
    return inter_area/union_area
     
from sort import *

tracker = Sort() # SORT bounding box tracker

tracked_bbox = {}

# Tracks the detected bounging boxes
for i in range(int(total_frames)):
    frame_index = total_data_result[i]["frame"]

    if "person" in total_data_result[i]:
        track_bbs_ids = tracker.update(np.array(total_data_result[i]["person"]))
    else:
        track_bbs_ids = tracker.update()
        pass
    
    # Associate the action with the person
    for j in range(track_bbs_ids.shape[0]):
        if not track_bbs_ids[j][4] in tracked_bbox:
            tracked_bbox[track_bbs_ids[j][4]] = {"action":[]}
        tracked_bbox[track_bbs_ids[j][4]][frame_index] = {}
        tracked_bbox[track_bbs_ids[j][4]][frame_index]["bounding_box"] = track_bbs_ids[j][:4]
        tracked_bbox[track_bbs_ids[j][4]][frame_index]["action"] = []


        for k in range(len( total_data_result[i]["person"])):
            if intersection_over_union(track_bbs_ids[j][:4], total_data_result[i]["person"][k]) > 0.8:
                tracked_bbox[track_bbs_ids[j][4]][frame_index]["action"].append(total_data_result[i]["action"][k])
                tracked_bbox[track_bbs_ids[j][4]]["action"].append(total_data_result[i]["action"][k])
# Shows the tracked bounding boxes
cap = cv.VideoCapture("One man band.mp4")
total_frames = cap.get(cv.CAP_PROP_FRAME_COUNT)

for i in range(int(total_frames)):
    cap.set(cv.CAP_PROP_POS_FRAMES, i)
    ret, frame = cap.read()

    for actor_id in tracked_bbox:
        if i in tracked_bbox[actor_id]:
            bbox = tracked_bbox[actor_id][i]["bounding_box"].astype(int)
            cv.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
            

            actions, counts = np.unique(tracked_bbox[actor_id]["action"], return_counts=True)
            text = actions[np.argmax(counts)] 

            cv.putText(frame, text, (bbox[0],bbox[1]), cv.FONT_HERSHEY_SIMPLEX,0.7,(0,255,0),2,cv.LINE_AA)
    
    cv.imshow("frame", cv.resize(frame, (1920//2, 1080//2)))
    if cv.waitKey(1) == ord('q'):
        break
        
        
cv.destroyAllWindows()
from movienet.tools.detector.persondetector import resources_dir
import ntpath

weight_path = ntpath.join(resources_dir, 'resnet50_csm.pth')

extractor = PersonExtractor(weight_path) # Extractor to compare the detected people with the characters


# Extract the feature of each character
actors_feat = np.ndarray((3,256), np.float32)
for i in range(3):
    actor_img = cv.imread("actor"+str(i+1)+".png")
    feat = extractor.extract(actor_img)

    feat /= np.linalg.norm(feat)

    actors_feat[i] = feat
# Prints the match matrix to evaluate the threashold of matchs
match_matrix = np.ndarray((3,3))

for i in range(3):
    for j in range(3):
        match_matrix[i][j] = actors_feat[i].dot(actors_feat[j])

print(match_matrix)
# Run the person feature extractor on every tracked bounding box, and associate the corresponding actor
cap = cv.VideoCapture("One man band.mp4")
total_frames = cap.get(cv.CAP_PROP_FRAME_COUNT)

for actor_id in list(tracked_bbox.keys()):
    n_score = 0
    scores = np.zeros(3)
    
    frame_index = list(tracked_bbox[actor_id].keys())[1]
    for frame_index in list(tracked_bbox[actor_id].keys())[1:]:

        cap.set(cv.CAP_PROP_POS_FRAMES, frame_index)
        ret, frame = cap.read()

        bbox = tracked_bbox[actor_id][frame_index]["bounding_box"]
        bbox[bbox<0] = 0

        mask = frame[int(bbox[1]):int(bbox[3]), int(bbox[0]):int(bbox[2])]

        mask_feature = extractor.extract(mask)
        mask_feature /= np.linalg.norm(mask_feature)

        for i in range(3):
            scores[i] += actors_feat[i].dot(mask_feature)

        n_score += 1

    scores /= n_score
    actor_index = np.argmax(scores)
    actor_score = scores[actor_index]

    tracked_bbox[actor_id]["actor_id"] = actor_index
    tracked_bbox[actor_id]["score"] = actor_score
    tracked_bbox[actor_id]["scores"] = scores

    # If the best match is bad, delete the bounding box
    if actor_score < 0.65:
        del tracked_bbox[actor_id]
        pass

cap.release()<jupyter_output><empty_output><jupyter_text>Step 3: Export

Export the results to JSON format.<jupyter_code># Separate detections by character
final_result = {0:{}, 1:{}, 2:{}}

for actor_id in tracked_bbox:
    real_actor_id = tracked_bbox[actor_id]["actor_id"]
    score = tracked_bbox[actor_id]["score"]

    frames_index = list(tracked_bbox[actor_id].keys())[1:]
    frames_index.remove("actor_id")
    frames_index.remove('score')
    frames_index.remove('scores')

    actions, counts = np.unique(tracked_bbox[actor_id]["action"], return_counts=True)
    action = actions[np.argmax(counts)] 

    for index in frames_index:
        data = tracked_bbox[actor_id][index]
        data["actor_score"] = score
        data["action"] = action

        if index in final_result[real_actor_id]:
            if final_result[real_actor_id][index]["actor_score"] < score:
                final_result[real_actor_id][index] = data
        else:
            final_result[real_actor_id][index] = data

# Shows the detections

cap = cv.VideoCapture("One man band.mp4")
total_frames = cap.get(cv.CAP_PROP_FRAME_COUNT)

for i in range(int(total_frames)):
    cap.set(cv.CAP_PROP_POS_FRAMES, i)
    ret, frame = cap.read()

    for actor_id in final_result:
        if i in final_result[actor_id]:
            bbox = final_result[actor_id][i]["bounding_box"].astype(int)
            cv.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2) 

            actor_text = str(actor_id)+ " " +"{0:.2f}".format(final_result[actor_id][i]["actor_score"])

            action_text = final_result[actor_id][i]["action"]
            
            cv.putText(frame, actor_text, (bbox[0],bbox[1]), cv.FONT_HERSHEY_SIMPLEX,0.7,(0,255,0),2,cv.LINE_AA)
            cv.putText(frame, action_text, (bbox[0],bbox[3]), cv.FONT_HERSHEY_SIMPLEX,0.7,(0,255,0),2,cv.LINE_AA)
    
    cv.imshow("frame", cv.resize(frame, (1920//2, 1080//2)))
    
    if cv.waitKey(1) == ord('q'):
        break
        
        
cv.destroyAllWindows()
# Export the data

def encoder(obj):
    if isinstance(obj, np.integer):
        return int(obj)
    elif isinstance(obj, np.floating):
        return float(obj)
    elif isinstance(obj, np.ndarray):
        return obj.tolist()
    
    return obj.__dict__

result_file = open("one_man_band_events_filtered.json", "w")
json.dump(final_result, result_file, default=lambda o: encoder(o))
result_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17587150090205436, 'fraction_numerical': 0.02065995460629692, 'mean_word_length': 3.8790459965928448, 'pattern_counts': {'":': 242, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '139132', 'license': 'MIT', 'n_tokens_mistral': 6849, 'n_tokens_neox': 6211, 'n_words': 1416, 'path': 'Event/event_detector.ipynb', 'repo_name': 'EltonCN/Emotion-estimation-applied-to-storytelling-reaction', 'size': '17194'}