<jupyter_start><jupyter_text>Using Flow of center point and manual confidence update<jupyter_code># import the necessary packages
from imutils.video import VideoStream
import argparse
import datetime
import imutils
import time
import cv2
import time
import json
from bounding_box_utils.bounding_box_utils import iou
no_tracking_res = [] 
tracking_res = []
vs = cv2.VideoCapture('../modd.avi')
# initialize the first frame in the video stream
firstFrame = None
frameCount =0
step = 0.05
total_objects_no_tracking=0
total_objects_tracking=0
frame_width = int(vs.get(3))
frame_height = int(vs.get(4))
print(frame_width)
print(frame_height)
preds = []
pred = None
tracks=[]
out_tracking = cv2.VideoWriter('mod_ssd_tracking_all.avi',cv2.VideoWriter_fourcc('M','J','P','G'), 30, (frame_width,frame_height))
started = False
# your code
print("prev_count   current_count   entered   exited")
multiplier=0
cc=0
prev_frame=None
while frameCount<641:
    # grab the current frame and initialize the occupied/unoccupied
    # text
    frame = vs.read()[1]
    
 
    # if the frame could not be grabbed, then we have reached the end
    # of the video
    if frame is None:
        break
    frameCount+=1
    if(frameCount<0):
        continue
    elif started==False:
        start_time = time.time()
        started=True
    # resize the frame, convert it to grayscale, and blur it
    cv2.imwrite('temp.jpg',frame)
    img = image.load_img('temp.jpg', target_size=(img_height, img_width))
    img = image.img_to_array(img)
    input_images = []
    
    input_images.append(img)
    input_images = np.array(input_images)
    y_pred = model.predict(input_images)
    
    confidence_threshold = 0.1
    current_pred = []
    y_pred_thresh = [y_pred[k][y_pred[k,:,1] > confidence_threshold] for k in range(y_pred.shape[0])]
    f = frame
    
        
    # loop over the contours
    preds = []
    for box in y_pred_thresh[0]:
        if(box[0]!=4):
            continue
       
        preds.append(np.insert(box,0,0))
    
    #print(len(prev_pred),len(temp),len(entered_pred),len(exited_pred))
    print(tracks)
    for pred in preds:
        
        if(box_matches_list(pred,tracks)):
            
                
            index = box_in_list(pred,tracks)
            multiplier = tracks[index][0]
            org_conf =tracks[index][2]
            if(multiplier<0):
                multiplier=1
            else:
                multiplier +=1
            
             
            del tracks[index]
            
            pred[2] += multiplier*step
            
            pred[0] = multiplier
            #print(pred)
            tracks.append(pred)
        else:
            
            temp_pred = np.copy(pred)
           
            multiplier = 1
            
          
            temp_pred[0] = multiplier
            tracks.append(temp_pred)
         
            cc = frameCount
    print(tracks)
    for track in tracks:
        if(not box_matches_list(track,preds)):
            #print('not found in current frame')
            track[0]-=1
            #print('multiplier decreased to ',track[0])
            track[2]+=(step*track[0])
            frame_grey = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
            prev_frame_grey = cv2.cvtColor(prev_frame,cv2.COLOR_BGR2GRAY)
            flow = cv2.calcOpticalFlowFarneback(prev_frame_grey,frame_grey, None, 0.5, 3, 15, 3, 5, 1.2, 0)
            xmin = int(track[3] * frame.shape[1] / img_width)
            ymin = int(track[4] * frame.shape[0] / img_height)
            xmax =int(track[5] * frame.shape[1] / img_width)
            ymax =int(track[6] * frame.shape[0] / img_height)
            center_flow = flow[np.minimum(int((xmin+xmax)/2),flow.shape[0]-1),np.minimum(int((ymin+ymax)/2),flow.shape[1]-1)]
            #print('item is')
            #print(track)
            #print('center in image is')
            #print(int((xmin+xmax)/2),int((ymin+ymax)/2))
            #print('flow in image is')
           # print(center_flow)
            center_flow_min_x = int(center_flow[0] * img_width / frame.shape[1])
            center_flow_min_y = int(center_flow[1] * img_height / frame.shape[0])
            center_min_x = int((xmin+xmax)/2*img_width/frame.shape[1])
            center_min_y = int((ymin+ymax)/2*img_height/frame.shape[0])
            #print('center in min image is')
            #print(center_min_x,center_min_y)
            track[3] += center_flow_min_x
            track[5]+=center_flow_min_x
            track[4]+=center_flow_min_y
            track[6]+=center_flow_min_y
            #print('flow in min image is')
            #print(center_flow)
            
    to_display = [track for track in tracks if track[2]>0.4]
    #print(tracks)
    for box in to_display:
    # Transform the predicted bounding boxes for the 512x512 image to the original image dimensions.
    
        xmin = int(box[3] * frame.shape[1] / img_width)
        ymin = int(box[4] * frame.shape[0] / img_height)
        xmax =int(box[5] * frame.shape[1] / img_width)
        ymax =int(box[6] * frame.shape[0] / img_height)
        cv2.rectangle(frame, (int(xmin), int(ymin)), (int(xmax),int(ymax)), (0, 255, 0), 2)
        cv2.putText(frame,'{}: {:.2f}'.format(classes[int(box[1])], box[2]), (xmin, ymin),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
        
        total_objects_tracking+=1
        tracking_res.append({"image_id" : frameCount, "category_id" : 1, "bbox" : [float(xmin),float(ymin),float(xmax-xmin),float(ymax-ymin)], "score" : int(box[1])})
    
    # if the `q` key is pressed, break from the lop
    
    out_tracking.write(frame)
    prev_frame=frame
# cleanup the camera and close any open windows
elapsed_time = time.time() - start_time
print(elapsed_time)
vs.release()
cv2.destroyAllWindows()

out_tracking.release()

print(total_objects_tracking)
with open('res_tracking.json', 'w') as outfile:  
    json.dump(tracking_res, outfile)
with open('res_no_tracking.json', 'w') as outfile:  
    json.dump(no_tracking_res, outfile)
with open('res_tracking.json', 'w') as outfile:  
    json.dump(tracking_res, outfile)<jupyter_output><empty_output><jupyter_text>Using Key-Point Flow<jupyter_code># import the necessary packages
from imutils.video import VideoStream
import argparse
import datetime
import imutils
import time
import cv2 as cv
import time
import json
from bounding_box_utils.bounding_box_utils import iou
feature_params = dict( maxCorners = 25,
                       qualityLevel = 0.3,
                       minDistance = 7,
                       blockSize = 7 )
lk_params = dict( winSize  = (15,15),
                  maxLevel = 2,
                  criteria = (cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 0.03))
no_tracking_res = [] 
tracking_res = []
vs = cv.VideoCapture('../1.avi')
# initialize the first frame in the video stream
firstFrame = None
frameCount =0
step = 0.05
total_objects_no_tracking=0
total_objects_tracking=0
frame_width = int(vs.get(3))
frame_height = int(vs.get(4))
print(frame_width)
print(frame_height)
preds = []
pred = None
tracks=[]
out_tracking = cv.VideoWriter('1_tracking_keypoi.avi',cv.VideoWriter_fourcc('M','J','P','G'), 30, (frame_width,frame_height))
started = False
# your code
print("prev_count   current_count   entered   exited")
multiplier=0
cc=0
prev_frame=None
while frameCount<540:
    # grab the current frame and initialize the occupied/unoccupied
    # text
    frame = vs.read()[1]
    
 
    # if the frame could not be grabbed, then we have reached the end
    # of the video
    if frame is None:
        break
    frameCount+=1
    if(frameCount<0):
        continue
    elif started==False:
        start_time = time.time()
        started=True
    # resize the frame, convert it to grayscale, and blur it
    cv.imwrite('temp.jpg',frame)
    img = image.load_img('temp.jpg', target_size=(img_height, img_width))
    img = image.img_to_array(img)
    input_images = []
    
    input_images.append(img)
    input_images = np.array(input_images)
    y_pred = model.predict(input_images)
    
    confidence_threshold = 0.1
    current_pred = []
    y_pred_thresh = [y_pred[k][y_pred[k,:,1] > confidence_threshold] for k in range(y_pred.shape[0])]
    f = frame
    
        
    # loop over the contours
    preds = []
    for box in y_pred_thresh[0]:
        if(box[0]!=4):
            continue
        temp_pred = np.insert(box,0,0)
        temp_pred=np.insert(temp_pred,7,0)
        preds.append(temp_pred)
    for track in tracks:
        track[7]=0
    #print(len(prev_pred),len(temp),len(entered_pred),len(exited_pred))
    
    for pred in preds:
        index = box_matches_list(pred,tracks)
        if(index>-1):
            
            multiplier = tracks[index][0]
            org_conf =tracks[index][2]
            if(multiplier<0):
                multiplier=1
            else:
                multiplier +=1
            
             
            del tracks[index]
            
            pred[2] += multiplier*step
            
            pred[0] = multiplier
            #print(pred)
            tracks.append(pred)
        else:
            
            temp_pred = np.copy(pred)
           
            multiplier = 1
            
          
            temp_pred[0] = multiplier
            tracks.append(temp_pred)
         
            cc = frameCount

    for track in tracks:
        if(box_matches_list(track,preds)==-1):
            #print('not found in current frame')
            track[0]-=1
            #print('multiplier decreased to ',track[0])
            track[2]+=(step*track[0])
            frame_grey = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
            prev_frame_grey = cv.cvtColor(prev_frame,cv.COLOR_BGR2GRAY)
            mask = np.zeros(frame_grey.shape, dtype = "uint8")


            cv.rectangle(mask, (track[3], track[4]), (track[5], track[6]), (255, 255, 255), -1)
            p0 = cv.goodFeaturesToTrack(prev_frame_grey, mask = mask, **feature_params)
            if(not p0 is None ):
                p1, st, err = cv.calcOpticalFlowPyrLK(prev_frame_grey, frame_grey, p0, None, **lk_params)
                
                average_flow = np.average(p1-p0,0)[0]
            else:
                average_flow=[0,0]
            xmin = int(track[3] * frame.shape[1] / img_width)
            ymin = int(track[4] * frame.shape[0] / img_height)
            xmax =int(track[5] * frame.shape[1] / img_width)
            ymax =int(track[6] * frame.shape[0] / img_height)
            flow_x_box_coords = int(average_flow[0] * img_width / frame.shape[1])
            flow_y_box_coords = int(average_flow[1] * img_height / frame.shape[0])
            track[3] += flow_x_box_coords
            track[5]+=flow_x_box_coords
            track[4]+=flow_y_box_coords
            track[6]+=flow_y_box_coords
            
            
    to_display = [track for track in tracks if track[2]>0.4]
    #print(tracks)
    for box in to_display:
    # Transform the predicted bounding boxes for the 512x512 image to the original image dimensions.
    
        xmin = int(box[3] * frame.shape[1] / img_width)
        ymin = int(box[4] * frame.shape[0] / img_height)
        xmax =int(box[5] * frame.shape[1] / img_width)
        ymax =int(box[6] * frame.shape[0] / img_height)
        cv.rectangle(frame, (int(xmin), int(ymin)), (int(xmax),int(ymax)), (0, 255, 0), 2)
        cv.putText(frame,'{}: {:.2f}'.format(classes[int(box[1])], box[2]), (xmin, ymin),cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
        
        total_objects_tracking+=1
        tracking_res.append({"image_id" : frameCount, "category_id" : 1, "bbox" : [float(xmin),float(ymin),float(xmax-xmin),float(ymax-ymin)], "score" : int(box[1])})
    #print('preds%d'%len(preds))
    #print('tracks%d'%len(tracks))
    #print('displayed%d'%len(to_display))
    # if the `q` key is pressed, break from the lop
    #plt.figure()
    #plt.imshow(frame)
    out_tracking.write(frame)
    prev_frame=frame
# cleanup the camera and close any open windows
elapsed_time = time.time() - start_time
print(elapsed_time)
vs.release()
cv.destroyAllWindows()

out_tracking.release()

print(total_objects_tracking)
with open('1_res_keypoint.json', 'w') as outfile:  
    json.dump(tracking_res, outfile)
# import the necessary packages
from imutils.video import VideoStream
import argparse
import datetime
import imutils
import time
import cv2 as cv
import time
import json
from bounding_box_utils.bounding_box_utils import iou
feature_params = dict( maxCorners = 25,
                       qualityLevel = 0.3,
                       minDistance = 7,
                       blockSize = 7 )
lk_params = dict( winSize  = (15,15),
                  maxLevel = 2,
                  criteria = (cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 0.03))
no_tracking_res = [] 
tracking_res = []
vs = cv.VideoCapture('../modd.avi')
# initialize the first frame in the video stream
firstFrame = None
frameCount =0
step = 0.05
total_objects_no_tracking=0
total_objects_tracking=0
frame_width = int(vs.get(3))
frame_height = int(vs.get(4))
print(frame_width)
print(frame_height)
preds = []
pred = None
tracks=[]

out_tracking = cv.VideoWriter('1_tracking_keypoi.avi',cv.VideoWriter_fourcc('M','J','P','G'), 30, (frame_width,frame_height))
started = False
# your code
print("prev_count   current_count   entered   exited")
multiplier=0
cc=0
prev_frame=None

while frameCount<540:
    # grab the current frame and initialize the occupied/unoccupied
    # text
    frame = vs.read()[1]
    
 
    # if the frame could not be grabbed, then we have reached the end
    # of the video
    if frame is None:
        break
    frameCount+=1
    if(frameCount<150):
        continue
    elif started==False:
        start_time = time.time()
        started=True
    # resize the frame, convert it to grayscale, and blur it
    cv.imwrite('temp.jpg',frame)
    img = image.load_img('temp.jpg', target_size=(img_height, img_width))
    img = image.img_to_array(img)
    input_images = []
    
    input_images.append(img)
    input_images = np.array(input_images)
    y_pred = model.predict(input_images)
    
    confidence_threshold = 0.1
    current_pred = []
    y_pred_thresh = [y_pred[k][y_pred[k,:,1] > confidence_threshold] for k in range(y_pred.shape[0])]
    f = frame
    
    prediction = kalman.predict()
    process_noise = sqrt(kalman.processNoiseCov[0,0]) * np.random.randn(4, 1)
    state = np.dot(kalman.transitionMatrix, state) + process_noise
    #print(state)
    # loop over the contours
    preds = []
    for box in y_pred_thresh[0]:
        if(box[0]!=4):
            continue
        temp_pred = np.insert(box,0,0)
        temp_pred=np.insert(temp_pred,7,0)
        preds.append(temp_pred)
        
        print('correcting with ',box[2:6])
        kalman.correct(box[2:6])
        
    for track in tracks:
        track[7]=0
    #print(len(prev_pred),len(temp),len(entered_pred),len(exited_pred))
    
    for pred in preds:
        index = box_matches_list(pred,tracks)
        if(index>-1):
            
            multiplier = tracks[index][0]
            org_conf =tracks[index][2]
            if(multiplier<0):
                multiplier=1
            else:
                multiplier +=1
            
             
            del tracks[index]
            
            pred[2] += multiplier*step
            
            pred[0] = multiplier
            #print(pred)
            tracks.append(pred)
        else:
            
            temp_pred = np.copy(pred)
           
            multiplier = 1
            
          
            temp_pred[0] = multiplier
            tracks.append(temp_pred)
         
            cc = frameCount
    print('prediction:',kalman.predict())
    for track in tracks:
        if(tracks[7]==0 and box_matches_list(track,preds)==-1):
            #print('not found in current frame')
            track[0]-=1
            #print('multiplier decreased to ',track[0])
            track[2]+=(step*track[0])
            frame_grey = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
            prev_frame_grey = cv.cvtColor(prev_frame,cv.COLOR_BGR2GRAY)
            mask = np.zeros(frame_grey.shape, dtype = "uint8")


            cv.rectangle(mask, (track[3], track[4]), (track[5], track[6]), (255, 255, 255), -1)
            p0 = cv.goodFeaturesToTrack(prev_frame_grey, mask = mask, **feature_params)
            if(not p0 is None ):
                p1, st, err = cv.calcOpticalFlowPyrLK(prev_frame_grey, frame_grey, p0, None, **lk_params)
                
                average_flow = np.average(p1-p0,0)[0]
            else:
                average_flow=[0,0]
            xmin = int(track[3] * frame.shape[1] / img_width)
            ymin = int(track[4] * frame.shape[0] / img_height)
            xmax =int(track[5] * frame.shape[1] / img_width)
            ymax =int(track[6] * frame.shape[0] / img_height)
            flow_x_box_coords = int(average_flow[0] * img_width / frame.shape[1])
            flow_y_box_coords = int(average_flow[1] * img_height / frame.shape[0])
            track[3] += flow_x_box_coords
            track[5]+=flow_x_box_coords
            track[4]+=flow_y_box_coords
            track[6]+=flow_y_box_coords
            
            
    to_display = [track for track in tracks if track[2]>0.4]
    #print(tracks)
    for box in to_display:
    # Transform the predicted bounding boxes for the 512x512 image to the original image dimensions.
    
        xmin = int(box[3] * frame.shape[1] / img_width)
        ymin = int(box[4] * frame.shape[0] / img_height)
        xmax =int(box[5] * frame.shape[1] / img_width)
        ymax =int(box[6] * frame.shape[0] / img_height)
        cv.rectangle(frame, (int(xmin), int(ymin)), (int(xmax),int(ymax)), (0, 255, 0), 2)
        cv.putText(frame,'{}: {:.2f}'.format(classes[int(box[1])], box[2]), (xmin, ymin),cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
        
        total_objects_tracking+=1
        tracking_res.append({"image_id" : frameCount, "category_id" : 1, "bbox" : [float(xmin),float(ymin),float(xmax-xmin),float(ymax-ymin)], "score" : int(box[1])})
    #print('preds%d'%len(preds))
    #print('tracks%d'%len(tracks))
    #print('displayed%d'%len(to_display))
    # if the `q` key is pressed, break from the lop
    #plt.figure()
    #plt.imshow(frame)
    out_tracking.write(frame)
    prev_frame=frame
# cleanup the camera and close any open windows
elapsed_time = time.time() - start_time
print(elapsed_time)
vs.release()
cv.destroyAllWindows()

out_tracking.release()

print(total_objects_tracking)
with open('1_res_keypoint.json', 'w') as outfile:  
    json.dump(tracking_res, outfile)

#!/usr/bin/env python
"""
   Tracking of rotating point.
   Rotation speed is constant.
   Both state and measurements vectors are 1D (a point angle),
   Measurement is the real point angle + gaussian noise.
   The real and the estimated points are connected with yellow line segment,
   the real and the measured points are connected with red line segment.
   (if Kalman filter works correctly,
    the yellow segment should be shorter than the red one).
   Pressing any key (except ESC) will reset the tracking with a different speed.
   Pressing ESC will stop the program.
"""
# Python 2/3 compatibility
import sys
PY3 = sys.version_info[0] == 3

if PY3:
    long = int

import cv2 as cv
from math import cos, sin, sqrt
import numpy as np

if __name__ == "__main__":

    img_height = 500
    img_width = 500
    kalman = cv.KalmanFilter(4, 1, 0)

    code = long(-1)

    cv.namedWindow("Kalman")

    while True:
        state = 0.1 * np.random.randn(4, 1)

        kalman.transitionMatrix = np.array([[1., 1.,1.,1.], [0., 1.,1.,1.],[0,1,1,0],[1,0,0,1]])
        kalman.measurementMatrix = 1. * np.ones((1, 4))
        kalman.processNoiseCov = 1e-5 * np.eye(4)
        kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1))
        kalman.errorCovPost = 1. * np.ones((4, 4))
        kalman.statePost = 0.1 * np.random.randn(4, 1)

        while True:
            def calc_point(angle):
                return (np.around(img_width/2 + img_width/3*cos(angle), 0).astype(int),
                        np.around(img_height/2 - img_width/3*sin(angle), 1).astype(int))

            state_angle = state[0, 0]
            state_pt = calc_point(state_angle)

            prediction = kalman.predict()
            predict_angle = prediction[0, 0]
            predict_pt = calc_point(predict_angle)

            measurement = kalman.measurementNoiseCov * np.random.randn(1, 1)

            # generate measurement
            measurement = np.dot(kalman.measurementMatrix, state) + measurement

            measurement_angle = measurement[0, 0]
            measurement_pt = calc_point(measurement_angle)

            # plot points
            def draw_cross(center, color, d):
                cv.line(img,
                         (center[0] - d, center[1] - d), (center[0] + d, center[1] + d),
                         color, 1, cv.LINE_AA, 0)
                cv.line(img,
                         (center[0] + d, center[1] - d), (center[0] - d, center[1] + d),
                         color, 1, cv.LINE_AA, 0)

            img = np.zeros((img_height, img_width, 3), np.uint8)
            draw_cross(np.int32(state_pt), (255, 255, 255), 3)
            draw_cross(np.int32(measurement_pt), (0, 0, 255), 3)
            draw_cross(np.int32(predict_pt), (0, 255, 0), 3)

            cv.line(img, state_pt, measurement_pt, (0, 0, 255), 3, cv.LINE_AA, 0)
            cv.line(img, state_pt, predict_pt, (0, 255, 255), 3, cv.LINE_AA, 0)

            kalman.correct(measurement)

            process_noise = sqrt(kalman.processNoiseCov[0,0]) * np.random.randn(2, 1)
            state = np.dot(kalman.transitionMatrix, state) + process_noise

            cv.imshow("Kalman", img)

            code = cv.waitKey(100)
            if code != -1:
                break

        if code in [27, ord('q'), ord('Q')]:
            break

    cv.destroyWindow("Kalman")

kalman.correct??<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12341255442670537, 'fraction_numerical': 0.02603410740203193, 'mean_word_length': 2.3958108732481134, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '509451', 'license': 'Apache-2.0', 'n_tokens_mistral': 7764, 'n_tokens_neox': 7345, 'n_words': 1564, 'path': 'SSD300-Video.ipynb', 'repo_name': 'issamouawad/object_tracking', 'size': '22048'}