<jupyter_start><jupyter_text>========================================================   File name   : YOLOv3_colab_training.ipynb   Author      : PyLessons   Created date: 2020-09-30   Website     : https://pylessons.com/YOLOv3-TF2-GoogleColab   GitHub      : https://github.com/pythonlessons/TensorFlow-2.x-YOLOv3   Description : Train custom model on Google colab tutorial================================================================**Open this notebook from google drive****Go to "Edit" -> "Notebook settings" and enable GPU.**<jupyter_code># Check if NVIDIA GPU is enabled
!nvidia-smi<jupyter_output>Thu Oct 15 04:07:05 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   65C    P8    11W /  70W |      0MiB / 15079MiB |      0%      Default |
|                               |                      |                 ERR! |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>**Connect and authorize google drive with google colab:**<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')
!ls<jupyter_output>Mounted at /content/gdrive
gdrive	sample_data<jupyter_text>**Open our project "TensorFlow-2.x-YOLOv3" direcotry in google drive:**<jupyter_code>%cd gdrive/My\ Drive/TensorFlow-2.x-YOLOv3/
!ls<jupyter_output>/content/gdrive/My Drive/TensorFlow-2.x-YOLOv3
'Beltech Assignment Results'   mnist
 checkpoints		       model_data
 Collect_training_data.py      object_tracker.py
 deep_sort		       README.md
 detection_custom.py	       requirements.txt
 detection_demo.py	       tools
 detect_mnist.py	       train.py
 evaluate_mAP.py	       yolov3
 final-detection3.mp4	       YOLOv3_colab_training.ipynb
 IMAGES			       YOLOv3_detection.ipynb
 LICENSE<jupyter_text>**Install all required libraries for our project:**<jupyter_code>!pip install -r ./requirements.txt<jupyter_output><empty_output><jupyter_text>**Download yolov3.weights if you don't have it:**<jupyter_code>!wget -P model_data https://pjreddie.com/media/files/yolov3.weights
import tensorflow as tf
print(tf.__version__)
tf.test.gpu_device_name()<jupyter_output>2.3.0<jupyter_text>**Import requirements:**<jupyter_code>import cv2
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline  
import tensorflow as tf
from yolov3.yolov4 import Create_Yolo
from yolov3.configs import *
from yolov3.utils import Load_Yolo_model, image_preprocess, postprocess_boxes, nms, draw_bbox, read_class_names, load_yolo_weights
from yolov3.configs import *
import time
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
from google.colab.patches import cv2_imshow
from deep_sort import generate_detections as gdet
from collections import Counter
from collections import deque
import datetime
import math
def ccw(A, B, C):
  return (C[1] - A[1]) * (B[0] - A[0]) > (B[1] - A[1]) * (C[0] - A[0])

def intersect(A, B, C, D):
  return ccw(A, C, D) != ccw(B, C, D) and ccw(A, B, C) != ccw(A, B, D)

def vector_angle(midpoint, previous_midpoint):
  x = midpoint[0] - previous_midpoint[0]
  y = midpoint[1] - previous_midpoint[1]
  return math.degrees(math.atan2(y, x))<jupyter_output><empty_output><jupyter_text>**RTSP URL:**<jupyter_code>video_path   = 'rtsp://admin:admin123@172.16.58.3:5554/cam/realmonitor?channel=1&subtype=1'<jupyter_output><empty_output><jupyter_text>**Start training custom model:**<jupyter_code>def Object_tracking(Yolo, video_path, output_path, input_size=416, show=False, CLASSES=YOLO_COCO_CLASSES, score_threshold=0.3, iou_threshold=0.45, rectangle_colors='', Track_only = []):
    # Definition of the parameters
    max_cosine_distance = 0.7
    nn_budget = None
    
    #initialize deep sort object
    model_filename = 'model_data/mars-small128.pb'
    encoder = gdet.create_box_encoder(model_filename, batch_size=1)
    metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
    tracker = Tracker(metric)

    times, times_2 = [], []

    if video_path:
        vid = cv2.VideoCapture(video_path) # detect on video
    else:
        vid = cv2.VideoCapture(0) # detect from webcam

    # by default VideoCapture returns float instead of int
    width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
    assert (width==int(vid.get(3))), "Not matching - width"
    height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
    assert (height==int(vid.get(4))), "Not matching - height"
    fps = int(vid.get(cv2.CAP_PROP_FPS))
    codec = cv2.VideoWriter_fourcc(*'XVID')
    out = cv2.VideoWriter(output_path, codec, fps, (width,height)) # output_path must be .mp4
    print("Frame size - width: {}\t height: {}".format(width,height))
    NUM_CLASS = read_class_names(CLASSES)
    key_list = list(NUM_CLASS.keys()) 
    val_list = list(NUM_CLASS.values())

    current_date = datetime.datetime.now().date()
    count_dict = {}  # initiate dict for storing counts

    total_counter = 0
    up_count = 0
    down_count = 0
    frame_count = 0
    car_count = 0
    bus_count = 0
    truck_count = 0
    bike_count = 0
    class_counter = Counter()  # store counts of each detected class
    already_counted = deque(maxlen=50)  # temporary memory for storing counted IDs
    intersect_info = []  # initialise intersection list

    memory = {}

    while True:
        ret , frame = vid.read()
        print(ret)
        if cv2.waitKey(1) & 0xFF == ord('q'):
          break
        frame_count = frame_count + 1
        # line = [(0, int(0.55 * frame.shape[0])), (int(frame.shape[1]), int(0.55 * frame.shape[0]))]
        line = [(int(0.3*frame.shape[1]), int(0.65 * frame.shape[0])), (int(frame.shape[1]), int(0.65 * frame.shape[0]))]
        line1 = [(0, int(0.31 * frame.shape[0])), (int(0.3*frame.shape[1]), int(0.31 * frame.shape[0]))]
        cv2.line(frame, line[0], line[1], (0, 255, 255), 2)
        cv2.line(frame, line1[0], line1[1], (0, 255, 255), 2)
        try:
            original_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            original_frame = cv2.cvtColor(original_frame, cv2.COLOR_BGR2RGB)
        except:
            break
        image_data = image_preprocess(np.copy(original_frame), [input_size, input_size])
        #image_data = tf.expand_dims(image_data, 0)
        image_data = image_data[np.newaxis, ...].astype(np.float32)

        t1 = time.time()
        if YOLO_FRAMEWORK == "tf":
            pred_bbox = Yolo.predict(image_data)
        elif YOLO_FRAMEWORK == "trt":
            batched_input = tf.constant(image_data)
            result = Yolo(batched_input)
            pred_bbox = []
            for key, value in result.items():
                value = value.numpy()
                pred_bbox.append(value)
        
        #t1 = time.time()
        #pred_bbox = Yolo.predict(image_data)
        t2 = time.time()
        
        pred_bbox = [tf.reshape(x, (-1, tf.shape(x)[-1])) for x in pred_bbox]
        pred_bbox = tf.concat(pred_bbox, axis=0)

        bboxes = postprocess_boxes(pred_bbox, original_frame, input_size, score_threshold)
        bboxes = nms(bboxes, iou_threshold, method='nms')

        # extract bboxes to boxes (x, y, width, height), scores and names
        boxes, scores, names = [], [], []
        for bbox in bboxes:
            if len(Track_only) !=0 and NUM_CLASS[int(bbox[5])] in Track_only or len(Track_only) == 0:
                boxes.append([bbox[0].astype(int), bbox[1].astype(int), bbox[2].astype(int)-bbox[0].astype(int), bbox[3].astype(int)-bbox[1].astype(int)])
                scores.append(bbox[4])
                names.append(NUM_CLASS[int(bbox[5])])

        # Obtain all the detections for the given frame.
        boxes = np.array(boxes) 
        names = np.array(names)
        scores = np.array(scores)
        features = np.array(encoder(original_frame, boxes))
        detections = [Detection(bbox, score, class_name, feature) for bbox, score, class_name, feature in zip(boxes, scores, names, features)]

        # Pass detections to the deepsort object and obtain the track information.
        tracker.predict()
        tracker.update(detections)

        # Obtain info from the tracks
        tracked_bboxes = []
        for track in tracker.tracks:
            if not track.is_confirmed() or track.time_since_update > 5:
                continue 
            bbox = track.to_tlbr() # Get the corrected/predicted bounding box
            # track_cls = track.cls  # most common detection class for track
            class_name = track.get_class() #Get the class name of particular object
            tracking_id = track.track_id # Get the ID for the particular track
            index = key_list[val_list.index(class_name)] # Get predicted object index by object name
            tracked_bboxes.append(bbox.tolist() + [tracking_id, index]) # Structure data, that we could use it with our draw_bbox function
            midpoint = track.tlbr_midpoint(bbox)
            origin_midpoint = (midpoint[0], original_frame.shape[0] - midpoint[1])  # get midpoint respective to botton-left
            if track.track_id not in memory:
              print("memory!!")
              memory[track.track_id] = deque(maxlen=2)
            memory[track.track_id].append(midpoint)
            previous_midpoint = memory[track.track_id][0]
            origin_previous_midpoint = (previous_midpoint[0], original_frame.shape[0] - previous_midpoint[1])
            cv2.line(original_frame, midpoint, previous_midpoint, (0, 255, 0), 2)
            if intersect(midpoint, previous_midpoint, line[0], line[1]) and track.track_id not in already_counted:
              print("Here!!")
              class_counter[class_name] += 1
              total_counter += 1
              # draw red line
              cv2.line(original_frame, line[0], line[1], (0, 0, 255), 2)
              already_counted.append(track.track_id)  # Set already counted for ID to true.
              intersection_time = datetime.datetime.now() - datetime.timedelta(microseconds=datetime.datetime.now().microsecond)
              angle = vector_angle(origin_midpoint, origin_previous_midpoint)
              intersect_info.append([class_name, origin_midpoint, angle, intersection_time])
              if angle > 0:
                up_count += 1
              if angle < 0:
                down_count += 1
                 
            if intersect(midpoint, previous_midpoint, line1[0], line1[1]) and track.track_id not in already_counted:
              print("Here1!!")
              class_counter[class_name] += 1
              total_counter += 1
              # draw red line
              cv2.line(original_frame, line1[0], line1[1], (0, 0, 255), 2)
              already_counted.append(track.track_id)  # Set already counted for ID to true.
              intersection_time = datetime.datetime.now() - datetime.timedelta(microseconds=datetime.datetime.now().microsecond)
              angle = vector_angle(origin_midpoint, origin_previous_midpoint)
              intersect_info.append([class_name, origin_midpoint, angle, intersection_time])
              if angle > 0:
                up_count += 1
              if angle < 0:
                down_count += 1

        if len(memory) > 100:
          del memory[list(memory)[0]]

        print("Truck Class counter: ", class_counter["truck"])
        print("Car Class counter: ", class_counter["car"])
        print("Bus Class counter: ", class_counter["bus"])
        print("Motorbike Class counter: ", class_counter["motorbike"])

        cv2.putText(original_frame, "Bus: {}".format(str(class_counter["bus"])), (150,25), 0, 0.63, (0, 0, 255), 2)
        cv2.putText(original_frame, "Car: {}".format(str(class_counter["car"])), (150,50), 0, 0.63, (0, 0, 255), 2)
        cv2.putText(original_frame, "Motorbike: {}".format(str(class_counter["motorbike"])), (150,75), 0, 0.63, (0, 0, 255), 2)
        cv2.putText(original_frame, "Truck: {}".format(str(class_counter["truck"])), (150,100), 0, 0.63, (0, 0, 255), 2)
        # Draw total count.
        cv2.putText(original_frame, "Total: {}".format(str(total_counter)), (25,25), 0, 0.63, (255, 0, 0), 2)
        cv2.putText(original_frame, "up: {}".format(str(up_count)), (25,50), 0, 0.63, (255, 0, 0), 2)
        cv2.putText(original_frame, "down: {}".format(str(down_count)), (25,75), 0, 0.63, (255, 0, 0), 2)

        print("Total: {} ({} up, {} down)".format(str(total_counter), str(up_count),
                        str(down_count)))
        # draw detection on frame
        image = draw_bbox(original_frame, tracked_bboxes, CLASSES=CLASSES, tracking=True)

        t3 = time.time()
        times.append(t2-t1)
        times_2.append(t3-t1)
        
        times = times[-20:]
        times_2 = times_2[-20:]

        ms = sum(times)/len(times)*1000
        fps = 1000 / ms
        fps2 = 1000 / (sum(times_2)/len(times_2)*1000)
        
        # image = cv2.putText(image, "Time: {:.1f} FPS".format(fps), (0, 400), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)

        # draw original yolo detection
        # image = draw_bbox(image, bboxes, CLASSES=CLASSES, show_label=False, rectangle_colors=rectangle_colors, tracking=True)

        # print("Time: {:.2f}ms, Detection FPS: {:.1f}, total FPS: {:.1f}".format(ms, fps, fps2))
        if output_path != '': 
          image = cv2.resize(image, (width,height))
          h,w,c = image.shape
          assert (width==w), "Not matching - width"
          assert (height==h), "Not matching - height"
          out.write(image.astype('uint8'))
          print("Input shape: {}\nWritten frame! at {}".format(image.shape,output_path))
        # if show:
        #     cv2_imshow(image)
            
        #     if cv2.waitKey(25) & 0xFF == ord("q"):
        #         cv2.destroyAllWindows()
        #         break
    
    out.release() 
    # vid.release()       
    # cv2.destroyAllWindows()<jupyter_output><empty_output><jupyter_text>**Create Yolo v3 custom model and load custom trained weights**<jupyter_code>yolo = Load_Yolo_model()
Object_tracking(yolo, video_path, "final-detection46-XVID.mp4", input_size=YOLO_INPUT_SIZE, show=True, iou_threshold=0.1, rectangle_colors=(255,0,0), Track_only = ["car","truck","motorbike","bus"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15207064962868802, 'fraction_numerical': 0.032514885930287014, 'mean_word_length': 2.6529814271749754, 'pattern_counts': {'":': 2, '<': 35, '<?xml version=': 0, '>': 39, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '581276', 'license': 'MIT', 'n_tokens_mistral': 5169, 'n_tokens_neox': 4817, 'n_words': 1048, 'path': 'YOLOv3_detection.ipynb', 'repo_name': 'adisam007/Vehicle-Counting-Application', 'size': '14950'}