<jupyter_start><jupyter_text>Processing Cityscapes videos with Keras/tf segmentation model and OpenCV  Imports :<jupyter_code>import cv2
import numpy as np
import os
import tensorflow as tf
os.environ["SM_FRAMEWORK"] = "tf.keras"
from segmentation_models.losses import bce_jaccard_loss
from segmentation_models.metrics import iou_score
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.vgg16 import preprocess_input
import time<jupyter_output>Segmentation Models: using `tf.keras` framework.<jupyter_text>Functions and configuration :<jupyter_code>model_path = 'C://Users//Lewin//Downloads//OC//Projet_8//Flask_webapp//azureml-models//Unet_vgg16_fullaugm_cpu//1//model'
image_size = (224, 224)

# Custom metric used by model : 
def dice_coeff(y_true, y_pred):
    smooth = 1.
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
    return score

# Grouping object class labels into main categories
cats = {'void': [0, 1, 2, 3, 4, 5, 6],
 'flat': [7, 8, 9, 10],
 'construction': [11, 12, 13, 14, 15, 16],
 'object': [17, 18, 19, 20],
 'nature': [21, 22],
 'sky': [23],
 'human': [24, 25],
 'vehicle': [26, 27, 28, 29, 30, 31, 32, 33, -1]}

# Configuring colors for each object category
category_colors = [[0.,0.,0.],
              [128.,128.,128.],
              [255.,128.,0.],
              [255.,255.,0.],
              [0.,255.,0.],
              [0.,0.,255.],
              [255.,0.,0.],
              [255.,255.,255.]
              ]
# Building color image from predicted mask
def colorize_predicted_mask(mask, category_colors=category_colors): 
    color_mask = np.zeros((mask.shape[0], mask.shape[1], 3))
    for frame, cat in enumerate(cats.keys()) :     
        color_mask[np.where(np.argmax(mask,axis=-1) == frame)] = category_colors[frame]
    return color_mask<jupyter_output><empty_output><jupyter_text>Load model :<jupyter_code>model = tf.keras.models.load_model(model_path, custom_objects={
    'binary_crossentropy_plus_jaccard_loss':bce_jaccard_loss,
    'dice_coeff':dice_coeff,
    'iou_score': iou_score})<jupyter_output><empty_output><jupyter_text>Predict object classes for each frame, blend image and predicted mask then write resulting images to video file while monitoring frames in cv2 imshow window :<jupyter_code># Instantiate video capture and writer : 
vid = cv2.VideoCapture('C://Users//Lewin//Downloads//OC//post-training//busycityintersection.mp4')
# t = time.time()
fourcc = cv2.VideoWriter_fourcc('M','P','4','V')
out = cv2.VideoWriter('C://Users//Lewin//Downloads//OC//post-training//segmented.mp4', fourcc, 30.0, image_size, True)

# predict object class and create blended image for each frame : 
while vid.isOpened():
    # read and resize frame : 
    success, frame = vid.read()
    if not success:
        print("Ignoring empty camera frame.")
        break
    img = cv2.resize(frame, image_size)
    # predict segmented image and blend with original video frame : 
    y_pred = model.predict(preprocess_input(img[tf.newaxis, ...]))
    predicted_mask = cv2.cvtColor(np.float32(colorize_predicted_mask(y_pred[0])), cv2.COLOR_RGB2BGR)
    alpha = 0.25
    annotated_image = cv2.addWeighted(image.img_to_array(img), alpha, image.img_to_array(predicted_mask), 1-alpha, 0)      
    # Save blended image to video file
    out.write(annotated_image.astype('uint8'))
    # Monitor while processing : 
    cv2.imshow('Segmented Video', annotated_image)
    if cv2.waitKey(5) & 0xFF == 27:
        cv2.destroyWindow('Segmented Video')
        break
    # print (f"{t-time.time()}")
out.release()
vid.release()
cv2.destroyAllWindows()<jupyter_output>Ignoring empty camera frame.<jupyter_text>Checking generated video :<jupyter_code>vid = cv2.VideoCapture('C://Users//Lewin//Downloads//OC//post-training//segmented.mp4')

if vid.isOpened() == False:
    print("ERROR OPENING FILE")

while vid.isOpened():
    success, frame = vid.read()
    if success:
        cv2.imshow('Segmented Video', frame)
        if cv2.waitKey(25) & 0xFF == 27:
            cv2. destroyWindow('Segmented Video')
            break
    else:    
        print("Ignoring empty camera frame.")
        break    
vid.release()<jupyter_output>Ignoring empty camera frame.
{'fraction_non_alphanumeric': 0.1372189077558513, 'fraction_numerical': 0.03900871959614502, 'mean_word_length': 3.7073434125269977, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '563401', 'license': 'MIT', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1486, 'n_words': 336, 'path': 'Predicting_on_video.ipynb', 'repo_name': 'Lewince/P8', 'size': '4358'}