Summary

app.css

@@ -3,8 +3,8 @@ div.app-flex-container {
     align-items: left;
 }
 
-div.app-flex-container > img {
-    margin-right: 6px;
+div.app-flex-container > a {
+    margin-left: 6px;
 }
 
 div.dl1 div.upload-container {
@@ -20,6 +20,22 @@ div.dl2 img {
     max-height: 200px;
 }
 
+div.video1 div.video-container {
+    height: 500px;
+}
+
+div.video2 {
+    height: 200px;
+}
+
+div.video3 {
+    height: 200px;
+}
+
+div.stat {
+    height: 286px;
+}
+
 .submit {
     display: inline-block;
     padding: 10px 20px;

app.py

@@ -10,21 +10,52 @@ License: MIT License
 import gradio as gr
 
 # Importing necessary components for the Gradio app
-from app.description import DESCRIPTION
+from app.description import DESCRIPTION_STATIC, DESCRIPTION_DYNAMIC
 from app.authors import AUTHORS
-from app.app_utils import preprocess_and_predict
+from app.app_utils import preprocess_image_and_predict, preprocess_video_and_predict
 
 
-def clear():
+def clear_static_info():
     return (
         gr.Image(value=None, type="pil"),
         gr.Image(value=None, scale=1, elem_classes="dl2"),
         gr.Label(value=None, num_top_classes=3, scale=1, elem_classes="dl3"),
     )
 
+def clear_dynamic_info():
+    return (
+        gr.Video(value=None),
+        gr.Video(value=None),
+        gr.Video(value=None),
+        gr.Plot(value=None),
+    )
+
 with gr.Blocks(css="app.css") as demo:
-    with gr.Tab("App"):
-        gr.Markdown(value=DESCRIPTION)
+    with gr.Tab("Dynamic App"):
+        gr.Markdown(value=DESCRIPTION_DYNAMIC)
+        with gr.Row():
+            with gr.Column(scale=2):
+                input_video = gr.Video(elem_classes="video1")
+                with gr.Row():
+                    clear_btn_dynamic = gr.Button(
+                        value="Clear", interactive=True, scale=1
+                    )
+                    submit_dynamic = gr.Button(
+                        value="Submit", interactive=True, scale=1, elem_classes="submit"
+                    )
+            with gr.Column(scale=2, elem_classes="dl4"):
+                with gr.Row():
+                    output_video = gr.Video(label="Original video", scale=2, elem_classes="video2")
+                    output_face = gr.Video(label="Pre-processed video", scale=1, elem_classes="video3")
+                output_statistics = gr.Plot(label="Statistics of emotions", elem_classes="stat")
+        gr.Examples(
+            ["videos/video1.mp4",
+            "videos/video2.mp4"],
+            [input_video],
+        )
+
+    with gr.Tab("Static App"):
+        gr.Markdown(value=DESCRIPTION_STATIC)
         with gr.Row():
             with gr.Column(scale=2, elem_classes="dl1"):
                 input_image = gr.Image(type="pil")
@@ -54,17 +85,39 @@ with gr.Blocks(css="app.css") as demo:
         gr.Markdown(value=AUTHORS)
 
     submit.click(
-        fn=preprocess_and_predict,
+        fn=preprocess_image_and_predict,
         inputs=[input_image],
         outputs=[output_image, output_label],
         queue=True,
     )
     clear_btn.click(
-        fn=clear,
+        fn=clear_static_info,
         inputs=[],
         outputs=[input_image, output_image, output_label],
         queue=True,
     )
 
+    submit_dynamic.click(
+        fn=preprocess_video_and_predict,
+        inputs=input_video,
+        outputs=[
+            output_video,
+            output_face, 
+            output_statistics
+        ],
+        queue=True,
+    )
+    clear_btn_dynamic.click(
+        fn=clear_dynamic_info,
+        inputs=[],
+        outputs=[
+            input_video,
+            output_video,
+            output_face, 
+            output_statistics
+        ],
+        queue=True,
+    )
+
 if __name__ == "__main__":
     demo.queue(api_open=False).launch(share=False)

app/app_utils.py

@@ -9,17 +9,19 @@ import torch
 import numpy as np
 import mediapipe as mp
 from PIL import Image
+import cv2
 
 # Importing necessary components for the Gradio app
-from app.model import pth_model, pth_processing
-from app.face_utils import get_box
+from app.model import pth_model_static, pth_model_dynamic, pth_processing
+from app.face_utils import get_box, display_info
 from app.config import DICT_EMO
+from app.plot import statistics_plot
 
 
 mp_face_mesh = mp.solutions.face_mesh
 
 
-def preprocess_and_predict(inp):
+def preprocess_image_and_predict(inp):
     inp = np.array(inp)
 
     if inp is None:
@@ -43,10 +45,91 @@ def preprocess_and_predict(inp):
                 cur_face = inp[startY:endY, startX:endX]
                 cur_face_n = pth_processing(Image.fromarray(cur_face))
                 prediction = (
-                    torch.nn.functional.softmax(pth_model(cur_face_n), dim=1)
+                    torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
                     .detach()
                     .numpy()[0]
                 )
                 confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
 
     return cur_face, confidences
+
+
+def preprocess_video_and_predict(video):
+
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video = 'result.mp4'
+    vid_writer = cv2.VideoWriter(path_save_video, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    probs = []
+    frames = []
+    last_output = None 
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if (count_frame-1)%5 == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                        elif last_output is None:
+                            output = np.zeros((7))
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+            else:
+                empty = np.empty((7))
+                empty[:] = np.nan
+                probs.append(empty)
+                frames.append(count_frame)                        
+
+            cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+            cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+
+            cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+            count_frame += 1
+            vid_writer.write(cur_face)
+
+        vid_writer.release()
+
+        stat = statistics_plot(frames, probs)
+
+        if not stat:
+            return None, None, None
+        
+    return video, path_save_video, stat

app/config.py

@@ -37,3 +37,13 @@ DICT_EMO = {
     5: "Disgust",
     6: "Anger",
 }
+
+COLORS = {
+    0: 'blue', 
+    1: 'orange', 
+    2: 'green', 
+    3: 'red', 
+    4: 'purple', 
+    5: 'brown', 
+    6: 'pink'
+}

app/description.py

@@ -8,10 +8,20 @@ License: MIT License
 # Importing necessary components for the Gradio app
 from app.config import config_data
 
-DESCRIPTION = f"""\
+DESCRIPTION_STATIC = f"""\
 # Static Facial Expression Recognition
 <div class="app-flex-container">
     <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
     <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
-</div>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_DYNAMIC = f"""\
+# Dynamic Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
 """

app/face_utils.py

@@ -7,6 +7,7 @@ License: MIT License
 
 import numpy as np
 import math
+import cv2
 
 
 def norm_coordinates(normalized_x, normalized_y, image_width, image_height):
@@ -31,3 +32,36 @@ def get_box(fl, w, h):
     (endX, endY) = (min(w - 1, endX), min(h - 1, endY))
 
     return startX, startY, endX, endY
+
+def display_info(img, text, margin=1.0, box_scale=1.0):
+    img_h, img_w, _ = img.shape
+    line_width = int(min(img_h, img_w) * 0.001)
+    thickness = max(int(line_width / 3), 1)
+
+    font_face = cv2.FONT_HERSHEY_SIMPLEX
+    font_color = (0, 0, 0)
+    font_scale = thickness / 1.5
+
+    t_w, t_h = cv2.getTextSize(text, font_face, font_scale, None)[0]
+
+    margin_n = int(t_h * margin)
+    sub_img = img[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+              img_w - t_w - margin_n - int(2 * t_h * box_scale): img_w - margin_n]
+
+    white_rect = np.ones(sub_img.shape, dtype=np.uint8) * 255
+
+    img[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+    img_w - t_w - margin_n - int(2 * t_h * box_scale):img_w - margin_n] = cv2.addWeighted(sub_img, 0.5, white_rect, .5, 1.0)
+
+    cv2.putText(img=img,
+                text=text,
+                org=(img_w - t_w - margin_n - int(2 * t_h * box_scale) // 2,
+                     0 + margin_n + t_h + int(2 * t_h * box_scale) // 2),
+                fontFace=font_face,
+                fontScale=font_scale,
+                color=font_color,
+                thickness=thickness,
+                lineType=cv2.LINE_AA,
+                bottomLeftOrigin=False)
+
+    return img

app/model.py

@@ -27,7 +27,10 @@ def load_model(model_url, model_path):
         return None
 
 
-pth_model = load_model(config_data.model_url, config_data.model_path)
+pth_model_static = load_model(config_data.model_static_url, config_data.model_static_path)
+
+pth_model_dynamic = load_model(config_data.model_dynamic_url, config_data.model_dynamic_path)
+
 
 
 def pth_processing(fp):

app/plot.py

@@ -0,0 +1,29 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Plotting statistical information.
+License: MIT License
+"""
+import matplotlib.pyplot as plt
+import numpy as np 
+
+# Importing necessary components for the Gradio app
+from app.config import DICT_EMO, COLORS
+
+
+def statistics_plot(frames, probs):
+    fig, ax = plt.subplots(figsize=(10, 4))
+    fig.subplots_adjust(left=0.07, bottom=0.14, right=0.98, top=0.8, wspace=0, hspace=0)
+    # Установка параметров left, bottom, right, top, чтобы выделить место для легенды и названий осей
+    probs = np.array(probs)
+    for i in range(7):
+        try:
+            ax.plot(frames, probs[:, i], label=DICT_EMO[i], c=COLORS[i]) 
+        except Exception:
+            return None
+            
+    ax.legend(loc='upper center', bbox_to_anchor=(0.47, 1.2), ncol=7, fontsize=12) 
+    ax.set_xlabel('Frames', fontsize=12)  # Добавляем подпись к оси X
+    ax.set_ylabel('Probability', fontsize=12)  # Добавляем подпись к оси Y
+    ax.grid(True)
+    return plt

config.toml

@@ -1,5 +1,9 @@
-APP_VERSION = "0.1.0"
+APP_VERSION = "0.2.0"
 
-[model]
+[model_static]
 url = "https://huggingface.co/ElenaRyumina/face_emotion_recognition/resolve/main/FER_static_ResNet50_AffectNet.pth"
 path = "FER_static_ResNet50_AffectNet.pth"
+
+[model_dynamic]
+url = "https://huggingface.co/ElenaRyumina/face_emotion_recognition/resolve/main/FER_dinamic_LSTM_IEMOCAP.pth"
+path = "FER_dinamic_LSTM_IEMOCAP.pth"