pre-launch

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Elena Ryumina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,40 @@
 ---
-title: MMESA ZeroGPU
-emoji: 📈
-colorFrom: red
-colorTo: yellow
+title: Multi-Modal for Emotion and Sentiment Analysis (MMESA)
+emoji: 😀😲😐😥🥴😱😡
+colorFrom: blue
+colorTo: pink
 sdk: gradio
-sdk_version: 4.39.0
+sdk_version: 4.24.0
 app_file: app.py
 pinned: false
-license: apache-2.0
+license: mit
+short_description: A tool to detect Stress, Anxiety and Depression
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Technologies
+
+This project utilizes various Python scripts for different aspects of analysis and recognition:
+
+- `blink_detection.py`: Detects and analyzes blinking patterns.
+- `body_movement_analysis.py`: Analyzes body movements.
+- `emotion_analysis.py`: Analyzes emotional states.
+- `face_expressions.py`: Recognizes facial expressions.
+- `FACS_analysis_sad.py`: Performs Facial Action Coding System analysis for sadness.
+- `gaze_estimation.py`: Estimates gaze direction.
+- `head_posture_detection.py`: Detects head posture.
+- `heart_rate_variability.py`: Analyzes heart rate variability.
+- `posture_analysis.py`: Analyzes posture.
+- `roberta_chatbot.py`: Chatbot using the RoBERTa model.
+- `sentiment_analysis.py`: Performs sentiment analysis.
+- `skin_analysis.py`: Analyzes skin conditions.
+- `sleep_quality.py`: Evaluates sleep quality.
+- `speech_emotion_recognition.py`: Recognizes emotions from speech.
+- `speech_stress_analysis.py`: Analyzes stress levels from speech.
+
+These scripts combine to provide comprehensive analysis capabilities for various aspects of human behavior and physiology.
+
+## Upload Trick to HG
+
+git lfs track "_.dat" && git lfs track "_.pt" && git add .gitattributes && git add assets/models/shape_predictor_68_face_landmarks.dat && git add assets/models/FER_dinamic_LSTM_IEMOCAP.pt && git add assets/models/FER_static_ResNet50_AffectNet.pt && git commit -m 'bigfiles' && git push origin main --force
+
+git add . && git commit -m 'pre-launch' && git push

app.py

@@ -0,0 +1,66 @@
+import gradio as gr
+from tabs.heart_rate_variability import create_hrv_tab
+from tabs.blink_detection import create_blink_tab
+from tabs.gaze_estimation import create_gaze_estimation_tab
+from tabs.speech_stress_analysis import create_voice_stress_tab
+from tabs.head_posture_detection import create_head_posture_tab
+from tabs.face_expressions import create_face_expressions_tab
+from tabs.speech_emotion_recognition import create_emotion_recognition_tab
+from tabs.sleep_quality import create_sleep_quality_tab
+from tabs.sentiment_analysis import create_sentiment_tab
+from tabs.emotion_analysis import create_emotion_tab
+from tabs.body_movement_analysis import create_body_movement_tab
+from tabs.posture_analysis import create_posture_analysis_tab
+from tabs.skin_analysis import create_skin_conductance_tab
+from tabs.FACS_analysis_sad import create_facs_analysis_sad_tab
+from tabs.roberta_chatbot import create_roberta_chatbot_tab
+import spaces
+
+# Import the UI components
+from ui_components import CUSTOM_CSS, HEADER_HTML, DISCLAIMER_HTML
+
+TAB_STRUCTURE = [
+    ("Visual Analysis", [
+        ("Emotional Face Expressions", create_face_expressions_tab),
+        ("FACS for Stress, Anxiety, Depression", create_facs_analysis_sad_tab),
+        ("Gaze Estimation", create_gaze_estimation_tab),
+        ("Head Posture", create_head_posture_tab),
+        ("Blink Rate", create_blink_tab),
+        ("Sleep Quality", create_sleep_quality_tab),
+        ("Heart Rate Variability", create_hrv_tab),
+        ("Body Movement", create_body_movement_tab),
+        ("Posture", create_posture_analysis_tab),
+        ("Skin", create_skin_conductance_tab)
+    ]),
+    ("Speech Analysis", [
+        ("Speech Stress", create_voice_stress_tab),
+        ("Speech Emotion", create_emotion_recognition_tab)
+    ]),
+    ("Text Analysis", [
+        ("Sentiment", create_sentiment_tab),
+        ("Emotion", create_emotion_tab),
+        ("Roberta Mental Health Chatbot", create_roberta_chatbot_tab)
+    ]),
+    ("Brain Analysis (coming soon)", [
+    ])
+]
+
+@spaces.GPU
+def create_demo():
+    with gr.Blocks(css=CUSTOM_CSS) as demo:
+        gr.Markdown(HEADER_HTML)
+        with gr.Tabs(elem_classes=["main-tab"]):
+            for main_tab, sub_tabs in TAB_STRUCTURE:
+                with gr.Tab(main_tab):
+                    with gr.Tabs():
+                        for sub_tab, create_fn in sub_tabs:
+                            with gr.Tab(sub_tab):
+                                create_fn()
+        gr.HTML(DISCLAIMER_HTML)
+    return demo
+
+# Create the demo instance
+demo = create_demo()
+
+if __name__ == "__main__":
+    demo.queue(api_open=True).launch(share=False)

app/app_utils.py

@@ -0,0 +1,321 @@
+"""
+File: app_utils.py
+Author: Elena Ryumina and Dmitry Ryumin (modified by Assistant)
+Description: This module contains utility functions for facial expression recognition application, including FACS Analysis for SAD.
+License: MIT License
+"""
+
+import torch
+import numpy as np
+import mediapipe as mp
+from PIL import Image
+import cv2
+from pytorch_grad_cam.utils.image import show_cam_on_image
+import matplotlib.pyplot as plt
+
+# Importing necessary components for the Gradio app
+from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing
+from app.face_utils import get_box, display_info
+from app.config import DICT_EMO, config_data
+from app.plot import statistics_plot
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_image_and_predict(inp):
+    inp = np.array(inp)
+
+    if inp is None:
+        return None, None, None
+
+    try:
+        h, w = inp.shape[:2]
+    except Exception:
+        return None, None, None
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as face_mesh:
+        results = face_mesh.process(inp)
+        if results.multi_face_landmarks:
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = inp[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                with torch.no_grad():
+                    prediction = (
+                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
+                        .detach()
+                        .numpy()[0]
+                    )
+                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face,(224,224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+
+    return cur_face, heatmap, confidences
+
+def preprocess_frame_and_predict_aus(frame):
+    if len(frame.shape) == 2:
+        frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
+    elif frame.shape[2] == 4:
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB)
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5
+    ) as face_mesh:
+        results = face_mesh.process(frame)
+        
+        if results.multi_face_landmarks:
+            h, w = frame.shape[:2]
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = frame[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                
+                with torch.no_grad():
+                    features = pth_model_static(cur_face_n)
+                    au_intensities = features_to_au_intensities(features)
+                
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face, (224, 224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+                
+                return cur_face, au_intensities, heatmap
+    
+    return None, None, None
+
+def features_to_au_intensities(features):
+    features_np = features.detach().cpu().numpy()[0]
+    au_intensities = (features_np - features_np.min()) / (features_np.max() - features_np.min())
+    return au_intensities[:24]  # Assuming we want 24 AUs
+
+def preprocess_video_and_predict(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_face = 'result_face.mp4'
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    path_save_video_hm = 'result_hm.mp4'
+    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    count_face = 0
+    probs = []
+    frames = []
+    au_intensities_list = []
+    last_output = None
+    last_heatmap = None 
+    last_au_intensities = None
+    cur_face = None
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        with torch.no_grad():
+                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+                            au_intensities = features_to_au_intensities(pth_model_static(cur_face_copy))
+
+                        grayscale_cam = cam(input_tensor=cur_face_copy)
+                        grayscale_cam = grayscale_cam[0, :]
+                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)
+                        cur_face_hm = np.float32(cur_face_hm) / 255
+                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)
+                        last_heatmap = heatmap
+                        last_au_intensities = au_intensities
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        with torch.no_grad():
+                            output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+
+                        if count_face == 0:
+                            count_face += 1
+
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                            heatmap = last_heatmap
+                            au_intensities = last_au_intensities
+
+                        elif last_output is None:
+                            output = np.empty((1, 7))
+                            output[:] = np.nan
+                            au_intensities = np.empty(24)
+                            au_intensities[:] = np.nan
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+                    au_intensities_list.append(au_intensities)
+            else:
+                if last_output is not None:
+                    lstm_features = []
+                    empty = np.empty((7))
+                    empty[:] = np.nan
+                    probs.append(empty)
+                    frames.append(count_frame)
+                    au_intensities_list.append(np.full(24, np.nan))
+
+            if cur_face is not None:
+                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)
+
+                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+                vid_writer_face.write(cur_face)
+                vid_writer_hm.write(heatmap_f)
+
+            count_frame += 1
+            if count_face != 0:
+                count_face += 1
+
+        vid_writer_face.release()
+        vid_writer_hm.release()
+
+        stat = statistics_plot(frames, probs)
+        au_stat = au_statistics_plot(frames, au_intensities_list)
+
+        if not stat or not au_stat:
+            return None, None, None, None, None
+        
+    return video, path_save_video_face, path_save_video_hm, stat, au_stat
+
+def au_statistics_plot(frames, au_intensities_list):
+    fig, ax = plt.subplots(figsize=(12, 6))
+    au_intensities_array = np.array(au_intensities_list)
+    
+    for i in range(au_intensities_array.shape[1]):
+        ax.plot(frames, au_intensities_array[:, i], label=f'AU{i+1}')
+    
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('AU Intensity')
+    ax.set_title('Action Unit Intensities Over Time')
+    ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
+    plt.tight_layout()
+    return fig
+
+def preprocess_video_and_predict_sleep_quality(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_original = 'result_original.mp4'
+    path_save_video_face = 'result_face.mp4'
+    path_save_video_sleep = 'result_sleep.mp4'
+    
+    vid_writer_original = cv2.VideoWriter(path_save_video_original, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+    vid_writer_sleep = cv2.VideoWriter(path_save_video_sleep, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    frames = []
+    sleep_quality_scores = []
+    eye_bags_images = []
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_rgb)
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY = get_box(fl, w, h)
+                    cur_face = frame_rgb[startY:endY, startX:endX]
+                    
+                    sleep_quality_score, eye_bags_image = analyze_sleep_quality(cur_face)
+                    sleep_quality_scores.append(sleep_quality_score)
+                    eye_bags_images.append(cv2.resize(eye_bags_image, (224, 224)))
+
+                    sleep_quality_viz = create_sleep_quality_visualization(cur_face, sleep_quality_score)
+                    
+                    cur_face = cv2.resize(cur_face, (224, 224))
+                    
+                    vid_writer_face.write(cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR))
+                    vid_writer_sleep.write(sleep_quality_viz)
+
+            vid_writer_original.write(frame)
+            frames.append(len(frames) + 1)
+
+    cap.release()
+    vid_writer_original.release()
+    vid_writer_face.release()
+    vid_writer_sleep.release()
+
+    sleep_stat = sleep_quality_statistics_plot(frames, sleep_quality_scores)
+    
+    if eye_bags_images:
+        average_eye_bags_image = np.mean(np.array(eye_bags_images), axis=0).astype(np.uint8)
+    else:
+        average_eye_bags_image = np.zeros((224, 224, 3), dtype=np.uint8)
+
+    return (path_save_video_original, path_save_video_face, path_save_video_sleep, 
+            average_eye_bags_image, sleep_stat)
+
+def analyze_sleep_quality(face_image):
+    # Placeholder function - implement your sleep quality analysis here
+    sleep_quality_score = np.random.random()
+    eye_bags_image = cv2.resize(face_image, (224, 224))
+    return sleep_quality_score, eye_bags_image
+
+def create_sleep_quality_visualization(face_image, sleep_quality_score):
+    viz = face_image.copy()
+    cv2.putText(viz, f"Sleep Quality: {sleep_quality_score:.2f}", (10, 30), 
+                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
+    return cv2.cvtColor(viz, cv2.COLOR_RGB2BGR)
+
+def sleep_quality_statistics_plot(frames, sleep_quality_scores):
+    # Placeholder function - implement your statistics plotting here
+    fig, ax = plt.subplots()
+    ax.plot(frames, sleep_quality_scores)
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('Sleep Quality Score')
+    ax.set_title('Sleep Quality Over Time')
+    return fig

app/au_processing.py

@@ -0,0 +1,64 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2
+import torch
+from PIL import Image
+from app.model import pth_model_static, cam, pth_processing
+from app.face_utils import get_box
+import mediapipe as mp
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_frame_and_predict_aus(frame):
+    if len(frame.shape) == 2:
+        frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
+    elif frame.shape[2] == 4:
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB)
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5
+    ) as face_mesh:
+        results = face_mesh.process(frame)
+        
+        if results.multi_face_landmarks:
+            h, w = frame.shape[:2]
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = frame[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                
+                with torch.no_grad():
+                    features = pth_model_static(cur_face_n)
+                    au_intensities = features_to_au_intensities(features)
+                
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face, (224, 224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+                
+                return cur_face, au_intensities, heatmap
+    
+    return None, None, None
+
+def features_to_au_intensities(features):
+    features_np = features.detach().cpu().numpy()[0]
+    au_intensities = (features_np - features_np.min()) / (features_np.max() - features_np.min())
+    return au_intensities[:24]  # Assuming we want 24 AUs
+
+def au_statistics_plot(frames, au_intensities_list):
+    fig, ax = plt.subplots(figsize=(12, 6))
+    au_intensities_array = np.array(au_intensities_list)
+    
+    for i in range(au_intensities_array.shape[1]):
+        ax.plot(frames, au_intensities_array[:, i], label=f'AU{i+1}')
+    
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('AU Intensity')
+    ax.set_title('Action Unit Intensities Over Time')
+    ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
+    plt.tight_layout()
+    return fig

app/authors.py

@@ -0,0 +1,34 @@
+"""
+File: authors.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: About the authors.
+License: MIT License
+"""
+
+
+AUTHORS = """
+    Authors: [Elena Ryumina](https://github.com/ElenaRyumina), [Dmitry Ryumin](https://github.com/DmitryRyumin), [Denis Dresvyanskiy](https://www.uni-ulm.de/en/nt/staff/research-assistants/dresvyanskiy/), [Maxim Markitantov](https://hci.nw.ru/en/employees/10) and [Alexey Karpov](https://hci.nw.ru/en/employees/1)
+
+    Authorship contribution:
+
+    App developers: ``Elena Ryumina`` and ``Dmitry Ryumin``
+
+    Methodology developers: ``Elena Ryumina``, ``Denis Dresvyanskiy`` and ``Alexey Karpov``
+
+    Model developer: ``Elena Ryumina``
+
+    TensorFlow to PyTorch model converters: ``Maxim Markitantov`` and ``Elena Ryumina``
+
+    Citation
+
+    If you are using EMO-AffectNetModel in your research, please consider to cite research [paper](https://www.sciencedirect.com/science/article/pii/S0925231222012656). Here is an example of BibTeX entry:
+
+    <div class="highlight highlight-text-bibtex notranslate position-relative overflow-auto" dir="auto"><pre><span class="pl-k">@article</span>{<span class="pl-en">RYUMINA2022</span>,
+        <span class="pl-s">title</span>        = <span class="pl-s"><span class="pl-pds">{</span>In Search of a Robust Facial Expressions Recognition Model: A Large-Scale Visual Cross-Corpus Study<span class="pl-pds">}</span></span>,
+        <span class="pl-s">author</span>       = <span class="pl-s"><span class="pl-pds">{</span>Elena Ryumina and Denis Dresvyanskiy and Alexey Karpov<span class="pl-pds">}</span></span>,
+        <span class="pl-s">journal</span>      = <span class="pl-s"><span class="pl-pds">{</span>Neurocomputing<span class="pl-pds">}</span></span>,
+        <span class="pl-s">year</span>         = <span class="pl-s"><span class="pl-pds">{</span>2022<span class="pl-pds">}</span></span>,
+        <span class="pl-s">doi</span>          = <span class="pl-s"><span class="pl-pds">{</span>10.1016/j.neucom.2022.10.013<span class="pl-pds">}</span></span>,
+        <span class="pl-s">url</span>          = <span class="pl-s"><span class="pl-pds">{</span>https://www.sciencedirect.com/science/article/pii/S0925231222012656<span class="pl-pds">}</span></span>,
+    }</div>
+"""

app/config.py

@@ -0,0 +1,49 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Configuration file.
+License: MIT License
+"""
+
+import toml
+from typing import Dict
+from types import SimpleNamespace
+
+
+def flatten_dict(prefix: str, d: Dict) -> Dict:
+    result = {}
+
+    for k, v in d.items():
+        if isinstance(v, dict):
+            result.update(flatten_dict(f"{prefix}{k}_", v))
+        else:
+            result[f"{prefix}{k}"] = v
+
+    return result
+
+
+config = toml.load("config.toml")
+
+config_data = flatten_dict("", config)
+
+config_data = SimpleNamespace(**config_data)
+
+DICT_EMO = {
+    0: "Neutral",
+    1: "Happiness",
+    2: "Sadness",
+    3: "Surprise",
+    4: "Fear",
+    5: "Disgust",
+    6: "Anger",
+}
+
+COLORS = {
+    0: 'blue', 
+    1: 'orange', 
+    2: 'green', 
+    3: 'red', 
+    4: 'purple', 
+    5: 'brown', 
+    6: 'pink'
+}

app/description.py

@@ -0,0 +1,46 @@
+"""
+File: description.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Project description for the Gradio app.
+License: MIT License
+"""
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+
+DESCRIPTION_STATIC = f"""\
+# Static Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_DYNAMIC = f"""\
+# Dynamic Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_SLEEP_QUALITY = """
+# Sleep Quality Analysis
+
+This tab analyzes sleep quality based on facial features, focusing on skin tone and eye bags. 
+
+## How to use:
+1. Upload a video of a person's face.
+2. Click 'Submit' to process the video.
+3. View the results, including:
+   - Original video
+   - Processed face video
+   - Sleep quality analysis video
+   - Eye bags detection image
+   - Sleep quality statistics over time
+
+The analysis provides insights into potential sleep issues based on visual cues.
+
+Note: This analysis is for informational purposes only and should not be considered a medical diagnosis. Always consult with a healthcare professional for sleep-related concerns.
+"""

app/face_utils.py

@@ -0,0 +1,68 @@
+"""
+File: face_utils.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module contains utility functions related to facial landmarks and image processing.
+License: MIT License
+"""
+
+import numpy as np
+import math
+import cv2
+
+
+def norm_coordinates(normalized_x, normalized_y, image_width, image_height):
+    x_px = min(math.floor(normalized_x * image_width), image_width - 1)
+    y_px = min(math.floor(normalized_y * image_height), image_height - 1)
+    return x_px, y_px
+
+
+def get_box(fl, w, h):
+    idx_to_coors = {}
+    for idx, landmark in enumerate(fl.landmark):
+        landmark_px = norm_coordinates(landmark.x, landmark.y, w, h)
+        if landmark_px:
+            idx_to_coors[idx] = landmark_px
+
+    x_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 0])
+    y_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 1])
+    endX = np.max(np.asarray(list(idx_to_coors.values()))[:, 0])
+    endY = np.max(np.asarray(list(idx_to_coors.values()))[:, 1])
+
+    (startX, startY) = (max(0, x_min), max(0, y_min))
+    (endX, endY) = (min(w - 1, endX), min(h - 1, endY))
+
+    return startX, startY, endX, endY
+
+def display_info(img, text, margin=1.0, box_scale=1.0):
+    img_copy = img.copy()
+    img_h, img_w, _ = img_copy.shape
+    line_width = int(min(img_h, img_w) * 0.001)
+    thickness = max(int(line_width / 3), 1)
+
+    font_face = cv2.FONT_HERSHEY_SIMPLEX
+    font_color = (0, 0, 0)
+    font_scale = thickness / 1.5
+
+    t_w, t_h = cv2.getTextSize(text, font_face, font_scale, None)[0]
+
+    margin_n = int(t_h * margin)
+    sub_img = img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+              img_w - t_w - margin_n - int(2 * t_h * box_scale): img_w - margin_n]
+
+    white_rect = np.ones(sub_img.shape, dtype=np.uint8) * 255
+
+    img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+    img_w - t_w - margin_n - int(2 * t_h * box_scale):img_w - margin_n] = cv2.addWeighted(sub_img, 0.5, white_rect, .5, 1.0)
+
+    cv2.putText(img=img_copy,
+                text=text,
+                org=(img_w - t_w - margin_n - int(2 * t_h * box_scale) // 2,
+                     0 + margin_n + t_h + int(2 * t_h * box_scale) // 2),
+                fontFace=font_face,
+                fontScale=font_scale,
+                color=font_color,
+                thickness=thickness,
+                lineType=cv2.LINE_AA,
+                bottomLeftOrigin=False)
+
+    return img_copy

app/image_processing.py

@@ -0,0 +1,49 @@
+import numpy as np
+import cv2
+from PIL import Image
+import torch
+from app.model import pth_model_static, cam, pth_processing
+from app.face_utils import get_box
+from app.config import DICT_EMO
+from pytorch_grad_cam.utils.image import show_cam_on_image
+import mediapipe as mp
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_image_and_predict(inp):
+    inp = np.array(inp)
+
+    if inp is None:
+        return None, None, None
+
+    try:
+        h, w = inp.shape[:2]
+    except Exception:
+        return None, None, None
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as face_mesh:
+        results = face_mesh.process(inp)
+        if results.multi_face_landmarks:
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = inp[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                with torch.no_grad():
+                    prediction = (
+                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
+                        .detach()
+                        .numpy()[0]
+                    )
+                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face,(224,224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+
+    return cur_face, heatmap, confidences

app/model.py

@@ -0,0 +1,64 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides functions for loading and processing a pre-trained deep learning model
+             for facial expression recognition.
+License: MIT License
+"""
+
+import torch
+import requests
+from PIL import Image
+from torchvision import transforms
+from pytorch_grad_cam import GradCAM
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+from app.model_architectures import ResNet50, LSTMPyTorch
+
+
+def load_model(model_url, model_path):
+    try:
+        with requests.get(model_url, stream=True) as response:
+            with open(model_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+        return model_path
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        return None
+
+path_static = load_model(config_data.model_static_url, config_data.model_static_path)   
+pth_model_static = ResNet50(7, channels=3)
+pth_model_static.load_state_dict(torch.load(path_static))
+pth_model_static.eval()
+
+path_dynamic = load_model(config_data.model_dynamic_url, config_data.model_dynamic_path) 
+pth_model_dynamic = LSTMPyTorch()
+pth_model_dynamic.load_state_dict(torch.load(path_dynamic))
+pth_model_dynamic.eval()
+
+target_layers = [pth_model_static.layer4]
+cam = GradCAM(model=pth_model_static, target_layers=target_layers)
+
+def pth_processing(fp):
+    class PreprocessInput(torch.nn.Module):
+        def init(self):
+            super(PreprocessInput, self).init()
+
+        def forward(self, x):
+            x = x.to(torch.float32)
+            x = torch.flip(x, dims=(0,))
+            x[0, :, :] -= 91.4953
+            x[1, :, :] -= 103.8827
+            x[2, :, :] -= 131.0912
+            return x
+
+    def get_img_torch(img, target_size=(224, 224)):
+        transform = transforms.Compose([transforms.PILToTensor(), PreprocessInput()])
+        img = img.resize(target_size, Image.Resampling.NEAREST)
+        img = transform(img)
+        img = torch.unsqueeze(img, 0)
+        return img
+
+    return get_img_torch(fp)

app/model_architectures.py

@@ -0,0 +1,150 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides model architectures.
+License: MIT License
+"""
+
+import torch
+import torch.nn as  nn
+import torch.nn.functional as F
+import math
+
+class Bottleneck(nn.Module):
+    expansion = 4
+    def __init__(self, in_channels, out_channels, i_downsample=None, stride=1):
+        super(Bottleneck, self).__init__()
+        
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, padding=0, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding='same', bias=False)
+        self.batch_norm2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv3 = nn.Conv2d(out_channels, out_channels*self.expansion, kernel_size=1, stride=1, padding=0, bias=False)
+        self.batch_norm3 = nn.BatchNorm2d(out_channels*self.expansion, eps=0.001, momentum=0.99)
+        
+        self.i_downsample = i_downsample
+        self.stride = stride
+        self.relu = nn.ReLU()
+        
+    def forward(self, x):
+        identity = x.clone()
+        x = self.relu(self.batch_norm1(self.conv1(x)))
+        
+        x = self.relu(self.batch_norm2(self.conv2(x)))
+        
+        x = self.conv3(x)
+        x = self.batch_norm3(x)
+        
+        #downsample if needed
+        if self.i_downsample is not None:
+            identity = self.i_downsample(identity)
+        #add identity
+        x+=identity
+        x=self.relu(x)
+        
+        return x
+
+class Conv2dSame(torch.nn.Conv2d):
+
+    def calc_same_pad(self, i: int, k: int, s: int, d: int) -> int:
+        return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        ih, iw = x.size()[-2:]
+
+        pad_h = self.calc_same_pad(i=ih, k=self.kernel_size[0], s=self.stride[0], d=self.dilation[0])
+        pad_w = self.calc_same_pad(i=iw, k=self.kernel_size[1], s=self.stride[1], d=self.dilation[1])
+
+        if pad_h > 0 or pad_w > 0:
+            x = F.pad(
+                x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]
+            )
+        return F.conv2d(
+            x,
+            self.weight,
+            self.bias,
+            self.stride,
+            self.padding,
+            self.dilation,
+            self.groups,
+        )
+
+class ResNet(nn.Module):
+    def __init__(self, ResBlock, layer_list, num_classes, num_channels=3):
+        super(ResNet, self).__init__()
+        self.in_channels = 64
+
+        self.conv_layer_s2_same = Conv2dSame(num_channels, 64, 7, stride=2, groups=1, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(64, eps=0.001, momentum=0.99)
+        self.relu = nn.ReLU()
+        self.max_pool = nn.MaxPool2d(kernel_size = 3, stride=2)
+        
+        self.layer1 = self._make_layer(ResBlock, layer_list[0], planes=64, stride=1)
+        self.layer2 = self._make_layer(ResBlock, layer_list[1], planes=128, stride=2)
+        self.layer3 = self._make_layer(ResBlock, layer_list[2], planes=256, stride=2)
+        self.layer4 = self._make_layer(ResBlock, layer_list[3], planes=512, stride=2)
+        
+        self.avgpool = nn.AdaptiveAvgPool2d((1,1))
+        self.fc1 = nn.Linear(512*ResBlock.expansion, 512)
+        self.relu1 = nn.ReLU()
+        self.fc2 = nn.Linear(512, num_classes)
+
+    def extract_features(self, x):
+        x = self.relu(self.batch_norm1(self.conv_layer_s2_same(x)))
+        x = self.max_pool(x)
+        # print(x.shape)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        
+        x = self.avgpool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+        return x
+        
+    def forward(self, x):
+        x = self.extract_features(x)
+        x = self.relu1(x)
+        x = self.fc2(x)
+        return x
+        
+    def _make_layer(self, ResBlock, blocks, planes, stride=1):
+        ii_downsample = None
+        layers = []
+        
+        if stride != 1 or self.in_channels != planes*ResBlock.expansion:
+            ii_downsample = nn.Sequential(
+                nn.Conv2d(self.in_channels, planes*ResBlock.expansion, kernel_size=1, stride=stride, bias=False, padding=0),
+                nn.BatchNorm2d(planes*ResBlock.expansion, eps=0.001, momentum=0.99)
+            )
+            
+        layers.append(ResBlock(self.in_channels, planes, i_downsample=ii_downsample, stride=stride))
+        self.in_channels = planes*ResBlock.expansion
+        
+        for i in range(blocks-1):
+            layers.append(ResBlock(self.in_channels, planes))
+            
+        return nn.Sequential(*layers)
+        
+def ResNet50(num_classes, channels=3):
+    return ResNet(Bottleneck, [3,4,6,3], num_classes, channels)
+
+
+class LSTMPyTorch(nn.Module):
+    def __init__(self):
+        super(LSTMPyTorch, self).__init__()
+        
+        self.lstm1 = nn.LSTM(input_size=512, hidden_size=512, batch_first=True, bidirectional=False)
+        self.lstm2 = nn.LSTM(input_size=512, hidden_size=256, batch_first=True, bidirectional=False)
+        self.fc = nn.Linear(256, 7)
+        self.softmax = nn.Softmax(dim=1)
+
+    def forward(self, x):
+        x, _ = self.lstm1(x)
+        x, _ = self.lstm2(x)        
+        x = self.fc(x[:, -1, :])
+        x = self.softmax(x)
+        return x

app/plot.py

@@ -0,0 +1,29 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Plotting statistical information.
+License: MIT License
+"""
+import matplotlib.pyplot as plt
+import numpy as np 
+
+# Importing necessary components for the Gradio app
+from app.config import DICT_EMO, COLORS
+
+
+def statistics_plot(frames, probs):
+    fig, ax = plt.subplots(figsize=(10, 4))
+    fig.subplots_adjust(left=0.07, bottom=0.14, right=0.98, top=0.8, wspace=0, hspace=0)
+    # Установка параметров left, bottom, right, top, чтобы выделить место для легенды и названий осей
+    probs = np.array(probs)
+    for i in range(7):
+        try:
+            ax.plot(frames, probs[:, i], label=DICT_EMO[i], c=COLORS[i]) 
+        except Exception:
+            return None
+            
+    ax.legend(loc='upper center', bbox_to_anchor=(0.47, 1.2), ncol=7, fontsize=12) 
+    ax.set_xlabel('Frames', fontsize=12)  # Добавляем подпись к оси X
+    ax.set_ylabel('Probability', fontsize=12)  # Добавляем подпись к оси Y
+    ax.grid(True)
+    return plt

app/sleep_quality_processing.py

@@ -0,0 +1,94 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import mediapipe as mp
+from app.face_utils import get_box
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_video_and_predict_sleep_quality(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_original = 'result_original.mp4'
+    path_save_video_face = 'result_face.mp4'
+    path_save_video_sleep = 'result_sleep.mp4'
+    
+    vid_writer_original = cv2.VideoWriter(path_save_video_original, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+    vid_writer_sleep = cv2.VideoWriter(path_save_video_sleep, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    frames = []
+    sleep_quality_scores = []
+    eye_bags_images = []
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_rgb)
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY = get_box(fl, w, h)
+                    cur_face = frame_rgb[startY:endY, startX:endX]
+                    
+                    sleep_quality_score, eye_bags_image = analyze_sleep_quality(cur_face)
+                    sleep_quality_scores.append(sleep_quality_score)
+                    eye_bags_images.append(cv2.resize(eye_bags_image, (224, 224)))
+
+                    sleep_quality_viz = create_sleep_quality_visualization(cur_face, sleep_quality_score)
+                    
+                    cur_face = cv2.resize(cur_face, (224, 224))
+                    
+                    vid_writer_face.write(cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR))
+                    vid_writer_sleep.write(sleep_quality_viz)
+
+            vid_writer_original.write(frame)
+            frames.append(len(frames) + 1)
+
+    cap.release()
+    vid_writer_original.release()
+    vid_writer_face.release()
+    vid_writer_sleep.release()
+
+    sleep_stat = sleep_quality_statistics_plot(frames, sleep_quality_scores)
+    
+    if eye_bags_images:
+        average_eye_bags_image = np.mean(np.array(eye_bags_images), axis=0).astype(np.uint8)
+    else:
+        average_eye_bags_image = np.zeros((224, 224, 3), dtype=np.uint8)
+
+    return (path_save_video_original, path_save_video_face, path_save_video_sleep, 
+            average_eye_bags_image, sleep_stat)
+
+def analyze_sleep_quality(face_image):
+    # Placeholder function - implement your sleep quality analysis here
+    sleep_quality_score = np.random.random()
+    eye_bags_image = cv2.resize(face_image, (224, 224))
+    return sleep_quality_score, eye_bags_image
+
+def create_sleep_quality_visualization(face_image, sleep_quality_score):
+    viz = face_image.copy()
+    cv2.putText(viz, f"Sleep Quality: {sleep_quality_score:.2f}", (10, 30), 
+                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
+    return cv2.cvtColor(viz, cv2.COLOR_RGB2BGR)
+
+def sleep_quality_statistics_plot(frames, sleep_quality_scores):
+    fig, ax = plt.subplots()
+    ax.plot(frames, sleep_quality_scores)
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('Sleep Quality Score')
+    ax.set_title('Sleep Quality Over Time')
+    plt.tight_layout()
+    return fig

app/video_processing.py

@@ -0,0 +1,132 @@
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+import mediapipe as mp
+from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing
+from app.face_utils import get_box, display_info
+from app.config import config_data
+from app.plot import statistics_plot
+from .au_processing import features_to_au_intensities, au_statistics_plot
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_video_and_predict(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_face = 'result_face.mp4'
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    path_save_video_hm = 'result_hm.mp4'
+    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    count_face = 0
+    probs = []
+    frames = []
+    au_intensities_list = []
+    last_output = None
+    last_heatmap = None 
+    last_au_intensities = None
+    cur_face = None
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        with torch.no_grad():
+                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+                            au_intensities = features_to_au_intensities(pth_model_static(cur_face_copy))
+
+                        grayscale_cam = cam(input_tensor=cur_face_copy)
+                        grayscale_cam = grayscale_cam[0, :]
+                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)
+                        cur_face_hm = np.float32(cur_face_hm) / 255
+                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)
+                        last_heatmap = heatmap
+                        last_au_intensities = au_intensities
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        with torch.no_grad():
+                            output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+
+                        if count_face == 0:
+                            count_face += 1
+
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                            heatmap = last_heatmap
+                            au_intensities = last_au_intensities
+
+                        elif last_output is None:
+                            output = np.empty((1, 7))
+                            output[:] = np.nan
+                            au_intensities = np.empty(24)
+                            au_intensities[:] = np.nan
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+                    au_intensities_list.append(au_intensities)
+            else:
+                if last_output is not None:
+                    lstm_features = []
+                    empty = np.empty((7))
+                    empty[:] = np.nan
+                    probs.append(empty)
+                    frames.append(count_frame)
+                    au_intensities_list.append(np.full(24, np.nan))
+
+            if cur_face is not None:
+                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)
+
+                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+                vid_writer_face.write(cur_face)
+                vid_writer_hm.write(heatmap_f)
+
+            count_frame += 1
+            if count_face != 0:
+                count_face += 1
+
+        vid_writer_face.release()
+        vid_writer_hm.release()
+
+        stat = statistics_plot(frames, probs)
+        au_stat = au_statistics_plot(frames, au_intensities_list)
+
+        if not stat or not au_stat:
+            return None, None, None, None, None
+        
+    return video, path_save_video_face, path_save_video_hm, stat, au_stat

app_gpuzero.py

@@ -0,0 +1,64 @@
+import gradio as gr
+from tabs.heart_rate_variability import create_hrv_tab
+from tabs.blink_detection import create_blink_tab
+from tabs.gaze_estimation import create_gaze_estimation_tab
+from tabs.speech_stress_analysis import create_voice_stress_tab
+from tabs.head_posture_detection import create_head_posture_tab
+from tabs.face_expressions import create_face_expressions_tab
+from tabs.speech_emotion_recognition import create_emotion_recognition_tab
+from tabs.sleep_quality import create_sleep_quality_tab
+from tabs.sentiment_analysis import create_sentiment_tab
+from tabs.emotion_analysis import create_emotion_tab
+from tabs.body_movement_analysis import create_body_movement_tab
+from tabs.posture_analysis import create_posture_analysis_tab
+from tabs.skin_analysis import create_skin_conductance_tab
+from tabs.FACS_analysis_sad import create_facs_analysis_sad_tab
+from tabs.roberta_chatbot import create_roberta_chatbot_tab
+
+# Import the UI components
+from ui_components import CUSTOM_CSS, HEADER_HTML, DISCLAIMER_HTML
+
+TAB_STRUCTURE = [
+    ("Visual Analysis", [
+        ("Emotional Face Expressions", create_face_expressions_tab),
+        ("FACS for Stress, Anxiety, Depression", create_facs_analysis_sad_tab),
+        ("Gaze Estimation", create_gaze_estimation_tab),
+        ("Head Posture", create_head_posture_tab),
+        ("Blink Rate", create_blink_tab),
+        ("Sleep Quality", create_sleep_quality_tab),
+        ("Heart Rate Variability", create_hrv_tab),
+        ("Body Movement", create_body_movement_tab),
+        ("Posture", create_posture_analysis_tab),
+        ("Skin", create_skin_conductance_tab)
+    ]),
+    ("Speech Analysis", [
+        ("Speech Stress", create_voice_stress_tab),
+        ("Speech Emotion", create_emotion_recognition_tab)
+    ]),
+    ("Text Analysis", [
+        ("Sentiment", create_sentiment_tab),
+        ("Emotion", create_emotion_tab),
+        ("Roberta Mental Health Chatbot", create_roberta_chatbot_tab)
+    ]),
+    ("Brain Analysis (coming soon)", [
+    ])
+]
+
+def create_demo():
+    with gr.Blocks(css=CUSTOM_CSS) as demo:
+        gr.Markdown(HEADER_HTML)
+        with gr.Tabs(elem_classes=["main-tab"]):
+            for main_tab, sub_tabs in TAB_STRUCTURE:
+                with gr.Tab(main_tab):
+                    with gr.Tabs():
+                        for sub_tab, create_fn in sub_tabs:
+                            with gr.Tab(sub_tab):
+                                create_fn()
+        gr.HTML(DISCLAIMER_HTML)
+    return demo
+
+# Create the demo instance
+demo = create_demo()
+
+if __name__ == "__main__":
+    demo.queue(api_open=True).launch(share=False)