Upload 8 files

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 incept_v3_10fps_full_dp0.4.keras filter=lfs diff=lfs merge=lfs -text
 res10_300x300_ssd_iter_140000.caffemodel filter=lfs diff=lfs merge=lfs -text
 shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text
+incept_v3_10fps_full_0.4.keras filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,131 +1,281 @@
-
-import os
-os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
-
-import streamlit as st
-import cv2
-from tqdm import tqdm
-import numpy as np
-import tensorflow as tf
-import pandas as pd
-from tempfile import NamedTemporaryFile
-from functions import *
-
-threshold = [
-    0.6827917, 0.7136434, 0.510756, 0.56771123, 0.49417764, 0.45892453,
-    0.32996163, 0.5038406, 0.44855, 0.32959282, 0.45619836, 0.4969851
-]
-
-au_to_movements = {
-    'au1': 'inner brow raiser',
-    'au2': 'outer brow raiser',
-    'au4': 'brow lowerer',
-    'au5': 'upper lid raiser',
-    'au6': 'cheek raiser',
-    'au9': 'nose wrinkler',
-    'au12': 'lip corner puller',
-    'au15': 'lip corner depressor',
-    'au17': 'chin raiser',
-    'au20': 'lip stretcher',
-    'au25': 'lips part',
-    'au26': 'jaw drop'
-}
-
-au_labels = [
-    "au1", "au12", "au15", "au17", "au2", "au20",
-    "au25", "au26", "au4", "au5", "au6", "au9"
-]
-
-col = [au_to_movements[i] for i in au_labels]
-
-def binary_focal_loss(gamma=2.0, alpha=0.25):
-    def focal_loss(y_true, y_pred):
-        epsilon = tf.keras.backend.epsilon()
-        y_pred = tf.clip_by_value(y_pred, epsilon, 1.0 - epsilon)
-        fl = - alpha * (y_true * (1 - y_pred)**gamma * tf.math.log(y_pred)
-                       + (1 - y_true) * (y_pred**gamma) * tf.math.log(1 - y_pred))
-        return tf.reduce_mean(fl, axis=-1)
-    return focal_loss
-
-loss = binary_focal_loss(gamma=2.0, alpha=0.25)
-
-# Function to read video frames into a list and get timestamps
-def read_video_frames(video_path):
-    cap = cv2.VideoCapture(video_path)
-    frames = []
-    faces=[]
-    timestamps = []
-    fps = cap.get(cv2.CAP_PROP_FPS)
-    while True:
-        ret, frame = cap.read()
-        if not ret:
-            break
-        face=get_face(frame)
-        if face is not None:
-            faces.append(face)
-            frames.append(frame)
-            timestamps.append(cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0)  # Time in seconds
-
-    cap.release()
-    return frames,faces, timestamps
-
-# Function to process frames and make predictions
-def process_frames(frames, model):
-    frame_array = np.array(frames)
-    preds = model.predict(frame_array)
-    predicted_labels = np.zeros_like(preds, dtype='int')
-    for i in range(12):
-        predicted_labels[:, i] = (preds[:, i] > threshold[i]).astype(int)
-    return preds
-
-# Function to save predictions to a CSV file with timestamps
-def save_predictions_to_csv(predictions, timestamps, filename="predictions.csv"):
-    df = pd.DataFrame(predictions, columns=col)
-    df['timestamp'] = timestamps
-    df.set_index('timestamp', inplace=True)
-    df.to_csv(filename)
-    return filename
-
-# Load your Keras model
-def load_model():
-    model = tf.keras.models.load_model('incept_v3_10fps_full_dp0.4.keras',
-                                       custom_objects={'binary_focal_loss': binary_focal_loss})
-    return model
-
-# Streamlit app
-def main():
-    st.title("Facial action unit detection")
-    
-    uploaded_file = st.file_uploader("Upload a video file", type=["mp4", "avi", "mov"])
-
-    if uploaded_file is not None:
-        with NamedTemporaryFile(delete=False) as tmp_file:
-            tmp_file.write(uploaded_file.read())
-            video_path = tmp_file.name
-
-        model = load_model()
-
-        if st.button("Predict"):
-            st.text("Reading video frames...")
-            frames,faces, timestamps = read_video_frames(video_path)
-            st.text(f"Total frames in which faces found: {len(faces)}")
-            
-            st.text("Processing frames and making predictions...")
-            predictions = process_frames(faces, model)
-            st.text("Predictions completed!")
-
-            csv_file_path = save_predictions_to_csv(predictions, timestamps)
-            st.text("Predictions saved to CSV!")
-
-            with open(csv_file_path, "rb") as f:
-                st.download_button(
-                    label="Download CSV",
-                    data=f,
-                    file_name="predictions.csv",
-                    mime="text/csv"
-                )
-
-            os.remove(video_path)
-
-if __name__ == "__main__":
-    main()
+# import os
+# os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+
+# import streamlit as st
+# import cv2
+# from tqdm import tqdm
+# import numpy as np
+# import tensorflow as tf
+# import pandas as pd
+# from tempfile import NamedTemporaryFile
+# from functions import *
+
+# threshold=[0.6827917,
+#  0.7136434,
+#  0.510756,
+#  0.56771123,
+#  0.49417764,
+#  0.45892453,
+#  0.32996163,
+#  0.5038406,
+#  0.44855,
+#  0.32959282,
+#  0.45619836,
+#  0.4969851]
+# au_to_movements= {
+#     'au1': 'inner brow raiser',
+#     'au2': 'outer brow raiser',
+#     'au4': 'brow lowerer',
+#     'au5': 'upper lid raiser',
+#     'au6': 'cheek raiser',
+#     'au9': 'nose wrinkler',
+#     'au12': 'lip corner puller',
+#     'au15': 'lip corner depressor',
+#     'au17': 'chin raiser',
+#     'au20': 'lip stretcher',
+#     'au25': 'lips part',
+#     'au26': 'jaw drop'
+# }
+# au_labels = [
+#     "au1",
+#     "au12",
+#     "au15",
+#     "au17",
+#     "au2",
+#     "au20",
+#     "au25",
+#     "au26",
+#     "au4",
+#     "au5",
+#     "au6",
+#     "au9"
+# ]
+# col=[au_to_movements[i] for i in au_labels]
+# def binary_focal_loss(gamma=2.0, alpha=0.25):
+#     def focal_loss(y_true, y_pred):
+#         # Define epsilon to avoid log(0)
+#         epsilon = tf.keras.backend.epsilon()
+#         # Clip predictions to prevent log(0) and log(1 - 0)
+#         y_pred = tf.clip_by_value(y_pred, epsilon, 1.0 - epsilon)
+#         # Compute the focal loss
+#         fl = - alpha * (y_true * (1 - y_pred)**gamma * tf.math.log(y_pred)
+#                        + (1 - y_true) * (y_pred**gamma) * tf.math.log(1 - y_pred))
+#         return tf.reduce_mean(fl, axis=-1)
+#     return focal_loss
+
+# loss = binary_focal_loss(gamma=2.0, alpha=0.25)
+
+# # Function to read video frames into a list
+# def read_video_frames(video_path):
+#     cap = cv2.VideoCapture(video_path)
+#     frames = []
+#     while True:
+#         ret, frame = cap.read()
+#         if not ret:
+#             break
+#         frames.append(frame)
+
+#     cap.release()
+#     return frames
+
+# # Function to process frames and make predictions
+# def process_frames(frames, model):
+#     frames = [get_face(frame) for frame in tqdm(frames[:len(frames)-1])]
+#     st.text(f"face shape : {frames[0].shape}")
+#     frame_array = np.array(frames)
+#     preds = model.predict(frame_array)
+#     print(preds[0])
+#     predicted_labels = np.zeros_like(preds,dtype='int')
+#     for i in range(12):
+#         predicted_labels[:, i] = (preds[:, i] > threshold[i]).astype(int)
+#     return predicted_labels
+
+# # Function to save predictions to a CSV file
+# def save_predictions_to_csv(predictions, filename="predictions.csv"):
+#     df = pd.DataFrame(predictions,columns=col)
+#     df.to_csv(filename, index=False)
+#     return filename
+
+# # Load your Keras model
+# def load_model():
+#     model = tf.keras.models.load_model('incept_v3_10fps_full_0.4.keras',
+#                                        custom_objects={'binary_focal_loss': binary_focal_loss})
+#     return model
+
+# # Streamlit app
+# def main():
+#     st.title("Video Frame Prediction App")
+    
+#     # Upload video file
+#     uploaded_file = st.file_uploader("Upload a video file", type=["mp4", "avi", "mov"])
+
+#     if uploaded_file is not None:
+#         with NamedTemporaryFile(delete=False) as tmp_file:
+#             tmp_file.write(uploaded_file.read())
+#             video_path = tmp_file.name
+
+#         # Load the model
+#         model = load_model()
+
+#         # Predict button
+#         if st.button("Predict"):
+#             # Read frames from video
+#             st.text("Reading video frames...")
+#             frames = read_video_frames(video_path)
+#             st.text(f"Total frames read: {len(frames)}")
+
+#             # Process frames and make predictions
+#             st.text("Processing frames and making predictions...")
+#             predictions = process_frames(frames, model)
+#             st.text("Predictions completed!")
+
+#             # Save predictions to CSV
+#             csv_file_path = save_predictions_to_csv(predictions)
+#             st.text("Predictions saved to CSV!")
+
+#             # Make CSV downloadable
+#             with open(csv_file_path, "rb") as f:
+#                 st.download_button(
+#                     label="Download CSV",
+#                     data=f,
+#                     file_name="predictions.csv",
+#                     mime="text/csv"
+#                 )
+
+#             # Clean up the temporary file
+#             os.remove(video_path)
+
+# if __name__ == "__main__":
+#     main()
+
+
+import os
+os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+
+import streamlit as st
+import cv2
+from tqdm import tqdm
+import numpy as np
+import tensorflow as tf
+import pandas as pd
+from tempfile import NamedTemporaryFile
+from functions import *
+
+threshold = [
+    0.6827917, 0.7136434, 0.510756, 0.56771123, 0.49417764, 0.45892453,
+    0.32996163, 0.5038406, 0.44855, 0.32959282, 0.45619836, 0.4969851
+]
+
+au_to_movements = {
+    'au1': 'inner brow raiser',
+    'au2': 'outer brow raiser',
+    'au4': 'brow lowerer',
+    'au5': 'upper lid raiser',
+    'au6': 'cheek raiser',
+    'au9': 'nose wrinkler',
+    'au12': 'lip corner puller',
+    'au15': 'lip corner depressor',
+    'au17': 'chin raiser',
+    'au20': 'lip stretcher',
+    'au25': 'lips part',
+    'au26': 'jaw drop'
+}
+
+au_labels = [
+    "au1", "au12", "au15", "au17", "au2", "au20",
+    "au25", "au26", "au4", "au5", "au6", "au9"
+]
+
+col = [au_to_movements[i] for i in au_labels]
+
+def binary_focal_loss(gamma=2.0, alpha=0.25):
+    def focal_loss(y_true, y_pred):
+        epsilon = tf.keras.backend.epsilon()
+        y_pred = tf.clip_by_value(y_pred, epsilon, 1.0 - epsilon)
+        fl = - alpha * (y_true * (1 - y_pred)**gamma * tf.math.log(y_pred)
+                       + (1 - y_true) * (y_pred**gamma) * tf.math.log(1 - y_pred))
+        return tf.reduce_mean(fl, axis=-1)
+    return focal_loss
+
+loss = binary_focal_loss(gamma=2.0, alpha=0.25)
+
+# Function to read video frames into a list and get timestamps
+def read_video_frames(video_path):
+    cap = cv2.VideoCapture(video_path)
+    frames = []
+    faces=[]
+    timestamps = []
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+        face=get_face(frame)
+        if face is not None:
+            faces.append(face)
+            frames.append(frame)
+            timestamps.append(cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0)  # Time in seconds
+
+    cap.release()
+    return frames,faces, timestamps
+
+# Function to process frames and make predictions
+def process_frames(frames, model):
+    frame_array = np.array(frames)
+    preds = model.predict(frame_array)
+    predicted_labels = np.zeros_like(preds, dtype='int')
+    for i in range(12):
+        predicted_labels[:, i] = (preds[:, i] > threshold[i]).astype(int)
+    return preds
+
+# Function to save predictions to a CSV file with timestamps
+def save_predictions_to_csv(predictions, timestamps, filename="predictions.csv"):
+    df = pd.DataFrame(predictions, columns=col)
+    df['timestamp'] = timestamps
+    df.set_index('timestamp', inplace=True)
+    df.to_csv(filename)
+    return filename
+
+# Load your Keras model
+def load_model():
+    model = tf.keras.models.load_model('incept_v3_10fps_full_0.4.keras',
+                                       custom_objects={'binary_focal_loss': binary_focal_loss})
+    return model
+
+# Streamlit app
+def main():
+    st.title("Facial action unit detection")
+    
+    uploaded_file = st.file_uploader("Upload a video file", type=["mp4", "avi", "mov"])
+
+    if uploaded_file is not None:
+        with NamedTemporaryFile(delete=False) as tmp_file:
+            tmp_file.write(uploaded_file.read())
+            video_path = tmp_file.name
+
+        model = load_model()
+
+        if st.button("Predict"):
+            st.text("Reading video frames...")
+            frames,faces, timestamps = read_video_frames(video_path)
+            st.text(f"Total frames in which faces found: {len(faces)}")
+            
+            st.text("Processing frames and making predictions...")
+            predictions = process_frames(faces, model)
+            st.text("Predictions completed!")
+
+            csv_file_path = save_predictions_to_csv(predictions, timestamps)
+            st.text("Predictions saved to CSV!")
+
+            with open(csv_file_path, "rb") as f:
+                st.download_button(
+                    label="Download CSV",
+                    data=f,
+                    file_name="predictions.csv",
+                    mime="text/csv"
+                )
+
+            os.remove(video_path)
+
+if __name__ == "__main__":
+    main()

functions.py

@@ -7,11 +7,6 @@ dnn_net = cv2.dnn.readNetFromCaffe("deploy.prototxt", "res10_300x300_ssd_iter_14
 # Initialize dlib's facial landmark predictor
 predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
 
-dnn_net = cv2.dnn.readNetFromCaffe("deploy.prototxt", "res10_300x300_ssd_iter_140000.caffemodel")
-
-# Initialize dlib's facial landmark predictor
-predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
-
 def get_face(image, net=dnn_net, predictor=predictor):
     # Prepare the image for DNN face detection
     (h, w) = image.shape[:2]
@@ -34,8 +29,8 @@ def get_face(image, net=dnn_net, predictor=predictor):
             landmarks = predictor(gray, dlib_rect)
 
             # Visualize landmarks
-            for p in landmarks.parts():
-                cv2.circle(image, (p.x, p.y), 2, (0, 255, 0), -1)
+            # for p in landmarks.parts():
+            #     cv2.circle(image, (p.x, p.y), 2, (0, 255, 0), -1)
 
             # Get the bounding box for the face based on landmarks
             landmarks_np = np.array([[p.x, p.y] for p in landmarks.parts()])
@@ -49,9 +44,8 @@ def get_face(image, net=dnn_net, predictor=predictor):
             y = max(0, y)
             w = min(w, image.shape[1] - x)
             h = min(h, image.shape[0] - y)
-
             # Crop and resize the face
-            face_crop = image[y-h//2:y+h, x:x+w]
+            face_crop = image[max(y-h//2,0):y+h, x:x+w]
             face_crop = cv2.resize(face_crop, (224, 224))
             return face_crop
 

incept_v3_10fps_full_0.4.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c76de29d4085ca4b5f903e7a24c9afe8e836e0b125cc2ae39e6dc59a214ed800
+size 296199144