<jupyter_start><jupyter_text>ASL Gesture Detection using OpenCV  This system detects some basic ASL gestures using OpenCV & Mediapipe.  Initial Imports<jupyter_code>import mediapipe as mp
import time
import cv2
import numpy as np
from matplotlib import pyplot as plt
import os
# Holistic detetction API (Hands, Face and Pose)
mp_holistic = mp.solutions.holistic

# Drawing Utilities for keypoint and landmark detection
mp_drawings = mp.solutions.drawing_utils<jupyter_output><empty_output><jupyter_text>Camera openCV detects the colors as BGR so have to convert to RGB for mediapipe for detect<jupyter_code>def mediapipe_detection(image, model):
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image.flags.writeable = False
    results = model.process(image)
    image.flags.writeable = True
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
    return image, results<jupyter_output><empty_output><jupyter_text>MediaPipe offers drawing utilities to change color, thickness and radius of landmarks and keypoints.<jupyter_code>def draw_landmarks(image, results):
    # Face Landmarks
    mp_drawings.draw_landmarks(image, 
                               results.face_landmarks, 
                               mp_holistic.FACEMESH_TESSELATION, 
                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), 
                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))
    # Pose Landmarks
    mp_drawings.draw_landmarks(image, 
                               results.pose_landmarks, 
                               mp_holistic.POSE_CONNECTIONS, 
                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), 
                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))
    # Left Hand Landmarks
    mp_drawings.draw_landmarks(image, 
                               results.left_hand_landmarks, 
                               mp_holistic.HAND_CONNECTIONS, 
                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), 
                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))
    # Right Hand 
    mp_drawings.draw_landmarks(image, 
                               results.right_hand_landmarks, 
                               mp_holistic.HAND_CONNECTIONS, 
                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), 
                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))<jupyter_output><empty_output><jupyter_text>Testing Camera and MediaPipe Detection<jupyter_code>cap = cv2.VideoCapture(0)
with mp_holistic.Holistic(min_detection_confidence=0.7, min_tracking_confidence=0.7) as holistic:
    while cap.isOpened():
        ret, frame = cap.read()

        image, results = mediapipe_detection(frame, holistic)
        draw_landmarks(image, results)
        
        cv2.imshow('OpenCV Feed', image)

        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()<jupyter_output><empty_output><jupyter_text>Keypoint ExtractionMediapipe returns results with X, Y, Z axis values. X & Y being offsets on 2D Plane and Z being distance from the camera.All keypoints of face, hands and pose are concatenated to a single array.<jupyter_code>def extract_keypoints(results):
    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
    face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)
    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)
    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
    return np.concatenate([pose, face, lh, rh])<jupyter_output><empty_output><jupyter_text>Creating Dataset  Initiating Variables<jupyter_code># Path to save keypoints Array
DATA_PATH = os.path.join('Keypoints_DATA')

# Actions to be detected
actions = np.array(['Hello', 'ThankYou'])

# actions are used to store and make directories hence cant have space
words = np.array(['Hello', 'Thank You'])

# no of videos for a single action
no_sequence = 30

# frames for a single action
sequence_length = 30<jupyter_output><empty_output><jupyter_text>Making Directories to store keypoint arrays for dataset<jupyter_code># for action in actions:
#     for sequence in range(no_sequence):
#         try:
#             os.makedirs(os.path.join(DATA_PATH, action, str(sequence)))
#         except:
#             pass<jupyter_output><empty_output><jupyter_text>Detecting and storing keypoints to directories<jupyter_code># cap = cv2.VideoCapture(0)

# with mp_holistic.Holistic(min_detection_confidence=0.7, min_tracking_confidence=0.7) as holistic:
#     for action in actions:
#         for sequence in range(no_sequence):
#             for frame_num in range(sequence_length):
#                 ret, frame = cap.read()

#                 image, results = mediapipe_detection(frame, holistic)
#                 # print(results)
#                 draw_landmarks(image, results)

#                 if frame_num == 0: 
#                     cv2.putText(image, 'STARTING COLLECTION', (120,200), 
#                                cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255, 0), 4, cv2.LINE_AA)
#                     cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,12), 
#                                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)
#                     cv2.imshow('OpenCV Feed', image)
#                     cv2.waitKey(2000)
#                 else: 
#                     cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,12), 
#                                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)
#                     cv2.imshow('OpenCV Feed', image)
                
#                 keypoints = extract_keypoints(results)
#                 npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))
#                 np.save(npy_path, keypoints)

#                 if cv2.waitKey(10) & 0xFF == ord('q'):
#                     break
                
#     cap.release()
#     cv2.destroyAllWindows()   
    cap.release()
    cv2.destroyAllWindows()<jupyter_output><empty_output><jupyter_text>Data Preprocessing and Model Generation  Imports for train test split and categorise data<jupyter_code>from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical
# Numbers are assigned to each action
label_map = {label: num for num, label in enumerate(actions)}
label_map
# Saved numpy keypoint arrays are loaded into arrays
sequences, labels = [], []
for action in actions:
    for sequence in range(no_sequence):
        window = []
        for frame_num in range(sequence_length):
            res = np.load(os.path.join(DATA_PATH, action, str(sequence), "{}.npy".format(frame_num)))
            window.append(res)
        sequences.append(window)
        labels.append(label_map[action])
X = np.array(sequences)
y = to_categorical(labels).astype(int)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05)<jupyter_output><empty_output><jupyter_text>Model Imports<jupyter_code>from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
from tensorflow.keras.callbacks import TensorBoard
# logs to check model train progress on tensor board
## Open terminal in Logs folder and type: tensorboard --logdir=. 

log_dir = os.path.join('Logs_Gestures')
tb_callbacks = TensorBoard(log_dir=log_dir)<jupyter_output><empty_output><jupyter_text>Training Model<jupyter_code>model = Sequential()
model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(30,1662)))
model.add(LSTM(128, return_sequences=True, activation='relu'))
model.add(LSTM(64, return_sequences=False, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dense(actions.shape[0], activation='softmax'))
model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])<jupyter_output><empty_output><jupyter_text>Fitting Model with train data<jupyter_code>model.fit(X_train, y_train, epochs=2000, callbacks=[tb_callbacks])<jupyter_output><empty_output><jupyter_text>Testing the model<jupyter_code>res = model.predict(X_test)
if actions[np.argmax(res[0])] == actions[np.argmax(y_test[0])]:
    print('Correct Prediction')
else:
    print('Incorrect Prediction')<jupyter_output>Correct Prediction<jupyter_text>Saving Model WeightsModel weights are saved to a file so that model does not need to be trained everytime.<jupyter_code>model.save('ASL_Gestures.h5')
del model
# model is loaded using this method
model.load_weights('ASL_Gestures.h5')<jupyter_output><empty_output><jupyter_text>Multilevel Confusion Matrix and Prediction Accuracy Score  Imports<jupyter_code>from sklearn.metrics import multilabel_confusion_matrix, accuracy_score
yhat = model.predict(X_test)
ytrue = np.argmax(y_test, axis=1).tolist()
yhat = np.argmax(yhat, axis=1).tolist()<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>import pandas as pd
import seaborn as sns
confusion_matrix = multilabel_confusion_matrix(ytrue, yhat)
confusion_matrix
def generate_heatmap(confusion_matrix, axes, class_label, class_names):
    df_cm = pd.DataFrame(confusion_matrix, index=class_names, columns=class_names)
    try:
        heatmap = sns.heatmap(df_cm, annot=True, fmt="d", cbar=False, ax=axes)
    except ValueError:
        raise ValueError("Confusion matrix values must be integers.")
    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=14)
    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=14)
    axes.set_ylabel('True label')
    axes.set_xlabel('Predicted label')
    axes.set_title("Confusion Matrix for the action - " + class_label)
fig, ax = plt.subplots(2, 1, figsize=(10, 5))
for axes, cfs_matrix, label in zip(ax.flatten(), confusion_matrix, actions):
        generate_heatmap(cfs_matrix, axes, label, ["True", "False"])
fig.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Accuracy Score<jupyter_code>accuracy_score(ytrue, yhat)<jupyter_output><empty_output><jupyter_text>Real Time Prediction Testing<jupyter_code>import pyttsx3 as px
# text-to-speech initiation
engine = px.init()
sequence = []
sentence = []
predictions = []
threshold = 0.5

cap = cv2.VideoCapture(0)

with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
    while cap.isOpened():
        ret, frame = cap.read()
        
        image, results = mediapipe_detection(frame, holistic)

        keypoints = extract_keypoints(results)
        sequence.append(keypoints)
        sequence = sequence[-30:]
        
        if len(sequence) == 30:
            res = model.predict(np.expand_dims(sequence, axis=0))[0]
            predictions.append(np.argmax(res))
            
            if np.unique(predictions[-10:])[0]==np.argmax(res): 
                if res[np.argmax(res)] > threshold: 
                    if len(sentence) > 0: 
                        if actions[np.argmax(res)] != sentence[-1]:
                            sentence.append(actions[np.argmax(res)])
                            engine.say(words[np.argmax(res)])
                            engine.runAndWait()
                    else:
                        sentence.append(actions[np.argmax(res)])
                        engine.say(words[np.argmax(res)])
                        engine.runAndWait()

            if len(sentence) > 5: 
                sentence = sentence[-5:]
            
        cv2.rectangle(image, (0,0), (640, 40), (245, 117, 16), -1)
        cv2.putText(image, ' '.join(sentence), (3,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
        
        cv2.imshow('OpenCV Feed', image)

        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11027948085038618, 'fraction_numerical': 0.022215144517875627, 'mean_word_length': 3.2375168690958165, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '71840', 'license': 'MIT', 'n_tokens_mistral': 4263, 'n_tokens_neox': 3947, 'n_words': 828, 'path': 'ASL_Gesture_Detection.ipynb', 'repo_name': 'yashas-hm/ASL-Gesture-Sensing', 'size': '12559'}