<jupyter_start><jupyter_text>Importing required Libraries<jupyter_code># importing required liberaries
import requests
import numpy as np
import cv2
from mpl_toolkits import mplot3d
from sklearn.cluster import KMeans
import itertools
from IPython.display import clear_output

import pandas as pd 
import numpy as np
import matplotlib.pyplot as plt 
import tensorflow as tf 
from tensorflow.keras import Sequential
from tensorflow.keras.layers import *
from tensorflow.keras.models import Model,load_model
from tensorflow.keras.initializers import glorot_uniform
from tensorflow.keras.utils import plot_model
from IPython.display import display
from tensorflow.keras.preprocessing.image import img_to_array, array_to_img
import copy
# this gives a track of the latest frame displayed by open cv
var_p1 = [] 
image_with_filter = [] 

# setting up our highper parameters for displaying facial key points if needed
color = (0, 0, 255) 
thickness = 2
radius = 2<jupyter_output><empty_output><jupyter_text>Loading  architechture and weights of our Trained model We are loading the architecture of our pretrained model from .json fileLoading weights from the .hdf5 file<jupyter_code>with open('KeyPointDetector.json','r') as json_file:

    json_SavedModel = json_file.read()
model = tf.keras.models.model_from_json(json_SavedModel)
model.load_weights('weights.hdf5')<jupyter_output><empty_output><jupyter_text>Getting the names of the facial key_points<jupyter_code>facialpoints_df = pd.read_csv('training.csv')
columns = facialpoints_df.columns[:-1]
del facialpoints_df
columns
len(columns)<jupyter_output><empty_output><jupyter_text>Using Haarcascade classifier to detect the frountal face<jupyter_code>face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')<jupyter_output><empty_output><jupyter_text>K Means clustering Since, we have the hardware restrictions.Therefore the model is trained on image shape of (96 X 96 X 1). The key points predicted are also on the image of size (96 X 96 X 1). In order to get keypoints on higher image dimensions we use the Kmeans clustering Furthur detailed visuals are given at the end<jupyter_code>kmeans = KMeans(
    init="random",
    n_clusters=1,
    n_init=1,
    max_iter=1,
    random_state=42
    )<jupyter_output><empty_output><jupyter_text>Here we will be calculating the key points on much higher image dimensions<jupyter_code>def enlarged_point(df_predict_copy,frame_width,frame_hight):
    
#       we will take all the points
        df_predict_copy = df_predict_copy.astype(np.int16)
        df_predict_copy = np.array(df_predict_copy)
        key_points = []
        for i in range(1,31,2):
            
#           creating a black frame of the same size as our model inputs 
            b2_frame = np.zeros((96,96,1))
            
#           marking the key_points on the black frame
    
            b2_frame[df_predict_copy[0,i],df_predict_copy[0,i-1]] = 255
            x_y = []
            
#           enlarging the black frame to the original size of the face 
#           as we enlarge the frame the points we marked become a clustur of points and we want the center of that cluster

            b2_frame = cv2.resize(b2_frame,(og_width,og_hight))
    
            hx ,hy = b2_frame.shape
            for i in range(hx):
                for j in range(hy):
                    if b2_frame[i,j] :

                        x_y.append([i,j])
            x_y = np.array(x_y)
            
#           here we use KMeans to fid the centroid of the cluster.
#           Since I am locating one cluster or one key point on image at a time, I used max_iters,n_init,n_clusters = 1

            kmeans.fit(x_y[:,:2])
            
#            getting the coordinates of key_points on the original image

            key_points.append(kmeans.cluster_centers_)

        return key_points<jupyter_output><empty_output><jupyter_text>These are the list of keypoints which we have on our enlarged image or the original image<jupyter_code># 0 : left_eye_center
# 1 : right_eye_center
# 2 : left_eye_inner_corner
# 3 : left_eye_outer_corner
# 4 : right_eye_inner_corner
# 5 : right_eye_outer_corner
# 6 : left_eyebrow_inner_end
# 7 : left_eyebrow_outer_end
# 8 : right_eyebrow_inner_end
# 9 : right_eyebrow_outer_end
# 10 : nose_tip
# 11 : mouth_left_corner
# 12 : mouth_right_corner
# 13 : mouth_center_top_lip
# 14 : mouth_center_bottom_lip
check_coordinates = []
# we use the ip webcam for using mobile camera for displaing on our pc 
while True:

    # getting the address for our mobile camera
    
    # here you write your own ip given by ip webcam for your android
    image = requests.get("http://19x.xxx.xx.x:8080/shot.jpg") 
    
    # conveerting into numpy array
    video = np.array(bytearray(image.content),dtype = np.uint8)
    
    # decoding our numpy array
    render = cv2.imdecode(video,-1)
    
    #cv2.imshow('frame',render)
    
    # loading our haar cascade file 
    
    
    # converting into gray scale from rgb
    gray = cv2.cvtColor(render, cv2.COLOR_BGR2GRAY)
    
    # detecting faces
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
  
    # making a rectangle box on the detected faces
    for (x,y,w,h) in faces:
        
#       getting the image of the face 
        var_p1 = render[ y:y+h ,x:x+w,1]
#       resizing  the image to the shape of model inputs 
    
        og_width,og_hight = var_p1.shape
        image_sample = cv2.resize(var_p1,(96,96))
        img = array_to_img(image_sample.reshape((96,96,1)))
        arr = img_to_array(img)
        
        background_image = arr
        arr = arr/255
        arr = arr.reshape((1,96,96,1))
        
#       prdecting the key points on the image of the face        
        df_predict = model.predict(arr)
#         print('model_predicted')
        df_enlarged_points = []
        df_sample = df_predict
#         key_points = np.zeros((15,2))
        
#       getting the key points on the original size of the image of the face         
        try: 
            key_points = enlarged_point(df_sample,og_width,og_hight)
            key_points = np.array(key_points)
            key_points = np.reshape(key_points,(15,2))
            key_points = key_points.astype(np.int16)
        except:
            continue
        
        

#       getting the coordinates of all key points on the frame displayed by open cv     
        try:
            true_coordinates = []
            for i in range(15):
                center_coordinates = (x+key_points[i,1],y+key_points[i,0])
                
                true_coordinates.append(center_coordinates)
                
                render = cv2.circle(render, center_coordinates, radius, color, thickness)  
        except:
            continue
        
         
            
        roi_gray = gray[y:y+h, x:x+w]
        roi_color = render[y:y+h, x:x+w]
        
    # displaying the video frame with detected faces
    cv2.imshow('frame',render)
    
    # setting our terminating commands
    if (cv2.waitKey(1)==ord('q')):
        cv2.destroyAllWindows()
        break<jupyter_output><empty_output><jupyter_text>Grayscale visualization  Here is a grayscale image of the detected face and the key points on it<jupyter_code>fig = plt.figure(figsize=(20,20))

for i in range(1):
    ax = fig.add_subplot(4,2,i+1)

    plt.imshow(var_p1,cmap='gray')

    for j in range(15):
        plt.plot(key_points[j,1], key_points[j , 0],'rx')<jupyter_output><empty_output><jupyter_text>These are the values of x's and y's of the key points predicted by model on image(96 x 96 x 1)<jupyter_code>df_sample<jupyter_output><empty_output><jupyter_text>generating the points on small image<jupyter_code># This is the image of face having the original size 
import matplotlib.pyplot as plt
plt.imshow(var_p1,cmap='gray')<jupyter_output><empty_output><jupyter_text>Resizing the image<jupyter_code>og_width,og_hight = var_p1.shape
image_sample = cv2.resize(var_p1,(96,96))
img = array_to_img(image_sample.reshape((96,96,1)))
arr = img_to_array(img)

background_image1 = arr


arr = arr/255
arr = arr.reshape((1,96,96,1))<jupyter_output><empty_output><jupyter_text>Now, This image is perfect to feed into our model<jupyter_code>img
df_predict = model.predict(arr)<jupyter_output><empty_output><jupyter_text>Here is a dataframe for the predicted values of the key_points<jupyter_code>df_predict = pd.DataFrame(df_predict,columns = columns)
df_predict.head()
fig = plt.figure(figsize=(20,20))

# ploting the key_points on the resized image of face
for i in range(1):
    ax = fig.add_subplot(4,2,i+1)

    plt.imshow(arr.squeeze(),cmap='gray')

    for j in range(1,31,2):
        plt.plot(df_predict.loc[i][j-1], df_predict.loc[i][j],'rx')<jupyter_output><empty_output><jupyter_text>Creating a black plane for locating points<jupyter_code>b_frame = np.zeros((96,96,1))
b_frame.shape
plt.imshow(b_frame.reshape((96,96)))
# ploting the key_points on the black frame 

df_predict = df_predict.astype(np.int16)
for j in range(1,31,2):
    b_frame[df_predict.loc[0][j],df_predict.loc[0][j-1]] = 255
plt.imshow(b_frame.reshape((96,96)))<jupyter_output><empty_output><jupyter_text>resizing the frame to the original size of the image of face After enlarging the points we ploted will be zoomed and became a cluster of points<jupyter_code>b_frame = cv2.resize(b_frame,(og_width,og_hight))
plt.imshow(b_frame)
# getting all the points with non zero pixel value

x_y = []
hx ,hy = b_frame.shape
for i in range(hx):
    for j in range(hy):
        if b_frame[i,j] :
            
            x_y.append([i,j,b_frame[i,j]])
            
x_y = np.array(x_y)
# Plotting the cluster of points

plt.scatter(x_y[:,0],x_y[:,1])
x_y[i,0]
# The peaks in the graph are the points in which we are interested.These are the centroids of each clusters

fig = plt.figure()
ax = fig.gca(projection='3d')
ax.plot_trisurf(x_y[:,0], x_y[:,1],x_y[:,2], linewidth=0, antialiased=True)
plt.show()
np.array(x_y).shape
np.max(b_frame)<jupyter_output><empty_output><jupyter_text>using Kmeans clustering for locating the centers of points in enlarged image<jupyter_code># Here I have used the parameters higher than given above because here i am locating all centers of clusters at once
kmeans = KMeans(
            init="random",
            n_clusters=15,
            n_init=500,
            max_iter=1000,
            random_state=42
            )
kmeans.fit(x_y[:,:2])<jupyter_output><empty_output><jupyter_text>Getting all the key_points on original image at once<jupyter_code>key_points = kmeans.cluster_centers_
key_points
key_points = key_points.astype(np.int16)
key_points[1,1]<jupyter_output><empty_output><jupyter_text>Here The red points are the key_points that we obtained by using Kmeans once and finding allAnd the black points are the points obtained by one-by-one strategy and finding one key_point at a time by Kmeans<jupyter_code>fig = plt.figure(figsize=(20,20))

for i in range(1):
    ax = fig.add_subplot(4,2,i+1)

    plt.imshow(var_p1,cmap='gray')

    for j in range(15):
        plt.plot(key_points[j,1], key_points[j , 0],'rx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09006099748833872, 'fraction_numerical': 0.022246142805884463, 'mean_word_length': 3.4154455445544554, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '592164', 'license': 'MIT', 'n_tokens_mistral': 3918, 'n_tokens_neox': 3720, 'n_words': 986, 'path': 'detecting_facial_points.ipynb', 'repo_name': 'HarshKhandelwal1552/Snapchat_filter_using_facial_key_points', 'size': '11148'}