<jupyter_start><jupyter_text>Data Preprocessing and creating Eigen Images with PCA Import required libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2
from PIL import Image

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load previously saved pickle dataset<jupyter_code>import pickle
df = pickle.load(open('./Data/Dataframe_images_100_100.pickle','rb'))
df.head()
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 5444 entries, 0 to 6052
Columns: 10001 entries, Gender to 9999
dtypes: object(1), uint8(10000)
memory usage: 52.0+ MB<jupyter_text>Checking for missing values<jupyter_code>df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Removing missing values<jupyter_code>df.dropna(axis=0,inplace=True)
df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Split Data into Dependant and Independant variables<jupyter_code>X = df.iloc[:,1:].values
y = df.iloc[:,0].values
X.shape, y.shape
X.min(), X.max()<jupyter_output><empty_output><jupyter_text>Normalizing data<jupyter_code>from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
X
X_norm = (X-X.min())/(X.max()-X.min())
X_norm
X_norm.shape
y
y_enc = np.where(y=='Male',0,1)
y_enc<jupyter_output><empty_output><jupyter_text>Save X and y in numpy zip<jupyter_code>np.savez('./Data/Data_norm.npz', X_norm,y_enc)<jupyter_output><empty_output><jupyter_text>Loading Data<jupyter_code>from glob import glob
data = np.load('./Data/Data_norm.npz')
data.files
X_comp = data['arr_0']#Independent features
y_comp = data['arr_1']#target value
X_comp.shape
y_comp.shape<jupyter_output><empty_output><jupyter_text>Create Eigen Images<jupyter_code>X1 = abs(X_comp - X_comp.mean(axis=0))
from sklearn.decomposition import PCA
pca = PCA(n_components=None,whiten=True,svd_solver='auto')
x_pca =pca.fit_transform(X1)
x_pca.shape
eigen_ratio = pca.explained_variance_ratio_
eigen_ratio_cum = np.cumsum(eigen_ratio)
'''plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.plot(eigen_ratio[:200], 'r>--')
plt.subplot(1,2,2)
plt.plot(eigen_ratio_cum[:200], 'r>--')'''<jupyter_output><empty_output><jupyter_text>Forming Eigen images with 50 components<jupyter_code>pca_50 = PCA(n_components=50,whiten=True)
x_pca_50 = pca_50.fit_transform(X1)
x_pca_50.shape<jupyter_output><empty_output><jupyter_text>Save PCA using pickle<jupyter_code>import pickle
pickle.dump(pca_50,open('./Model/pca_50.pickle','wb'))
x_pca_inv = pca_50.inverse_transform(x_pca_50)
x_pca_inv.shape<jupyter_output><empty_output><jupyter_text>Check output for a single Image<jupyter_code>eig_img = x_pca_inv[0,:]
eig_img = eig_img.reshape(100,100)
plt.imshow(eig_img)
plt.imshow(eig_img,cmap='gray')<jupyter_output><empty_output><jupyter_text>Save Data<jupyter_code>np.savez('./Data/Data_pca_50_y_mean.pickle',x_pca_50,y_comp,X.mean())<jupyter_output><empty_output><jupyter_text>Load Data and remove warnings<jupyter_code>import warnings
warnings.filterwarnings('ignore')
final_data = np.load('./Data/Data_pca_50_y_mean.pickle.npz')
final_data.files<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13733075435203096, 'fraction_numerical': 0.030947775628626693, 'mean_word_length': 9.84965034965035, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '422346', 'license': 'MIT', 'n_tokens_mistral': 1335, 'n_tokens_neox': 1263, 'n_words': 176, 'path': 'Data-Preprocessing.ipynb', 'repo_name': 'souparnabose99/Gender-Classification', 'size': '3102'}