<jupyter_start><jupyter_text>PCA (Part 2)CPSC 340: Machine Learning and Data MiningThe University of British Columbia2017 Winter Term 2<NAME><jupyter_code>import numpy as np
import pickle
import os.path
import random
import matplotlib.pyplot as plt
%matplotlib inline
from sklearn.decomposition import PCA
with open(os.path.join("img","faces.pkl"), "rb") as f:
    faces = pickle.load(f)

# helper functions
def display_face(f, cmap='Greys_r', vmin=None, vmax=None):
    plt.imshow(np.reshape(f, (32,32)).T, cmap=cmap, vmin=vmin, vmax=vmax)
    plt.tick_params(axis='both', which='both', bottom='off', left='off', labelbottom='off', labelleft='off')
    plt.grid(False)    
def reconstruct(model, X):
    Z = model.transform(X)
    try:
        return model.inverse_transform(Z)
    except AttributeError: # no inverse_transform function
        W = model.components_
        return Z@W
    # Note on the above: you can't just do Z@W for PCA because of the centring of the data, 
    # which needs to be undone for the reconstruction


plt.figure()
plt.subplot(1,2,1)
display_face(random.choice(faces))
plt.title("A random face")
plt.subplot(1,2,2)
display_face(np.mean(faces,axis=0))
plt.title("The average face")

N,D = faces.shape
print("We have %d faces, each 32x32 pixels." % N)<jupyter_output>We have 2414 faces, each 32x32 pixels.<jupyter_text>Testing it out with scikit-learn's PCA<jupyter_code>pca = PCA()
pca.fit(faces)
cumulative_var_expl = np.cumsum(pca.explained_variance_ratio_)
for i in range(20):
    print("With %2d components, variance explained = %.0f%%" % (i+1, cumulative_var_expl[i]*100))<jupyter_output>With  1 components, variance explained = 36%
With  2 components, variance explained = 71%
With  3 components, variance explained = 76%
With  4 components, variance explained = 79%
With  5 components, variance explained = 80%
With  6 components, variance explained = 82%
With  7 components, variance explained = 83%
With  8 components, variance explained = 84%
With  9 components, variance explained = 85%
With 10 components, variance explained = 85%
With 11 components, variance explained = 86%
With 12 components, variance explained = 86%
With 13 components, variance explained = 87%
With 14 components, variance explained = 87%
With 15 components, variance explained = 88%
With 16 components, variance explained = 88%
With 17 components, variance explained = 88%
With 18 components, variance explained = 89%
With 19 components, variance explained = 89%
With 20 components, variance explained = 89%<jupyter_text>Our own implementation<jupyter_code>pixel_means = np.mean(faces,axis=0)
faces_centered = faces - pixel_means
u,s,W = np.linalg.svd(faces_centered, full_matrices=False)
Z = u@np.diag(s)
pixel_means.shape
Z.shape
W.shape
for i in range(1,20):
    reconstr = Z[:,:i]@W[:i]
    var_expl = 1-np.sum((reconstr-faces_centered)**2)/np.sum(faces_centered**2)
    print("With %2d components, variance explained = %.0f%%" % (i, var_expl*100))<jupyter_output>With  1 components, variance explained = 36%
With  2 components, variance explained = 71%
With  3 components, variance explained = 76%
With  4 components, variance explained = 79%
With  5 components, variance explained = 80%
With  6 components, variance explained = 82%
With  7 components, variance explained = 83%
With  8 components, variance explained = 84%
With  9 components, variance explained = 85%
With 10 components, variance explained = 85%
With 11 components, variance explained = 86%
With 12 components, variance explained = 86%
With 13 components, variance explained = 87%
With 14 components, variance explained = 87%
With 15 components, variance explained = 88%
With 16 components, variance explained = 88%
With 17 components, variance explained = 88%
With 18 components, variance explained = 89%
With 19 components, variance explained = 89%
{'fraction_non_alphanumeric': 0.10442708333333334, 'fraction_numerical': 0.049479166666666664, 'mean_word_length': 5.17524115755627, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '15726', 'license': 'MIT', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1165, 'n_words': 414, 'path': 'lectures/L26demo.ipynb', 'repo_name': 'mitul23/mitul_cpsc.github.io', 'size': '3846'}