<jupyter_start><jupyter_text>Machine Learning - <NAME> ( Python Implementation) K-means Clustering  Loading of Data<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from scipy.io import loadmat 
mat = loadmat("ex7data2.mat")
X = mat["X"]<jupyter_output><empty_output><jupyter_text>Finding closest centroids$c^{(i)} := j$ that minimizes $\|x^{(i)} - u_j\|^2$<jupyter_code>def findClosestCentroids(X, centroids):
    """
    Returns the closest centroids in idx for a dataset X where each row is a single example.
    """
    K = centroids.shape[0]
    idx = np.zeros((X.shape[0],1))
    temp = np.zeros((centroids.shape[0],1))
    
    for i in range(X.shape[0]):
        for j in range(K):
            dist = X[i,:] - centroids[j,:]
            length = np.sum(dist**2)
            temp[j] = length
        idx[i] = np.argmin(temp)+1
    return idx
# Select an initial set of centroids
K = 3
initial_centroids = np.array([[3,3],[6,2],[8,5]])
idx = findClosestCentroids(X, initial_centroids)
print("Closest centroids for the first 3 examples:\n",idx[0:3])<jupyter_output>Closest centroids for the first 3 examples:
 [[1.]
 [3.]
 [2.]]<jupyter_text>Computing centroid means$u_k := \frac{1}{C_k} \Sigma_{i\in C_k}x^{(i)}$<jupyter_code>def computeCentroids(X, idx, K):
    """
    returns the new centroids by computing the means of the data points assigned to each centroid.
    """
    m, n = X.shape[0],X.shape[1]
    centroids = np.zeros((K,n))
    count = np.zeros((K,1))
    
    for i in range(m):
        index = int((idx[i]-1)[0])
        centroids[index,:]+=X[i,:]
        count[index]+=1
    
    return centroids/count
centroids = computeCentroids(X, idx, K)
print("Centroids computed after initial finding of closest centroids:\n", centroids)<jupyter_output>Centroids computed after initial finding of closest centroids:
 [[2.42830111 3.15792418]
 [5.81350331 2.63365645]
 [7.11938687 3.6166844 ]]<jupyter_text>Visualizing K-means Clustering<jupyter_code>def plotKmeans(X, centroids, idx, K, num_iters):
    """
    plots the data points with colors assigned to each centroid
    """
    m,n = X.shape[0],X.shape[1]
    
    fig, ax = plt.subplots(nrows=num_iters,ncols=1,figsize=(6,36))
    
    for i in range(num_iters):    
        # Visualisation of data
        color = "rgb"
        for k in range(1,K+1):
            grp = (idx==k).reshape(m,1)
            ax[i].scatter(X[grp[:,0],0],X[grp[:,0],1],c=color[k-1],s=15)

        # visualize the new centroids
        ax[i].scatter(centroids[:,0],centroids[:,1],s=120,marker="x",c="black",linewidth=3)
        title = "Iteration Number " + str(i)
        ax[i].set_title(title)
        
        # Compute the centroids mean
        centroids = computeCentroids(X, idx, K)
        
        # assign each training example to the nearest centroid
        idx = findClosestCentroids(X, centroids)
    
    plt.tight_layout()
m,n = X.shape[0],X.shape[1]
plotKmeans(X, initial_centroids,idx, K,10)<jupyter_output><empty_output><jupyter_text>Random Initialization<jupyter_code>def kMeansInitCentroids(X, K):
    """
    This function initializes K centroids that are to beused in K-Means on the dataset X
    """
    m,n = X.shape[0], X.shape[1]
    centroids = np.zeros((K,n))
    
    for i in range(K):
        centroids[i] = X[np.random.randint(0,m+1),:]
        
    return centroids
centroids = kMeansInitCentroids(X, K)
idx = findClosestCentroids(X, centroids)
plotKmeans(X, centroids,idx, K,10)<jupyter_output><empty_output><jupyter_text>Image Compression with K-means<jupyter_code>mat2 = loadmat("bird_small.mat")
A = mat2["A"]

# preprocess and reshape the image
X2 = (A/255).reshape(128*128,3)
def runKmeans(X, initial_centroids,num_iters,K):
    
    idx = findClosestCentroids(X, initial_centroids)
    
    for i in range(num_iters):
        
        # Compute the centroids mean
        centroids = computeCentroids(X, idx, K)

        # assign each training example to the nearest centroid
        idx = findClosestCentroids(X, initial_centroids)

        
    return centroids, idx
# Running K-means algorithm on the data
K2 = 16
num_iters = 10
initial_centroids2 = kMeansInitCentroids(X2, K2)
centroids2, idx2 = runKmeans(X2, initial_centroids2, num_iters,K2)
m2,n2 = X.shape[0],X.shape[1]
X2_recovered = X2.copy()
for i in range(1,K2+1):
    X2_recovered[(idx2==i).ravel(),:] = centroids2[i-1]

# Reshape the recovered image into proper dimensions
X2_recovered = X2_recovered.reshape(128,128,3)

# Display the image
import matplotlib.image as mpimg
fig, ax = plt.subplots(1,2)
ax[0].imshow(X2.reshape(128,128,3))
ax[1].imshow(X2_recovered)<jupyter_output><empty_output><jupyter_text>Principal Component Analysis<jupyter_code>mat3 = loadmat("ex7data1.mat")
X3 = mat3["X"]<jupyter_output><empty_output><jupyter_text>Plotting the data<jupyter_code>plt.scatter(X3[:,0],X3[:,1],marker="o",facecolors="none",edgecolors="b")<jupyter_output><empty_output><jupyter_text>Implementing PCACovariance matrix is computed by $\sum = \frac{1}{m} X^TX$<jupyter_code>def featureNormalize(X):
    """
    Returns a normalized version of X where the mean value of each feature is 0 and the standard deviation is 1.
    """
    mu = np.mean(X,axis=0)
    sigma = np.std(X,axis=0)
    
    X_norm = (X - mu)/sigma
    
    return X_norm, mu , sigma
def pca(X):
    """
    Computes eigenvectors of the covariance matrix of X
    """
    m,n = X.shape[0], X.shape[1]
    
    sigma = 1/m * X.T @ X
    
    U,S,V = svd(sigma)
    
    return U,S,V
from numpy.linalg import svd
X_norm,mu,std = featureNormalize(X3)
U,S = pca(X_norm)[:2]<jupyter_output><empty_output><jupyter_text>Visualizing PCA<jupyter_code>plt.scatter(X3[:,0],X3[:,1],marker="o",facecolors="none",edgecolors="b")
plt.plot([mu[0],(mu+1.5*S[0]*U[:,0].T)[0]],[mu[1],(mu+1.5*S[0]*U[:,0].T)[1]],color="black",linewidth=3)
plt.plot([mu[0],(mu+1.5*S[1]*U[:,1].T)[0]],[mu[1],(mu+1.5*S[1]*U[:,1].T)[1]],color="black",linewidth=3)
plt.xlim(-1,7)
plt.ylim(2,8)
print("Top eigenvector U(:,1) =:",U[:,0])<jupyter_output>Top eigenvector U(:,1) =: [-0.70710678 -0.70710678]<jupyter_text>Dimensionality reduction with PCA<jupyter_code>def projectData(X, U, K):
    """
    Computes the reduced data representation when projecting only on to the top k eigenvectors
    """
    m = X.shape[0]
    U_reduced = U[:,:K]
    Z = np.zeros((m,K))
    
    for i in range(m):
        for j in range(K):
            Z[i,j] = X[i,:] @ U_reduced[:,j]
    
    return Z
# Project the data onto K=1 dimension
K=1
Z = projectData(X_norm, U, K)
print("Projection of the first example:",Z[0][0])<jupyter_output>Projection of the first example: 1.4963126084578515<jupyter_text>Reconstructing an approximation of the data<jupyter_code>def recoverData(Z, U, K):
    """
    Recovers an approximation of the original data when using the projected data
    """
    m,n = Z.shape[0],U.shape[0]
    X_rec = np.zeros((m,n))
    U_reduced = U[:,:K]
    
    for i in range(m):
        X_rec[i,:] = Z[i,:] @ U_reduced.T
    
    return X_rec
X_rec  = recoverData(Z, U, K)
print("Approximation of the first example:",X_rec[0,:])<jupyter_output>Approximation of the first example: [-1.05805279 -1.05805279]<jupyter_text>Visualizing the projections<jupyter_code>plt.scatter(X_norm[:,0],X_norm[:,1],marker="o",label="Original",facecolors="none",edgecolors="b",s=15)
plt.scatter(X_rec[:,0],X_rec[:,1],marker="o",label="Approximation",facecolors="none",edgecolors="r",s=15)
plt.title("The Normalized and Projected Data after PCA")
plt.legend()<jupyter_output><empty_output><jupyter_text>Face Image Dataset<jupyter_code>mat4 = loadmat("ex7faces.mat")
X4 = mat4["X"]<jupyter_output><empty_output><jupyter_text>Visualizing the dataset<jupyter_code>fig, ax = plt.subplots(nrows=10,ncols=10,figsize=(8,8))
for i in range(0,100,10):
    for j in range(10):
        ax[int(i/10),j].imshow(X4[i+j,:].reshape(32,32,order="F"),cmap="gray")
        ax[int(i/10),j].axis("off")<jupyter_output><empty_output><jupyter_text>PCA on face<jupyter_code>X_norm2 = featureNormalize(X4)[0]

# Run PCA
U2 =pca(X_norm2)[0]

#Visualize the top 36 eigenvectors found
U_reduced = U2[:,:36].T
fig2, ax2 = plt.subplots(6,6,figsize=(8,8))
for i in range(0,36,6):
    for j in range(6):
        ax2[int(i/6),j].imshow(U_reduced[i+j,:].reshape(32,32,order="F"),cmap="gray")
        ax2[int(i/6),j].axis("off")
K2 = 100
Z2 = projectData(X_norm2, U2, K2)
print("The projected data Z has a size of:",Z2.shape)
# Data reconstruction
X_rec2  = recoverData(Z2, U2, K2)

# Visualize the reconstructed data
fig3, ax3 = plt.subplots(10,10,figsize=(8,8))
for i in range(0,100,10):
    for j in range(10):
        ax3[int(i/10),j].imshow(X_rec2[i+j,:].reshape(32,32,order="F"),cmap="gray")
        ax3[int(i/10),j].axis("off")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16113636363636363, 'fraction_numerical': 0.045454545454545456, 'mean_word_length': 4.273217495506291, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '102738', 'license': 'MIT', 'n_tokens_mistral': 3646, 'n_tokens_neox': 3495, 'n_words': 675, 'path': 'KmeansClustering_PCA/K-means Clustering & PCA.ipynb', 'repo_name': 'GuiCamargoX/Starting_Machine_Learning', 'size': '8803'}