<jupyter_start><jupyter_text>**Author: <NAME> Roll: 2019121004** Excercise 2In Excercise 1, we computed the LDA for a multi-class problem, the IRIS dataset. In this excercise, we will now compare the LDA and PCA for the IRIS dataset.To revisit, the iris dataset contains measurements for 150 iris flowers from three different species.The three classes in the Iris dataset:1. Iris-setosa (n=50)2. Iris-versicolor (n=50)3. Iris-virginica (n=50)The four features of the Iris dataset:1. sepal length in cm2. sepal width in cm3. petal length in cm4. petal width in cm<jupyter_code>from sklearn.datasets import make_classification
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns; sns.set();
import pandas as pd
from sklearn.model_selection import train_test_split
from numpy import pi<jupyter_output><empty_output><jupyter_text>Importing the dataset<jupyter_code>url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'Class']
dataset = pd.read_csv(url, names=names)

dataset.tail()<jupyter_output><empty_output><jupyter_text>Data preprocessingOnce dataset is loaded into a pandas data frame object, the first step is to divide dataset into features and corresponding labels and then divide the resultant dataset into training and test sets. The following code divides data into labels and feature set:<jupyter_code>X = dataset.iloc[:, 0:4].values
y = dataset.iloc[:, 4].values<jupyter_output><empty_output><jupyter_text>The above script assigns the first four columns of the dataset i.e. the feature set to X variable while the values in the fifth column (labels) are assigned to the y variable.The following code divides data into training and test sets:<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)<jupyter_output><empty_output><jupyter_text>Feature ScalingWe will now perform feature scaling as part of data preprocessing too. For this task, we will be using scikit learn `StandardScalar`.<jupyter_code>from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)<jupyter_output><empty_output><jupyter_text>Write your code belowWrite your code to compute the PCA and LDA on the IRIS dataset below.<jupyter_code>### WRITE YOUR CODE HERE ####
from sklearn.preprocessing import LabelEncoder
enc = LabelEncoder()
label_encoder = enc.fit(y_train)
y_train = label_encoder.transform(y_train) + 1  #labels are done in alphabetical order
# 1: 'Iris-setosa', 2: 'Iris-versicolor', 3:'Iris-virginica'
label_encoder = enc.fit(y_test)
y_test = label_encoder.transform(y_test) + 1
labels = ['setosa', 'Versicolor', 'Virginica']
# LDA

num_classes = 3
num_classes_plus1 = num_classes + 1
def find_mean(X_train,y_train,num_classes_plus1):
    mean_arr = []
    for cl in range(1,num_classes_plus1):
        mean_arr.append(np.mean(X_train[y_train==cl], axis=0))
    return mean_arr

mean_arr =  find_mean(X_train,y_train,num_classes_plus1)

def within_classScatter(mean_arr,X_train,y_train,num_classes_plus1):
    S_w = np.zeros((num_classes_plus1,num_classes_plus1))
    for cl, mv in zip(range(1,num_classes_plus1),mean_arr):
        temp_s = np.zeros((num_classes_plus1,num_classes_plus1))
        for data in X_train[y_train==cl]:
            data, mv = data.reshape(num_classes_plus1,1), mv.reshape(num_classes_plus1,1) ### making them vertical vectors
            temp_s += (data-mv)@((data-mv).T)
        S_w += temp_s
    return S_w
S_w = within_classScatter(mean_arr,X_train,y_train,num_classes_plus1)      
print("within class scatter matrix S_w:\n")
print(S_w)

def btw_clasScatter(mean_arr,X_train,y_train,num_classes_plus1):
    total_mean = np.mean(X_train, axis=0).reshape(num_classes_plus1,1)
    S_b = np.zeros((num_classes_plus1,num_classes_plus1))
    for cl, mv in zip(range(1,num_classes_plus1), mean_arr):
        n = X_train[y_train==cl].shape[0]
        class_mean = mv.reshape(num_classes_plus1,1)
        S_b += n*((class_mean - total_mean)@(class_mean - total_mean).T)
    return S_b
S_b = btw_clasScatter(mean_arr,X_train,y_train,num_classes_plus1)
print("between class scatter matrix S_b:\n")
print(S_b) 

def takeTopEigen(S_w, S_b,k):
    eigen_vals, eigen_vecs = np.linalg.eig(np.linalg.inv(S_w).dot(S_b))
    eigs_sorted_in = np.argsort(eigen_vals)[::-1]
    eigen_vals = eigen_vals[eigs_sorted_in]
    eigen_vecs = eigen_vecs[:,eigs_sorted_in]
    weights = eigen_vecs[:,:k]
    return weights

def lda_vecs(X_train, y_train,weights):
    Xtrain_lda = X_train@weights
    Xtest_lda = X_test@weights
    return Xtrain_lda, Xtest_lda 

weights = takeTopEigen(S_w, S_b,2)
Xtrain_lda, Xtest_lda  = lda_vecs(X_train, y_train,weights)

def centroid(Xtrain_lda,y_train):
    centroids = []
    for i in range(1,num_classes_plus1):
        centroids.append(np.mean(Xtrain_lda[y_train == i], axis = 0))
    return centroids

centroids = centroid(Xtrain_lda,y_train)


def pred(X_lda,centroids):
    y_pred = []
    for i in range(len(X_lda)):
        y_pred.append(np.argmin([ np.linalg.norm(centroids[0]-X_lda[i]), np.linalg.norm(centroids[1]-X_lda[i]), np.linalg.norm(centroids[2]-X_lda[i]) ])+1)
    return np.array(y_pred)

def accuracy(X_lda,y,centroids):
    y_pred = pred(X_lda,centroids)
    err = y-y_pred
    accuracy = len(err[err == 0])/len(err)
    return accuracy*100

acc = accuracy(Xtrain_lda,y_train,centroids)
print("Accuracy on train set",acc)

acc = accuracy(Xtest_lda,y_test,centroids)
print("Accuracy on test set:",acc)

def calc_class(Xtrain_lda,centroids):
    x_r, y_r = np.meshgrid(np.linspace(np.min(Xtrain_lda[:,0])-0.2, np.max(Xtrain_lda[:,1])+0.2,200), np.linspace(np.min(Xtrain_lda[:,1])-0.2, np.max(Xtrain_lda[:,1])+0.2,200))
    cl = np.zeros(x_r.shape)
    # finding which class the sample belongs to
    # cl is label vector of predicted class
    for i in range(len(x_r)):
        for j in range(len(y_r)):
            pt = [x_r[i,j], y_r[i,j]]
            clas = []
            for l in range(3):
                clas.append(np.linalg.norm(centroids[l]-pt))
            cl[i,j] = np.argmin(clas)+1
    return cl,x_r,y_r

def plot(X_lda,y,cl,title,strr):  
  
    for clas in range(1,num_classes_plus1):
        plt.scatter(X_lda[y == clas][:,0],X_lda[y == clas][:,1],label=labels[clas-1])
    
    plt.xlabel(strr+"1")
    plt.ylabel(strr+"2")
    plt.title(title)
    
    plt.legend(loc='upper right')
    plt.contour(x_r,y_r,cl)
    
    plt.show()
z,x_r,y_r = calc_class(Xtrain_lda,centroids)
plot(Xtrain_lda,y_train,z,"Training set","LDA")
plot(Xtest_lda,y_test,z,"Test set","LDA")

# PCA
u, s, vt = np.linalg.svd(X_train, full_matrices=False)
w_pca = vt.T[:,:2]
Xtrain_pca = X_train@w_pca
Xtest_pca = X_test@w_pca
cntr = centroid(Xtrain_pca,y_train)

acc = accuracy(Xtrain_pca,y_train,cntr)
print("Accuracy on train set",acc)
cl,x_r,y_r = calc_class(Xtrain_pca,cntr)
plot(Xtrain_pca,y_train,cl,"training set","PCA")

acc = accuracy(Xtest_pca,y_test,centroids)
print("Accuracy on test set:",acc)

plot(Xtest_pca,y_test,cl,"test set","PCA")<jupyter_output>Accuracy on train set 85.0
{'fraction_non_alphanumeric': 0.12253073628954275, 'fraction_numerical': 0.01588617212322144, 'mean_word_length': 5.063651591289783, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '668443', 'license': 'MIT', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2815, 'n_words': 579, 'path': 'HW9/2019121004_LDAExcercise2.ipynb', 'repo_name': 'avani17101/SMAI-Assignments', 'size': '7244'}