<jupyter_start><jupyter_text>HW1, <NAME> (USC ID: 2446752777)<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

df2 = pd.read_csv("column_2C.dat", sep=" ", header = None) 
print(df2)
print(df2.iloc[208:212, :])

#  pelvic incidence, pelvic tilt, lumbar lordosis angle, sacral slope, pelvic radius and grade of spondylolisthesis<jupyter_output>0      1      2      3       4      5   6
0    63.03  22.55  39.61  40.48   98.67  -0.25  AB
1    39.06  10.06  25.02  29.00  114.41   4.56  AB
2    68.83  22.22  50.09  46.61  105.99  -3.53  AB
3    69.30  24.65  44.31  44.64  101.87  11.21  AB
4    49.71   9.65  28.32  40.06  108.17   7.92  AB
5    40.25  13.92  25.12  26.33  130.33   2.23  AB
6    53.43  15.86  37.17  37.57  120.57   5.99  AB
7    45.37  10.76  29.04  34.61  117.27 -10.68  AB
8    43.79  13.53  42.69  30.26  125.00  13.29  AB
9    36.69   5.01  41.95  31.68   84.24   0.66  AB
10   49.71  13.04  31.33  36.67  108.65  -7.83  AB
11   31.23  17.72  15.50  13.52  120.06   0.50  AB
12   48.92  19.96  40.26  28.95  119.32   8.03  AB
13   53.57  20.46  33.10  33.11  110.97   7.04  AB
14   57.30  24.19  47.00  33.11  116.81   5.77  AB
15   44.32  12.54  36.10  31.78  124.12   5.42  AB
16   63.83  20.36  54.55  43.47  112.31  -0.62  AB
17   31.28   3.14  32.56  28.13  129.01   3.62  AB
18   38.70  13.44  31.00  25.25[...]<jupyter_text>(b) i. scatterplots of the independent variables.(NO=blue and AB=red) 1. x: column 0(pelvic incidence), y: column 1(pelvic tilt) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,0], df2.iloc[:210,1], color='red')    # 'AB': red
plt.scatter(df2.iloc[210:,0], df2.iloc[210:,1], color='blue')   # 'NO': Blue

plt.title('scatter 1')
plt.xlabel('pelvic incidence')
plt.ylabel('pelvic tilt')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>2. x: column 0(pelvic incidence), y: column 2(lumbar lordosis angle) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,0], df2.iloc[:210,2], color='red')
plt.scatter(df2.iloc[210:,0], df2.iloc[210:,2], color='blue')

plt.title('scatter 2')
plt.xlabel('pelvic incidence')
plt.ylabel('lumbar lordosis angle')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>3. x: column 0(pelvic incidence), y: column 3(sacral slope) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,0], df2.iloc[:210,3], color='red')
plt.scatter(df2.iloc[210:,0], df2.iloc[210:,3], color='blue')

plt.title('scatter 3')
plt.xlabel('pelvic incidence')
plt.ylabel('sacral slope')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>4. x: column 0(pelvic incidence), y: column 4(pelvic radius) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,0], df2.iloc[:210,4], color='red')
plt.scatter(df2.iloc[210:,0], df2.iloc[210:,4], color='blue')

plt.title('scatter 4')
plt.xlabel('pelvic incidence')
plt.ylabel('pelvic radius')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>5. x: column 0(pelvic incidence), y: column 5(grade of spondylolisthesis) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,0], df2.iloc[:210,5], color='red')
plt.scatter(df2.iloc[210:,0], df2.iloc[210:,5], color='blue')

plt.title('scatter 5')
plt.xlabel('pelvic incidence')
plt.ylabel('grade of spondylolisthesis')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>6. x: column 1(pelvic tilt), y: column 2(lumbar lordosis angle) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,1], df2.iloc[:210,2], color='red')
plt.scatter(df2.iloc[210:,1], df2.iloc[210:,2], color='blue')

plt.title('scatter 6')
plt.xlabel('pelvic tilt')
plt.ylabel('lumbar lordosis angle')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>7. x: column 1(pelvic tilt), y: column 3(sacral slope) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,1], df2.iloc[:210,3], color='red')
plt.scatter(df2.iloc[210:,1], df2.iloc[210:,3], color='blue')

plt.title('scatter 7')
plt.xlabel('pelvic tilt')
plt.ylabel('sacral slope')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>8. x: column 1(pelvic tilt), y: column 4(pelvic radius) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,1], df2.iloc[:210,4], color='red')
plt.scatter(df2.iloc[210:,1], df2.iloc[210:,4], color='blue')

plt.title('scatter 8')
plt.xlabel('pelvic tilt')
plt.ylabel('pelvic radius')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>9. x: column 1(pelvic tilt), y: column 5(grade of spondylolisthesis) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,1], df2.iloc[:210,5], color='red')
plt.scatter(df2.iloc[210:,1], df2.iloc[210:,5], color='blue')

plt.title('scatter 9')
plt.xlabel('pelvic tilt')
plt.ylabel('grade of spondylolisthesis')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>10. x: column 2(lumbar lordosis angle), y: column 3(sacral slope) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,2], df2.iloc[:210,3], color='red')
plt.scatter(df2.iloc[210:,2], df2.iloc[210:,3], color='blue')

plt.title('scatter 10')
plt.xlabel('lumbar lordosis angle')
plt.ylabel('sacral slope')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>11. x: column 2(lumbar lordosis angle), y: column 4(pelvic radius) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,2], df2.iloc[:210,4], color='red')
plt.scatter(df2.iloc[210:,2], df2.iloc[210:,4], color='blue')

plt.title('scatter 11')
plt.xlabel('lumbar lordosis angle')
plt.ylabel('pelvic radius')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>12. x: column 2(lumbar lordosis angle), y: column 5(grade of spondylolisthesis) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,2], df2.iloc[:210,5], color='red')
plt.scatter(df2.iloc[210:,2], df2.iloc[210:,5], color='blue')

plt.title('scatter 12')
plt.xlabel('lumbar lordosis angle')
plt.ylabel('grade of spondylolisthesis')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>13. x: column 3(sacral slope), y: column 4(pelvic radius) (NO=blue and AB=red)) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,3], df2.iloc[:210,4], color='red')
plt.scatter(df2.iloc[210:,3], df2.iloc[210:,4], color='blue')

plt.title('scatter 13')
plt.xlabel('sacral slope')
plt.ylabel('pelvic radius')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>14. x: column 3(sacral slope), y: column 5(grade of spondylolisthesis) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,3], df2.iloc[:210,5], color='red')
plt.scatter(df2.iloc[210:,3], df2.iloc[210:,5], color='blue')

plt.title('scatter 14')
plt.xlabel('sacral slope')
plt.ylabel('grade of spondylolisthesis')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>15. x: column 4(pelvic radius), y: column 5(grade of spondylolisthesis) (NO=blue and AB=red)<jupyter_code>plt.scatter(df2.iloc[:210,4], df2.iloc[:210,5], color='red')
plt.scatter(df2.iloc[210:,4], df2.iloc[210:,5], color='blue')

plt.title('scatter 15')
plt.xlabel('pelvic radius')
plt.ylabel('grade of spondylolisthesis')
plt.legend(['AB', 'NO'])
plt.show()<jupyter_output><empty_output><jupyter_text>(b) ii. Boxplots for each of the independent variables. (NO=blue and AB=red) 1. y: column 0(pelvic incidence), x: AB, NO<jupyter_code>import seaborn as sns

sns.boxplot(df2.iloc[:,6], df2.iloc[:,0])

plt.xlabel('class')
plt.ylabel('pelvic incidence')<jupyter_output><empty_output><jupyter_text>2. y: column 1(pelvic tilt), x: AB, NO<jupyter_code>sns.boxplot(df2.iloc[:,6], df2.iloc[:,1])

plt.xlabel('class')
plt.ylabel('pelvic tilt')<jupyter_output><empty_output><jupyter_text>3. y: column 2(lumbar lordosis angle), x: AB, NO<jupyter_code>sns.boxplot(df2.iloc[:,6], df2.iloc[:,2])

plt.xlabel('class')
plt.ylabel('lumbar lordosis angle')<jupyter_output><empty_output><jupyter_text>4. y: column 3(sacral slope), x: AB, NO<jupyter_code>sns.boxplot(df2.iloc[:,6], df2.iloc[:,3])

plt.xlabel('class')
plt.ylabel('sacral slope')<jupyter_output><empty_output><jupyter_text>5. y: column 4(pelvic radius), x: AB, NO<jupyter_code>sns.boxplot(df2.iloc[:,6], df2.iloc[:,4])

plt.xlabel('class')
plt.ylabel('pelvic radius')<jupyter_output><empty_output><jupyter_text>6. y: column 5(grade of spondylolisthesis), x: AB, NO<jupyter_code>sns.boxplot(df2.iloc[:,6], df2.iloc[:,5])

plt.xlabel('class')
plt.ylabel('grade of spondylolisthesis')<jupyter_output><empty_output><jupyter_text>(b) iii. Select the ﬁrst 70 rows of Class 0 and the ﬁrst 140 rows of Class 1 as the training set and the rest of the data as the test set.1. training setClass 0(NO): row[210:280], Class 1(AB): row[0:140]2. test setClass 0(NO): row[280:],    Class 1(AB): row[140:210]<jupyter_code>train1 = df2.iloc[0:140, :] # 140 rows
train2 = df2.iloc[210:280, :] # 70 rows

train = pd.concat([train1, train2]) # 210 rows

test1 = df2.iloc[140:210, :] # 70 rows
test2 = df2.iloc[280:, :] # 30 rows

test = pd.concat([test1, test2]) # 100 rows<jupyter_output><empty_output><jupyter_text>(c) Classiﬁcation using KNN on Vertebral Column Data Set        i. Write code for k-nearest neighbors with Euclidean metric (or use a software package).<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

train_X = train.iloc[:, :6]     # train data
train_y = train.iloc[:, 6]      # class 

def KNN_dtm (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k)  # defaulted p = 2, defaulted metric = ‘minkowski’: Euclidean_distance    
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred<jupyter_output><empty_output><jupyter_text>c. ii. Test all the data in the test database with k nearest neighbors. Take decisions by majority polling.<jupyter_code>test_X = test.iloc[:, :6]   # test data
test_y = test.iloc[:, 6]    # original class of test data

for i in range(70):
    k = 3*i+1    # k = 1,4,7,..,208
    print(KNN_dtm (train_X, train_y, k, test_X)) # defaluted weights=’uniform’: majority polling<jupyter_output>['AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'NO' 'AB' 'NO'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'NO' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'AB' 'AB' 'NO' 'AB'
 'NO' 'NO' 'AB' 'AB' 'NO' 'NO' 'NO' 'NO' 'AB' 'NO' 'NO' 'AB' 'AB' 'AB'
 'NO' 'NO']
['AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'AB' 'AB' 'NO' 'AB' 'AB' 'NO' 'NO'
 'NO' 'NO' 'AB' 'AB' 'NO' 'AB' 'NO' 'NO' 'AB' 'NO' 'NO' 'AB' 'AB'[...]<jupyter_text>(c) ii. Plot train and test errors in terms of k for k ∈{208,205,...,7,4,1} (in reverse order). You are welcome to use smaller increments of k.<jupyter_code># train_X = train.iloc[:, :6]     # train data
# train_y = train.iloc[:, 6]      # class 
# test_X  = test.iloc[:, :6]      # test data
# test_y  = test.iloc[:, 6]       # original class of test data


from sklearn.metrics import accuracy_score  # to get error rate

# training error rate: trainER

dic = dict ()
for i in range(70):
    k = 3*i+1    # k = 1,4,7,..,208
    predtr = KNN_dtm (train_X, train_y, k, train_X)   # testing train data
    
    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
    
trainER = pd.DataFrame(dic).T

# print(trainER)

# test error rate: testER

dic = dict ()
for i in range(70):
    k = 3*i+1    # k = 1,4,7,..,208
    predte = KNN_dtm (train_X, train_y, k, test_X)   # testing test data
    
    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testER = pd.DataFrame(dic).T

# print(testER)<jupyter_output><empty_output><jupyter_text>(c) ii. Plot train and test errors in terms of k for k ∈{208,205,...,7,4,1} (in reverse order)<jupyter_code>plt.plot(trainER.iloc[:,0], trainER.iloc[:,1])  # train error rate
plt.plot(testER.iloc[:,0], testER.iloc[:,1])    # test error rate
plt.gca().invert_xaxis()

plt.title('Error Rate per k')
plt.xlabel('k')
plt.ylabel('Error Rate')
plt.legend(['train error rate', 'test error rate'])
plt.show()<jupyter_output><empty_output><jupyter_text>(c) ii. Which k∗ is the most suitable k among those values?<jupyter_code>s_testER = testER.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k

# select best k with the least test error rate 
# If many k's give the same error, select the least k except for k=1 to avoid overfitting and biased

if s_testER.iloc[0, 0] == 1 and round(s_testER.iloc[0, 1], 3) == round(s_testER.iloc[1, 1], 3):
    bst_k = int(s_testER.iloc[1, 0])
    bst_er = s_testER.iloc[1, 1]
else:
    bst_k = int(s_testER.iloc[0, 0])
    bst_er = s_testER.iloc[0, 1]

print("k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testER)  # k, test error
y_pred = KNN_dtm (train_X, train_y, 16, test_X) # Testing test data with k* = 16
print(y_pred)<jupyter_output>['AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB' 'AB'
 'AB' 'NO' 'NO' 'NO' 'NO' 'NO' 'NO' 'AB' 'NO' 'NO' 'AB' 'AB' 'NO' 'NO'
 'NO' 'NO' 'NO' 'AB' 'NO' 'AB' 'NO' 'AB' 'AB' 'NO' 'NO' 'AB' 'AB' 'AB'
 'NO' 'NO']<jupyter_text>(c) ii. confusion matrix when k = k∗ = 16<jupyter_code>from sklearn.metrics import confusion_matrix

# test_y  = test.iloc[:, 6]       # original class of test data
# y_pred  : predicted test data with k* = 16

cm = confusion_matrix(test_y, y_pred)
print(cm)<jupyter_output>[[70  0]
 [11 19]]<jupyter_text>When k*=16, all 70 'AB' patients are precited as 'AB', but 11 out of 30 normal people are predicted as 'AB.'  (c) ii. true positive rate when k = k∗=16, the rate of that actual patients (AB) are precited as 'AB'<jupyter_code>TPR = cm[0][0]/(cm[0][0]+cm[0][1])
print(TPR)<jupyter_output>1.0<jupyter_text>(c) ii. true negative rate when k = k∗=10,  the rate of that normal people (NO) are precited as 'NO'<jupyter_code>TNR = cm[1][1]/(cm[1][0]+cm[1][1])

print(TNR)<jupyter_output>0.6333333333333333<jupyter_text>(c) ii. precision when k = k∗=10<jupyter_code>def prec (TP, FP):      # TP: 'AB' people predicted as 'AB'
    PPV = TP /(TP+FP)   # FP: 'NO' people predicted as 'AB'
    return PPV

PPV = prec (cm[0][0], cm[1][0]) # precision
print(PPV)<jupyter_output>0.8641975308641975<jupyter_text>(c) ii. F-score when k = k∗=10<jupyter_code>f_scr = 2*PPV*TPR/(PPV+TPR)    # TPR: the rate of that actual patients (AB) are precited as 'AB'
print(f_scr)<jupyter_output>0.9271523178807948<jupyter_text>(c) iii. a subset of the training set N ∈ {10,20,30,...,210}for each N, select your training set by choosing the ﬁrst  [N/3] rows of Class 0 and the ﬁrst N −[N/3] rows of Class 1 in the training set you creatd in 1(b)iii.<jupyter_code># train1 = df2.iloc[0:140, :] # 140 rows
# train2 = df2.iloc[210:280, :] # 70 rows

# train = pd.concat([train1, train2]) # 210 rows

# test1 = df2.iloc[140:210, :] # 70 rows
# test2 = df2.iloc[280:, :] # 30 rows

# test = pd.concat([test1, test2]) # 100 rows

# train_X = train.iloc[:, :6]     # train data
# train_y = train.iloc[:, 6]      # class 
# test_X  = test.iloc[:, :6]      # test data
# test_y  = test.iloc[:, 6]       # original class of test data

# function for making new train sets

import math  # to use floor function

def mkntrain (N):
    n = math.floor(N/3)
    
    ntrain1 = df2.iloc[0:N-n, :]     # the first N-n rows of Class 1 (AB)
    ntrain2 = df2.iloc[210:210+n, :] # the first n rows of Class 0 (NO)

    ntrain = pd.concat([ntrain1, ntrain2]) # N rows of trains
    
    return ntrain<jupyter_output><empty_output><jupyter_text>(c) iii. the optimal k is selected from {1,6,11,...,196}.<jupyter_code># function for KNN
# def KNN_dtm (X, y, k, tX):    # tX: test data set
#    neigh = KNeighborsClassifier(n_neighbors=k)  # defaulted p = 2, defaulted metric = ‘minkowski’: Euclidean_distance    
#    neigh.fit(X, y)
#    pred = neigh.predict(tX)
    
#    return pred

dt = dict()

for n in range(21):    
    N = 10*(n+1)      # N ∈ {10,20,30,...,210}
    
    ntrain = mkntrain (N)
    tr_X = ntrain.iloc[:, :6]     # subset train data
    tr_y = ntrain.iloc[:, 6]      # class

    dic = dict ()
    
    testerr = None
    s_testerr = None
    
    for i in range(int(N/5)):  
        k = 5*i+1    # k = 1,6,...
        predte = KNN_dtm (tr_X, tr_y, k, test_X)   # testing all test data for N, k

        # calculating error rate
        ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
        dic[i] = k, 1-ac
          
    testerr = pd.DataFrame(dic).T   # test error rate for N & all k
    
    # the best test error rate, and the best k for N
    
    s_testerr = testerr.sort_values(by=[1, 0])  # sorting testerr(k, test error) by test error and k

# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased 

    if s_testerr.iloc[0, 0] == 1 and round(s_testerr.iloc[0, 1],3) == round(s_testerr.iloc[1, 1], 3):
        bst_k = int(s_testerr.iloc[1, 0])
        bst_er = s_testerr.iloc[1, 1]
    else:
        bst_k = int(s_testerr.iloc[0, 0])
        bst_er = s_testerr.iloc[0, 1]
     
    dt[n]= N, bst_k, bst_er # N, opt_k, best test error rate
    
nbo = pd.DataFrame(dt).T

print(nbo) # N, opt_k, best error rate<jupyter_output>0     1     2
0    10.0   1.0  0.25
1    20.0   1.0  0.22
2    30.0   1.0  0.22
3    40.0  11.0  0.25
4    50.0  16.0  0.27
5    60.0  21.0  0.29
6    70.0  26.0  0.29
7    80.0  26.0  0.28
8    90.0  36.0  0.29
9   100.0   6.0  0.23
10  110.0   6.0  0.22
11  120.0  11.0  0.21
12  130.0  11.0  0.20
13  140.0   1.0  0.18
14  150.0  21.0  0.15
15  160.0  11.0  0.14
16  170.0  11.0  0.13
17  180.0  16.0  0.11
18  190.0  16.0  0.11
19  200.0  16.0  0.11
20  210.0  21.0  0.10<jupyter_text>(c) iii. Plot the best test error rate. Obviously, use the test data you created in 1(b)iii. This plot is called a Learning Curve.<jupyter_code>plt.plot(nbo.iloc[:,0], nbo.iloc[:,2])
plt.title('Learning Curve')
plt.xlabel('N')
plt.ylabel('Best test error rate')
plt.show()<jupyter_output><empty_output><jupyter_text>(d) Replace the Euclidean metric with the following metrics and test them (sklearn.neighbors.DistanceMetric). Summarize the test errors (i.e., when k = k∗) in a table. Use all of your training data and select the best k when {1,6,11,...,196}. i. Minkowski Distance: A. which becomes Manhattan Distance with p = 1.<jupyter_code># train_X = train.iloc[:, :6]     # train data
# train_y = train.iloc[:, 6]      # class 
# test_X  = test.iloc[:, :6]      # test data
# test_y  = test.iloc[:, 6]       # original class of test data

def KNN_dtm2 (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, p=1)  # defaulted metric = ‘minkowski’    
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtm2 (train_X, train_y, k, test_X)  # testing test data with all train data

    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testER2 = pd.DataFrame(dic).T

s_testER2 = testER2.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased  

if s_testER2.iloc[0, 0] == 1 and round(s_testER2.iloc[0, 1], 3) == round(s_testER2.iloc[1, 1], 3):
    bst_k = int(s_testER2.iloc[1, 0])
    bst_er = s_testER2.iloc[1, 1]
else:
    bst_k = int(s_testER2.iloc[0, 0])
    bst_er = s_testER2.iloc[0, 1]

print("Manhattan Distance, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testER2)  # k, test error<jupyter_output>Manhattan Distance, k*= 11 with best test error: 0.11
        0     1
2    11.0  0.11
0     1.0  0.12
4    21.0  0.12
5    26.0  0.12
6    31.0  0.12
7    36.0  0.13
8    41.0  0.13
1     6.0  0.14
3    16.0  0.14
9    46.0  0.14
10   51.0  0.14
14   71.0  0.14
11   56.0  0.15
12   61.0  0.15
15   76.0  0.15
13   66.0  0.16
16   81.0  0.16
17   86.0  0.17
18   91.0  0.20
19   96.0  0.21
20  101.0  0.22
24  121.0  0.22
21  106.0  0.23
22  111.0  0.23
23  116.0  0.23
25  126.0  0.27
26  131.0  0.28
27  136.0  0.30
28  141.0  0.30
29  146.0  0.30
30  151.0  0.30
31  156.0  0.30
32  161.0  0.30
33  166.0  0.30
34  171.0  0.30
35  176.0  0.30
36  181.0  0.30
37  186.0  0.30
38  191.0  0.30
39  196.0  0.30<jupyter_text>(d) i. B. Minkowski Distance with log10(p) ∈ {0.1,0.2,0.3,...,1}. In this case, use the k∗ you found for the Manhattan distance in 1(d)iA. What is the best log10(p)?<jupyter_code># k∗ = 11
# p = 10**0.1, 10**0.2,...10**1

def KNN_dtm3 (X, y, k, tX, p_in):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, p=10**(0.1*p_in))  # defaulted metric = ‘minkowski’    
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()

for i in range(10):
    p_in = i+1    #  p = 10**0.1, 10**0.2,...10**1 (p=10**(0.1*p_in)
    predte = KNN_dtm3 (train_X, train_y, 11, test_X, p_in) # testing test data with all train data
    
    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = 0.1*p_in, 1-ac    # test error rate for log10(p) = log10(10**(0.1*p_in)) = 0.1*p_in 
          
testER3 = pd.DataFrame(dic).T

# sorting testER(log10(p), test error) by test error and log10(p)

s_testER3 = testER3.sort_values(by=[1, 0])  

print(s_testER3)  # log10(p), test error rate<jupyter_output>0     1
0  0.1  0.11
1  0.2  0.11
7  0.8  0.11
8  0.9  0.11
9  1.0  0.11
2  0.3  0.12
3  0.4  0.12
4  0.5  0.12
5  0.6  0.12
6  0.7  0.12<jupyter_text>the best log10(p) = 0.1, 0.2, 0.8, 0.9, 1.0 with best test error = 0.11 and k*=11  (d) Replace the Euclidean metric with the following metrics and test them (sklearn.neighbors.DistanceMetric). Summarize the test errors (i.e., when k = k∗) in a table. Use all of your training data and select the best k when {1,6,11,...,196}. i. C. which becomes Chebyshev Distance with p →∞<jupyter_code>def KNN_dtmc (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, metric = 'chebyshev')  # metric = Chebyshev    
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtmc (train_X, train_y, k, test_X)  # testing test data with all train data
    
    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testERc = pd.DataFrame(dic).T

s_testERc = testERc.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased  

if s_testERc.iloc[0, 0] == 1 and round(s_testERc.iloc[0, 1], 3) == round(s_testERc.iloc[1, 1], 3):
    bst_k = int(s_testERc.iloc[1, 0])
    bst_er = s_testERc.iloc[1, 1]
else:
    bst_k = int(s_testERc.iloc[0, 0])
    bst_er = s_testERc.iloc[0, 1]

print("Chebyshev Distance, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testERc)  # k, test error<jupyter_output>Chebyshev Distance, k*= 21 with best test error: 0.11
        0     1
4    21.0  0.11
2    11.0  0.12
3    16.0  0.12
6    31.0  0.12
10   51.0  0.12
0     1.0  0.13
1     6.0  0.13
7    36.0  0.13
8    41.0  0.13
9    46.0  0.13
13   66.0  0.13
5    26.0  0.14
11   56.0  0.14
12   61.0  0.14
14   71.0  0.14
16   81.0  0.14
15   76.0  0.15
18   91.0  0.15
17   86.0  0.16
19   96.0  0.17
22  111.0  0.18
20  101.0  0.19
21  106.0  0.19
23  116.0  0.20
24  121.0  0.21
25  126.0  0.22
26  131.0  0.25
27  136.0  0.29
28  141.0  0.30
29  146.0  0.30
30  151.0  0.30
31  156.0  0.30
32  161.0  0.30
33  166.0  0.30
34  171.0  0.30
35  176.0  0.30
36  181.0  0.30
37  186.0  0.30
38  191.0  0.30
39  196.0  0.30<jupyter_text>(d) Replace the Euclidean metric with the following metrics and test them. Summarize the test errors (i.e., when k = k∗) in a table. Use all of your training data and select the best k when {1,6,11,...,196}. ii. Mahalanobis Distance.(Mahalanobis Distance requires inverting the covariance matrix of the data. When the covariance matrix is singular or ill-conditioned, the data live in a linear subspace of the feature space. In this case, the features have to be transformed into a reduced feature set in the linear subspace, which is equivalent to using a pseudoinverse instead of an inverse.)<jupyter_code>cov=np.cov(train_X, rowvar=False)  # cov = covariance matrix of the train data

def KNN_dtmd (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, metric = 'mahalanobis', metric_params={'V': cov})
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtmd (train_X, train_y, k, test_X)  # testing test data with all train data

    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testERd = pd.DataFrame(dic).T

s_testERd = testERd.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased  

if s_testERd.iloc[0, 0] == 1 and round(s_testERd.iloc[0, 1], 3) == round(s_testERd.iloc[1, 1], 3):
    bst_k = int(s_testERd.iloc[1, 0])
    bst_er = s_testERd.iloc[1, 1]
else:
    bst_k = int(s_testERd.iloc[0, 0])
    bst_er = s_testERd.iloc[0, 1]

print("Mahalanobis Distance, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testERd)  # k, test error<jupyter_output>Mahalanobis Distance, k*= 1 with best test error: 0.17
        0     1
0     1.0  0.17
2    11.0  0.20
3    16.0  0.20
4    21.0  0.21
1     6.0  0.23
5    26.0  0.23
6    31.0  0.23
7    36.0  0.24
8    41.0  0.24
10   51.0  0.24
16   81.0  0.24
9    46.0  0.25
12   61.0  0.25
14   71.0  0.26
15   76.0  0.26
18   91.0  0.26
11   56.0  0.27
13   66.0  0.27
17   86.0  0.27
19   96.0  0.27
20  101.0  0.28
21  106.0  0.29
22  111.0  0.29
23  116.0  0.30
24  121.0  0.30
25  126.0  0.30
26  131.0  0.30
27  136.0  0.30
28  141.0  0.30
29  146.0  0.30
30  151.0  0.30
31  156.0  0.30
32  161.0  0.30
33  166.0  0.30
34  171.0  0.30
35  176.0  0.30
36  181.0  0.30
37  186.0  0.30
38  191.0  0.30
39  196.0  0.30<jupyter_text>(d) Summarized table<jupyter_code>from pandas import DataFrame as df

sum_table=df(data={'k*': [11, 11, 21, 1], 'best log10(p)':['N/A', '0.1, 0.2, 0.8, 0.9, 1.0', 'N/A', 'N/A'],
    'best test ER': [0.11, 0.11, 0.11, 0.17]}, columns=['k*', 'best test ER','best log10(p)' ], index=['Manhattan Distance', 
 'Minkowski Distance', 'Chebyshev Distance', 'Mahalanobis Distance' ])
print(sum_table)<jupyter_output>k*  best test ER            best log10(p)
Manhattan Distance    11          0.11                      N/A
Minkowski Distance    11          0.11  0.1, 0.2, 0.8, 0.9, 1.0
Chebyshev Distance    21          0.11                      N/A
Mahalanobis Distance   1          0.17                      N/A<jupyter_text>(e) The majority polling decision can be replaced by weighted decision, in which the weight of each point in voting is proportional to its distance from the query/test data point. In this case, closer neighbors of a query point will have a greater inﬂuence than neighbors which are further away. Use weighted voting with Euclidean, Manhattan, and Chebyshev distances and report the best test errors when k ∈{1,6,11,16,...,196}.  weighted voting with Euclidean, the best test error when k ∈{1,6,11,16,...,196}<jupyter_code>def KNN_dtm_we (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, weights = 'distance') # defaulted: Euclidean_distance    
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtm_we (train_X, train_y, k, test_X)  # testing test data with all train data

    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testERwe = pd.DataFrame(dic).T

s_testERwe = testERwe.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased  

if s_testERwe.iloc[0, 0] == 1 and round(s_testERwe.iloc[0, 1], 3) == round(s_testERwe.iloc[1, 1], 3):
    bst_k = int(s_testERwe.iloc[1, 0])
    bst_er = s_testERwe.iloc[1, 1]
else:
    bst_k = int(s_testERwe.iloc[0, 0])
    bst_er = s_testERwe.iloc[0, 1]

print("weighted Euclidean, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testERwe)  # k, test error<jupyter_output>weighted Euclidean, k*= 6 with best test error: 0.1
        0     1
1     6.0  0.10
3    16.0  0.11
4    21.0  0.11
5    26.0  0.11
6    31.0  0.11
7    36.0  0.11
8    41.0  0.11
9    46.0  0.11
10   51.0  0.11
11   56.0  0.11
12   61.0  0.11
13   66.0  0.11
14   71.0  0.11
2    11.0  0.12
15   76.0  0.12
20  101.0  0.12
0     1.0  0.13
16   81.0  0.13
17   86.0  0.13
23  116.0  0.13
24  121.0  0.13
25  126.0  0.13
18   91.0  0.14
19   96.0  0.14
21  106.0  0.14
22  111.0  0.14
26  131.0  0.14
27  136.0  0.14
28  141.0  0.17
29  146.0  0.19
30  151.0  0.19
31  156.0  0.20
32  161.0  0.23
33  166.0  0.25
34  171.0  0.27
35  176.0  0.27
36  181.0  0.27
37  186.0  0.27
38  191.0  0.28
39  196.0  0.28<jupyter_text>weighted voting with Manhattan, the best test error when k ∈{1,6,11,16,...,196}<jupyter_code>def KNN_dtm_wm (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, weights ='distance', p=1)  # Manhattan (p=1)   
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtm_wm (train_X, train_y, k, test_X)  # testing test data with all train data

    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testERwm = pd.DataFrame(dic).T

s_testERwm = testERwm.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased 

if s_testERwm.iloc[0, 0] == 1 and round(s_testERwm.iloc[0, 1], 3) == round(s_testERwm.iloc[1, 1], 3):
    bst_k = int(s_testERwm.iloc[1, 0])
    bst_er = s_testERwm.iloc[1, 1]
else:
    bst_k = int(s_testERwm.iloc[0, 0])
    bst_er = s_testERwm.iloc[0, 1]

print("weighted Manhattan, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testERwm)  # k, test error<jupyter_output>weighted Manhattan, k*= 26 with best test error: 0.1
        0     1
5    26.0  0.10
1     6.0  0.11
2    11.0  0.11
4    21.0  0.11
6    31.0  0.11
7    36.0  0.11
8    41.0  0.11
9    46.0  0.11
14   71.0  0.11
0     1.0  0.12
3    16.0  0.12
10   51.0  0.12
11   56.0  0.12
15   76.0  0.12
12   61.0  0.13
13   66.0  0.13
18   91.0  0.13
19   96.0  0.13
16   81.0  0.14
17   86.0  0.14
20  101.0  0.15
21  106.0  0.15
22  111.0  0.15
23  116.0  0.15
24  121.0  0.15
25  126.0  0.15
26  131.0  0.15
27  136.0  0.16
28  141.0  0.17
29  146.0  0.18
30  151.0  0.19
31  156.0  0.21
32  161.0  0.24
33  166.0  0.25
34  171.0  0.27
35  176.0  0.27
36  181.0  0.27
37  186.0  0.27
38  191.0  0.28
39  196.0  0.29<jupyter_text>weighted voting with Chebyshev, the best test error when k ∈{1,6,11,16,...,196}<jupyter_code>def KNN_dtm_wc (X, y, k, tX):    # tX: test data set
    neigh = KNeighborsClassifier(n_neighbors=k, weights = 'distance', metric = 'chebyshev')
    neigh.fit(X, y)
    pred = neigh.predict(tX)
    
    return pred

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predte = KNN_dtm_wc (train_X, train_y, k, test_X)  # testing test data with all train data

    # calculating error rate
    ac = accuracy_score(test.iloc[:, 6], predte)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
testERwc = pd.DataFrame(dic).T

s_testERwc = testERwc.sort_values(by=[1, 0])  # sorting testER(k, test error) by test error and k
    
# select best k with the least test error rate 
# If many k's give the same error, 
# select the least k except for k=1 to avoid overfitting and biased 

if s_testERwc.iloc[0, 0] == 1 and round(s_testERwc.iloc[0, 1], 3) == round(s_testERwc.iloc[1, 1],3):
    bst_k = int(s_testERwc.iloc[1, 0])
    bst_er = s_testERwc.iloc[1, 1]
else:
    bst_k = int(s_testERwc.iloc[0, 0])
    bst_er = s_testERwc.iloc[0, 1]

print("weighted Chebyshev, k*=", bst_k, "with best test error:", round(bst_er, 3))    
print(s_testERwc)  # k, test error<jupyter_output>weighted Chebyshev, k*= 16 with best test error: 0.11
        0     1
3    16.0  0.11
6    31.0  0.11
7    36.0  0.11
8    41.0  0.11
12   61.0  0.11
2    11.0  0.12
4    21.0  0.12
5    26.0  0.12
9    46.0  0.12
10   51.0  0.12
11   56.0  0.12
14   71.0  0.12
15   76.0  0.12
16   81.0  0.12
0     1.0  0.13
1     6.0  0.13
13   66.0  0.13
18   91.0  0.13
17   86.0  0.14
19   96.0  0.14
20  101.0  0.14
21  106.0  0.14
22  111.0  0.14
23  116.0  0.14
25  126.0  0.14
24  121.0  0.15
26  131.0  0.16
27  136.0  0.16
28  141.0  0.18
29  146.0  0.19
30  151.0  0.21
31  156.0  0.21
32  161.0  0.23
33  166.0  0.26
34  171.0  0.26
35  176.0  0.26
36  181.0  0.27
37  186.0  0.27
38  191.0  0.27
39  196.0  0.28<jupyter_text>(e) summarized answer:weighted Euclidean, k*= 6 with best test error: 0.1weighted Manhattan, k*= 26 with best test error: 0.1weighted Chebyshev, k*= 16 with best test error: 0.11 (f) What is the lowest training error rate you achieved in this exercise? A. Training error rate with Euclidean metric, k ∈{208,205,...,7,4,1,} ((c). ii)<jupyter_code># trainER : obtained in (c) ii. 

s_trainER = trainER.sort_values(by=[1, 0])  # sorting trainER(k, train error) by train error and k
    
bst_k = int(s_trainER.iloc[0, 0])
bst_er = round(s_trainER.iloc[0, 1], 3)  # the lowest train error rate
s_k = int(s_trainER.iloc[1, 0])
s_er = round(s_trainER.iloc[1, 1], 3)  # the second lowest train error rate

print("Euclidean, best train error rate:", bst_er, "with k=", bst_k)
print("Euclidean, the second best train error rate:", s_er, "with k=", s_k)
print(s_trainER)  # k, train error rate<jupyter_output>Euclidean, best train error rate: 0.0 with k= 1
Euclidean, the second best train error rate: 0.129 with k= 22
        0         1
0     1.0  0.000000
7    22.0  0.128571
8    25.0  0.138095
2     7.0  0.142857
4    13.0  0.142857
1     4.0  0.147619
5    16.0  0.147619
6    19.0  0.147619
10   31.0  0.147619
15   46.0  0.147619
3    10.0  0.152381
11   34.0  0.152381
12   37.0  0.152381
31   94.0  0.152381
32   97.0  0.152381
34  103.0  0.157143
9    28.0  0.161905
13   40.0  0.161905
14   43.0  0.161905
16   49.0  0.161905
20   61.0  0.161905
21   64.0  0.161905
27   82.0  0.161905
22   67.0  0.166667
24   73.0  0.166667
30   91.0  0.166667
33  100.0  0.166667
17   52.0  0.171429
18   55.0  0.171429
29   88.0  0.171429
..    ...       ...
40  121.0  0.214286
41  124.0  0.242857
42  127.0  0.247619
43  130.0  0.276190
44  133.0  0.290476
45  136.0  0.323810
46  139.0  0.333333
47  142.0  0.333333
48  145.0  0.333333
49  148.0  0.333333
50  151.0  0.333333
51  154.0  0.333333
52  157.0 [...]<jupyter_text>B. Training error rate with Euclidean metric, N ∈ {10,20,30,...,210}, k = 1, increasing by 5 ((c). iii)<jupyter_code># function for KNN: def KNN_dtm (X, y, k, tX)

dt = dict()

for n in range(21):    
    N = 10*(n+1)      # N ∈ {10,20,30,...,210}
    
    ntrain = mkntrain (N)
    tr_X = ntrain.iloc[:, :6]     # subset train data
    tr_y = ntrain.iloc[:, 6]      # class

    dic = dict ()
    trainerr = None
    s_trainerr = None
    for i in range(int(N/5)):  
        k = 5*i+1    # k = 1,6,11,...,196
        predtr = KNN_dtm (tr_X, tr_y, k, train_X)   # testing train data for N, k

        # calculating error rate
        ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
        dic[i] = k, 1-ac
          
    trainerr = pd.DataFrame(dic).T   # train error rate for N & all k
    
    # the best train error rate, and the best k for N
    
    s_trainerr = trainerr.sort_values(by=[1, 0])  # sorting trainerr(k, train error) by train error and k

    # select the least train error rate with k 
    bst_k = s_trainerr.iloc[0, 0]
    bst_er = s_trainerr.iloc[0, 1]
     
    dt[n]= N, bst_k, bst_er # N, opt_k, best test error rate
    
ntr = pd.DataFrame(dt).T
s_ntr = ntr.sort_values(by=[2, 0, 1]) 
print(s_ntr) # N, opt_k, best error rate<jupyter_output>0     1         2
19  200.0   1.0  0.000000
20  210.0   1.0  0.000000
17  180.0   1.0  0.004762
18  190.0   1.0  0.004762
15  160.0   1.0  0.009524
16  170.0   1.0  0.009524
14  150.0   1.0  0.033333
13  140.0   1.0  0.038095
12  130.0   1.0  0.061905
11  120.0   1.0  0.066667
10  110.0   1.0  0.100000
9   100.0   1.0  0.161905
1    20.0   6.0  0.200000
2    30.0  11.0  0.204762
3    40.0  11.0  0.247619
0    10.0   1.0  0.280952
5    60.0  21.0  0.290476
7    80.0  21.0  0.290476
6    70.0  26.0  0.300000
4    50.0  11.0  0.304762
8    90.0  31.0  0.304762<jupyter_text>Euclidean, best train error rate: 0.0 with k= 1 and N = 200, 210 C. Training error rate with Manhattan, k ∈{1,6,11,16,...,196} ((d). i. A)<jupyter_code># function for KNN: KNN_dtm2 (train_X, train_y, k, train_X)

dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtm2 (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerr2 = pd.DataFrame(dic).T

s_trainerr2 = trainerr2.sort_values(by=[1, 0])  # sorting trainER(k, train error) by train error and k
    
bst_k = int(s_trainerr2.iloc[0, 0])
bst_er = round(s_trainerr2.iloc[0, 1], 3)  # the lowest train error rate
s_k = int(s_trainerr2.iloc[1, 0])
s_er = round(s_trainerr2.iloc[1, 1], 3)  # the second lowest train error rate

print("Manhattan, best train error rate:", bst_er, "with k=", bst_k)
print("Manhattan, the second best train error rate:", s_er, "with k=", s_k)
print(s_trainerr2)  # k, train error rate<jupyter_output>Manhattan, best train error rate: 0.0 with k= 1
Manhattan, the second best train error rate: 0.143 with k= 11
        0         1
0     1.0  0.000000
2    11.0  0.142857
1     6.0  0.147619
3    16.0  0.147619
4    21.0  0.147619
7    36.0  0.161905
6    31.0  0.166667
8    41.0  0.166667
16   81.0  0.166667
17   86.0  0.166667
5    26.0  0.171429
9    46.0  0.171429
10   51.0  0.171429
11   56.0  0.171429
12   61.0  0.176190
13   66.0  0.176190
14   71.0  0.176190
15   76.0  0.180952
18   91.0  0.180952
19   96.0  0.195238
20  101.0  0.200000
21  106.0  0.209524
22  111.0  0.219048
23  116.0  0.238095
24  121.0  0.242857
25  126.0  0.290476
26  131.0  0.304762
27  136.0  0.333333
28  141.0  0.333333
29  146.0  0.333333
30  151.0  0.333333
31  156.0  0.333333
32  161.0  0.333333
33  166.0  0.333333
34  171.0  0.333333
35  176.0  0.333333
36  181.0  0.333333
37  186.0  0.333333
38  191.0  0.333333
39  196.0  0.333333<jupyter_text>D. Training error rate with Minkowski Distance, log10(p) ∈ {0.1,0.2,0.3,...,1}, k* = 11 ((d). i. B)<jupyter_code># k∗ = 11
# p = 10**0.1, 10**0.2,...10**1.0

dic = dict ()
for i in range(10):
    p_in = i+1    #  p = 10**0.1, 10**0.2,...10**1 (p=10**(0.1*p_in))
    predtr = KNN_dtm3 (train_X, train_y, 11, train_X, p_in)  # test train data

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = 0.1*p_in, 1-ac    # train error rate for log10(p) = log10(10**(0.1*p_in)) = 0.1*p_in 
          
trainER3 = pd.DataFrame(dic).T
s_trainER3 = trainER3.sort_values(by=[1, 0])  # sorting trainER(log10(p), train error) by train error and log10(p)

bst_k = s_trainER3.iloc[0, 0]    # best log10(p)
bst_er = round(s_trainER3.iloc[0, 1], 3)  # the lowest train error rate

print("Minkowski, best train error rate:", bst_er, "with k = 11 and log10(p)=", bst_k)
print(s_trainER3)  # log10(p), train error rate<jupyter_output>Minkowski, best train error rate: 0.129 with k = 11 and log10(p)= 0.1
     0         1
0  0.1  0.128571
1  0.2  0.138095
2  0.3  0.147619
3  0.4  0.147619
5  0.6  0.161905
9  1.0  0.161905
4  0.5  0.166667
7  0.8  0.171429
6  0.7  0.176190
8  0.9  0.180952<jupyter_text>E. Training error rate with Chebyshev Distance, k ∈{1,6,11,16,...,196} ((d). i. C)<jupyter_code>dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtmc (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerrc = pd.DataFrame(dic).T

s_trainerrc = trainerrc.sort_values(by=[1, 0])  # sorting trainER(k, train error) by train error and k
    
bst_k = int(s_trainerrc.iloc[0, 0])
bst_er = round(s_trainerrc.iloc[0, 1], 3)  # the lowest train error rate
s_k = int(s_trainerrc.iloc[1, 0])
s_er = round(s_trainerrc.iloc[1, 1], 3)  # the second lowest train error rate

print("Chebyshev, best train error rate:", bst_er, "with k=", bst_k)
print("Chebyshev, the second best train error rate:", s_er, "with k=", s_k)
print(s_trainerrc)  # k, train error rate<jupyter_output>Chebyshev, best train error rate: 0.0 with k= 1
Chebyshev, the second best train error rate: 0.133 with k= 21
        0         1
0     1.0  0.000000
4    21.0  0.133333
5    26.0  0.142857
6    31.0  0.142857
8    41.0  0.142857
7    36.0  0.147619
1     6.0  0.152381
2    11.0  0.152381
3    16.0  0.152381
18   91.0  0.157143
9    46.0  0.166667
10   51.0  0.166667
19   96.0  0.166667
12   61.0  0.171429
16   81.0  0.171429
17   86.0  0.171429
20  101.0  0.171429
21  106.0  0.171429
13   66.0  0.176190
15   76.0  0.176190
23  116.0  0.176190
11   56.0  0.180952
14   71.0  0.180952
22  111.0  0.180952
24  121.0  0.209524
25  126.0  0.238095
26  131.0  0.247619
27  136.0  0.328571
28  141.0  0.333333
29  146.0  0.333333
30  151.0  0.333333
31  156.0  0.333333
32  161.0  0.333333
33  166.0  0.333333
34  171.0  0.333333
35  176.0  0.333333
36  181.0  0.333333
37  186.0  0.333333
38  191.0  0.333333
39  196.0  0.333333<jupyter_text>F. Training error rate with Mahalanobis Distance, k ∈{1,6,11,16,...,196} ((d). ii)<jupyter_code>dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtmd (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerrd = pd.DataFrame(dic).T

s_trainerrd = trainerrd.sort_values(by=[1, 0])  # sorting trainER(k, train error) by train error and k
    
bst_k = int(s_trainerrd.iloc[0, 0])
bst_er = round(s_trainerrd.iloc[0, 1], 3)  # the lowest train error rate
s_k = int(s_trainerrd.iloc[1, 0])
s_er = round(s_trainerrd.iloc[1, 1], 3)  # the second lowest train error rate

print("Mahalanobis, best train error rate:", bst_er, "with k=", bst_k)
print("Mahalanobis, the second best train error rate:", s_er, "with k=", s_k)
print(s_trainerrd)  # k, train error rate<jupyter_output>Mahalanobis, best train error rate: 0.0 with k= 1
Mahalanobis, the second best train error rate: 0.133 with k= 6
        0         1
0     1.0  0.000000
1     6.0  0.133333
3    16.0  0.176190
4    21.0  0.176190
6    31.0  0.180952
2    11.0  0.185714
8    41.0  0.185714
5    26.0  0.190476
10   51.0  0.190476
7    36.0  0.200000
9    46.0  0.204762
12   61.0  0.204762
11   56.0  0.209524
13   66.0  0.209524
14   71.0  0.219048
15   76.0  0.233333
16   81.0  0.238095
18   91.0  0.252381
17   86.0  0.257143
19   96.0  0.280952
20  101.0  0.285714
22  111.0  0.295238
21  106.0  0.300000
23  116.0  0.323810
24  121.0  0.328571
25  126.0  0.333333
26  131.0  0.333333
27  136.0  0.333333
28  141.0  0.333333
29  146.0  0.333333
30  151.0  0.333333
31  156.0  0.333333
32  161.0  0.333333
33  166.0  0.333333
34  171.0  0.333333
35  176.0  0.333333
36  181.0  0.333333
37  186.0  0.333333
38  191.0  0.333333
39  196.0  0.333333<jupyter_text>G. Training error rate with weighted Euclidean, k ∈{1,6,11,16,...,196} (e)<jupyter_code>dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtm_we (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerr_we = pd.DataFrame(dic).T

s_trainerr_we = trainerr_we.sort_values(by=[1, 0])  # sorting trainER(k, train error) by train error and k
    
print(s_trainerr_we)  # k, train error rate<jupyter_output>0    1
0     1.0  0.0
1     6.0  0.0
2    11.0  0.0
3    16.0  0.0
4    21.0  0.0
5    26.0  0.0
6    31.0  0.0
7    36.0  0.0
8    41.0  0.0
9    46.0  0.0
10   51.0  0.0
11   56.0  0.0
12   61.0  0.0
13   66.0  0.0
14   71.0  0.0
15   76.0  0.0
16   81.0  0.0
17   86.0  0.0
18   91.0  0.0
19   96.0  0.0
20  101.0  0.0
21  106.0  0.0
22  111.0  0.0
23  116.0  0.0
24  121.0  0.0
25  126.0  0.0
26  131.0  0.0
27  136.0  0.0
28  141.0  0.0
29  146.0  0.0
30  151.0  0.0
31  156.0  0.0
32  161.0  0.0
33  166.0  0.0
34  171.0  0.0
35  176.0  0.0
36  181.0  0.0
37  186.0  0.0
38  191.0  0.0
39  196.0  0.0<jupyter_text>H. Training error rate with weighted Manhattan , k ∈{1,6,11,16,...,196} (e)<jupyter_code>dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtm_wm (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerr_wm = pd.DataFrame(dic).T

print(trainerr_wm)<jupyter_output>0    1
0     1.0  0.0
1     6.0  0.0
2    11.0  0.0
3    16.0  0.0
4    21.0  0.0
5    26.0  0.0
6    31.0  0.0
7    36.0  0.0
8    41.0  0.0
9    46.0  0.0
10   51.0  0.0
11   56.0  0.0
12   61.0  0.0
13   66.0  0.0
14   71.0  0.0
15   76.0  0.0
16   81.0  0.0
17   86.0  0.0
18   91.0  0.0
19   96.0  0.0
20  101.0  0.0
21  106.0  0.0
22  111.0  0.0
23  116.0  0.0
24  121.0  0.0
25  126.0  0.0
26  131.0  0.0
27  136.0  0.0
28  141.0  0.0
29  146.0  0.0
30  151.0  0.0
31  156.0  0.0
32  161.0  0.0
33  166.0  0.0
34  171.0  0.0
35  176.0  0.0
36  181.0  0.0
37  186.0  0.0
38  191.0  0.0
39  196.0  0.0<jupyter_text>I. Training error rate with weighted Chebyshev , k ∈{1,6,11,16,...,196} (e)<jupyter_code>dic = dict ()
for i in range(40):
    k = 5*i+1    # k = 1,6,11,...,196
    predtr = KNN_dtm_wc (train_X, train_y, k, train_X) # test train data(train_X)

    # calculating error rate
    ac = accuracy_score(train.iloc[:, 6], predtr)     # comparing true class and predicted class
    dic[i] = k, 1-ac
          
trainerr_wc = pd.DataFrame(dic).T

print(trainerr_wc)<jupyter_output>0    1
0     1.0  0.0
1     6.0  0.0
2    11.0  0.0
3    16.0  0.0
4    21.0  0.0
5    26.0  0.0
6    31.0  0.0
7    36.0  0.0
8    41.0  0.0
9    46.0  0.0
10   51.0  0.0
11   56.0  0.0
12   61.0  0.0
13   66.0  0.0
14   71.0  0.0
15   76.0  0.0
16   81.0  0.0
17   86.0  0.0
18   91.0  0.0
19   96.0  0.0
20  101.0  0.0
21  106.0  0.0
22  111.0  0.0
23  116.0  0.0
24  121.0  0.0
25  126.0  0.0
26  131.0  0.0
27  136.0  0.0
28  141.0  0.0
29  146.0  0.0
30  151.0  0.0
31  156.0  0.0
32  161.0  0.0
33  166.0  0.0
34  171.0  0.0
35  176.0  0.0
36  181.0  0.0
37  186.0  0.0
38  191.0  0.0
39  196.0  0.0
{'fraction_non_alphanumeric': 0.15078126583304624, 'fraction_numerical': 0.1562329003100744, 'mean_word_length': 2.804178552154807, 'pattern_counts': {'":': 0, '<': 188, '<?xml version=': 0, '>': 188, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '53', 'dir': 'jupyter-structured-clean-dedup', 'id': '362330', 'license': 'CC0-1.0', 'n_tokens_mistral': 26087, 'n_tokens_neox': 22224, 'n_words': 6159, 'path': 'HW1_Myunghee_ID_2446752777.ipynb', 'repo_name': 'Myunghee13/MachineLearning', 'size': '49349'}