<jupyter_start><jupyter_text>KNN<jupyter_code>Ks = np.arange(1, 57, 2)
Ks
%%time
pipe = Pipeline([('knn', KNeighborsClassifier())])

search_space = [{'knn': [KNeighborsClassifier()],
                 'knn__weights': ['uniform', 'distance'],
                 'knn__n_neighbors': Ks}
                ]

search = GridSearchCV(pipe, search_space, cv = StratifiedKFold(n_splits=5), 
                   scoring = ['accuracy'], refit=False, return_train_score=True,
                   verbose = 0, n_jobs = -1)


best_knn = search.fit(X_train, Y_train)
best_knn.cv_results_['params'][np.argmin(best_knn.cv_results_['rank_test_accuracy'])]
knn = KNeighborsClassifier(weights = best_knn.cv_results_['params'][np.argmin(best_knn.cv_results_['rank_test_accuracy'])].get('knn__weights'),
                               n_neighbors = best_knn.cv_results_['params'][np.argmin(best_knn.cv_results_['rank_test_accuracy'])].get('knn__n_neighbors')
                              )
    
knn.fit(X_train, Y_train)
    
train_acc = accuracy_score(Y_train, knn.predict(X_train))
test_acc = accuracy_score(Y_test, knn.predict(X_test))

train_acc, test_acc
labels = list(string.ascii_uppercase)
con_mat = confusion_matrix(Y_test, knn.predict(X_test), labels);

f, ax = plt.subplots(figsize=(12, 10), dpi = 80)
ax = sns.heatmap(con_mat, annot=True, cmap="YlGnBu", fmt='g')
ax.set_xticklabels(labels)
ax.set_yticklabels(labels)
ax.set_xlabel('Predicted Letters')
ax.set_ylabel('True Letters')
ax.set_title('Confusion Matrix')
knn = KNeighborsClassifier(weights = best_knn.cv_results_['params'][np.argmin(best_knn.cv_results_['rank_test_accuracy'])].get('knn__weights'),
                               n_neighbors = best_knn.cv_results_['params'][np.argmin(best_knn.cv_results_['rank_test_accuracy'])].get('knn__n_neighbors')
                              )
    
knn.fit(X, Y)
    
acc_all = accuracy_score(Y, knn.predict(X))

acc_all
best_knn.cv_results_
best_knn.cv_results_['mean_test_accuracy']
from scipy import stats
knn_3_dis = [best_knn.cv_results_['split0_test_accuracy'][3],
            best_knn.cv_results_['split1_test_accuracy'][3],
            best_knn.cv_results_['split2_test_accuracy'][3],
            best_knn.cv_results_['split3_test_accuracy'][3],
            best_knn.cv_results_['split4_test_accuracy'][3]]
knn_5_dis = [best_knn.cv_results_['split0_test_accuracy'][5],
            best_knn.cv_results_['split1_test_accuracy'][5],
            best_knn.cv_results_['split2_test_accuracy'][5],
            best_knn.cv_results_['split3_test_accuracy'][5],
            best_knn.cv_results_['split4_test_accuracy'][5]]
knn_1_uni = [best_knn.cv_results_['split0_test_accuracy'][0],
            best_knn.cv_results_['split1_test_accuracy'][0],
            best_knn.cv_results_['split2_test_accuracy'][0],
            best_knn.cv_results_['split3_test_accuracy'][0],
            best_knn.cv_results_['split4_test_accuracy'][0]]
knn_1_dis = [best_knn.cv_results_['split0_test_accuracy'][1],
            best_knn.cv_results_['split1_test_accuracy'][1],
            best_knn.cv_results_['split2_test_accuracy'][1],
            best_knn.cv_results_['split3_test_accuracy'][1],
            best_knn.cv_results_['split4_test_accuracy'][1]]
knn_7_dis = [best_knn.cv_results_['split0_test_accuracy'][7],
            best_knn.cv_results_['split1_test_accuracy'][7],
            best_knn.cv_results_['split2_test_accuracy'][7],
            best_knn.cv_results_['split3_test_accuracy'][7],
            best_knn.cv_results_['split4_test_accuracy'][7]]
knn_9_dis = [best_knn.cv_results_['split0_test_accuracy'][9],
            best_knn.cv_results_['split1_test_accuracy'][9],
            best_knn.cv_results_['split2_test_accuracy'][9],
            best_knn.cv_results_['split3_test_accuracy'][9],
            best_knn.cv_results_['split4_test_accuracy'][9]]
knn_3_uni = [best_knn.cv_results_['split0_test_accuracy'][2],
            best_knn.cv_results_['split1_test_accuracy'][2],
            best_knn.cv_results_['split2_test_accuracy'][2],
            best_knn.cv_results_['split3_test_accuracy'][2],
            best_knn.cv_results_['split4_test_accuracy'][2]]
knn_11_dis = [best_knn.cv_results_['split0_test_accuracy'][11],
            best_knn.cv_results_['split1_test_accuracy'][11],
            best_knn.cv_results_['split2_test_accuracy'][11],
            best_knn.cv_results_['split3_test_accuracy'][11],
            best_knn.cv_results_['split4_test_accuracy'][11]]


print(stats.ttest_rel(knn_3_dis, knn_5_dis))
print(stats.ttest_rel(knn_3_dis, knn_1_uni))
print(stats.ttest_rel(knn_3_dis, knn_1_dis))
print(stats.ttest_rel(knn_3_dis, knn_7_dis))
print(stats.ttest_rel(knn_3_dis, knn_9_dis))
print(stats.ttest_rel(knn_3_dis, knn_3_uni))
print(stats.ttest_rel(knn_3_dis, knn_11_dis))
d = {'Model': ['k=3 distance', 'k=5 distance','k=1 uniform','k=1 distance',
             'k=7 distance', 'k=9 distance', 'k=3 uniform', 'k=11 distance'],
    'P value': [stats.ttest_rel(knn_3_dis, knn_3_dis)[1],
                 stats.ttest_rel(knn_3_dis, knn_5_dis)[1],
                 stats.ttest_rel(knn_3_dis, knn_1_uni)[1],
                 stats.ttest_rel(knn_3_dis, knn_1_dis)[1],
                 stats.ttest_rel(knn_3_dis, knn_7_dis)[1],
                 stats.ttest_rel(knn_3_dis, knn_9_dis)[1],
                 stats.ttest_rel(knn_3_dis, knn_3_uni)[1],
                 stats.ttest_rel(knn_3_dis, knn_11_dis)[1]]}
df_t_test = pd.DataFrame(d)
df_t_test<jupyter_output><empty_output><jupyter_text>K=3 uniform and K=11 distance is significantly different from the best model, k=3 distance.<jupyter_code>best_knn.cv_results_['mean_test_accuracy']
best_knn.cv_results_
uni_avg_acc = []
dis_avg_acc = []

for i in np.arange(0, 56, 2):
    uni_avg_acc.append(best_knn.cv_results_['mean_test_accuracy'][i])

for i in np.arange(1, 56, 2):
    dis_avg_acc.append(best_knn.cv_results_['mean_test_accuracy'][i])
    
x_axis = np.arange(1, 56, 2)

fig = plt.figure(figsize = (5, 5), dpi = 90)
uni, = plt.plot(x_axis, uni_avg_acc, 'r-', label = 'Uniform')
dis, = plt.plot(x_axis, dis_avg_acc, 'b-', label = 'Distance')
plt.xlabel('K values')
plt.ylabel('Accuracy Scores')
plt.title('Average Testing Performance of Cross-Validation')
plt.grid(True)
plt.legend()
uni_avg_acc_tr = []
dis_avg_acc_tr = []

for i in np.arange(0, 56, 2):
    uni_avg_acc_tr.append(best_knn.cv_results_['mean_train_accuracy'][i])

for i in np.arange(1, 56, 2):
    dis_avg_acc_tr.append(best_knn.cv_results_['mean_train_accuracy'][i])
    
x_axis = np.arange(1, 56, 2)

fig = plt.figure(figsize = (5, 5), dpi = 90)
uni, = plt.plot(x_axis, uni_avg_acc_tr, 'r-', label = 'Uniform')
dis, = plt.plot(x_axis, dis_avg_acc_tr, 'b-', label = 'Distance')
plt.xlabel('K values')
plt.ylabel('Accuracy Scores')
plt.title('Average Training Performance of Cross-Validation')
plt.grid(True)
plt.legend()
for i in range(28):
    if uni_avg_acc[i] < uni_avg_acc_tr[i]:
        print('True')
    else:
        print('False')
for i in range(28):
    if dis_avg_acc[i] < dis_avg_acc_tr[i]:
        print('True')
    else:
        print('False')
dis_avg_acc, uni_avg_acc<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14518393709632127, 'fraction_numerical': 0.027520359449592812, 'mean_word_length': 4.241353936718175, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '523022', 'license': 'MIT', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2880, 'n_words': 324, 'path': 'Yuan.ipynb', 'repo_name': 'MChengW/cogs109_finalproject_2021', 'size': '7122'}