<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.neighbors import KNeighborsClassifier<jupyter_output><empty_output><jupyter_text>Loading<jupyter_code>df = pd.read_csv("./Programming-Language-Classifier/dataset.csv")
df.head()
df = df.values
X, Y = df.T[0], df.T[1]<jupyter_output><empty_output><jupyter_text>Vectorizing<jupyter_code>vectorizer = CountVectorizer()
X_vec = vectorizer.fit_transform(X)
X_vec = X_vec.toarray()
print(X_vec.shape)
print("Dic Size:", len(vectorizer.get_feature_names()))<jupyter_output>Dic Size: 10916<jupyter_text>Creating a KNN Classifier<jupyter_code>neigh = KNeighborsClassifier(n_neighbors=5)
neigh.fit(X_vec[:int(0.8*len(Y))], Y[:int(0.8*len(Y))])<jupyter_output><empty_output><jupyter_text>Measuring Performance Training<jupyter_code>neigh.score(X_vec[:int(0.8*len(Y))], Y[:int(0.8*len(Y))])<jupyter_output><empty_output><jupyter_text>Valdidation<jupyter_code>neigh.score(X_vec[int(0.8*len(Y)):], Y[int(0.8*len(Y)):])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15784671532846714, 'fraction_numerical': 0.01824817518248175, 'mean_word_length': 12.886075949367088, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '499737', 'license': 'MIT', 'n_tokens_mistral': 468, 'n_tokens_neox': 457, 'n_words': 45, 'path': 'knn_classifier.ipynb', 'repo_name': 'vedant130701/Programming-Language-Classifier', 'size': '1096'}