<jupyter_start><jupyter_text>Objective Determine whether the given reviews are positive or negative using all features except SCORE and then check the accuracy using the Score feature using KNN classifier  1. Loading data from dataset<jupyter_code>%matplotlib inline

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sqlite3
import nltk
import seaborn as sns
# Loading data from sqlite3 database ignoring those having rewiew score = 3

connection = sqlite3.connect('./database.sqlite')
dataset = pd.read_sql_query("SELECT * FROM Reviews WHERE Score != 3", connection)
print("The shape of the data in dataset is:", dataset.shape)
print('==' * 60)
dataset.columns                # This will list all the features name in dataset column-wise
# Assigning Positive value to reviews with score>3 and Negative value to reviews with score<3

def assign_pn(score):
    if (score > 3):
        score = 'Positive'
    elif (score < 3):
        score = 'Negative'
    return score

pn = dataset['Score']
pn_assign = pn.map(assign_pn)
dataset['Score'] = pn_assign
print(dataset['Score'].value_counts() )                       # Printing the number of each classes (positive and negative reviews)
print('==' * 60)
print("The shape of the data in dataset is:", dataset.shape)
# Printing first few rows as a sample of the dataset
# You may pass the number of rows to be displayed as an argument to the head() method

dataset.head()<jupyter_output><empty_output><jupyter_text>2. Pre-processing of data  2.1 De-duplication of data<jupyter_code># Sorting data based on ProductId

sorted_data = dataset.sort_values('ProductId', ascending=True, axis=0)
sorted_data.head()
# Remove all the data points that exactly match the field values mention below

data = sorted_data.drop_duplicates(subset={'UserId', 'Time', 'Text'}, keep='first', inplace=False)
data.shape
# Check what percentage of original data is remaining

percentage = (data.shape[0] / dataset.shape[0]) * 100
print(percentage)
# Removing the reviews having HelpfulnessNumerator greater than HelpfulnessDenominator

data = data[data.HelpfulnessDenominator >= data.HelpfulnessNumerator]
data.shape
# Getting the number of positive and negative reviews

data['Score'].value_counts().plot(kind='bar')
data['Score'].value_counts()<jupyter_output><empty_output><jupyter_text>2.2 Data Cleaning<jupyter_code>import re
import string
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
# Find sentences containing html tags

i = 0
for sentence in data['Text'].values:
    if (len(re.findall('<.*?>', sentence))):
        print("Total number of html tags present:", i)
        print("=="*60)
        print(sentence)
        break
    i += 1
# Function to remove texts with html tags(<...>)

def cleanHtml(sentence):
    cleanr = re.compile('<.*?>')                  # Select ANY text with < > tags
    cleanText = re.sub(cleanr, ' ', sentence)     # Replace texts enclosed in < > with white space
    return cleanText
# Function to remove punctuation marks(.,?/:;)

def cleanPunctuation(sentence):
    cleanp = re.sub(r'[?|$|.|!]',r'', sentence)
    cleanp = re.sub(r'[.|,|)|(|\|/]',r' ', cleanp)
    return cleanp
# Printing the set of stop words present in english language
# You need to download stopword first before using it(import nltk; nltk.download("stopwords");)

stopWords = set(stopwords.words('english'))
print(stopWords)
# Initializing SnowBall Stemmer

snowball = SnowballStemmer('english')
# Printing a base-word using snowball stemmer

print(snowball.stem('tasty'))
print(snowball.stem('delicious'))
print(snowball.stem('little'))
# Core Pre-processing code step-by-step(removing html tags, punctuations, stopwords, etc)
# This may take a few minutes to execute as it needs to execute all the sentences present in data['Text']

i = 0
initial_str = ''
final_str = []          # Store final string of clean words
positive_words = []     # Store words from positive reviews here
negative_words = []     # Store words from negative reviews here
s = ''

for sentence in data['Text'].values:
    filtered_sentence = []
    sentence = cleanHtml(sentence)          # Remove html tags
    for words in sentence.split():
        for word in cleanPunctuation(words).split():
            if ((word.isalpha()) & (len(word) > 2)):
                if (word.lower() not in stopWords):
                    s = (snowball.stem(word.lower()).encode('utf-8'))
                    filtered_sentence.append(s)
                    if ((data['Score'].values)[i] == 'Positive'):
                        positive_words.append(s)
                    if ((data['Score'].values)[i] == 'Negative'):
                        negative_words.append(s)
                else:
                    continue
            else:
                continue
    initial_str = b" ".join(filtered_sentence)      # Final string of clean words
    final_str.append(initial_str)
    i += 1
# Adding a column in dataset and storing cleaned text in it.

data['CleanedText'] = final_str
data['CleanedText'] = data['CleanedText'].str.decode("utf-8")
data.head()<jupyter_output><empty_output><jupyter_text>3. Time-based splitting of data<jupyter_code># Sorting data based on time

time_sort = data.sort_values('Time', ascending=True, axis=0, na_position='last')       # na_postion='last' puts NaNs at the end
time_sort.head()
from sklearn.model_selection import train_test_split
# Splitting data into Train, Test and Cross-validation

X = time_sort['CleanedText']
Y = time_sort['Score']

x_training, x_test, y_training, y_test= train_test_split(X, Y, test_size=0.25)
x_train, x_cv, y_train, y_cv = train_test_split(x_training, y_training, test_size=0.3)
print("The shape of the training data is:", x_train.shape, y_train.shape)
print("The shape of the CV data is:", x_cv.shape, y_cv.shape)
print("The shape of the test data is:", x_test.shape, y_test.shape)<jupyter_output>The shape of the training data is: (191168,) (191168,)
The shape of the CV data is: (81930,) (81930,)
The shape of the test data is: (91033,) (91033,)<jupyter_text>4. Creating a function to Train and Test KNN models<jupyter_code>import warnings
warnings.filterwarnings("ignore")
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score<jupyter_output><empty_output><jupyter_text>4.1 Train funcrion for KNN models<jupyter_code>def knn_train(x_train, x_cv, y_train, y_cv, algo):
    k = []
    accuracy_cv = []

    for i in range(2, 25, 2):
        neighbor = KNeighborsClassifier(n_neighbors=i, algorithm=algo)
        neighbor.fit(x_train, y_train)
        
        prediction = neighbor.predict(x_cv)
        
        acc_cv = accuracy_score(y_cv, prediction)
        f1_cv = f1_score(y_cv, prediction, average='macro')
        precision_cv = precision_score(y_cv, prediction, average='macro')
        recall_cv = recall_score(y_cv, prediction, average='macro')
        
        print("FOR K =", i)
        print("The accuracy of the model is:", acc_cv*100)
        print("The f1_score of the model is:", f1_cv*100)
        print("The precision_score of the model is:", precision_cv*100)
        print("The recall_score of the model is:", recall_cv*100)
        print("=="*60)
        
        k.append(i)
        accuracy_cv.append(acc_cv)
        
    # Plot k vs auc_score graph
    plt.plot(k, accuracy_cv, 'b-', label="Cross Validation Data")
    plt.legend(loc='lower right')
    plt.title("K v/s Auc Score")
    plt.ylabel('Auc Score')
    plt.xlabel('K')
    plt.show()<jupyter_output><empty_output><jupyter_text>4.2 Test funcrion for KNN models<jupyter_code>from sklearn.metrics import classification_report
from scikitplot.metrics import plot_confusion_matrix
def knn_test(x_train, y_train, x_test, y_test, algo, optimal_k):
    neighbor = KNeighborsClassifier(n_neighbors=optimal_k, algorithm=algo) 
    neighbor.fit(x_train, y_train)
    
    prediction = neighbor.predict(x_test)

    print("The accuracy of the model is:", accuracy_score(y_test, prediction)*100)
    print("The macro f1_score of the model is:", f1_score(y_test, prediction, average = 'macro')*100)
    print("The micro f1_score of the model is:", f1_score(y_test, prediction, average = 'micro')*100)
    print("The precision_score of the model is:", precision_score(y_test, prediction, average='macro')*100)
    print("The recall_score of the model is:", recall_score(y_test, prediction, average='macro')*100)
    print("The precision recall report for data:\n", classification_report(y_test, prediction))
    
    plot_confusion_matrix(y_test, prediction)
    plt.show()<jupyter_output><empty_output><jupyter_text>5. Vectorization of splitted of data  5.1 Bag of Words (BoW)<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
count_vect = CountVectorizer(ngram_range=(2,2))
count_vect.fit(x_train)
# Fit should be done only to train data
# After fitting count_vect only learns the vocabulary of x_train i.e unique words(unique n-grams)

# To convert the SENTENCE into numerical vectors, we call transform()
x_train_bow = count_vect.transform(x_train)
x_cv_bow = count_vect.transform(x_cv)
x_test_bow = count_vect.transform(x_test)
print("The shape of the training data is:", x_train_bow.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_bow.shape, y_cv.shape)
print("The shape of the test data is:", x_test_bow.shape, y_test.shape)<jupyter_output>The shape of the training data is: (191168, 1863703) (191168,)
The shape of the CV data is: (81930, 1863703) (81930,)
The shape of the test data is: (91033, 1863703) (91033,)<jupyter_text>5.2 Term Frequency - Inverse Document Frequency (tf-idf)<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vect = TfidfVectorizer(stop_words=None, ngram_range=(1,2))
tfidf_vect.fit(x_train)
# Fit should be done only to train data
# After fitting tfidf_vect only learnss the vocabulary of x_train i.e unique words(unique n-grams)

# To convert the SENTENCE into numerical vectors, we call transform()
x_train_tfidf = tfidf_vect.transform(x_train)
x_cv_tfidf = tfidf_vect.transform(x_cv)
x_test_tfidf = tfidf_vect.transform(x_test)
print("The shape of the training data is:", x_train_tfidf.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_tfidf.shape, y_cv.shape)
print("The shape of the test data is:", x_test_tfidf.shape, y_test.shape)<jupyter_output>The shape of the training data is: (191168, 1925085) (191168,)
The shape of the CV data is: (81930, 1925085) (81930,)
The shape of the test data is: (91033, 1925085) (91033,)<jupyter_text>5.3 Word2Vec<jupyter_code>from gensim.models import Word2Vec
from gensim.models import KeyedVectors
from tqdm import tqdm
sentence_list = []
for sentence in x_train:
    sentence_list.append(sentence.split())
sentence_list_cv = []
for sentence in x_cv:
    sentence_list_cv.append(sentence.split())
sentence_list_test = []
for sentence in x_test:
    sentence_list_test.append(sentence.split())
# Train your own Word2Vec model using your own train text corpus 

w2v_train_model = Word2Vec(sentence_list, min_count=5, size=50, workers=4)          # Considers only those words that occured atleast 5 times
print(type(w2v_train_model))
w2v_words = list(w2v_train_model.wv.vocab)
print("The number of words that occured minimum 5 times:", len(w2v_words))
print("These are the sample words from index 0 to 50:/n", w2v_words[0:50])<jupyter_output>The number of words that occured minimum 5 times: 16438
These are the sample words from index 0 to 50:/n ['spent', 'year', 'look', 'low', 'calori', 'fat', 'dress', 'like', 'ago', 'found', 'vidalia', 'brand', 'sweet', 'onion', 'raspberri', 'vinaigrett', 'knew', 'immedi', 'search', 'yummi', 'healthi', 'veget', 'could', 'longer', 'find', 'local', 'store', 'start', 'order', 'onlin', 'want', 'without', 'help', 'weight', 'health', 'goal', 'great', 'snack', 'desert', 'plus', 'bonus', 'realli', 'tell', 'differ', 'chocol', 'tough', 'eat', 'whole', 'bag', 'one']<jupyter_text>5.4 Average Word2Vec<jupyter_code>avg_train_vector = []                         # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list):          # For each sentence/review
    sentence_vector = np.zeros(50)            # As word vectors are of zero length
    count_words = 0                           # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            sentence_vector += vect
            count_words += 1
        except:
            pass
    if count_words != 0:
        sentence_vector /= count_words
    avg_train_vector.append(sentence_vector)
avg_train_vector = np.array(avg_train_vector)
print(avg_train_vector.shape)
print(avg_train_vector[0])
avg_cv_vector = []                            # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list_cv):       # For each sentence/review
    sentence_vector = np.zeros(50)            # As word vectors are of zero length
    count_words = 0                           # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            sentence_vector += vect
            count_words += 1
        except:
            pass
    if count_words != 0:
        sentence_vector /= count_words
    avg_cv_vector.append(sentence_vector)
avg_cv_vector = np.array(avg_cv_vector)
print(avg_cv_vector.shape)
print(avg_cv_vector[0])
avg_test_vector = []                          # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list_test):     # For each sentence/review
    sentence_vector = np.zeros(50)            # As word vectors are of zero length
    count_words = 0                           # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            sentence_vector += vect
            count_words += 1
        except:
            pass
    if count_words != 0:
        sentence_vector /= count_words
    avg_test_vector.append(sentence_vector)
avg_test_vector = np.array(avg_test_vector)
print(avg_test_vector.shape)
print(avg_test_vector[0])
# Checking if the model has NULL values ins the list avg_train_vector

pd.isnull(avg_train_vector)
x_train_avgw2v = avg_train_vector
x_cv_avgw2v = avg_cv_vector
x_test_avgw2v = avg_test_vector
print("The shape of the training data is:", x_train_avgw2v.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_avgw2v.shape, y_cv.shape)
print("The shape of the test data is:", x_test_avgw2v.shape, y_test.shape)<jupyter_output>The shape of the training data is: (191168, 50) (191168,)
The shape of the CV data is: (81930, 50) (81930,)
The shape of the test data is: (91033, 50) (91033,)<jupyter_text>5.5 TF-IDF Weighed Word2Vec<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vect = TfidfVectorizer(ngram_range=(1,2), min_df=10, max_features=500)
tfidf_vect.fit_transform(x_train)
tfidf_feat = tfidf_vect.get_feature_names()
dictionary = dict(zip(tfidf_feat, list(tfidf_vect.idf_)))
tfidf_train_vector = []                         # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list):            # For each sentence/review
    sentence_vector = np.zeros(50)              # As word vectors are of zero length
    weight_sum = 0                              # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            tfidf_weight = dictionary[word]*(sentence.count(word)/len(sentence))
            sentence_vector += (vect*tfidf_weight)
            weight_sum += tfidf_weight
        except:
            pass
    if weight_sum != 0:
        sentence_vector /= weight_sum
    tfidf_train_vector.append(sentence_vector)
tfidf_train_vector = np.array(tfidf_train_vector)
print(tfidf_train_vector.shape)
print(tfidf_train_vector[0])
tfidf_cv_vector = []                            # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list_cv):         # For each sentence/review
    sentence_vector = np.zeros(50)              # As word vectors are of zero length
    weight_sum = 0                              # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            tfidf_weight = dictionary[word]*(sentence.count(word)/len(sentence))
            sentence_vector += (vect*tfidf_weight)
            weight_sum += tfidf_weight
        except:
            pass
    if weight_sum != 0:
        sentence_vector /= weight_sum
    tfidf_cv_vector.append(sentence_vector)
tfidf_cv_vector = np.array(tfidf_cv_vector)
print(tfidf_cv_vector.shape)
print(tfidf_cv_vector[0])
tfidf_test_vector = []                          # Average w2v for each review/sentence is stored in this array
for sentence in tqdm(sentence_list_test):       # For each sentence/review
    sentence_vector = np.zeros(50)              # As word vectors are of zero length
    weight_sum = 0                              # Num of words with a valid vector in a sentence/review
    for word in sentence:
        try:
            vect = w2v_train_model.wv[word]
            tfidf_weight = dictionary[word]*(sentence.count(word)/len(sentence))
            sentence_vector += (vect*tfidf_weight)
            weight_sum += tfidf_weight
        except:
            pass
    if weight_sum != 0:
        sentence_vector /= weight_sum
    tfidf_test_vector.append(sentence_vector)
tfidf_test_vector = np.array(tfidf_test_vector)
print(tfidf_test_vector.shape)
print(tfidf_test_vector[0])
x_train_tfidfw2v = tfidf_train_vector
x_cv_tfidfw2v = tfidf_cv_vector
x_test_tfidfw2v = tfidf_test_vector
print("The shape of the training data is:", x_train_tfidfw2v.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_tfidfw2v.shape, y_cv.shape)
print("The shape of the test data is:", x_test_tfidfw2v.shape, y_test.shape)<jupyter_output>The shape of the training data is: (191168, 50) (191168,)
The shape of the CV data is: (81930, 50) (81930,)
The shape of the test data is: (91033, 50) (91033,)<jupyter_text>6. Applying KNN Brute Force on Our Model  6.1 Bag of Words (BoW)  6.1.1 BoW KNN Brute Force on Train Data<jupyter_code># Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_bow, x_cv_bow, y_train, y_cv, algo='brute')<jupyter_output>FOR K = 2
The accuracy of the model is: 17.966556816794824
The f1_score of the model is: 16.55390703402162
The precision_score of the model is: 52.07945851493865
The recall_score of the model is: 50.4262316559973
========================================================================================================================
FOR K = 4
The accuracy of the model is: 70.5883070914195
The f1_score of the model is: 50.51809166624709
The precision_score of the model is: 50.64007096640525
The recall_score of the model is: 50.78313050051708
========================================================================================================================
FOR K = 6
The accuracy of the model is: 84.12181130233127
The f1_score of the model is: 47.204062741284716
The precision_score of the model is: 64.55145626155593
The recall_score of the model is: 50.607418303078035
======================================================================================================================[...]<jupyter_text>6.1.2 BoW KNN Brute Force on Test Data<jupyter_code># Testing the final model

knn_test(x_train_bow, y_train, x_test_bow, y_test, algo='brute', optimal_k=18)<jupyter_output>The accuracy of the model is: 84.32875990025596
The macro f1_score of the model is: 45.84633982647454
The micro f1_score of the model is: 84.32875990025596
The precision_score of the model is: 73.98506274466324
The recall_score of the model is: 50.043836105667914
The precision recall report for data:
               precision    recall  f1-score   support

    Negative       0.64      0.00      0.00     14272
    Positive       0.84      1.00      0.91     76761

    accuracy                           0.84     91033
   macro avg       0.74      0.50      0.46     91033
weighted avg       0.81      0.84      0.77     91033<jupyter_text>6.2 Term Frequency - Inverse Document Frequency (tf-idf)  6.2.1 TF-IDF KNN Brute Force on Train Data<jupyter_code># Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_tfidf, x_cv_tfidf, y_train, y_cv, algo='brute')<jupyter_output>FOR K = 2
The accuracy of the model is: 84.65397290369829
The f1_score of the model is: 47.21906685176988
The precision_score of the model is: 84.54425296936958
The recall_score of the model is: 50.679831531283234
========================================================================================================================
FOR K = 4
The accuracy of the model is: 85.40583424874893
The f1_score of the model is: 58.44503648151475
The precision_score of the model is: 74.10447380893291
The recall_score of the model is: 56.935799837915425
========================================================================================================================
FOR K = 6
The accuracy of the model is: 85.9160258757476
The f1_score of the model is: 59.36443318112903
The precision_score of the model is: 78.14917554192029
The recall_score of the model is: 57.532975186835976
=====================================================================================================================[...]<jupyter_text>6.2.2 TF-IDF KNN Brute Force on Test Data<jupyter_code># Testing the final model

knn_test(x_train_tfidf, y_train, x_test_tfidf, y_test, algo='brute', optimal_k=12)<jupyter_output>The accuracy of the model is: 86.0676897388859
The macro f1_score of the model is: 58.72234766583724
The micro f1_score of the model is: 86.0676897388859
The precision_score of the model is: 83.98849962108666
The recall_score of the model is: 57.112345123169085
The precision recall report for data:
               precision    recall  f1-score   support

    Negative       0.82      0.15      0.25     14337
    Positive       0.86      0.99      0.92     76696

    accuracy                           0.86     91033
   macro avg       0.84      0.57      0.59     91033
weighted avg       0.85      0.86      0.82     91033<jupyter_text>6.3 Average Word2Vec  6.3.1 Avg W2V KNN Brute Force on Train Data<jupyter_code># Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_avgw2v, x_cv_avgw2v, y_train, y_cv, algo='brute')<jupyter_output>FOR K = 2
The accuracy of the model is: 81.84547784694252
The f1_score of the model is: 70.45003859713259
The precision_score of the model is: 68.49759635697535
The recall_score of the model is: 74.1074481845198
========================================================================================================================
FOR K = 4
The accuracy of the model is: 86.760649334798
The f1_score of the model is: 73.95664984743577
The precision_score of the model is: 75.06293974088554
The recall_score of the model is: 73.00803744471519
========================================================================================================================
FOR K = 6
The accuracy of the model is: 88.02636396924204
The f1_score of the model is: 74.48358428693248
The precision_score of the model is: 78.59480407955321
The recall_score of the model is: 71.85710733129395
========================================================================================================================
F[...]<jupyter_text>6.3.2 Avg W2V KNN Brute Force on Test Data<jupyter_code># Testing the final model

knn_test(x_train_avgw2v, y_train, x_test_avgw2v, y_test, algo='brute', optimal_k=14)<jupyter_output>The accuracy of the model is: 88.81394659079675
The macro f1_score of the model is: 73.72023421612728
The micro f1_score of the model is: 88.81394659079675
The precision_score of the model is: 83.32099730189277
The recall_score of the model is: 69.54314754143651
The precision recall report for data:
               precision    recall  f1-score   support

    Negative       0.77      0.41      0.54     14312
    Positive       0.90      0.98      0.94     76721

    accuracy                           0.89     91033
   macro avg       0.83      0.70      0.74     91033
weighted avg       0.88      0.89      0.87     91033<jupyter_text>6.4 TF-IDF Weighed Word2Vec  6.4.1 TF_IDF W2V KNN Brute Force on Train Data<jupyter_code># Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_tfidfw2v, x_cv_tfidfw2v, y_train, y_cv, algo='brute')<jupyter_output>FOR K = 2
The accuracy of the model is: 77.042597339192
The f1_score of the model is: 63.70237186887714
The precision_score of the model is: 62.42854571647249
The recall_score of the model is: 66.98115364997567
========================================================================================================================
FOR K = 4
The accuracy of the model is: 83.45905040888564
The f1_score of the model is: 66.5982247095232
The precision_score of the model is: 67.93022360885297
The recall_score of the model is: 65.58895000354859
========================================================================================================================
FOR K = 6
The accuracy of the model is: 85.07262297082876
The f1_score of the model is: 66.0780834872641
The precision_score of the model is: 71.17678226966227
The recall_score of the model is: 63.73504329898155
========================================================================================================================
FO[...]<jupyter_text>6.4.2 TF_IDF W2V KNN Brute Force on Test Data<jupyter_code># Testing the final model

knn_test(x_train_tfidfw2v, y_train, x_test_tfidfw2v, y_test, algo='brute', optimal_k=12)<jupyter_output>The accuracy of the model is: 85.9501499456241
The macro f1_score of the model is: 63.221535731700264
The micro f1_score of the model is: 85.9501499456241
The precision_score of the model is: 76.08696562461792
The recall_score of the model is: 60.48069276692581
The precision recall report for data:
               precision    recall  f1-score   support

    Negative       0.65      0.23      0.34     14329
    Positive       0.87      0.98      0.92     76704

    accuracy                           0.86     91033
   macro avg       0.76      0.60      0.63     91033
weighted avg       0.84      0.86      0.83     91033<jupyter_text>7. Applying KNN KD - Tree on Our Model<jupyter_code>from sklearn.decomposition import TruncatedSVD<jupyter_output><empty_output><jupyter_text>7.1 Bag of Words (BoW)  7.1.1 BoW KNN KD Tree on Train Data<jupyter_code># KD_Tree algorithm does not work well with sparse metrix.
# So, to make the metrix dense(i.e. to reduce dimension of the metrix) we use TruncatedSVD.
# Here we are using TruncatedSVD with 1000 features.
# This may take several minutes to run

tsvd = TruncatedSVD(n_components=1000)

x_train_bow = tsvd.fit_transform(x_train_bow)
x_cv_bow = tsvd.fit_transform(x_cv_bow)
x_test_bow = tsvd.fit_transform(x_test_bow)
print("The shape of the training data is:", x_train_bow.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_bow.shape, y_cv.shape)
print("The shape of the test data is:", x_test_bow.shape, y_test.shape)
tsvd_plot = TruncatedSVD(n_components=999).fit(x_train_bow)
plt.plot(np.cumsum(tsvd_plot.explained_variance_ratio_))
plt.xlim(0, 999, 10)
plt.xlabel('Number of Components/Features')
plt.ylabel('Cumulative Explained Variance')
# Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_bow, x_cv_bow, y_train, y_cv, algo='kd_tree')<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.1.2 BoW KNN KD Tree on Test Data<jupyter_code># Testing the final model

knn_test(x_train_bow, y_train, x_test_bow, y_test, algo='kd_tree', optimal_k=k)<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.2 Term Frequency - Inverse Document Frequency (TF-IDF)  7.2.1 TF-IDF KNN KD Tree on Train Data<jupyter_code>tsvd_plot = TruncatedSVD(n_components=999).fit(x_train_tfidf)
plt.plot(np.cumsum(tsvd_plot.explained_variance_ratio_))
plt.xlim(0, 999, 10)
plt.xlabel('Number of Components/Features')
plt.ylabel('Cumulative Explained Variance')
tsvd = TruncatedSVD(n_components=1000)
# KD_Tree algorithm does not work well with sparse metrix.
# So, to make the metrix dense(i.e. to reduce dimension of the metrix) we use TruncatedSVD.
# Here we are using TruncatedSVD with 1000 features.

x_train_tfidf = tsvd.fit_transform(x_train_tfidf)
x_cv_tfidf = tsvd.fit_transform(x_cv_tfidf)
x_test_tfidf = tsvd.fit_transform(x_test_tfidf)
print("The shape of the training data is:", x_train_tfidf.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_tfidf.shape, y_cv.shape)
print("The shape of the test data is:", x_test_tfidf.shape, y_test.shape)
# Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_tfidf, x_cv_tfidf, y_train, y_cv, algo='kd_tree')<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.2.2 TF-IDF KNN KD Tree on Test Data<jupyter_code># Testing the final model

knn_test(x_train_tfidf, y_train, x_test_tfidf, y_test, algo='kd_tree', optimal_k=k)<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.3 Average Word2Vec  7.3.1 Avg W2V KNN KD Tree on Train Data<jupyter_code>tsvd_plot = TruncatedSVD(n_components=49).fit(x_train_avgw2v)
plt.plot(np.cumsum(tsvd_plot.explained_variance_ratio_))
plt.xlim(0, 49, 10)
plt.xlabel('Number of Components/Features')
plt.ylabel('Cumulative Explained Variance')
tsvd = TruncatedSVD(n_components=49)
# KD_Tree algorithm does not work well with sparse metrix.
# So, to make the metrix dense(i.e. to reduce dimension of the metrix) we use TruncatedSVD.
# Here we are using TruncatedSVD with 1000 features.

x_train_avgw2v = tsvd.fit_transform(x_train_avgw2v)
x_cv_avgw2v = tsvd.fit_transform(x_cv_avgw2v)
x_test_avgw2v = tsvd.fit_transform(x_test_avgw2v)
print("The shape of the training data is:", x_train_avgw2v.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_avgw2v.shape, y_cv.shape)
print("The shape of the test data is:", x_test_avgw2v.shape, y_test.shape)
# Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_avgw2v, x_cv_avgw2v, y_train, y_cv, algo='kd_tree')<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.3.2 Avg W2V KNN KD Tree on Test Data<jupyter_code># Testing the final model

knn_test(x_train_avgw2v, y_train, x_test_avgw2v, y_test, algo='kd_tree', optimal_k=k)<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.4 TF-IDF Weighed Word2Vec  7.4.1 TF-IDF Weighed W2V KNN KD Tree on Train Data<jupyter_code>tsvd_plot = TruncatedSVD(n_components=49).fit(x_train_tfidfw2v)
plt.plot(np.cumsum(tsvd_plot.explained_variance_ratio_))
plt.xlim(0, 50, 5)
plt.xlabel('Number of Components/Features')
plt.ylabel('Cumulative Explained Variance')
tsvd = TruncatedSVD(n_components=49)
# KD_Tree algorithm does not work well with sparse metrix.
# So, to make the metrix dense(i.e. to reduce dimension of the metrix) we use TruncatedSVD.
# Here we are using TruncatedSVD with 1000 features.

x_train_tfidfw2v = tsvd.fit_transform(x_train_tfidfw2v)
x_cv_tfidfw2v = tsvd.fit_transform(x_cv_tfidfw2v)
x_test_tfidfw2v = tsvd.fit_transform(x_test_tfidfw2v)
print("The shape of the training data is:", x_train_tfidfw2v.shape, y_train.shape)
print("The shape of the CV data is:", x_cv_tfidfw2v.shape, y_cv.shape)
print("The shape of the test data is:", x_test_tfidfw2v.shape, y_test.shape)
# Find the optimal value of K
# This may take several minutes(even hour/s) to run

knn_train(x_train_tfidfw2v, x_cv_tfidfw2v, y_train, y_cv, algo='kd_tree')<jupyter_output><empty_output><jupyter_text>This cell was left unexecuted because it took quite long to execute in my machine. However, the code is completely good to go with.  7.4.2 TF-IDF Weighed W2V KNN KD Tree on Test Data<jupyter_code># Testing the final model

knn_test(x_train_tfidfw2v, y_train, x_test_tfidfw2v, y_test, algo='kd_tree', optimal_k=k)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12885885885885887, 'fraction_numerical': 0.05915915915915916, 'mean_word_length': 4.120867292019068, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 112, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '29', 'dir': 'jupyter-structured-clean-dedup', 'id': '97362', 'license': 'MIT', 'n_tokens_mistral': 12237, 'n_tokens_neox': 10862, 'n_words': 3251, 'path': 'KNN - Amazon fine food reviews .ipynb', 'repo_name': 'arunism/Sentiment-Analysis-for-Amazon-Fine-Food-Reviews', 'size': '33300'}