<jupyter_start><jupyter_text>Data Cleaning: Deduplication<jupyter_code>display= pd.read_sql_query("""
SELECT *
FROM Reviews
WHERE Score != 3 AND UserId="AR5J8UI46CURR"
ORDER BY ProductID
""", con)
display
#Sorting data according to ProductId in ascending order
sorted_data=filtered_data.sort_values('ProductId', axis=0, ascending=True, inplace=False, kind='quicksort', na_position='last')
#Deduplication of entries
final=sorted_data.drop_duplicates(subset={"UserId","ProfileName","Time","Text"}, keep='first', inplace=False)
#final=final.drop_duplicates(subset={"UserId","ProductId","Time"}, keep='first', inplace=False)
final.shape
#Checking to see how much % of data still remains
(final['Id'].size*1.0)/(filtered_data['Id'].size*1.0)*100
display= pd.read_sql_query("""
SELECT *
FROM Reviews
WHERE Score != 3 AND Id=44737 OR Id=64422
ORDER BY ProductID
""", con)
display

final=final[final.HelpfulnessNumerator<=final.HelpfulnessDenominator]

#Before starting the next phase of preprocessing lets see the number of entries left
print(final.shape)

#How many positive and negative reviews are present in our dataset?
final['Score'].value_counts()
Data-preprocessing
# find sentences containing HTML tags
import re

i=0;
for sent in final['Text'].values:
    if (len(re.findall('<.*?>', sent))):
        print(i)
        print(sent)
        break;
    i += 1;    

        
import re

import string
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem.wordnet import WordNetLemmatizer

stop = set(stopwords.words('english')) #set of stopwords
sno = nltk.stem.SnowballStemmer('english') #initialising the snowball stemmer

def cleanhtml(sentence): #function to clean the word of any html-tags
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, ' ', sentence)
    return cleantext
def cleanpunc(sentence): #function to clean the word of any punctuation or special characters
    cleaned = re.sub(r'[?|!|\'|"|#]',r'',sentence)
    cleaned = re.sub(r'[.|,|)|(|\|/]',r' ',cleaned)
    return  cleaned
print(stop)
print('************************************')
print(sno.stem('tasty'))
#Code for implementing step-by-step the checks mentioned in the pre-processing phase
i=0
str1=' '
final_string=[]
all_positive_words=[] # store words from +ve reviews here
all_negative_words=[] # store words from -ve reviews here.
s=''
for sent in final['Text'].values:
    filtered_sentence=[]
    #print(sent);
    sent=cleanhtml(sent) # remove HTMl tags
    for w in sent.split():
        for cleaned_words in cleanpunc(w).split():
            if((cleaned_words.isalpha()) & (len(cleaned_words)>2)):    
                if(cleaned_words.lower() not in stop):
                    s=(sno.stem(cleaned_words.lower())).encode('utf8')
                    filtered_sentence.append(s)
                    if (final['Score'].values)[i] == 'positive': 
                        all_positive_words.append(s) #list of all words used to describe positive reviews
                    if(final['Score'].values)[i] == 'negative':
                        all_negative_words.append(s) #list of all words used to describe negative reviews reviews
                else:
                    continue
            else:
                continue 
    #print(filtered_sentence)
    str1 = b" ".join(filtered_sentence) #final string of cleaned words
    #print("***********************************************************************")
    
    final_string.append(str1)
    i+=1
final['CleanedText']=final_string #adding a column of CleanedText which displays the data after pre-processing of the review 
# store final table into an SQlLite table for future.
conn = sqlite3.connect('final.sqlite')
c=conn.cursor()
conn.text_factory = str
final.to_sql('Reviews', conn, flavor=None, schema=None, if_exists='replace', index=True, index_label=None, chunksize=None, dtype=None)<jupyter_output><empty_output><jupyter_text>Bag of Words (BoW)<jupyter_code>final_df = pd.DataFrame(final) # converting table to a dataframe
random_final = final_df.sample(n=10000) # Sampling 10k rows 
random_final
random_final = random_final.sort_values('Time') # Sorting the dataframe based on  Time
random_final
#BoW
count_vect = CountVectorizer() 
final_counts = count_vect.fit_transform(random_final['CleanedText'].values)

type(final_counts) # sparse matrix
final_counts.get_shape() # shape of matrix after Vectorizing
# Standardizing the data with mean = 0 and std.dev = 1
from sklearn.preprocessing import StandardScaler
standardized_data = StandardScaler(with_mean=False).fit_transform(final_counts)
print(standardized_data.shape)
X = standardized_data.toarray() # converting the standardized data to dense array
y = np.array(random_final['Score']) # converting Review attribute to a numpy array
from sklearn.model_selection import train_test_split

# split the data set into train and test 70% train and 30% test 
X1, X_test, y1, y_test = train_test_split(X, y, test_size=0.3, shuffle=False, random_state=0)
from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import BernoulliNB
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import GridSearchCV
                               
tss = TimeSeriesSplit(n_splits=10) # TimeSeries Split with number of splits=10
N_BB = BernoulliNB() # applying Bernoulli Naive-Bayes

a = np.logspace(0.0, 10.0, num=50, base=1.25) # specifying the range of the hyperparameter alpha for GridSearch

NB_params = {'alpha':a} # hyperparameter to choose  

# GridSearch for finding right hyperparameter with with 10 fold CV on TimeSeriesSplit
gcv = GridSearchCV(N_BB, NB_params,cv=tss, refit=True) 

# Fit the training data
gcv.fit(X1, y1)

# find the mean score for Cross-Validation using 10 Fold Time Series CV after finding the right hyperparameter 
score_BOW = cross_val_score(gcv, X=X1, y=y1, cv=tss).mean()
print(score_BOW)
gcv.best_params_  # print the best hyperparameter alpha
from sklearn.metrics import recall_score, f1_score, confusion_matrix, accuracy_score, precision_score


# instantiate learning model for best alpha
N_BB_op = BernoulliNB(alpha=9.313225746154785)

# fitting the model
N_BB_op.fit(X1, y1)

# predict the response
pred_NB = N_BB_op.predict(X_test)

# print top 20 features for each class for BOW
print("Top 20 Positive Features for BOW and Top 20 Negative Features for BOW ")
print("\n\n")

# defining a function for finding the top features of each class
def top_most_features(vectorizer, clf, n=20):
    feature_names = vectorizer.get_feature_names() # getting all the feature names
    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names)) # retrieve the coefficient & sort them based on values 
    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1]) 
    for (coef_1, fn_1), (coef_2, fn_2) in top:
        print ("\t%-15s\t%.4f\t%-15s\t\t%-15s\t%.4f\t%-15s" % (fn_2, coef_1, 'positive', fn_1, coef_2, 'negative'))

        
top_most_features(count_vect, N_BB_op)

print()

# evaluate various performance measures:
print("Accuracy = ",accuracy_score(y_test, pred_NB)) # accuracy
print("Confusion Matrix\n", confusion_matrix(y_test, pred_NB)) # Confusion Matrix
tn, fn, fp, tp = confusion_matrix(y_test, pred_NB).ravel() # tn, fn, fp, tp
print("(tn, fn, fp, tp) =",(tn, fn, fp, tp))
print("Recall = ",recall_score(y_test, pred_NB, pos_label='positive')) # Recall
print("f1-Score = "  ,f1_score(y_test, pred_NB, pos_label='positive')) # f1-Score
print("Precision = ",precision_score(y_test, pred_NB, pos_label='positive'))# Precision<jupyter_output>Top 20 Positive Features for BOW and Top 20 Negative Features for BOW 



	like           	-6.4548	positive       		aaaaaahhhhh    	-1.1674	negative       
	tast           	-6.4548	positive       		aaah           	-1.2067	negative       
	love           	-6.4548	positive       		aafco          	-1.2192	negative       
	great          	-6.4548	positive       		aamzon         	-1.2453	negative       
	good           	-6.4548	positive       		abbazabba      	-1.2769	negative       
	flavor         	-6.4548	positive       		abbrevi        	-1.4353	negative       
	use            	-6.4548	positive       		abc            	-1.4707	negative       
	tri            	-6.4548	positive       		abhor          	-1.5158	negative       
	product        	-6.4548	positive       		abnorm         	-1.5220	negative       
	one            	-6.4548	positive       		abomin         	-1.5251	negative       
	make           	-6.4548	positive       		abouy          	-1.6025	negative       
	get            	-6.4548[...]<jupyter_text>TF-IDF<jupyter_code># Taking Tf-idf bigram and vectorizing it
tf_idf_vect = TfidfVectorizer(ngram_range=(1,2))
final_tf_idf = tf_idf_vect.fit_transform(random_final['CleanedText'].values)

final_tf_idf.get_shape() # Shape of tf-idf vectorizer

# Standardizing the data with mean=0 and std.dev=1
from sklearn.preprocessing import StandardScaler
standardized_data_tf = StandardScaler(with_mean=False).fit_transform(final_tf_idf)
print(standardized_data_tf.shape)
X_tf = standardized_data_tf.toarray() # storing the values after standardization in a dense array
y_tf = np.array(random_final['Score']) # storing the values of Scores in numpy array
# split the data set into train and test, 70% train and 30% test 
from sklearn.model_selection import train_test_split

X2, X_test2, y2, y_test2 = train_test_split(X_tf, y_tf, test_size=0.3, shuffle=False, random_state=0)
from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import BernoulliNB
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import GridSearchCV
                               
tss_tf = TimeSeriesSplit(n_splits=10) # TimeSeries Split with number of splits=10
N_BB_tf = BernoulliNB()  # applying Bernoulli Naive-Bayes

atf = np.logspace(0.0, 10.0, num=50, base=1.25) # specifying the range of the hyperparameter alpha for GridSearch

NB_params_tf = {'alpha':atf} # hyperparameter to choose  

# GridSearch for finding right hyperparameter with with 10 fold CV on TimeSeriesSplit
gcv_tf = GridSearchCV(N_BB_tf, NB_params_tf,cv=tss_tf, refit=True)

# Fit the training data
gcv_tf.fit(X2, y2)

# find the mean score for Cross-Validation using 10 Fold Time Series CV after finding the right hyperparameter 
score_tf = cross_val_score(gcv_tf, X=X2, y=y2, cv=tss_tf).mean()
print(score_tf)
gcv_tf.best_params_  # print the best hyperparameter alpha
from sklearn.metrics import recall_score, f1_score, confusion_matrix, accuracy_score, precision_score


# instantiate learning model for best alpha
N_BB_op_tf = BernoulliNB(alpha=1.0953555355153197)

# fitting the model
N_BB_op_tf.fit(X2, y2)

# predict the response
pred_NB_tf = N_BB_op_tf.predict(X_test2)


# print top 20 features for each class for tf-idf
print("Top 20 Positive Features for Tf-idf and Top 20 Negative Features for Tf-idf ")
print("\n\n")


# defining a function for finding the top features of each class
def show_most_informative_features(vectorizer, clf, n=20):
    feature_names = vectorizer.get_feature_names()  # getting all the feature names
    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names)) # retrieve the coefficient & sort them based on values 
    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])
    for (coef_1, fn_1), (coef_2, fn_2) in top:
        print ("\t%-10s\t%.4f\t%-15s\t\t%-10s\t%.4f\t%-15s" % (fn_2, coef_1, 'positive', fn_1, coef_2, 'negative'))
                


show_most_informative_features(tf_idf_vect, N_BB_op_tf)

print()

# evaluate various performance measures:
print("Accuracy = ",accuracy_score(y_test2, pred_NB_tf))# accuracy
print("Confusion Matrix\n", confusion_matrix(y_test2, pred_NB_tf))# Confusion matrix
tn, fn, fp, tp = confusion_matrix(y_test2, pred_NB_tf).ravel()# tn, fn, fp, tp
print("(tn, fn, fp, tp) =",(tn, fn, fp, tp))
print("Recall = ",recall_score(y_test2, pred_NB_tf, pos_label='positive')) # Recall
print("f1-Score = "  ,f1_score(y_test2, pred_NB_tf, pos_label='positive')) # f1-score
print("Precision = ",precision_score(y_test2, pred_NB_tf, pos_label='positive'))# precision

Summary: We saw from the data that the data was imbalanced but we still persisted with imbalanced data so as to gain 
    insights from the data and we sample the data to 10k for easy computations. Then, we standardized the data and then split it into 70% Train and 30%
    test data and then did Time Based Split for 10-fold Cross-validation and then applied naive Bayes to the BOW we first 
    found out the right alpha by Grid-Search and then we used the best hyperparameter found out by GridSerach for our 
    Bernoulli Naive Bayes and we chose Bernoulli Naive Bayes as our task was of classification of two classes and Bernoulli
    Naive Bayes was the ideal Naive Bayes variant. We found out the top/most important features by defining our own function.
    Then we found out the various performance measures such as accuracy, recall, confusion matrix, f1-score and precision.
    The f1-score was found out to be 0.91 which is a good score.

    Then, we repeated the same steps for tf-idf and found out the f1-score as 0.923.
    Since, word2vec is semantically dependent hence we cannot apply Naive Bayes on aver-w2v and tfidf-w2v. As, Naive Bayes
    works on independence among features.

    So, we can conclude that given the imbalanced data set f1-score, confusion-matrix, f1-score, precision and recall were 
    significantly high as accuracy cannot be only taken into consideration for imbaalnced data as it is not a good criteria 
    for imbalanced data. So, f1-score or recall or any other performance measure is taken into consideration.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10335094448867338, 'fraction_numerical': 0.026344358399073606, 'mean_word_length': 4.0375501275975205, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '24553', 'license': 'Apache-2.0', 'n_tokens_mistral': 4750, 'n_tokens_neox': 4425, 'n_words': 1335, 'path': 'Amazon_Fine_Food_Naive_Bayes.ipynb', 'repo_name': 'Dhruv5950/Amazon-Fine-Food-Review-Analysis', 'size': '13817'}