<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import nltk
from nltk.corpus import stopwords
from collections import Counter
import os
import glob as glob
import string
import re
import random
from bs4 import BeautifulSoup
from sklearn.model_selection import KFold
# Upload the file on drive and then copy that path
imdb_labelFile = open(r"/content/drive/MyDrive/Naive Bayes/imdb_labelled.txt", "r")
def getReviewTargetFromFile(file):
    lines=file.readlines()
    file.close()
    reviewsentimentList=[]
    for l in lines:
        x=l.split('\t')
        reviewsentimentList.append([str.lstrip(str.rstrip(x[0])),str.lstrip(str.rstrip(x[1]))])
    return reviewsentimentList
 
rtList=getReviewTargetFromFile(imdb_labelFile)
rtDF=pd.DataFrame(rtList,columns=['REVIEW','TARGET'])
rtDF.to_csv('imdb.csv', index = None)
imdb_data = pd.read_csv('imdb.csv')
imdb_data.head()
imdb_data.count()
#sentiment count
imdb_data['TARGET'].value_counts()
import nltk
nltk.download('stopwords')
nltk.download('wordnet')
df_train = rtDF.sample(frac=0.6)
df_dev = rtDF.sample(frac=0.2)
df_test = rtDF.sample(frac=0.2)
print("Size of Train Data :", len(df_train))
print("Size of Development Data :", len(df_dev))
print("Size of Test Data :", len(df_test))
def Word_Count(data):
    dev_review = data
    unique_words_dev = []
    for review in dev_review:
        for word in review.split(' '):
                unique_words_dev.append(word)
    unique_words_dev = Counter(unique_words_dev)
    unique_words_dev = Counter({k: n for k, n in unique_words_dev.items() if n > 5}) #removing rare words that occure more than 5
    return unique_words_dev
def probability_wordOccur(data,word,nOfdocs):
    idct_data=dict(data)
    if word in idct_data.keys():
        return idct_data[word] / nOfdocs
    else: return 0
wrdcnt = Word_Count(list(list(zip(*rtDF.values))[0]))
print("The P['the'] i.e the probablity of 'the' given no of docs is: ",probability_wordOccur(wrdcnt,'the',len(imdb_data)))
def conditional_probability(X,y,word,givenFlag):
    group = np.unique(y)
    separated = dict()
    for sentiment in group:
        separated[sentiment] = X[np.where(y == sentiment)]
    countpos = dict(Word_Count(separated['1']))
    countneg = dict(Word_Count(separated['0']))
    
    if givenFlag == '1':
        if word in countpos.keys():
            return countpos[word]/ len(separated['1'])
        else: return 0
    else:
        if word in countneg.keys():
            return countneg[word]/ len(separated['0'])
        else: return 0
X_review = df_train['REVIEW'].values
y_target = df_train['TARGET'].values

print(" The P['the'|positive] is:", conditional_probability(X_review,y_target,'the','1'))
class NaiveBayes:
    def __init__(self,X,y,test,alpha,flag):
        self.alpha = alpha
        self.X = X
        self.y = y
        self.test = test
        self.target = np.unique(y)
        self.flag = flag
        
    
    def conditional_prob(self,word,target):
        if target == '1':
            if word in self.countpositive.keys():
                return self.countpositive[word]/ len(self.separated['1'])
            else: return 0
        else:
            if word in self.countnegative.keys():
                return self.countnegative[word]/ len(self.separated['0'])
            else: return 0
            
    def probability_wordOcc(self,word):
        if word in self.wordcnt.keys():
            return self.wordcnt[word] / len(self.X)
        else: return 0
        
    def bayes(self,word,target):
        p_word_gvnSenti = self.conditional_prob(word,target)
        
        p_target = list(self.y).count(target) / len(list(self.y))
        
        p_word = self.probability_wordOcc(word)
        
        if p_word == 0:
            p_word = 0.0000001
        if self.alpha == 0:
            return (p_word_gvnSenti*p_target)/p_word  #without smoothing
        else:
            if word in self.wordcnt.keys():
                count = self.wordcnt[word]
            else: count = 1
            return ((p_word_gvnSenti*p_target) + self.alpha) / (p_word + count) #with smoothing
        
        
    def NB_predict(self):
        result = 1
        y_pred = []
        positive = 0
        negative = 0
        self.separated = dict()
        positivedi = dict()
        negativedi = dict()
        
        self.wordcnt = dict(Word_Count(self.X)) #vocab list
        
        for i in self.target:
            self.separated[i] = self.X[np.where(self.y == i)] 
        self.countpositive = dict(Word_Count(self.separated['1']))
        self.countnegative = dict(Word_Count(self.separated['0']))
        
        if self.flag != True:
            for review in self.test:
                for senti in self.target:
                    for word in review.split():
                        prob = self.bayes(word,senti)
                        result = result * prob
                    if senti == 1:
                        positive = result
                    else:
                        negative = result

                if positive > negative:
                    y_pred.append('1')
                else: y_pred.append('0')

            return y_pred
        else:
            for word in self.countpositive.keys(): 
                prob = self.bayes(word,1)
                positivedi[word] = prob
            for word in self.countnegative.keys(): 
                prob = self.bayes(word,0)
                negativedi[word] = prob
            return positivedi,negativedi
X_dev = df_dev['REVIEW'].values
y_dev = df_dev['TARGET'].values
NBC = NaiveBayes(X_review,y_target,X_dev,0,0)
ypred = NBC.NB_predict()
ypred = np.array(ypred)
def calculate_accuracy(actual,predicted):
    no_correctOfPredict = 0.0
    for y_true, y_pred in zip(actual,predicted):
        if y_true == y_pred:
            no_correctOfPredict += 1
    return no_correctOfPredict / float(len(actual)) * 100.0
print("Accuracy on the development Dataset without smoothing is:",calculate_accuracy(y_dev,ypred))
df_train_short = df_train.sample(frac=0.8)
len(df_train_short)
def cross_validation(data,k,alpha):
    dev = 1/k
    score = []
    for i in range(1, k + 1):
        df_dev_cv = data.sample(frac=dev, random_state=i, replace=False).copy(deep=True)
        df_train_cv = data.drop(df_dev_cv.index, axis=0).copy(deep=True)
        X_dev = df_dev_cv['REVIEW'].values
        y_dev = df_dev_cv['TARGET'].values
        X_train = df_train_cv['REVIEW'].values
        y_train = df_train_cv['TARGET'].values
        NBC = NaiveBayes(X_train,y_train,X_dev,alpha,flag= False)
        ypred = np.array(NBC.NB_predict())
        score.append(calculate_accuracy(y_dev,ypred))
    return score
score = cross_validation(df_train_short,5,1)
print("Accuracy on development dataset with smoothing with alpha 1 : {} %".format(np.mean(score)))
score12 = cross_validation(df_train_short,5,0.001)
print("Accuracy on development dataset with smoothing with alpha 0.001 : {} %".format(np.mean(score12)))
positive=dict()
wordcount = dict(wrdcnt)
NBC = NaiveBayes(X_train,y_train,0,1,flag=True)
positive, negative = NBC.NB_predict()
positive
negative
df_pos = pd.DataFrame.from_dict(positive, orient='index')
df_neg = pd.DataFrame.from_dict(negative, orient='index')
df_pos.sort_values(by=0,ascending=False).head(10)
df_neg.sort_values(by=0,ascending=False).head(10)
score = cross_validation(df_test,5,1)
print("Accuracy on test data with smoothing with alpha 1 : {} %".format(np.mean(score)))<jupyter_output>Accuracy on test data with smoothing with alpha 1 : 58.0 %
{'fraction_non_alphanumeric': 0.11903196106799947, 'fraction_numerical': 0.012363540707615415, 'mean_word_length': 2.775571002979146, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '16115', 'license': 'MIT', 'n_tokens_mistral': 2572, 'n_tokens_neox': 2443, 'n_words': 534, 'path': 'Assignment3_Shraddha.ipynb', 'repo_name': 'Shraddha-Ingale34/Shraddha-Ingale34.github.io', 'size': '7603'}