<jupyter_start><jupyter_text>Imports<jupyter_code># set up
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import glob
import re
import string
import cvxopt # <- installation via conda recommended
from collections import defaultdict
from sklearn import metrics
from tqdm import tqdm
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics.pairwise import rbf_kernel
import nltk
from nltk.stem.snowball import SnowballStemmer
from nltk.corpus import stopwords
import scipy.optimize as sopt
import scipy.stats as sstats<jupyter_output><empty_output><jupyter_text>Motivation The main goal of this project is to predict the sentiments of sentences. We work with IMDB movies reviews dataset. Movies are rated from 1 to 10. We consider only the reviews with rating 1 (the worst ones) and 10 (the best ones). The assumption is that movies with rating 1 have negative sentiment and movies with rating 10 have a positive sentiment.   Preparing data (IMDb movies' reviews)  Macro for windows / mac users<jupyter_code>system = 'Mac'
#system = 'Win'
train_pos_path = 'data_sets/aclImdb/train/pos/*'
train_neg_path = 'data_sets/aclImdb/train/neg/*'

train_pos = glob.glob(train_pos_path)
train_neg = glob.glob(train_neg_path)


test_pos_path = 'data_sets/aclImdb/test/pos/*'
test_neg_path = 'data_sets/aclImdb/test/neg/*'

test_pos = glob.glob(test_pos_path)
test_neg = glob.glob(test_neg_path)
train_df = []
test_df = []
def read_data(path, message):
    res = []
    for p in tqdm(path, desc=message, position=0):
        with open(p, encoding="utf8") as f:
            text = f.read()

    #         For win users
            if system == 'Win':
                beg = p.find('\\')

    #         For mac users
            if system == 'Mac':
                beg = re.search(r"\d",p).start()-1

            idx, rating = p[beg+1:-4].split('_')
            res.append([text, rating])
            
    return res
train_df += read_data(path=train_pos, message='Getting positive train data')
train_df += read_data(path=train_neg, message='Getting negative train data')

test_df += read_data(path=test_pos, message='Getting positive test data')
test_df += read_data(path=test_neg, message='Getting negative test data')
train_df = pd.DataFrame(train_df, columns=['text', 'rating'])
test_df = pd.DataFrame(test_df, columns=['text', 'rating'])
print('Records: ', train_df.size)
train_df.head()
for i in range(1, 11):
    print(f'Number of reviews with rating {i}: {train_df[train_df.rating == str(i)].shape[0]}')<jupyter_output>Number of reviews with rating 1: 5100
Number of reviews with rating 2: 2284
Number of reviews with rating 3: 2420
Number of reviews with rating 4: 2696
Number of reviews with rating 5: 0
Number of reviews with rating 6: 0
Number of reviews with rating 7: 2496
Number of reviews with rating 8: 3009
Number of reviews with rating 9: 2263
Number of reviews with rating 10: 4732<jupyter_text>Baseline As one can see there is 4732 reviews with rating 10 and 5100 reviews with rating 1. Simple classifier that always votes for one class gives us baseline accuracy of 51.87 %.  Preprocessing Apart from obvious procedures like removing punctuation and lowering all sentences we try different preprocessing techniques such as: 1. Stemming, (reducing words to their root form, we decrease the size of words dictionary).2. Stop words removal, (we assume that words like 'the', 'and', 'a/an' etc. don't give much information).3. N-grams, (considering contiguous sequences of n items from a given sample of text).  Remove punctuation and lower sentences<jupyter_code>def regex(text):
    text = re.sub(r'[^\w\s]', '', text.lower())
    return text
%%time
# Remove punctuaction and lower all texts
train_df.text = train_df.text.apply(lambda row: regex(row))
test_df.text = test_df.text.apply(lambda row: regex(row))
train_df.head()
# consider only rating 1 and 10
bayes_df_train = train_df[(train_df.rating == '1') | (train_df.rating == '10')]
bayes_df_test = test_df[(test_df.rating == '1') | (test_df.rating == '10')]<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>stemmer = SnowballStemmer("english")
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))
def stem_(text):
    return ' '.join([stemmer.stem(word) for word in text.split()])
%%time
stemmed_train_df = bayes_df_train.copy()
stemmed_test_df = bayes_df_test.copy()
stemmed_train_df.text = stemmed_train_df.text.apply(lambda row: stem_(row))
stemmed_test_df.text = stemmed_test_df.text.apply(lambda row: stem_(row))<jupyter_output>CPU times: user 49.8 s, sys: 0 ns, total: 49.8 s
Wall time: 49.8 s<jupyter_text>Stop words removal<jupyter_code>def remove_stop_words(text):
    return ' '.join([word for word in text.split() if word not in stop_words])
%%time
swr_train_df = bayes_df_train.copy()
swr_test_df = bayes_df_test.copy()
swr_train_df.text = swr_train_df.text.apply(lambda row: remove_stop_words(row))
swr_test_df.text = swr_test_df.text.apply(lambda row: remove_stop_words(row))
def stem_and_remove_stop_words(text):
    return ' '.join([stemmer.stem(word) for word in text.split() if word not in stop_words])
%%time
stemmed_swr_train_df = bayes_df_train.copy()
stemmed_swr_test_df = bayes_df_test.copy()
stemmed_swr_train_df.text = stemmed_swr_train_df.text.apply(lambda row: stem_and_remove_stop_words(row))
stemmed_swr_test_df.text = stemmed_swr_test_df.text.apply(lambda row: stem_and_remove_stop_words(row))<jupyter_output>CPU times: user 32.8 s, sys: 0 ns, total: 32.8 s
Wall time: 32.9 s<jupyter_text>Score visualization<jupyter_code>def print_score(preds,Y,name,prints=True):
    print(name)
    acc = np.mean(preds == Y)
    print(f"Acc: {acc}")
    M = metrics.confusion_matrix(preds,Y)
    N = np.sum(M)
    if prints:
        print('\nConfusion matrix:')
        print(M)
        print(f'\nTrue negative (rating = 1): {M[0][0]}')
        print(f'True positive (rating = 10): {M[1][1]}')
        print(f'False negative: {M[0][1]}')
        print(f'False positive: {M[1][0]}')
    return M,N,acc<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code>def plot_bar(X, Y1, Y2, title, x_title, width=0.02, a=0, b=-1):
    def autolabel(rects):
        for rect in rects:
            height = rect.get_height()
            ax.annotate('{:.2f}%'.format(height * 100),
                        xy=(rect.get_x() + rect.get_width() / 2, height),
                        xytext=(0, 3),  # 3 points vertical offset
                        textcoords="offset points",
                        ha='center', va='bottom')
        
    fig, ax = plt.subplots(figsize=(15,5))
    rects1 = ax.bar(X[a: b] - width/2, Y1[a: b], width, label='Train')
    rects2 = ax.bar(X[a: b] + width/2, Y2[a: b], width, label='Test')
    ax.set_ylabel('Accuracy')
    ax.set_xlabel(x_title)
    ax.set_xticks(X[a: b])
    ax.set_ylim([0, 0.7])
    ax.set_title(title)
    ax.legend(loc='lower right')
    autolabel(rects1)
    autolabel(rects2)
    
    
def plot(X, Y1, Y2, title, x_title):
    plt.plot(X, Y1, label='Train')
    plt.plot(X, Y2, label='Test')
    plt.title(title)
    plt.legend(loc='lower left')
    plt.xlabel(x_title)
    plt.ylabel('Accuracy')
    
    
def autolabel(rects):
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{:.2f}%'.format(height * 100),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')<jupyter_output><empty_output><jupyter_text>CountVectorizer class<jupyter_code>class MyCountVectorizer:
    def __init__(self, min_df=-1, max_df=1e18, binary=False):
        self.min_df = min_df
        self.max_df = max_df
        self.binary = binary
    
    def fit(self, df):
        words_cnt = defaultdict(int)
        col = df.columns[0]
        
        for i in range(len(df)):
            text = df.iloc[i][col]
            for word in text.split():
                words_cnt[word] += 1
                
        all_words = []
        for word, cnt in words_cnt.items():
            if self.min_df <= cnt <= self.max_df:
                all_words.append(word)
                
        self.all_words_ids = {w:i for i,w in enumerate(all_words)}
        self.width = len(all_words)
        
    
    def transform(self, df):
        col = df.columns[0]
        count_matrix = np.zeros([len(df), self.width], \
                                dtype=np.int32)
        
        for i in range(len(df)):
            text = df.iloc[i][col]
            words_cnt = defaultdict(int)
            
            for word in text.split():
                words_cnt[word] += 1
            
            for word, cnt in words_cnt.items():
                if word in self.all_words_ids:
                    pos = self.all_words_ids[word]
                    if self.binary:
                        count_matrix[i][pos] = 1
                    else:
                        count_matrix[i][pos] = cnt
                    
        return count_matrix<jupyter_output><empty_output><jupyter_text>Naive Bayes approach  Naive Bayes class<jupyter_code>class Naive_Bayes:
    def __init__(self,alpha=0,fit_prior=True,class_prior=None):
        self.alpha = alpha
        self.fit_prior = fit_prior
        self.class_prior_array = class_prior
        if class_prior:
            self.fit_prior = False
    
    
    def fit(self,X,y):
        self.classes,prior = np.unique(y,return_counts=True)
        self.N = len(y)
        
        # Setting class prior
        if self.fit_prior:
            self.class_prior = {class_ : np.log(prior[i]/self.N + 1e-100)
                                for i,class_ in enumerate(self.classes)}
        elif self.class_prior_array:
            self.class_prior = {class_ : np.log(self.class_prior_array[i] + 1e-100) 
                                for i,class_ in enumerate(self.classes)}
        else:
            self.class_prior = {class_ : np.log(1/len(self.classes) + 1e-100) 
                                for class_ in self.classes}
            
        # Creating words dictionaries
        self.class_words_counts = {class_ : defaultdict(lambda: 0) 
                                   for class_ in self.classes}
        for i,text in enumerate(X):
            target = y[i]
            for word in text.split():
                self.class_words_counts[target][word] += 1
        
        # Creating probabilities dictionaries
        self.class_words_probs = {class_ : defaultdict(lambda: np.log(self.alpha + 1e-100)) 
                                  for class_ in self.classes}
        for class_,dict_ in self.class_words_counts.items():
            for word,count in dict_.items():
                self.class_words_probs[class_][word] = np.log(count + 1e-100)
    
        self.class_words_amount = {class_ : np.log(sum(self.class_words_counts[class_].values())) 
                                   for class_ in self.classes}
    

    def get_class_log_probabilities(self,text):
        probs = {class_ : 0 for class_ in self.classes}
        for class_ in self.classes:
            for word in text.split():
                probs[class_] += self.class_words_probs[class_][word]
                probs[class_] -= self.class_words_amount[class_]
            probs[class_] += self.class_prior[class_]
        return probs
    
    
    def predict(self,X,return_probabilities = False):
        preds = []
        preds_probs = []
        for text in X:
            prob = self.get_class_log_probabilities(text)
            #prob = {class_ : np.exp(pbb) for class_,pbb in prob.items()}
            preds_probs.append(prob)
            pred = max(prob,key = prob.get)
            preds.append(pred)
        
        if return_probabilities:
            return preds,preds_probs
        return preds<jupyter_output><empty_output><jupyter_text>Train and test accuracy<jupyter_code>X_train,y_train = np.array(bayes_df_train['text']),np.array(bayes_df_train['rating'])
X_test,y_test = np.array(bayes_df_test['text']),np.array(bayes_df_test['rating'])
NBc_res = []
alpha = 1.5
NB = Naive_Bayes(fit_prior = False,alpha=alpha)
NB.fit(X_train,y_train)
predictions,ppb = NB.predict(X_train,return_probabilities=True)
M,N,acc = print_score(predictions,y_train,f"TRAIN, alpha : {alpha}")
NBc_res.append(['Org train data\na=1.5','Acc',acc])
NBc_res.append(['Org train data\na=1.5','FN',M[0][1]/N])
NBc_res.append(['Org train data\na=1.5','FP',M[1][0]/N])
predictions,ppb = NB.predict(X_test,return_probabilities=True)
M,N,acc = print_score(predictions,y_test,f"TEST, alpha : {alpha}")
NBc_res.append(['Org test data\na=1.5','Acc',acc])
NBc_res.append(['Org test data\na=1.5','FN',M[0][1]/N])
NBc_res.append(['Org test data\na=1.5','FP',M[1][0]/N])<jupyter_output>TEST, alpha : 1.5
Acc: 0.8893324019558926

Confusion matrix:
[[4651  738]
 [ 371 4261]]

True negative (rating = 1): 4651
True positive (rating = 10): 4261
False negative: 738
False positive: 371<jupyter_text>Now we can try alpha = 0<jupyter_code>alpha = 0.0
NB = Naive_Bayes(fit_prior = False,alpha=alpha)
NB.fit(X_train,y_train)
predictions,ppb = NB.predict(X_train,return_probabilities=True)
M,N,acc = print_score(predictions,y_train,f"TRAIN, alpha : {alpha}")
predictions,ppb = NB.predict(X_test,return_probabilities=True)
M,N,acc = print_score(predictions,y_test,f"TEST, alpha : {alpha}")<jupyter_output>TEST, alpha : 0.0
Acc: 0.7240794331903003

Confusion matrix:
[[3932 1675]
 [1090 3324]]

True negative (rating = 1): 3932
True positive (rating = 10): 3324
False negative: 1675
False positive: 1090<jupyter_text>Setting alpha to 0 improve our train accuracy, but decrease train accuracy, so it`s overfitting. We can try to find best alpha for our data.  Testing different alphas<jupyter_code>accuracy = []
alphas = [0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,3.0,4.0,5.0,10.0]
for alpha in alphas:
    NB = Naive_Bayes(fit_prior = False,alpha=alpha)
    NB.fit(X_train, y_train)
    predictions,ppb = NB.predict(X_test,return_probabilities=True)
    acc = np.mean(predictions == y_test)
    print(f'Alpha : {alpha}, test acc: {acc}')
    accuracy.append(acc)
plt.plot(alphas, accuracy)
plt.xlabel('Alpha')
plt.ylabel('Accuracy')
plt.title('Accuracy for different alphas')<jupyter_output><empty_output><jupyter_text>Stemming and stop words removing<jupyter_code># get best alpha
alpha = 1.5
NB = Naive_Bayes(fit_prior = False,alpha=alpha)
NB.fit(np.array(stemmed_train_df['text']),np.array(stemmed_train_df['rating']))
predictions = NB.predict(np.array(stemmed_test_df['text']))
M,N,acc = print_score(predictions,np.array(stemmed_test_df['rating']),
                      f"TEST, alpha : {alpha}, stemmed")
NBc_res.append(['Stemmed test data\na=1.5','Acc',acc])
NBc_res.append(['Stemmed test data\na=1.5','FN',M[0][1]/N])
NBc_res.append(['Stemmed test data\na=1.5','FP',M[1][0]/N])
NB = Naive_Bayes(fit_prior = False,alpha=alpha)
NB.fit(np.array(swr_train_df['text']),np.array(swr_test_df['rating']))
predictions = NB.predict(np.array(swr_test_df['text']))
M,N,acc = print_score(predictions,np.array(stemmed_test_df['rating']),
                      f"TEST, alpha : {alpha}, stop words removed")
NBc_res.append(['Test data\nwithout stopwords\na=1.5','Acc',acc])
NBc_res.append(['Test data\nwithout stopwords\na=1.5','FN',M[0][1]/N])
NBc_res.append(['Test data\nwithout stopwords\na=1.5','FP',M[1][0]/N])
NB = Naive_Bayes(fit_prior = False,alpha=alpha)
NB.fit(np.array(stemmed_swr_train_df['text']),np.array(stemmed_swr_train_df['rating']))
predictions = NB.predict(np.array(stemmed_swr_test_df['text']))
M,N,acc = print_score(predictions,np.array(stemmed_test_df['rating']),
                      f"TEST, alpha : {alpha}, stemmed and stop words removed")
NBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','Acc',acc])
NBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','FN',M[0][1]/N])
NBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','FP',M[1][0]/N])
NBc_res_df = pd.DataFrame(NBc_res,columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=NBc_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title('Our Naive Bayes')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Sklearn Naive Bayes<jupyter_code>NBb_res = []
CV_NB = CountVectorizer()
X_train,y_train = np.array(bayes_df_train['text']),np.array(bayes_df_train['rating'])
X_test,y_test = np.array(bayes_df_test['text']),np.array(bayes_df_test['rating'])
X_train_cv = CV_NB.fit_transform(X_train)
X_test_cv = CV_NB.transform(X_test)
alpha = 1.5
MNB = MultinomialNB(alpha = 1.5)
MNB.fit(X_train_cv,y_train)
predictions = MNB.predict(X_train_cv)
M,N,acc = print_score(predictions,y_train,f"TRAIN, alpha : {alpha}")
NBb_res.append(['Org train data\na=1.5','Acc',acc])
NBb_res.append(['Org train data\na=1.5','FN',M[0][1]/N])
NBb_res.append(['Org train data\na=1.5','FP',M[1][0]/N])
alpha = 1.5
MNB = MultinomialNB(alpha = 1.5)
MNB.fit(X_train_cv,y_train)
predictions = MNB.predict(X_test_cv)
M,N,acc = print_score(predictions,y_test,f"TEST, alpha : {alpha}")
NBb_res.append(['Org test data\na=1.5','Acc',acc])
NBb_res.append(['Org test data\na=1.5','FN',M[0][1]/N])
NBb_res.append(['Org test data\na=1.5','FP',M[1][0]/N])
CV_NB = CountVectorizer()
X_train,y_train = np.array(stemmed_train_df['text']),np.array(stemmed_train_df['rating'])
X_test,y_test = np.array(stemmed_test_df['text']),np.array(stemmed_test_df['rating'])
X_train_cv = CV_NB.fit_transform(X_train)
X_test_cv = CV_NB.transform(X_test)
alpha = 1.5
MNB = MultinomialNB(alpha = 1.5)
MNB.fit(X_train_cv,y_train)
predictions = MNB.predict(X_test_cv)
M,N,acc = print_score(predictions,y_test,
                      f"TEST, alpha : {alpha}, stemmed")
NBb_res.append(['Stemmed test data\na=1.5','Acc',acc])
NBb_res.append(['Stemmed test data\na=1.5','FN',M[0][1]/N])
NBb_res.append(['Stemmed test data\na=1.5','FP',M[1][0]/N])
CV_NB = CountVectorizer()
X_train,y_train = np.array(swr_train_df['text']),np.array(swr_train_df['rating'])
X_test,y_test = np.array(swr_test_df['text']),np.array(swr_test_df['rating'])
X_train_cv = CV_NB.fit_transform(X_train)
X_test_cv = CV_NB.transform(X_test)
alpha = 1.5
MNB = MultinomialNB(alpha = 1.5)
MNB.fit(X_train_cv,y_train)
predictions = MNB.predict(X_test_cv)
M,N,acc = print_score(predictions,y_test,
                      f"TEST, alpha : {alpha}, stemmed")
NBb_res.append(['Test data\nwithout stopwords\na=1.5','Acc',acc])
NBb_res.append(['Test data\nwithout stopwords\na=1.5','FN',M[0][1]/N])
NBb_res.append(['Test data\nwithout stopwords\na=1.5','FP',M[1][0]/N])
CV_NB = CountVectorizer()
X_train,y_train = np.array(stemmed_swr_train_df['text']),np.array(stemmed_swr_train_df['rating'])
X_test,y_test = np.array(stemmed_swr_test_df['text']),np.array(stemmed_swr_test_df['rating'])
X_train_cv = CV_NB.fit_transform(X_train)
X_test_cv = CV_NB.transform(X_test)
alpha = 1.5
MNB = MultinomialNB(alpha = 1.5)
MNB.fit(X_train_cv,y_train)
predictions = MNB.predict(X_test_cv)
M,N,acc = print_score(predictions,y_test,
                      f"TEST, alpha : {alpha}, stemmed")
NBb_res.append(['Stemmed test data\nwithout stop words\na=1.5','Acc',acc])
NBb_res.append(['Stemmed test data\nwithout stop words\na=1.5','FN',M[0][1]/N])
NBb_res.append(['Stemmed test data\nwithout stop words\na=1.5','FP',M[1][0]/N])
NBb_res_df = pd.DataFrame(NBb_res,columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=NBb_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title('SKLEARN Multinomial Naive Bayes')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare our NB with sklearn`s<jupyter_code>NBc_res_df_copyed = NBc_res_df.copy()
A_df = NBc_res_df[NBc_res_df.Y == 'Acc']
A_df.Y = ['Our'] * 5
NBb_res_df_copyed = NBb_res_df.copy()
B_df = NBb_res_df[NBb_res_df.Y == 'Acc']
B_df.Y = ['Sklearn'] * 5
compare_df = pd.concat([A_df,B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Our Naive Bayes class vs SKLEARN MultinomialNaiveBayes")
ax.legend(loc='lower right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Logistic Regression  Logistic Regression class<jupyter_code>class Logistic_Regression:
    def __init__(self, max_iter=500, solver_calls=5,lambda_ = 0.1,Theta=None, solver=sopt.fmin_l_bfgs_b, debug=False):
        self.Theta = Theta
        self.solver_calls = solver_calls
        self.max_iter = max_iter
        self.solver = solver
        self.debug = debug
        self.lambda_ = lambda_
    
    def __sigmoid(self,x):
        return 1 / (1 + np.exp(-x))    
    
    def __logreg_loss(self, Theta, X, Y):
        Theta = Theta.astype(np.float64)
        X = X.astype(np.float64)
        Y = Y.astype(np.float64)
        
        if self.debug:
            print(f"Loss calculating... ",end="")
        Z = np.dot(Theta,X.T)
        if self.debug:
            print(f" Z done... ",end="")
        SZ = self.__sigmoid(Z)
        Y_ = Y[:,np.newaxis]
        nll = -np.sum((Y_*np.log2(SZ+1e-50) + (1-Y_)*np.log2(1-SZ+1e-50)))
#         nll = -np.mean([
#                         y * np.log2(SZ + 1e-10) \
#                         + (1-y) * np.log2(1 - SZ + 1e-10) \
#                         for y in Y
#                         ])
        nll += (self.lambda_/2) * np.sum(Theta**2)
        if self.debug:
            print(f" nll done... ",end="")
        grad = np.dot(X.T, (SZ - Y).T ) / len(Y)
        grad = grad.reshape(Theta.shape) + self.lambda_ * Theta
        if self.debug:
            print(f" grad done... done ")
        return nll, grad
    
    def fit(self,X,y):
        Theta = self.Theta
        if Theta is None:
            Theta = np.ones(X.shape[1]+1)
        
        X_with_ones = np.hstack((np.ones((X.shape[0],1)),X))
      
        for i in tqdm(range(self.solver_calls), desc='Calculating Theta', position=0):
            Theta = self.solver(lambda th: self.__logreg_loss(th, X_with_ones, y), 
                                Theta, maxiter=self.max_iter)[0]
        self.Theta = Theta
    
#     def __logreg_classify(self, X):
#         return (self.Theta.T.dot(X) >= 0)

#     def predict(self, Xs):
#         return [self.__logreg_classify(x) for x in Xs]
    
    def predict(self,X):
        X_with_ones = np.hstack((np.ones((X.shape[0],1)),X))
        preds = (np.dot(self.Theta,X_with_ones.T) >= 0) * 1
        return preds<jupyter_output><empty_output><jupyter_text>Train and test accuracy<jupyter_code>LGc_res = []

y_train = np.array(list(map(int, bayes_df_train['rating']))) // 10
y_test = np.array(list(map(int, bayes_df_test['rating']))) // 10
%%time
cv = MyCountVectorizer(min_df=5)
cv.fit(bayes_df_train)

X_train = cv.transform(bayes_df_train) 
X_test = cv.transform(bayes_df_test)
Theta_beg = pd.read_csv('Thetas/Theta_opt_rawtext.csv')['Theta']
LR = Logistic_Regression(max_iter=1000, lambda_=0.1, Theta=Theta_beg, debug=False)
%%time
LR.fit(X_train, y_train)
pd.DataFrame(LR.Theta, columns=['Theta']).to_csv('Thetas/Theta_opt_rawtext.csv')
predictions = LR.predict(X_train)
M, N, acc = print_score(predictions, y_train, 'TRAIN')
LGc_res.append(['Org train data\n','Acc',acc])
LGc_res.append(['Org train data\n','FN',M[0][1]/N])
LGc_res.append(['Org train data\n','FP',M[1][0]/N])
predictions = LR.predict(X_test)
M, N, acc = print_score(predictions, y_test, 'TEST')
LGc_res.append(['Org test data\n','Acc',acc])
LGc_res.append(['Org test data\n','FN',M[0][1]/N])
LGc_res.append(['Org test data\n','FP',M[1][0]/N])<jupyter_output>TEST
Acc: 0.8678774573395869

Confusion matrix:
[[4393  695]
 [ 629 4304]]

True negative (rating = 1): 4393
True positive (rating = 10): 4304
False negative: 695
False positive: 629<jupyter_text>Stemming and stop words removal<jupyter_code>%%time
cv = MyCountVectorizer(min_df=5)
cv.fit(stemmed_swr_train_df)

X_train = cv.transform(stemmed_swr_train_df) 
X_test = cv.transform(stemmed_swr_test_df)
Theta_beg = pd.read_csv('Thetas/Theta_opt_stem_stop.csv')['Theta']
LR = Logistic_Regression(max_iter=1000, solver_calls=10,lambda_=0.1, Theta=Theta_beg, debug=False)
%%time
LR.fit(X_train, y_train)
pd.DataFrame(LR.Theta, columns=['Theta']).to_csv('Theta_opt_stem_stop.csv')
predictions = LR.predict(X_train)
M, N, acc = print_score(predictions, y_train, 'STEMMED TRAIN WITHOUT STOP WORDS')
LGc_res.append(['Stemmed train data\nwithout stop words\n','Acc',acc])
LGc_res.append(['Stemmed train data\nwithout stop words\n','FN',M[0][1]/N])
LGc_res.append(['Stemmed train data\nwithout stop words\n','FP',M[1][0]/N])
predictions = LR.predict(X_test)
M, N, acc = print_score(predictions, y_test, 'STEMMED TEST WITHOUT STOP WORDS')
LGc_res.append(['Stemmed test data\nwithout stop words\n','Acc',acc])
LGc_res.append(['Stemmed test data\nwithout stop words\n','FN',M[0][1]/N])
LGc_res.append(['Stemmed test data\nwithout stop words\n','FP',M[1][0]/N])
LGc_res_df = pd.DataFrame(LGc_res,columns=['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=LGc_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Our Logistic Regression class")
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Sklearn Linear Regression<jupyter_code>%%time
cv = CountVectorizer(min_df=5)
cv.fit(stemmed_swr_train_df.text)
cv2 = CountVectorizer(min_df=5)
cv2.fit(bayes_df_train.text)

X_train_og = cv2.transform(bayes_df_train.text)
X_test_og = cv2.transform(bayes_df_test.text)

X_train_stem_swr = cv.transform(stemmed_swr_train_df.text)
X_test_stem_swr = cv.transform(stemmed_swr_test_df.text)

y_train = np.array(list(map(int, bayes_df_train['rating']))) // 10
y_test = np.array(list(map(int, bayes_df_test['rating']))) // 10

LG2_res = []
%%time
for c in [0.01, 0.05, 0.25, 0.5, 1, 5]:    
    lr = LogisticRegression(C=c, max_iter=800, solver='lbfgs') 
    lr.fit(X_train_og, y_train)
    print ("Accuracy for C=%s: %s" 
           % (c, accuracy_score(y_test, lr.predict(X_test_og))))
best_C = 0.05
best_lr = LogisticRegression(C=best_C, max_iter=500, solver='lbfgs') 
best_lr.fit(X_train_og, y_train)
M, N, acc = print_score(best_lr.predict(X_train_og), y_train, 'TRAIN')
LG2_res.append(['Org train data\n','Acc',acc])
LG2_res.append(['Org train data\n','FN',M[0][1]/N])
LG2_res.append(['Org train data\n','FP',M[1][0]/N])
M, N, acc = print_score(best_lr.predict(X_test_og), y_test, 'TEST')
LG2_res.append(['Org test data\n','Acc',acc])
LG2_res.append(['Org test data\n','FN',M[0][1]/N])
LG2_res.append(['Org test data\n','FP',M[1][0]/N])
# stemmed data without stop words
best_C = 0.05
lrs = LogisticRegression(C=best_C, max_iter=500, solver='lbfgs') 
lrs.fit(X_train_stem_swr, y_train)
M, N, acc = print_score(lrs.predict(X_train_stem_swr), y_train, 'STEMMED TRAIN WITHOUT STOP WORDS')
LG2_res.append(['Stemmed train data\nwithout stop words\n','Acc',acc])
LG2_res.append(['Stemmed train data\nwithout stop words\n','FN',M[0][1]/N])
LG2_res.append(['Stemmed train data\nwithout stop words\n','FP',M[1][0]/N])
M, N, acc = print_score(lrs.predict(X_test_stem_swr), y_test, 'STEMMED TEST WITHOUT STOP WORDS')
LG2_res.append(['Stemmed test data\nwithout stop words\n','Acc',acc])
LG2_res.append(['Stemmed test data\nwithout stop words\n','FN',M[0][1]/N])
LG2_res.append(['Stemmed test data\nwithout stop words\n','FP',M[1][0]/N])
LG2_res_df  = pd.DataFrame(LG2_res, columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=LG2_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Sklearn Logistic Regression")
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare our LR with sklearn's<jupyter_code>A_df = LGc_res_df[LGc_res_df.Y == 'Acc']
A_df.Y = ['Our'] * 4

B_df = LG2_res_df[LG2_res_df.Y == 'Acc']
B_df.Y = ['Sklearn'] * 4
compare_df = pd.concat([A_df,B_df])

sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Our Logistic Regression vs Sklearn Logistic Regression")
ax.legend(loc='lower right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>N-gram  Testing different ranges<jupyter_code>y_train = np.array(list(map(int, bayes_df_train['rating']))) // 10
y_test = np.array(list(map(int, bayes_df_test['rating']))) // 10
print('Sklearn CV and LR, c=0.05, Stemmed test data without stop words')
accs = []
for r in range(1,11):
    cv_ngram = CountVectorizer(min_df=5,ngram_range=(1,r))
    cv_ngram.fit(stemmed_swr_train_df.text)
    X_train_ngram = cv_ngram.transform(stemmed_swr_train_df.text)
    X_test_ngram = cv_ngram.transform(stemmed_swr_test_df.text)
    lrngram = LogisticRegression(C=0.05, max_iter=500, solver='lbfgs') 
    lrngram.fit(X_train_ngram, y_train)
    preds = lrngram.predict(X_test_ngram)
    acc = np.mean(preds == y_test)
    accs.append(acc)
    print(f'N-gram range = (1,{r}), acc: {acc}')
plt.plot(range(1,11),accs)
plt.xlabel('N-gram upper range')
plt.ylabel('Accuracy')
plt.title('Sklearn CV and LR, c=0.05, Stemmed test data without stop words')<jupyter_output><empty_output><jupyter_text>Train and test accuracy<jupyter_code>cv_ngram = CountVectorizer(min_df=5,ngram_range=(1,5))
cv_ngram.fit(stemmed_swr_train_df.text)
cv_ngram2 = CountVectorizer(min_df=5,ngram_range=(1,5))
cv_ngram2.fit(bayes_df_train.text)

X_train_og_ngram = cv_ngram2.transform(bayes_df_train.text)
X_test_og_ngram = cv_ngram2.transform(bayes_df_test.text)

X_train_stem_swr_ngram = cv_ngram.transform(stemmed_swr_train_df.text)
X_test_stem_swr_ngram = cv_ngram.transform(stemmed_swr_test_df.text)

y_train = np.array(list(map(int, bayes_df_train['rating']))) // 10
y_test = np.array(list(map(int, bayes_df_test['rating']))) // 10

LG_ngram_res = []
lr_ngram = LogisticRegression(C=0.05, max_iter=500, solver='lbfgs') 
lr_ngram.fit(X_train_og_ngram, y_train)
M, N, acc = print_score(lr_ngram.predict(X_train_og_ngram), y_train, 'TRAIN')
LG_ngram_res.append(['Org train data\n','Acc',acc])
LG_ngram_res.append(['Org train data\n','FN',M[0][1]/N])
LG_ngram_res.append(['Org train data\n','FP',M[1][0]/N])
M, N, acc = print_score(lr_ngram.predict(X_test_og_ngram), y_test, 'TEST')
LG_ngram_res.append(['Org test data\n','Acc',acc])
LG_ngram_res.append(['Org test data\n','FN',M[0][1]/N])
LG_ngram_res.append(['Org test data\n','FP',M[1][0]/N])
lr_ngram2 = LogisticRegression(C=0.05, max_iter=500, solver='lbfgs') 
lr_ngram2.fit(X_train_stem_swr_ngram, y_train)
M, N, acc = print_score(lr_ngram2.predict(X_train_stem_swr_ngram), y_train, 'STEMMED TRAIN WITHOUT STOP WORDS')
LG_ngram_res.append(['Stemmed train data\nwithout stop words\n','Acc',acc])
LG_ngram_res.append(['Stemmed train data\nwithout stop words\n','FN',M[0][1]/N])
LG_ngram_res.append(['Stemmed train data\nwithout stop words\n','FP',M[1][0]/N])
M, N, acc = print_score(lr_ngram2.predict(X_test_stem_swr_ngram), y_test, 'STEMMED TEST WITHOUT STOP WORDS')
LG_ngram_res.append(['Stemmed test data\nwithout stop words\n','Acc',acc])
LG_ngram_res.append(['Stemmed test data\nwithout stop words\n','FN',M[0][1]/N])
LG_ngram_res.append(['Stemmed test data\nwithout stop words\n','FP',M[1][0]/N])
LG_ngram_res_df  = pd.DataFrame(LG_ngram_res, columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=LG_ngram_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Sklearn Linear Regression, n-gram range = (1,5)")
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare with default n-gram range<jupyter_code>C_df = LG_ngram_res_df[LG_ngram_res_df.Y == 'Acc']
C_df.Y = ['CV, N-gram (1,5)'] * 4

D_df = LG2_res_df[LG2_res_df.Y == 'Acc']
D_df.Y = ['CV, N-gram (1,1)'] * 4
compare_df2 = pd.concat([D_df,C_df])

sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_df2)
ax.figure.set_size_inches(15, 5)
ax.set_title("Sklearn Logistic Regression with CountVectorizer, ngram range (1,1) vs (1,5)")
ax.legend(loc='lower right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>TF-IDF  Train and test accuracy<jupyter_code>tf_idf = TfidfVectorizer(min_df=5,ngram_range=(1,5))
tf_idf.fit(stemmed_swr_train_df.text)
tf_idf2 = TfidfVectorizer(min_df=5,ngram_range=(1,5))
tf_idf2.fit(bayes_df_train.text)

X_train_og_tfidf = tf_idf2.transform(bayes_df_train.text)
X_test_og_tfidf = tf_idf2.transform(bayes_df_test.text)

X_train_stem_swr_tfidf = tf_idf.transform(stemmed_swr_train_df.text)
X_test_stem_swr_tfidf = tf_idf.transform(stemmed_swr_test_df.text)

y_train = np.array(list(map(int, bayes_df_train['rating']))) // 10
y_test = np.array(list(map(int, bayes_df_test['rating']))) // 10

LG_tfidf_res = []
lr_tfidf = LogisticRegression(C=0.05, max_iter=500, solver='lbfgs') 
lr_tfidf.fit(X_train_og_tfidf, y_train)
M, N, acc = print_score(lr_tfidf.predict(X_train_og_tfidf), y_train, 'TRAIN')
LG_tfidf_res.append(['Org train data\n','Acc',acc])
LG_tfidf_res.append(['Org train data\n','FN',M[0][1]/N])
LG_tfidf_res.append(['Org train data\n','FP',M[1][0]/N])
M, N, acc = print_score(lr_tfidf.predict(X_test_og_tfidf), y_test, 'TEST')
LG_tfidf_res.append(['Org test data\n','Acc',acc])
LG_tfidf_res.append(['Org test data\n','FN',M[0][1]/N])
LG_tfidf_res.append(['Org test data\n','FP',M[1][0]/N])
lr_tfidf2 = LogisticRegression(C=0.05, max_iter=500, solver='lbfgs') 
lr_tfidf2.fit(X_train_stem_swr_tfidf, y_train)
M, N, acc = print_score(lr_tfidf2.predict(X_train_stem_swr_tfidf), y_train, 'STEMMED TRAIN WITHOUT STOP WORDS')
LG_tfidf_res.append(['Stemmed train data\nwithout stop words\n','Acc',acc])
LG_tfidf_res.append(['Stemmed train data\nwithout stop words\n','FN',M[0][1]/N])
LG_tfidf_res.append(['Stemmed train data\nwithout stop words\n','FP',M[1][0]/N])
M, N, acc = print_score(lr_tfidf2.predict(X_test_stem_swr_tfidf), y_test, 'STEMMED TEST WITHOUT STOP WORDS')
LG_tfidf_res.append(['Stemmed test data\nwithout stop words\n','Acc',acc])
LG_tfidf_res.append(['Stemmed test data\nwithout stop words\n','FN',M[0][1]/N])
LG_tfidf_res.append(['Stemmed test data\nwithout stop words\n','FP',M[1][0]/N])
LG_tfidf_res_df  = pd.DataFrame(LG_tfidf_res, columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=LG_tfidf_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Sklearn Linear Regression, TF-IDF vectorizer")
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare with CountVectorizer<jupyter_code>E_df = LG_tfidf_res_df[LG_tfidf_res_df.Y == 'Acc']
E_df.Y = ['TF-IDF, N-gram (1,5)'] * 4
compare_df3 = pd.concat([D_df,C_df,E_df])

sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_df3)
ax.figure.set_size_inches(15, 5)
ax.set_title("Sklearn Logistic Regression CountVectorizer vs TF-IDFVectorizer")
ax.legend(loc='lower right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>SVM  SVM class<jupyter_code>class SVM:
    def __init__(self,C,gamma,th=1e-5):
        self.C = C
        self.gamma = gamma
        self.th = th
    def fit(self,X,y,info=False):
        m,n = X.shape
        
        # kernel matrix
        if info:
            print(f'Calculating kernel matrix...',end='')
        K = rbf_kernel(X,X,gamma=self.gamma)
        
        # setup solver
        if info:
            print(f'Setting solver...',end='')
        P = cvxopt.matrix(np.outer(y,y) * K)
        q = cvxopt.matrix(-np.ones((m, 1)))
        G = cvxopt.matrix(np.vstack((np.eye(m)*-1,np.eye(m))))
        h = cvxopt.matrix(np.hstack((np.zeros(m), np.ones(m) * self.C)))
        A = cvxopt.matrix(y.reshape(1, -1))
        b = cvxopt.matrix(np.zeros(1))
        if info:
            print(f'starting solver...')
        sol = cvxopt.solvers.qp(P, q, G, h, A, b)
        self.alphas = np.array(sol['x'])
        
        # getting support vectors
        self.S = (self.alphas > self.th).reshape(-1, )
        self.support_vextors = X[self.S]
        self.sv_target = y[self.S]
        
        # getting bias
        if info:
            print(f'Calculating bias...',end='')
        K_sv = rbf_kernel(self.support_vextors,self.support_vextors,gamma=self.gamma)
        B = self.sv_target.ravel() - np.sum(K_sv * self.sv_target.ravel() * self.alphas[self.S].ravel(),axis=1)
        self.bias = sstats.mode(B)[0][0]
        
        if info:
            print(f'done')
        
    def predict(self,X):
        K = rbf_kernel(X,self.support_vextors,gamma=self.gamma)
        preds = np.sign(np.sum(K * self.sv_target.ravel() * self.alphas[self.S].ravel(),axis=1) + self.bias)
        return preds<jupyter_output><empty_output><jupyter_text>Train and test accuracy<jupyter_code>CV = MyCountVectorizer(min_df = 5)
CV.fit(stemmed_swr_train_df)
X_train_SVM,X_test_SVM = CV.transform(stemmed_swr_train_df),CV.transform(stemmed_swr_test_df)
y_train = np.array([-1.0 if r == '1' else 1.0 for r in stemmed_swr_train_df['rating']]).reshape(-1,1)
y_test = np.array([-1.0 if r == '1' else 1.0 for r in stemmed_swr_test_df['rating']]).reshape(-1,1)
C = 1.25
gamma = 0.0025
svm_model = SVM(C,gamma)
svm_model.fit(X_train_SVM,y_train,info=True)
svm_predictions_train = svm_model.predict(X_train_SVM)
M,N,acc = print_score(svm_predictions_train,y_train.ravel(),
                      f'Train data, SVM, rbf kernel, C={C}, gamma={gamma}')
svm_predictions = svm_model.predict(X_test_SVM)
M,N,acc = print_score(svm_predictions,y_test.ravel(),
                      f'Test data, SVM, rbf kernel, C={C}, gamma={gamma}')<jupyter_output>Test data, SVM, rbf kernel, C=1.25, gamma=0.0025
Acc: 0.9209659714599341

Confusion matrix:
[[4566  336]
 [ 456 4663]]

True negative (rating = 1): 4566
True positive (rating = 10): 4663
False negative: 336
False positive: 456<jupyter_text>Testing different parameters<jupyter_code>SVMc_res_train = []
SVMc_res_test = []
cvxopt.solvers.options['show_progress'] = False
for c in [1.0,1.25,1.5]:
    for gamma_ in [0.001,0.0025,0.005]:
        svm_test_model = SVM(c,gamma_)
        print(f'C: {c}, gamma: {gamma_}')
        svm_test_model.fit(X_train_SVM,y_train,info=False)
        svm_preds_train = svm_test_model.predict(X_train_SVM)
        svm_preds_test = svm_test_model.predict(X_test_SVM)
        M_train,N_train,acc_train = print_score(svm_preds_train,y_train.ravel(),
                      f'Train data, SVM, rbf kernel, C={c}, gamma={gamma_}',prints=False)
        M_test,N_test,acc_test = print_score(svm_preds_test,y_test.ravel(),
                      f'Test data, SVM, rbf kernel, C={c}, gamma={gamma_}',prints=False)
        SVMc_res_train.append([f'C = {c}\ngamma = {gamma_}','Acc',acc_train])
        SVMc_res_train.append([f'C = {c}\ngamma = {gamma_}','FN',M_train[0][1]/N_train])
        SVMc_res_train.append([f'C = {c}\ngamma = {gamma_}','FP',M_train[1][0]/N_train])
        SVMc_res_test.append([f'C = {c}\ngamma = {gamma_}','Acc',acc_test])
        SVMc_res_test.append([f'C = {c}\ngamma = {gamma_}','FN',M_test[0][1]/N_test])
        SVMc_res_test.append([f'C = {c}\ngamma = {gamma_}','FP',M_test[1][0]/N_test])
        print()
SVMc_res_train_df = pd.DataFrame(SVMc_res_train,columns = ['Data','Y','Value'])
SVMc_res_test_df = pd.DataFrame(SVMc_res_test,columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=SVMc_res_train_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Stemmed train data without stop words, SVM with rbf kernel")
ax.legend(loc='center right')
autolabel(ax.patches)
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=SVMc_res_test_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Stemmed test data without stop words, SVM with rbf kernel")
ax.legend(loc='center right')
autolabel(ax.patches)
SVMc_res_train_df_copyed = SVMc_res_train_df.copy()
SVM_A_df = SVMc_res_train_df_copyed[SVMc_res_train_df_copyed.Y == 'Acc']
SVM_A_df.Y = ['Train'] * 9
SVMc_res_test_df_copyed = SVMc_res_test_df.copy()
SVM_B_df = SVMc_res_test_df_copyed[SVMc_res_test_df_copyed.Y == 'Acc']
SVM_B_df.Y = ['Test'] * 9
compare_svm_df = pd.concat([SVM_A_df,SVM_B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_svm_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Stemmed data without stop words, SVM with rbf kernel, train vs test accuracy")
ax.legend(loc='center right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Sklearn SVM<jupyter_code>CV = CountVectorizer(min_df=5)
CV.fit(stemmed_swr_train_df['text'])
X_train_SVM = CV.transform(stemmed_swr_train_df['text'])
X_test_SVM = CV.transform(stemmed_swr_test_df['text'])
SVMb_res_train = []
SVMb_res_test = []
for c in [1.0,1.25,1.5]:
    for gamma_ in [0.001,0.0025,0.005]:
        svm_test_model = SVC(C=c, kernel='rbf', gamma=gamma_)
        print(f'C: {c}, gamma: {gamma_}')
        svm_test_model.fit(X_train_SVM,y_train.ravel())
        svm_preds_train = svm_test_model.predict(X_train_SVM)
        svm_preds_test = svm_test_model.predict(X_test_SVM)
        M_train,N_train,acc_train = print_score(svm_preds_train,y_train.ravel(),
                      f'Train data, SVM, rbf kernel, C={c}, gamma={gamma_}',prints=False)
        M_test,N_test,acc_test = print_score(svm_preds_test,y_test.ravel(),
                      f'Test data, SVM, rbf kernel, C={c}, gamma={gamma_}',prints=False)
        SVMb_res_train.append([f'C = {c}\ngamma = {gamma_}','Acc',acc_train])
        SVMb_res_train.append([f'C = {c}\ngamma = {gamma_}','FN',M_train[0][1]/N_train])
        SVMb_res_train.append([f'C = {c}\ngamma = {gamma_}','FP',M_train[1][0]/N_train])
        SVMb_res_test.append([f'C = {c}\ngamma = {gamma_}','Acc',acc_test])
        SVMb_res_test.append([f'C = {c}\ngamma = {gamma_}','FN',M_test[0][1]/N_test])
        SVMb_res_test.append([f'C = {c}\ngamma = {gamma_}','FP',M_test[1][0]/N_test])
        print()
SVMb_res_train_df = pd.DataFrame(SVMb_res_train,columns = ['Data','Y','Value'])
SVMb_res_test_df = pd.DataFrame(SVMb_res_test,columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=SVMb_res_train_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Sklearn SVM, stemmed train data without stop words, rbf kernel")
ax.legend(loc='center right')
autolabel(ax.patches)
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=SVMb_res_test_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Sklearn SVM, stemmed test data without stop words, rbf kernel")
ax.legend(loc='center right')
autolabel(ax.patches)
SVMb_res_train_df_copyed = SVMb_res_train_df.copy()
SVMb_A_df = SVMb_res_train_df_copyed[SVMb_res_train_df_copyed.Y == 'Acc']
SVMb_A_df.Y = ['Train'] * 9
SVMb_res_test_df_copyed = SVMb_res_test_df.copy()
SVMb_B_df = SVMb_res_test_df_copyed[SVMb_res_test_df_copyed.Y == 'Acc']
SVMb_B_df.Y = ['Test'] * 9
compare_svmb_df = pd.concat([SVMb_A_df,SVMb_B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_svmb_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Sklearn SVM, stemmed data without stop words, rbf kernel, train vs test accuracy")
ax.legend(loc='center right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare our SVM with sklearn`s<jupyter_code>SVMc1_res_train_df_copyed = SVMc_res_train_df.copy()
SVMc1_A_df = SVMc1_res_train_df_copyed[SVMc1_res_train_df_copyed.Y == 'Acc']
SVMc1_A_df.Y = ['Our'] * 9
SVMb1_res_train_df_copyed = SVMb_res_train_df.copy()
SVMb1_B_df = SVMb1_res_train_df_copyed[SVMb1_res_train_df_copyed.Y == 'Acc']
SVMb1_B_df.Y = ['Sklearn'] * 9
compare_svm1_df = pd.concat([SVMc1_A_df,SVMb1_B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_svm1_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Our SVM class vs Sklearn SVC SVM, stemmed train data without stop words")
ax.legend(loc='center right')
autolabel(ax.patches)
SVMc2_res_test_df_copyed = SVMc_res_test_df.copy()
SVMc2_A_df = SVMc2_res_test_df_copyed[SVMc2_res_test_df_copyed.Y == 'Acc']
SVMc2_A_df.Y = ['Our'] * 9
SVMb2_res_test_df_copyed = SVMb_res_test_df.copy()
SVMb2_B_df = SVMb2_res_test_df_copyed[SVMb2_res_test_df_copyed.Y == 'Acc']
SVMb2_B_df.Y = ['Sklearn'] * 9
compare_svm2_df = pd.concat([SVMc2_A_df,SVMb2_B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_svm2_df)
ax.figure.set_size_inches(20, 5)
ax.set_title("Our SVM class vs Sklearn SVC SVM, stemmed test data without stop words")
ax.legend(loc='center right')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Heuristic algorithm Here we want to test some heuristics. 1. If we come across a negation word, we negate ${k}$ words after it. I.e. we change the next ${k}$ words' probability for probability from another class.2. For words that enhance sentiment we multiply the next ${k}$ words' probability.3. There are also lists with positive and negative words, because sometimes we have a positive word from the negative class and vice versa. We swap probability in this case.  Supplementary words sets<jupyter_code>#most popular negation, enhance, positive and negative words prepared for boosting naive bayes approach
negation_words = ['not','no','never','no','hardly','nobody','none','scarcely','nowhere','sparsely','scantly','seldom','sporadically','somewhat','infrequently','imperceptibly','rarely','comparatively','perceptibly','gradually','detectably','slightly','just','only','faintly','little']
enhance_words  = ['very', 'consistently', 'constantly', 'continually', 'inadvertently', 'mutually', 'simply', 'strongly', 'actively', 'energetically', 'firmly', 'fully', 'heartily', 'heavily', 'resolutely', 'robustly', 'solidly', 'staunchly', 'steadily', 'vigorously', 'completely', 'decidedly', 'forcibly', 'indomitably', 'invincibly', 'mightily', 'securely', 'stoutly', 'sturdily']
positive_words = ['accomplished', 'economical', 'instrumental', 'skilled', 'adept', 'expert', 'productive', 'skillful', 'advanced', 'fluent', 'proficient', 'strong', 'competent', 'ideal', 'profitable', 'superior', 'constructive', 'industrious', 'significant', 'qualified', 'costeffective', 'influential', 'smooth', 'quality', 'cuttingedge', 'imaginative', 'novel', 'sophisticated', 'elegant', 'ingenious', 'progressive', 'unique', 'firstclass', 'innovative', 'revolutionary', 'unprecedented', 'groundbreaking', 'inventive', 'robust', 'worldclass', 'astute', 'insightful', 'methodical', 'practical', 'calculating', 'intelligent', 'meticulous', 'shrewd', 'complex', 'investigative', 'objective', 'strategic', 'discerning', 'logical', 'perceptive', 'thoughtful', 'accept', 'favor', 'acclaim', 'admire', 'applaud', 'appreciate', 'approbate', 'commend', 'countenance', 'esteem', 'handle', 'like', 'praise', 'respect']
negative_words = ['only', 'coarse', 'dead', 'feeble', 'rely', 'faintly', 'bad', 'hate', 'enraged', 'fight', 'hard', 'practically', 'callous', 'ghastly', 'simply', 'banal', 'filthy', 'grave', 'grimace', 'harmful', 'annoy', 'damaging', 'perceptibly', 'deformed', 'infrequently', 'bemoan', 'faulty', 'fail', 'hardhearted', 'grim', 'sparsely', 'angry', 'gawky', 'abysmal', 'somewhat', 'evil', 'imperceptibly', 'coldhearted', 'sporadically', 'cold', 'decaying', 'fear', 'eroding', 'clumsy', 'rarely', 'adverse', 'gradually', 'slightly', 'scantly', 'just', 'damage', 'anxious', 'haggard', 'cant', 'barbed', 'little', 'comparatively', 'belligerent', 'dastardly', 'detectably', 'alarming', 'seldom', 'beneath']<jupyter_output><empty_output><jupyter_text>Heuristic Bayes<jupyter_code>class Heuristic_Bayes(Naive_Bayes):
    def __init__(self,alpha=1.5,fit_prior=True,class_prior=None,negation_range=1,enhance_range=3,
                 enhance_coef=5,negative_words = negative_words,positive_words = positive_words,
                 negations = negation_words,enhances = enhance_words):
        self.alpha = alpha
        self.fit_prior = fit_prior
        self.class_prior_array = class_prior
        self.negation_range = negation_range
        self.enhance_range = enhance_range
        self.enhance_coef = enhance_coef
        self.negations = negations
        self.positive_words = positive_words
        self.negative_words = negative_words
        self.enhances = enhances
        if class_prior:
            self.fit_prior = False
    
    def __word_class_log_prob(self,word,was_neg=False,was_enh=False):
        counts = {class_ : self.class_words_probs[class_][word] for class_ in self.classes}
        c1 = self.class_words_probs['1'][word]
        c10 = self.class_words_probs['10'][word]
        if word in self.positive_words:
            counts['10'],counts['1'] = max(c1,c10),min(c1,c10)
        if word in self.negative_words:
            counts['1'],counts['10'] = max(c1,c10),min(c1,c10)
        if was_neg:
            counts['1'],counts['10'] = counts['10'],counts['1'] 
        if was_enh:
            counts['1'],counts['10'] = self.enhance_coef*counts['1'],self.enhance_coef*counts['10']
        return counts
            
    def get_class_log_probabilities(self,text):
        probs = {class_ : 0 for class_ in self.classes}
        text_words = text.split()
        
        for class_ in self.classes:
            enh_ctz = 0
            neg_ctz = 0
            for i,word in enumerate(text_words):
                enh_ctz = (enh_ctz - 1) if enh_ctz>0 else 0
                neg_ctz = (neg_ctz - 1) if neg_ctz>0 else 0
                
                probs[class_] += self.__word_class_log_prob(word,was_neg=(neg_ctz>0),was_enh=(enh_ctz>0))[class_]
                probs[class_] -= self.class_words_amount[class_]
                
                if word in self.negations:
                    neg_ctz = self.negation_range+1
                if word in self.enhances:
                    enh_ctz = self.enhance_range+1
            
            probs[class_] += self.class_prior[class_]
        return probs
X_train,y_train = np.array(swr_train_df['text']),np.array(swr_train_df['rating'])
X_test,y_test = np.array(swr_test_df['text']),np.array(swr_test_df['rating'])<jupyter_output>test acc 0.9032032731264344<jupyter_text>Testing various parameters<jupyter_code>for alpha in [1.0,1.25,1.5,1.75]:
    print(f'alpha {alpha}:')
    for ec in [2,3,4,5]:
        print(f'    enhance coef {ec}:')
        for er in [1,2,3]:
            HB = Heuristic_Bayes(fit_prior = False,alpha=alpha,negation_range = 1,
                                 enhance_coef=ec,enhance_range=er,negative_words = negative_words,
                                enhances=enhance_words,negations=negation_words,positive_words=positive_words)
            HB.fit(X_train,y_train)
            predictions,ppb = HB.predict(X_test,return_probabilities=True)
            acc = np.mean(predictions == y_test)
            print(f'        enhance range {er} test acc {acc}')<jupyter_output>alpha 1.0:
    enhance coef 2:
        enhance range 1 test acc 0.9020057878455244
        enhance range 2 test acc 0.9023051591657519
        enhance range 3 test acc 0.9029039018062069
    enhance coef 3:
        enhance range 1 test acc 0.9025047400459036
        enhance range 2 test acc 0.9030036922462829
        enhance range 3 test acc 0.9029039018062069
    enhance coef 4:
        enhance range 1 test acc 0.9028041113661311
        enhance range 2 test acc 0.9032032731264344
        enhance range 3 test acc 0.9036024348867379
    enhance coef 5:
        enhance range 1 test acc 0.9028041113661311
        enhance range 2 test acc 0.9035026444466621
        enhance range 3 test acc 0.9039018062069654
alpha 1.25:
    enhance coef 2:
        enhance range 1 test acc 0.9023051591657519
        enhance range 2 test acc 0.9026045304859794
        enhance range 3 test acc 0.9027043209260552
    enhance coef 3:
        enhance range 1 test acc 0.9026045304859794
        enhance range 2 t[...]<jupyter_text>Train accuracy<jupyter_code>X_train,y_train = np.array(bayes_df_train['text']),np.array(bayes_df_train['rating'])
X_test,y_test = np.array(bayes_df_test['text']),np.array(bayes_df_test['rating'])
HBc_res = []
alpha = 1.5
HB = Heuristic_Bayes(fit_prior = False,alpha=alpha)
HB.fit(X_train,y_train)
predictions,ppb = HB.predict(X_train,return_probabilities=True)
M,N,acc = print_score(predictions,y_train,f"TRAIN, alpha : {alpha}")
HBc_res.append(['Org train data\na=1.5','Acc',acc])
HBc_res.append(['Org train data\na=1.5','FN',M[0][1]/N])
HBc_res.append(['Org train data\na=1.5','FP',M[1][0]/N])
predictions,ppb = HB.predict(X_test,return_probabilities=True)
M,N,acc = print_score(predictions,y_test,f"TEST, alpha : {alpha}")
HBc_res.append(['Org test data\na=1.5','Acc',acc])
HBc_res.append(['Org test data\na=1.5','FN',M[0][1]/N])
HBc_res.append(['Org test data\na=1.5','FP',M[1][0]/N])<jupyter_output>TEST, alpha : 1.5
Acc: 0.8889332401955893

Confusion matrix:
[[4582  673]
 [ 440 4326]]

True negative (rating = 1): 4582
True positive (rating = 10): 4326
False negative: 673
False positive: 440<jupyter_text>Stemming and stop words removing<jupyter_code># get best alpha
alpha = 1.5
HB = Heuristic_Bayes(fit_prior = False,alpha=alpha)
HB.fit(np.array(stemmed_train_df['text']),np.array(stemmed_train_df['rating']))
predictions = HB.predict(np.array(stemmed_test_df['text']))
M,N,acc = print_score(predictions,np.array(stemmed_test_df['rating']),
                      f"TEST, alpha : {alpha}, stemmed")
HBc_res.append(['Stemmed test data\na=1.5','Acc',acc])
HBc_res.append(['Stemmed test data\na=1.5','FN',M[0][1]/N])
HBc_res.append(['Stemmed test data\na=1.5','FP',M[1][0]/N])
HB = Heuristic_Bayes(fit_prior = False)
HB.fit(np.array(swr_train_df['text']),np.array(swr_train_df['rating']))
predictions = HB.predict(np.array(swr_test_df['text']))
M,N,acc = print_score(predictions,np.array(swr_test_df['rating']),
                      f"TEST, alpha : {alpha}, stop words removed")
HBc_res.append(['Test data\nwithout stopwords\na=1.5','Acc',acc])
HBc_res.append(['Test data\nwithout stopwords\na=1.5','FN',M[0][1]/N])
HBc_res.append(['Test data\nwithout stopwords\na=1.5','FP',M[1][0]/N])
HB = Heuristic_Bayes(fit_prior = False,alpha=alpha)
HB.fit(np.array(stemmed_swr_train_df['text']),np.array(stemmed_swr_train_df['rating']))
predictions = HB.predict(np.array(stemmed_swr_test_df['text']))
M,N,acc = print_score(predictions,np.array(stemmed_test_df['rating']),
                      f"TEST, alpha : {alpha}, stemmed and stop words removed")
HBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','Acc',acc])
HBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','FN',M[0][1]/N])
HBc_res.append(['Stemmed test data\nwithout stop words\na=1.5','FP',M[1][0]/N])
HBc_res_df = pd.DataFrame(HBc_res,columns = ['Data','Y','Value'])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=HBc_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title('Our Heuristic Bayes')
autolabel(ax.patches)<jupyter_output><empty_output><jupyter_text>Compare with our Naive Bayes<jupyter_code>sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=NBc_res_df)
ax.figure.set_size_inches(15, 5)
ax.set_title('Our Naive Bayes')
autolabel(ax.patches)
NBc_res_df_copyed = HBc_res_df.copy()
A_df = HBc_res_df[HBc_res_df.Y == 'Acc']
A_df.Y = ['Heuristic'] * 5
NBc_res_df_copyed = NBc_res_df.copy()
B_df = NBc_res_df[NBc_res_df.Y == 'Acc']
B_df.Y = ['Nive'] * 5
compare_df = pd.concat([A_df,B_df])
sns.set(style="darkgrid")
ax = sns.barplot(x='Data', y='Value', hue='Y', data=compare_df)
ax.figure.set_size_inches(15, 5)
ax.set_title("Heuristic Bayes class vs Naive Bayes class")
ax.legend(loc='lower right')
autolabel(ax.patches)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15587243231667053, 'fraction_numerical': 0.027680117073867184, 'mean_word_length': 4.461935861195048, 'pattern_counts': {'":': 0, '<': 138, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '36', 'dir': 'jupyter-structured-clean-dedup', 'id': '550536', 'license': 'MIT', 'n_tokens_mistral': 23097, 'n_tokens_neox': 21620, 'n_words': 3765, 'path': 'main.ipynb', 'repo_name': 'TheFebrin/Sentiment-Analysis', 'size': '56033'}