<jupyter_start><jupyter_text>Five-classes sentiment analysis using Naive Bayes (Categorical Classification approach)<jupyter_code># for visualization
import seaborn as sns
import matplotlib.pyplot as plt

%matplotlib inline
%config InlineBackend.figure_format='retina'

sns.set(style='whitegrid', palette='muted', font_scale=1.2)<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>import pandas as pd
reviews_dataset = pd.read_csv('reviews_preprocessed.csv')
reviews_dataset['content'] = reviews_dataset['content'].astype(str)
print(reviews_dataset.isna().sum())
len(reviews_dataset)
#reviews_dataset = reviews_dataset[reviews_dataset["word_count"]>2]
print("1 star rating count: " ,len(reviews_dataset[reviews_dataset["score"] == 0]),"\n")
print("2 stars rating count: ",len(reviews_dataset[reviews_dataset["score"] == 1]),"\n")
print("3 stars rating count: ",len(reviews_dataset[reviews_dataset["score"] == 2]),"\n")
print("4 stars rating count: ",len(reviews_dataset[reviews_dataset["score"] == 3]),"\n")
print("5 stars rating count: ",len(reviews_dataset[reviews_dataset["score"] == 4]),"\n")
sns.countplot(reviews_dataset.score)
plt.xlabel('review score');
reviews_dataset.head()
from sklearn.model_selection import train_test_split
# Split into training and testing data
x = reviews_dataset['content']
y = reviews_dataset['score']
x, x_test, y, y_test = train_test_split(x,y, stratify=y, test_size=0.16, random_state=42)<jupyter_output><empty_output><jupyter_text>The model<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
params={ 
        'count_vect__max_df': [0.2,0.4,0.6,0.8,1.0],
        'count_vect__min_df': [0.0,0.02,0.05,0.1,0.2],
        'count_vect__ngram_range':[(1,1),(1, 2)],
        'clf__alpha':[.01,.5,1.0],
        'clf__fit_prior':[True,False]
       }

classifier_pipline = Pipeline(
                      [
                       ('count_vect', CountVectorizer()), 
# if slow try to count the number of features extracted by count vectorizer and do feature selection
#                      ('reducer', SelectKBest(chi2, k=3000)),
                       ('clf', MultinomialNB())
                      ]
                    )
from sklearn.model_selection import GridSearchCV
grid = GridSearchCV(classifier_pipline, cv=5, n_jobs=-1, param_grid=params ,scoring='accuracy', verbose = 3, refit = True)
grid
grid.fit(x, y)
import pickle
# save the model to disk
filename = 'Naive_Bayes.sav'
pickle.dump(grid, open(filename, 'wb'))
# some time later...
filename = 'Naive_Bayes.sav'
# load the model from disk
grid = pickle.load(open(filename, 'rb'))
result = grid.score(x_test, y_test)
print(result)<jupyter_output>0.46291425565491845<jupyter_text>Accuracy<jupyter_code>best_model= grid.best_estimator_
best_model
best_model["clf"].class_count_
best_model["clf"].classes_
grid.best_score_
grid.best_params_
score = grid.score(x_test, y_test)
score
y_pred = grid.predict(x_test)
y_pred
from sklearn.metrics import accuracy_score
accuracy = accuracy_score(y_test, y_pred)
accuracy
y_pred_proba = grid.predict_proba(x_test)
print(y_pred_proba)
#grid.predict(vec.transform(['Love this app simply awesome!']))
grid.predict(['Love this app simply awesome!'])
# https://www.analyticsvidhya.com/blog/2021/07/performing-sentiment-analysis-with-naive-bayes-classifier/
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.MultinomialNB.html
# https://www.dataquest.io/blog/naive-bayes-tutorial/
# https://medium.com/analytics-vidhya/naive-bayes-classifier-for-text-classification-556fabaf252b<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>#from sklearn.model_selection import cross_val_predict
#y_train_pred = cross_val_predict(grid, x, y, cv=3)
from sklearn.metrics import confusion_matrix
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib import rc
from pylab import rcParams

%matplotlib inline
%config InlineBackend.figure_format='retina'
sns.set(style='whitegrid', palette='muted', font_scale=1.2)
HAPPY_COLORS_PALETTE = ["#01BEFE", "#FFDD00", "#FF7D00", "#FF006D", "#ADFF02", "#8F00FF"]
sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))
rcParams['figure.figsize'] = 12, 8
def show_confusion_matrix(confusion_matrix):
  hmap = sns.heatmap(confusion_matrix, annot=True, fmt="f", cmap="Blues")
  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')
  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')
  plt.ylabel('True sentiment')
  plt.xlabel('Predicted sentiment');

class_names = ['strongly_neg','neg','neutral','pos','strongly_pos']
conf_mx = confusion_matrix(y_test, y_pred)
conf_mx
df_cm = pd.DataFrame(conf_mx, index=class_names, columns=class_names)
import numpy as np
row_sums = conf_mx.sum(axis=1, keepdims=True)
print(row_sums)
norm_conf_mx = conf_mx / row_sums
norm_conf_mx
df_norm_cm = pd.DataFrame(norm_conf_mx, index=class_names, columns=class_names)
show_confusion_matrix(df_norm_cm)
import numpy as np
np.fill_diagonal(norm_conf_mx, 0)
df_norm_cm = pd.DataFrame(norm_conf_mx, index=class_names, columns=class_names)
show_confusion_matrix(df_norm_cm)<jupyter_output><empty_output><jupyter_text>Precison / Recall<jupyter_code>from sklearn.preprocessing import label_binarize
# Use label_binarize to be multi-label like settings
y_test_mult = label_binarize(y_test, classes=[0, 1, 2, 3, 4])
print(y_test_mult)
print(y_pred_proba)

n_classes = y_test_mult.shape[1]
n_classes
print(len(y_test_mult))
print(len(y_pred_proba))
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
# For each class
precision = dict()
recall = dict()
average_precision = dict()
for i in range(n_classes):
    precision[i], recall[i], _ = precision_recall_curve(y_test_mult[:, i],
                                                        y_pred_proba[:, i])
    
    curr_precision = precision[i]
    curr_recall = recall[i]

    precision[i] = precision[i][curr_precision+curr_recall > 0.0]
    recall[i] = recall[i][curr_precision+curr_recall > 0.0]

    average_precision[i] = average_precision_score(y_test_mult[:, i], y_pred_proba[:, i])
# A "macro-average": Averaging over classes
# ====> Cannot do macro-average by the below approach because:
# len(precision[i]) depend on i (the unique probas in column i)
# the thresholds that based on it the precision & recall are calculated is not the same at the corresponding indices
#precision_sum = precision[0] + precision[1] + precision[2] + precision[3] + precision[4]
#precision["macro"] = precision_sum / 5
#recall_sum = recall[0] + recall[1] + recall[2] + recall[3] + recall[4]
#recall["macro"] = recall_sum / 5

# A "micro-average": quantifying score on all classes jointly
precision["micro"], recall["micro"], _ = precision_recall_curve(y_test_mult.ravel(),
    y_pred_proba.ravel())


# compute F1 score
f1_score = dict()
# for each class
for i in range(n_classes):
  f1_score[i] = 2*((precision[i]*recall[i])/(precision[i]+recall[i]))

f1_score["micro"] = 2*((precision["micro"]*recall["micro"])/(precision["micro"]+recall["micro"]))
average_precision["micro"] = average_precision_score(y_test_mult, y_pred_proba,
                                                     average="micro")
average_precision["macro"] = average_precision_score(y_test_mult, y_pred_proba,
                                                     average="macro")
print('Average precision score, micro-averaged over all classes: {0:0.2f}'
      .format(average_precision["micro"]))
print('Average precision score, macro-averaged over all classes: {0:0.2f}'
      .format(average_precision["macro"]))<jupyter_output>Average precision score, micro-averaged over all classes: 0.49
Average precision score, macro-averaged over all classes: 0.47<jupyter_text>Plot the micro-averaged Precision-Recall curve<jupyter_code>plt.figure()
plt.step(recall['micro'], precision['micro'], where='post')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title(
    'Average precision score, micro-averaged over all classes: AP={0:0.2f}'
    .format(average_precision["micro"]))<jupyter_output><empty_output><jupyter_text>Plot Precision-Recall curve for each class and iso-f1 curves<jupyter_code>from itertools import cycle
# setup plot details
colors = cycle(['navy', 'turquoise', 'darkorange', 'cornflowerblue', 'teal'])

plt.figure(figsize=(12, 8))
f_scores = np.linspace(0.2, 0.8, num=4)
lines = []
labels = []
for f_score in f_scores:
    x = np.linspace(0.01, 1)
    y = f_score * x / (2 * x - f_score)
    l, = plt.plot(x[y >= 0], y[y >= 0], color='gray', alpha=0.2)
    plt.annotate('f1={0:0.1f}'.format(f_score), xy=(0.9, y[45] + 0.02))

lines.append(l)
labels.append('iso-f1 curves')
l, = plt.plot(recall["micro"], precision["micro"], color='gold', lw=2)
lines.append(l)
labels.append('micro-average Precision-recall (area = {0:0.2f})'
              ''.format(average_precision["micro"]))

for i, color in zip(range(n_classes), colors):
    l, = plt.plot(recall[i], precision[i], color=color, lw=2)
    lines.append(l)
    labels.append('Precision-recall for class {0} (area = {1:0.2f})'
                  ''.format(i, average_precision[i]))

fig = plt.gcf()
fig.subplots_adjust(bottom=0.25)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Extension of Precision-Recall curve to multi-class')
plt.legend(lines, labels, loc=(0, -.38), prop=dict(size=14))


plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14364468483461618, 'fraction_numerical': 0.022987310172664865, 'mean_word_length': 5.470390309555855, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '252076', 'license': 'MIT', 'n_tokens_mistral': 3614, 'n_tokens_neox': 3391, 'n_words': 653, 'path': 'NB_Nominal.ipynb', 'repo_name': 'omar73H/Sentiment-Analysis', 'size': '9614'}