<jupyter_start><jupyter_text>Sentiment Analysis of IMDB Movie Reviews  Problem Statement:* In this, we have to predict the number of positive and negative reviews based on sentiments by using different classification models. * IMDB Dataset having 50K Movie Reviews for Text Analytics.  Importing Libraries<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import nltk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from wordcloud import WordCloud,STOPWORDS
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize,sent_tokenize
from bs4 import BeautifulSoup
import spacy
import re,string,unicodedata
from nltk.tokenize.toktok import ToktokTokenizer
from nltk.stem import LancasterStemmer,WordNetLemmatizer
from sklearn.linear_model import LogisticRegression,SGDClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC
from textblob import TextBlob
from textblob import Word
from sklearn.metrics import classification_report,confusion_matrix,accuracy_score

import warnings
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Reading the Dataset<jupyter_code>imdb_data = pd.read_csv("IMDB Dataset.csv")
imdb_data.head()
imdb_data.shape<jupyter_output><empty_output><jupyter_text>Exploratery Data Analysis<jupyter_code>imdb_data.info()
imdb_data.describe().T
imdb_data['sentiment'].value_counts()<jupyter_output><empty_output><jupyter_text>* We have balanced Data (50% Positive & 50% Negative) * Dataset also don't have any missing values.  Spliting Data into Train & Test<jupyter_code>Train_reviews    = imdb_data.review[:40000]
Train_sentiments = imdb_data.sentiment[:40000]

Test_reviews     = imdb_data.review[40000:]
Test_sentiments  = imdb_data.sentiment[40000:]
print(" Train Dataset Shape : " ,Train_reviews.shape, Train_sentiments.shape)
print(" Test  Dataset Shape : " ,Test_reviews.shape, Test_sentiments.shape)<jupyter_output>Train Dataset Shape :  (40000,) (40000,)
 Test  Dataset Shape :  (10000,) (10000,)<jupyter_text>PreProcessing of Text  Text Normalization Tokenization<jupyter_code>tokenizer = ToktokTokenizer()

stopword_list = nltk.corpus.stopwords.words('english')<jupyter_output><empty_output><jupyter_text>Removing Html Strips & Noise Text<jupyter_code>#Removing the html strips
def strip_html(text):
    soup = BeautifulSoup(text, "html.parser")
    return soup.get_text()


#Removing the square brackets
def remove_between_square_brackets(text):
    return re.sub('\[[^]]*\]', '', text)


#Removing the noisy text
def denoise_text(text):
    text = strip_html(text)
    text = remove_between_square_brackets(text)
    return text

#Apply function on review column
imdb_data['review'] = imdb_data['review'].apply(denoise_text)<jupyter_output><empty_output><jupyter_text>Removing Special Characters<jupyter_code>def remove_special_characters(text, remove_digits = True):
    pattern = r'[^a-zA-z0-9\s]'
    text = re.sub(pattern,'',text)
    return text


#Apply function on review column
imdb_data['review'] = imdb_data['review'].apply(remove_special_characters)<jupyter_output><empty_output><jupyter_text>Text Stemming<jupyter_code>def simple_stemmer(text):
    ps = nltk.porter.PorterStemmer()
    text = ' '.join([ps.stem(word) for word in text.split()])
    return text


imdb_data['review'] = imdb_data['review'].apply(simple_stemmer)<jupyter_output><empty_output><jupyter_text>Removing Stopwords<jupyter_code>stop = set(stopwords.words('english'))

def remove_stopwords(text, is_lower_case = False):
    tokens = tokenizer.tokenize(text)
    tokens = [token.strip() for token in tokens]
    if is_lower_case:
        filtered_tokens = [token for token in tokens if token not in stopword_list]
    else:
        filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]
    filtered_text = ' '.join(filtered_tokens)    
    return filtered_text


imdb_data['review'] = imdb_data['review'].apply(remove_stopwords)<jupyter_output><empty_output><jupyter_text>Normalized Train Reviews<jupyter_code>norm_train_reviews = imdb_data.review[:40000]
norm_train_reviews[0]<jupyter_output><empty_output><jupyter_text>Normalized Test Reviews<jupyter_code>norm_test_reviews = imdb_data.review[40000:]
norm_test_reviews[45005]<jupyter_output><empty_output><jupyter_text>Bags of Words Model * It is used to convert text documents to numerical vectors or bag of words.<jupyter_code>cv = CountVectorizer(min_df = 0, max_df = 1, binary = False, ngram_range = (1,3))

cv_train_reviews = cv.fit_transform(norm_train_reviews)

cv_test_reviews = cv.transform(norm_test_reviews)

print('Bags of Words Model_cv_train:', cv_train_reviews.shape)
print('Bags of Words Model_cv_test :', cv_test_reviews.shape)<jupyter_output>Bags of Words Model_cv_train: (40000, 6209089)
Bags of Words Model_cv_test : (10000, 6209089)<jupyter_text>TF-IDF usage* Countvectorizer gives equal weightage to all the words, i.e. a word is converted to a column.* TF-IDF is different from countvectorizer.* vocabulary_ just gives a dict of indexes of the words.<jupyter_code>tv = TfidfVectorizer(min_df = 0, max_df = 1, use_idf = True, ngram_range = (1,3))

tv_train_reviews = tv.fit_transform(norm_train_reviews)

tv_test_reviews = tv.transform(norm_test_reviews)

print('Tfidf_train:',tv_train_reviews.shape)
print('Tfidf_test:',tv_test_reviews.shape)<jupyter_output>Tfidf_train: (40000, 6209089)
Tfidf_test: (10000, 6209089)<jupyter_text>Labeling the Text<jupyter_code>lb = LabelBinarizer()

sentiment_data = lb.fit_transform(imdb_data['sentiment'])
print(sentiment_data.shape)<jupyter_output>(50000, 1)<jupyter_text>Spliting the Sentiment Data<jupyter_code>train_sentiments = sentiment_data[:40000]
test_sentiments  = sentiment_data[40000:]

print(train_sentiments)
print(test_sentiments)<jupyter_output>[[1]
 [1]
 [1]
 ...
 [1]
 [0]
 [0]]
[[0]
 [0]
 [0]
 ...
 [0]
 [0]
 [0]]<jupyter_text>Modelling  1. Logistic Regression<jupyter_code>lr = LogisticRegression(penalty = 'l2', max_iter = 500, C = 1, random_state = 42)

#Fitting the model for Bag of words
lr_bow = lr.fit(cv_train_reviews, train_sentiments)
print(lr_bow)

#Fitting the model for tfidf features
lr_tfidf = lr.fit(tv_train_reviews, train_sentiments)
print(lr_tfidf)<jupyter_output>LogisticRegression(C=1, max_iter=500, random_state=42)
LogisticRegression(C=1, max_iter=500, random_state=42)<jupyter_text>Model Performane on Test Dataset<jupyter_code>#Predicting the model for bag of words
lr_bow_predict = lr.predict(cv_test_reviews)
print(lr_bow_predict)

#Predicting the model for tfidf features
lr_tfidf_predict = lr.predict(tv_test_reviews)
print(lr_tfidf_predict)<jupyter_output>[0 0 0 ... 0 1 1]
[0 0 0 ... 0 1 1]<jupyter_text>Accuracy<jupyter_code>lr_bow_score = accuracy_score(test_sentiments,lr_bow_predict)
print("Accuarcy Score for Bags of Words Model :", lr_bow_score)

#Accuracy score for tfidf features
lr_tfidf_score = accuracy_score(test_sentiments,lr_tfidf_predict)
print("Accuarcy Score for TF-IDF              :",lr_tfidf_score)
print("\n Classification report for Bags of Words Model features\n")
lr_bow_report = classification_report(test_sentiments, lr_bow_predict, target_names = ['Positive','Negative'])
print(lr_bow_report)

print("\n Classification report for TF_IDF features\n")
lr_tfidf_report = classification_report(test_sentiments, lr_tfidf_predict, target_names = ['Positive','Negative'])
print(lr_tfidf_report)<jupyter_output>Classification report for Bags of Words Model features

              precision    recall  f1-score   support

    Positive       0.75      0.75      0.75      4993
    Negative       0.75      0.75      0.75      5007

    accuracy                           0.75     10000
   macro avg       0.75      0.75      0.75     10000
weighted avg       0.75      0.75      0.75     10000


 Classification report for TF_IDF features

              precision    recall  f1-score   support

    Positive       0.74      0.77      0.75      4993
    Negative       0.76      0.73      0.75      5007

    accuracy                           0.75     10000
   macro avg       0.75      0.75      0.75     10000
weighted avg       0.75      0.75      0.75     10000<jupyter_text>Confusion matrix<jupyter_code>cm_bow = confusion_matrix(test_sentiments, lr_bow_predict, labels = [1,0])
cm_tfidf = confusion_matrix(test_sentiments, lr_tfidf_predict, labels = [1,0])
sns.heatmap(pd.DataFrame(cm_bow), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for Bags of Words Model features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)
sns.heatmap(pd.DataFrame(cm_tfidf), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for TF_IDF features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)<jupyter_output><empty_output><jupyter_text>2. Stochastic Gradient Descent or Linear Support Vector Machines<jupyter_code>#training the linear svm
svm = SGDClassifier(loss = 'hinge', max_iter = 500, random_state = 42)

#fitting the svm for bag of words
svm_bow = svm.fit(cv_train_reviews, train_sentiments)
print(svm_bow)

#fitting the svm for tfidf features
svm_tfidf = svm.fit(tv_train_reviews, train_sentiments)
print(svm_tfidf)<jupyter_output>SGDClassifier(max_iter=500, random_state=42)
SGDClassifier(max_iter=500, random_state=42)<jupyter_text>Model Performane on Test Dataset<jupyter_code>svm_bow_predict = svm.predict(cv_test_reviews)
print(svm_bow_predict)

svm_tfidf_predict = svm.predict(tv_test_reviews)
print(svm_tfidf_predict)<jupyter_output>[1 1 0 ... 1 1 1]
[1 1 1 ... 1 1 1]<jupyter_text>Accuracy<jupyter_code>svm_bow_score = accuracy_score(test_sentiments, svm_bow_predict)
print("svm_bow_score :" ,svm_bow_score)

svm_tfidf_score = accuracy_score(test_sentiments, svm_tfidf_predict)
print("svm_tfidf_score :" ,svm_tfidf_score)
print("\n Classification report for Bags of Words Model features\n")
svm_bow_report = classification_report(test_sentiments, svm_bow_predict, target_names = ['Positive','Negative'])
print(svm_bow_report)

print("\n Classification report for TF_IDF features\n")
svm_tfidf_report = classification_report(test_sentiments, svm_tfidf_predict, target_names = ['Positive','Negative'])
print(svm_tfidf_report)
cm_bow = confusion_matrix(test_sentiments, svm_bow_predict, labels = [1,0])
cm_tfidf = confusion_matrix(test_sentiments, svm_tfidf_predict, labels = [1,0])
sns.heatmap(pd.DataFrame(cm_bow), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for Bags of Words Model features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)
sns.heatmap(pd.DataFrame(cm_tfidf), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for TF_IDF features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)<jupyter_output><empty_output><jupyter_text>3. Naive Bayes<jupyter_code>mnb = MultinomialNB()
mnb_bow = mnb.fit(cv_train_reviews, train_sentiments)
print(mnb_bow)

mnb_tfidf = mnb.fit(tv_train_reviews, train_sentiments)
print(mnb_tfidf)<jupyter_output>MultinomialNB()
MultinomialNB()<jupyter_text>Model Performane on Test Dataset<jupyter_code>mnb_bow_predict = mnb.predict(cv_test_reviews)
print(mnb_bow_predict)

mnb_tfidf_predict = mnb.predict(tv_test_reviews)
print(mnb_tfidf_predict)<jupyter_output>[0 0 0 ... 0 1 1]
[0 0 0 ... 0 1 1]<jupyter_text>Accuracy<jupyter_code>mnb_bow_score = accuracy_score(test_sentiments, mnb_bow_predict)
print("mnb_bow_score :" ,mnb_bow_score)

mnb_tfidf_score = accuracy_score(test_sentiments, mnb_tfidf_predict)
print("mnb_tfidf_score :",mnb_tfidf_score)
print("\n Classification report for Bags of Words Model features\n")
mnb_bow_report = classification_report(test_sentiments, mnb_bow_predict, target_names = ['Positive','Negative'])
print(mnb_bow_report)

print("\n Classification report for TF_IDF features\n")
mnb_tfidf_report = classification_report(test_sentiments, mnb_tfidf_predict, target_names = ['Positive','Negative'])
print(mnb_tfidf_report)
cm_bow = confusion_matrix(test_sentiments, mnb_bow_predict, labels = [1,0])
cm_tfidf = confusion_matrix(test_sentiments, mnb_tfidf_predict, labels = [1,0])
sns.heatmap(pd.DataFrame(cm_bow), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for Bags of Words Model features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)
sns.heatmap(pd.DataFrame(cm_tfidf), annot = True, cmap = "YlGnBu" , fmt = 'g')
plt.tight_layout()
plt.title('Confusion matrix for TF_IDF features', y = 1.1, fontsize = 18)
plt.ylabel('Actual label', fontsize = 16)
plt.xlabel('Predicted label', fontsize = 16)
from sklearn.metrics import roc_curve
lr_bow_FPR,lr_bow_TPR,lr_bow_Thresholds = roc_curve(test_sentiments, lr_bow_predict)

fpr_series = pd.Series(lr_bow_FPR)
tpr_series = pd.Series(lr_bow_TPR)
thresholds_series = pd.Series(lr_bow_Thresholds)

df = pd.concat([fpr_series,tpr_series,thresholds_series], axis = 1, keys = ['FPR','TPR','THRESHOLD'])
df.sort_values(by = 'TPR', ascending = False)
from sklearn.metrics import roc_curve

df = pd.concat([fpr_series,tpr_series,thresholds_series], axis = 1, keys = ['FPR','TPR','THRESHOLD'])
lr_bow_FPR,lr_bow_TPR,lr_bow_Thresholds = roc_curve(test_sentiments, lr_bow_predict)
lr_tfidf_FPR,lr_tfidf_TPR,lr_tfidf_Thresholds = roc_curve(test_sentiments, lr_tfidf_predict)

svm_bow_FPR,svm_bow_TPR,svm_bow_Thresholds = roc_curve(test_sentiments, svm_bow_predict)
svm_tfidf_FPR,svm_tfidf_TPR,svm_tfidf_Thresholds = roc_curve(test_sentiments, svm_tfidf_predict)

mnb_bow_FPR,mnb_bow_TPR,mnb_bowf_Thresholds = roc_curve(test_sentiments, mnb_bow_predict)
mnb_tfidf_FPR,mnb_tfidf_TPR,mnb_tfidf_Thresholds = roc_curve(test_sentiments, mnb_tfidf_predict)


sns.set_style("darkgrid")
plt.figure(figsize = (20,8))

# Plot Logistic Regression ROC
plt.plot(lr_bow_FPR,lr_bow_TPR, label ='Logistic Regression Bags of Words Model')
plt.plot(lr_tfidf_FPR,lr_tfidf_TPR, label ='Logistic Regression TF_IDF')

# Plot SVM ROC
plt.plot(svm_bow_FPR,svm_bow_TPR, label = 'SVM  Bags of Words Model' )
plt.plot(svm_tfidf_FPR,svm_tfidf_TPR, label = 'SVM TF_IDF' )

# Plot Naive Bayes ROC
plt.plot(mnb_bow_FPR,mnb_bow_TPR, label = 'Naive Bayes  Bags of Words Model' )
plt.plot(mnb_tfidf_FPR,mnb_tfidf_TPR, label = 'Naive Bayes TF_IDF')

# Plot Base Rate ROC
plt.plot([0,1], [0,1], label = 'Base Rate')

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate', fontsize = 20)
plt.ylabel('True Positive Rate', fontsize = 20)
plt.title('ROC Graph', fontsize = 20)
plt.legend(loc = "lower right")
plt.show()<jupyter_output><empty_output><jupyter_text>Word Cloud<jupyter_code>plt.figure(figsize = (10,10))
positive_text = norm_train_reviews[1]
WC = WordCloud(width = 1000, height = 500, max_words = 500, min_font_size = 5)
positive_words = WC.generate(positive_text)
plt.imshow(positive_words, interpolation = 'bilinear')
plt.title("\n Word Cloud for Positive Review Words \n", fontsize = 20)
plt.show
plt.figure(figsize = (10,10))
negative_text = norm_train_reviews[8]
WC = WordCloud(width = 1000, height = 500, max_words = 500, min_font_size = 5)
negative_words = WC.generate(negative_text)
plt.imshow(negative_words, interpolation = 'bilinear')
plt.title("\n Word Cloud for Negative Review Words \n", fontsize = 20)
plt.show<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10684516291992928, 'fraction_numerical': 0.02955291740338469, 'mean_word_length': 6.348955916473318, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '401547', 'license': 'MIT', 'n_tokens_mistral': 6358, 'n_tokens_neox': 5826, 'n_words': 1186, 'path': 'Sentiment Analysis - IMDB Movie Reviews.ipynb', 'repo_name': 'ShivankUdayawal/Sentiment-Analysis-of-IMDB-Movie-Reviews', 'size': '15836'}