<jupyter_start><jupyter_text>Gensim Word2Vec Embeddings The goal for this analysis is to predict if a review rates the movie positively or negatively. Inside this dataset there are 25,000 labeled movies reviews for training, 50,000 unlabeled reviews for training, and 25,000 reviews for testing.- IMDB movie reviews dataset- http://ai.stanford.edu/~amaas/data/sentiment- https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews- Contains 25000 positive and 25000 negative reviews- Contains at most reviews per movie- At least 7 stars out of 10 $\rightarrow$ positive (label = 1)- At most 4 stars out of 10 $\rightarrow$ negative (label = 0)> **Here, we use Gensim Word2Vec vectors to train a bag of centroids model to train machine learning models on.**  Dependecies<jupyter_code>import os
import re
import nltk
import time
import logging  
import unicodedata
import numpy as np
import pandas as pd
import seaborn as sns
import multiprocessing
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from collections import defaultdict
from nltk.stem import SnowballStemmer
from nltk.tokenize import word_tokenize, sent_tokenize
from sklearn import preprocessing
from gensim.models import word2vec
from sklearn.cluster import KMeans
from sklearn.ensemble import VotingClassifier as vc
from sklearn.model_selection import cross_val_score
from sklearn.svm import LinearSVC as svc
from sklearn.linear_model import LogisticRegression as lr
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import roc_auc_score, accuracy_score, confusion_matrix
nltk.download('stopwords')
import warnings
warnings.filterwarnings('ignore')
# read from local
# movies = pd.read_csv('data/imdb_data.csv')
# movies.sample(7)
# for importing data to colab
from google.colab import drive 
drive.mount('/content/drive')

movies = pd.read_csv('/content/drive/My Drive/Colab Notebooks/imdb_data.csv')
movies.sample(7)
# Cateogrize positive and negative as 1 and 0 respectively

label_encoder = preprocessing.LabelEncoder()
movies['sentiment'] = label_encoder.fit_transform(movies['sentiment'])
movies.head()
X = movies['review']
y = (np.array(movies['sentiment']))

# Here we split data to training and testing parts
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=7)
print(f"Train dataset shape: {X_train.shape}, \nTest dataset shape: {X_test.shape}")
print<jupyter_output>Train dataset shape: (40000,), 
Test dataset shape: (10000,)<jupyter_text>Preprocessing<jupyter_code>!pip install contractions
!pip install textsearch
import contractions
def review_to_wordlist(review, custom = True, stem_words = True):
    # Clean the text, with the option to remove stopwords and stem words.
    
    # Strip html
    soup = BeautifulSoup(review, "html.parser")
    [s.extract() for s in soup(['iframe', 'script'])]
    stripped_text = soup.get_text()
    review_text = re.sub(r'[\r|\n|\r\n]+', '\n', stripped_text)
    
    # replace accents
    review_text = unicodedata.normalize('NFKD', review_text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
    review_text = re.sub(r"[^A-Za-z0-9!?\'\`]", " ", review_text) # remove special characters
    review_text = contractions.fix(review_text) # expand contractions
    
    review_text = review_text.lower()
    words = review_text.split()
    if custom:
        stop_words = set(nltk.corpus.stopwords.words('english'))
        stop_words.update(['movie', 'film', 'one', 'would', 'even', 
                           'movies', 'films', 'cinema',
                           'character', 'show', "'", "!", 'like'])
    else:
        stop_words = set(nltk.corpus.stopwords.words('english'))
    words = [w for w in words if not w in stop_words]
    review_text = " ".join(words)
        
    review_text = re.sub(r"!", " ! ", review_text)
    review_text = re.sub(r"\?", " ? ", review_text)
    review_text = re.sub(r"\s{2,}", " ", review_text)
    
    if stem_words:
        words = review_text.split()
        stemmer = SnowballStemmer('english')
        stemmed_words = [stemmer.stem(word) for word in words]
        review_text = " ".join(stemmed_words)
    
    # Return a list of words, with each word as its own string
    return review_text.split()
def review_to_sentences(review):
    # Split a review into parsed sentences
    
    raw_sentences = sent_tokenize(review.strip())
    
    sentences = []
    for raw_sentence in raw_sentences:
        if len(raw_sentence) > 0:
            sentences.append(review_to_wordlist(raw_sentence))
    
    # Each sentence is a list of words, so this returns a list of lists
    return sentences
sentences = [] 

print ("Parsing sentences ...")
for review in movies['review']:
    sentences += review_to_sentences(review)
# Check how many sentences we have in total 
print (len(sentences))
print()
print (sentences[0])
print()
print (sentences[1])
# Import the built-in logging module and configure it so that Word2Vec 
# creates nice output messages
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

num_features = 300      # Word vector dimensionality                      
min_word_count = 5      # Minimum word count                        
num_workers = 1         # Number of threads to run in parallel
context = 20            # Context window size                                                                                    
downsampling = 1e-4     # Downsample setting for frequent words


# Initialize and train the model
print ("Training model...")
model = word2vec.Word2Vec(sentences, 
                          workers = num_workers,
                          size = num_features,
                          min_count = min_word_count,
                          window = context, 
                          sample = downsampling)

# Call init_sims because we won't train the model any further 
# This will make the model much more memory-efficient.
model.init_sims(replace=True)

# save the model for potential, future use.
model_name = "{}features_{}minwords_{}context".format(num_features,min_word_count,context)
model.save(model_name)
# Load the model, if necessary
# model = Word2Vec.load("300features_5minwords_20context") 
model.most_similar("great")
model.most_similar("stori")
model.most_similar("bad")
model.wv.syn0.shape<jupyter_output><empty_output><jupyter_text>Bag of Centroids<jupyter_code># Set "k" (num_clusters) to be 1/5th of the vocabulary size, or an
# average of 5 words per cluster
word_vectors = model.wv.syn0 
num_clusters = int(word_vectors.shape[0] / 5)

# Initalize a k-means object and use it to extract centroids
kmeans_clustering = KMeans(n_clusters = num_clusters,
                           n_init = 5,
                           verbose = 2)
idx = kmeans_clustering.fit_predict(word_vectors)
# Create a Word / Index dictionary, mapping each vocabulary word to a cluster number                                                                                            
word_centroid_map = dict(zip(model.wv.index2word, idx))
X = movies['review']
y = movies['sentiment']
# (np.array(movies['sentiment']))

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=7)
print(f"Train dataset shape: {X_train.shape}, \nTest dataset shape: {X_test.shape}")
# Clean the training and testing reviews.
clean_train_reviews = []
for review in X_train:
    clean_train_reviews.append(review_to_wordlist(review))
    
print("Training reviews are clean")  

clean_test_reviews = []
for review in X_test:
    clean_test_reviews.append(review_to_wordlist(review))
    
print("Testing reviews are clean") 
def create_bag_of_centroids(wordlist, word_centroid_map):    
    # The number of clusters is equal to the highest cluster index
    # in the word / centroid map
    num_centroids = max(word_centroid_map.values()) + 1
    
    # Pre-allocate the bag of centroids vector (for speed)
    bag_of_centroids = np.zeros(num_centroids, dtype="float32")
    
    # Loop over the words in the review. If the word is in the vocabulary,
    # find which cluster it belongs to, and increment that cluster count 
    for word in wordlist:
        if word in word_centroid_map:
            index = word_centroid_map[word]
            bag_of_centroids[index] += 1
    
    return bag_of_centroids
# Pre-allocate an array for the training set bags of centroids (for speed)
train_centroids = np.zeros((X_train.size, num_clusters), dtype="float32")

# Transform the training set reviews into bags of centroids
counter = 0
for review in clean_train_reviews:
    train_centroids[counter] = create_bag_of_centroids(review, word_centroid_map)
    counter += 1

print("Training reviews are complete.")    
# Repeat for test reviews 
test_centroids = np.zeros((X_test.size, num_clusters), dtype="float32")

counter = 0
for review in clean_test_reviews:
    test_centroids[counter] = create_bag_of_centroids(review, word_centroid_map )
    counter += 1
    
print("Testing reviews are complete.")  
print(f"Train centroids shape: {train_centroids.shape}, \nTest centroids shape: {test_centroids.shape}")<jupyter_output>Train centroids shape: (37500, 5358), 
Test centroids shape: (12500, 5358)<jupyter_text>Modeling<jupyter_code>def plot_confusion_matrix(y_true, y_pred, ax, class_names  = ['Positive', 'Negative'], vmax=None,
                           normalized=True, title='Confusion matrix'):
    """
    Helper fuction to generate a clean Confusion Matrix using seaborn library.
    y_true: True labels, y_pred: Model Predictions, class_names: Override if needed
    normalized: True, gives the proportions instead of absolute numbers
    """
    matrix = confusion_matrix(y_true,y_pred)
    if normalized:
        matrix = matrix.astype('float') / matrix.sum(axis=1)[:, np.newaxis]
    annot_kws = {'fontsize':25, 
                'fontstyle': 'italic'}
    sns.heatmap(matrix, vmax=vmax, annot=True,  annot_kws = annot_kws,
                square=True, ax=ax, cbar=False,
                cmap=sns.diverging_palette(20, 250, as_cmap=True), 
                linecolor='black', linewidths=0.5,
                xticklabels=class_names)
    ax.set_title(title, y=1.20, fontsize=16)
    ax.set_ylabel('True labels', fontsize=12)
    ax.set_xlabel('Predicted labels', y=1.10, fontsize=12)
    ax.set_yticklabels(class_names, rotation=0)
lr_model = lr(C = 0.01,
              max_iter = 6,
              fit_intercept = True)

lr_model.fit(train_centroids, y_train)
lr_pred = lr_model.predict(test_centroids)
print("Test set Accuracy: ", accuracy_score(lr_pred, y_test)*100)
fig, axis1 = plt.subplots(nrows=1, ncols=1)
plot_confusion_matrix(y_test, lr_pred, ax=axis1,
                       title='Confusion matrix (lr)')
svc_model = svc()

svc_model.fit(train_centroids, y_train)
svc_pred = svc_model.predict(test_centroids)
print("Test set Accuracy: ", accuracy_score(svc_pred, y_test)*100)
fig, axis1 = plt.subplots(nrows=1, ncols=1)
plot_confusion_matrix(y_test, svc_pred, ax=axis1,
                       title='Confusion matrix (svc)')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09951586874663798, 'fraction_numerical': 0.015151515151515152, 'mean_word_length': 3.545639771801141, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '38268', 'license': 'MIT', 'n_tokens_mistral': 3647, 'n_tokens_neox': 3474, 'n_words': 959, 'path': 'word2vec_bag_of_centroids_predictions.ipynb', 'repo_name': 'SuryaPradeepM/Comprehensive-Sentiment-Analysis-of-Movie-Reviews-', 'size': '11154'}