<jupyter_start><jupyter_text>NLP: LDA & Sentiment Analysis on Movie Reviews<jupyter_code>import os
import pickle
use_pickled = input("Load pickled files/models? (y/n, defaults to y): ")
if use_pickled == 'n':
    use_pickled = False
else:
    use_pickled = True

pickled_dir = "pickled"

import pandas as pd

data = pd.read_csv('IMDB-Dataset.csv', error_bad_lines=False);

# split positive and negative sentiment reviews
pos_reviews = data[data.sentiment == "positive"]
neg_reviews = data[data.sentiment == "negative"]

pos_data = pos_reviews[['review']]
pos_data['index'] = pos_data.index
pos_documents = pos_data

neg_data = neg_reviews[['review']]
neg_data['index'] = neg_data.index
neg_documents = neg_data


# from nltk.corpus import stopwords
import re

# stop_words = list(set(stopwords.words('english')))

'''Positive Data'''
# Remove punctuation using regular expresssion
pos_documents['review_processed'] = pos_documents['review'].map(lambda x: re.sub('[,\.!?]', '', x))
# Lowercase the words using regular expresssion
pos_documents['review_processed'] = pos_documents['review'].map(lambda x: x.lower())
'''Negative Data'''
# Remove punctuation using regular expresssion
neg_documents['review_processed'] = neg_documents['review'].map(lambda x: re.sub('[,\.!?]', '', x))
# Lowercase the words using regular expresssion
neg_documents['review_processed'] = neg_documents['review'].map(lambda x: x.lower())

## Stemming and Lematizing
import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.stem.porter import *
import numpy as np
np.random.seed(2018)
import nltk

stemmer = SnowballStemmer("english")
def lemmatize_stemming(text):
    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))


def preprocess(text):
    result = []
    for token in gensim.utils.simple_preprocess(text):
        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:
            result.append(lemmatize_stemming(token))
    return result

pos_proc_docs_fname = os.path.join(pickled_dir, "pos_processed_docs")
neg_proc_docs_fname = os.path.join(pickled_dir, "neg_processed_docs")
if use_pickled:
    with open(pos_proc_docs_fname, 'rb') as f1:
        pos_processed_documents = pickle.load(f1)
    with open(neg_proc_docs_fname, 'rb') as f2:
        neg_processed_documents = pickle.load(f2)
else:
    pos_processed_documents = pos_documents['review_processed'].map(preprocess)
    neg_processed_documents = neg_documents['review_processed'].map(preprocess)
    with open(pos_proc_docs_fname, 'wb+') as f1:
        pickle.dump(pos_processed_documents, f1)
    with open(neg_proc_docs_fname, 'wb+') as f2:
        pickle.dump(neg_processed_documents, f2)
        
import gensim

# Making Positive and Negative Dictionaries
pos_dictionary = gensim.corpora.Dictionary(pos_processed_documents)
neg_dictionary = gensim.corpora.Dictionary(neg_processed_documents)        
#Removing Extreme Cases of Words
pos_dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)
pos_bow_corpus = [pos_dictionary.doc2bow(doc) for doc in pos_processed_documents] # corpus for topics that are seen as positive

neg_dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)
neg_bow_corpus = [neg_dictionary.doc2bow(doc) for doc in neg_processed_documents] # corpus for topics that are seen as negative<jupyter_output><empty_output><jupyter_text>Create the LDA Models<jupyter_code>num_topics = int(input("Number of topics to generate: "))

pos_lda_model_fname = os.path.join(pickled_dir, 'pos_lda_model_{}_topics'.format(num_topics))
neg_lda_model_fname = os.path.join(pickled_dir, 'neg_lda_model_{}_topics'.format(num_topics))
if use_pickled:
    pos_lda_model = gensim.models.LdaMulticore.load(pos_lda_model_fname)
    neg_lda_model = gensim.models.LdaMulticore.load(neg_lda_model_fname)

else:
    pos_lda_model = gensim.models.LdaMulticore(
        corpus=pos_bow_corpus, 
        num_topics=num_topics, 
        id2word=pos_dictionary, 
        passes=2, 
        workers=2)

    neg_lda_model = gensim.models.LdaMulticore(
        corpus=neg_bow_corpus, 
        num_topics=num_topics, 
        id2word=neg_dictionary, 
        passes=2, 
        workers=2)
    
    pos_lda_model.save(pos_lda_model_fname)
    neg_lda_model.save(neg_lda_model_fname)
import pyLDAvis
import pyLDAvis.gensim

# Visualize positive topic words
pyLDAvis.enable_notebook()
pos_vis = pyLDAvis.gensim.prepare(pos_lda_model, pos_bow_corpus, pos_dictionary, sort_topics=False)
pos_vis
# Visualize negative topic words
pyLDAvis.enable_notebook()
neg_vis = pyLDAvis.gensim.prepare(neg_lda_model, neg_bow_corpus, neg_dictionary, sort_topics=False)
neg_vis
def get_topic(text):
    bow_vector = pos_dictionary.doc2bow(preprocess(text))
    for idx, score in sorted(pos_lda_model[bow_vector], key=lambda tup:-1*tup[1]):
        pos_score = score
        pos_topic = "Topic: {}\nWords: {}".format(idx+1, pos_lda_model.print_topic(idx, 15))
        break           
    
    bow_vector = neg_dictionary.doc2bow(preprocess(text))
    for idx, score in sorted(neg_lda_model[bow_vector], key=lambda tup:-1*tup[1]):
        neg_score = score
        neg_topic = "Topic: {}\nWords: {}".format(idx+1, neg_lda_model.print_topic(idx, 15))
        break
        
    if pos_score>neg_score:
        return pos_topic
    else:
        return neg_topic
    
    
def get_mult_topics(text):
    bow_vector = pos_dictionary.doc2bow(preprocess(text))
    bow_vector = neg_dictionary.doc2bow(preprocess(text))
    
    print("\nPositive Topics:")
    for idx, score in sorted(pos_lda_model[bow_vector], key=lambda tup:-1*tup[1]):    
        print('Topic: {} Score: {}\nWords: {}'.format(idx+1, score, pos_lda_model.print_topic(idx, 15)))
        
    print("\nNegative Topics:")
    for idx, score in sorted(neg_lda_model[bow_vector], key=lambda tup:-1*tup[1]):    
        print('Topic: {} Score: {}\nWords: {}'.format(idx+1, score, neg_lda_model.print_topic(idx, 15)))


def get_sentiment(text):
    pos_score = 0
    neg_score = 0
    
    #get what positive topics might be related
    bow_vector = pos_dictionary.doc2bow(preprocess(text))
    count = 0
    for idx, score in sorted(pos_lda_model[bow_vector], key=lambda tup:-1*tup[1]):
        pos_score += score
        count += 1
        if count is 3:
            pos_score = pos_score/3
            break
    
    #get what negative topics might be related
    bow_vector = neg_dictionary.doc2bow(preprocess(text))
    count = 0
    for idx, score in sorted(neg_lda_model[bow_vector], key=lambda tup:-1*tup[1]):    
        neg_score += score
        count += 1
        if count is 3:
            neg_score = neg_score/3
            break
            
    if pos_score>neg_score:
        print("We predict that opinions on this movie are generally positive.")
        result = neg_score/pos_score
        result = 100 - (30*result)
        return result
    else:
        print("We predict that opinions on this movie are generally negative.")
        result = pos_score/neg_score
        result *= 60
        return result
unseen_movie_description = input("Please enter a movie description to analyze:")
print()
result = round(get_sentiment(unseen_movie_description), 3)
#print("\nWe predict that opinions on this movie are generally {}.".format(get_general_sentiment(unseen_movie_description)))
print("\nWe predict that this movie relates to the following topic:\n{}".format(get_topic(unseen_movie_description)))
#print("\nWe predict that this movie relates to the following topics:")
#get_mult_topics(unseen_movie_description)
print("\nWe predict that this movie has a rating of ~{}%.".format(result))<jupyter_output>Please enter a movie description to analyze:A young <NAME> (<NAME>) travels to the Far East, where he's trained in the martial arts by <NAME> (<NAME>), a member of the mysterious League of Shadows. When Ducard reveals the League's true purpose -- the complete destruction of Gotham City -- Wayne returns to Gotham intent on cleaning up the city without resorting to murder. With the help of Alfred (<NAME>), his loyal butler, and <NAME> (<NAME>), a tech expert at Wayne Enterprises, Batman is born.

We predict that opinions on this movie are generally positive.

We predict that this movie relates to the following topic:
Topic: 163
Words: 0.045*"batman" + 0.026*"burton" + 0.019*"return" + 0.019*"buffalo" + 0.017*"charact" + 0.013*"penguin" + 0.011*"dark" + 0.011*"like" + 0.010*"villain" + 0.010*"interest" + 0.009*"plot" + 0.008*"flaw" + 0.008*"problem" + 0.008*"stori" + 0.008*"taylor"

We predict that this movie has a rating of ~75.711%.
{'fraction_non_alphanumeric': 0.10974770642201835, 'fraction_numerical': 0.018004587155963304, 'mean_word_length': 4.376695437731196, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '414917', 'license': 'MIT', 'n_tokens_mistral': 3119, 'n_tokens_neox': 2896, 'n_words': 682, 'path': 'LDA_SA_Movie_Reviews_FINAL.ipynb', 'repo_name': 'cs3520-nlp/nlp', 'size': '8763'}