<jupyter_start><jupyter_text>1. Imports<jupyter_code># Required Imports
import re
import numpy as np
import pandas as pd
from pprint import pprint
import multiprocessing
import nltk
# Gensim for text preprocessing and LDA
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
# spacy for lemmatization ans Stemming
import spacy
# Plotting tools
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
%matplotlib inline
# Suppress all the warnings
import warnings
warnings.filterwarnings("ignore")
# Latent Dirichlet Allocation (LDA), using all CPU cores to parallelize and speed up model training
cores = multiprocessing.cpu_count()<jupyter_output><empty_output><jupyter_text>2. Load Data<jupyter_code># Read the data set from training file => train.csv
df = pd.read_csv('train.csv', encoding='latin1')
# Check the loaded data 
df.head(2)
# For LDA, we are only concerned with the text reviews
# therefore we will be selecting 'reviews.text' column as training data
# Converting data to a list for further text preprocessing
data = df['reviews.text'].tolist()
# Print the number of rows/ elements in list
len(data)<jupyter_output><empty_output><jupyter_text>3. Preprocess Data<jupyter_code># Remove new line characters
data = [re.sub('\s+', ' ', str(sent)) for sent in data]
# print the first element of the list
data[:1]
# Remove distracting single quotes
data = [re.sub("\'", "", sent) for sent in data]
# print the first element of the list
data[:1]
# Initialize spacy 'en' model, keeping only tagger component (for efficiency)
nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
# NLTK and Spacy Stop words
# The set of NLTK stopwords is limied.
# So we are adding the stopwords available in Spacy library with NLTK library to make a comprehensive set of stop words
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS
stop_words = list(set(stop_words + list(spacy_stopwords)))
# Check the total number of stop words in stop_words list
len(stop_words)
# function to convert each para into tokens after preprocessing
def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations
# clean and tokenize the data set
data_words = list(sent_to_words(data))
# Check the first element of preprocessed and tokenized list
data_words[:1]
# Function to remove stopwords
def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]
# Removing the reviews which has less than 10 words in the review.
# Output is the reduced list
data_reduced = [[x for x in data_words_1] for data_words_1 in data_words if len(data_words_1) > 10]
# Print the number of rows/ elements in reduced dataset/list
len(data_reduced)
# Create a list of sentences using tokenised words to be used in Final Results
data_reduced_sent = [" ".join(y) for y in data_reduced]
# Remove Stop Words
data_words_nostops = remove_stopwords(data_reduced)
# Check the training data with stop words removed
data_words_nostops[:1]
# Calculate the frequency of each word in dataset
from collections import defaultdict
frequency = defaultdict(int)
for text in data_words_nostops:
    for token in text:
        frequency[token] += 1
# Print the list of most frequent occuring words
import operator
most_frq = sorted(frequency.items(),key = operator.itemgetter(1), reverse=True)
most_frq[:30] # Top 10 most frquent words
# Selectively creating a list of most frequent words that are irrelevant for the topic models
most_frq_words = ['great','use','love',
                  'easy','good','bought',
                  'like','loves','best',
                  'buy']
# add the the most frequent words to the stopwords list from the above analysis
stop_words.extend(most_frq_words)
len(stop_words) # Check the length of updated stopwords list
# Again remove words from corpus using updated list
data_words_nostops = remove_stopwords(data_words_nostops)
# Print the first element of data list after removal of most frequent words
data_words_nostops[:1]
# Build the bigram model
bigram = gensim.models.Phrases(data_words_nostops, min_count=1, threshold=100) # higher threshold fewer phrases.
# Faster way to get a sentence clubbed as a trigram/bigram
bigram_mod = gensim.models.phrases.Phraser(bigram)
# Function to make bigrams
def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]
# Import lemmatizer
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
def lemmatization_stemming(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent))
        texts_out.append([lemmatizer.lemmatize(token.lemma_) for token in doc if token.pos_ in allowed_postags and len(token) > 1])
    return texts_out
# Form Bigrams
data_words_bigrams = make_bigrams(data_words_nostops)
# Do lemmatization keeping only noun, adj, vb, adv
data_lemmatized = lemmatization_stemming(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])
# Print first element of preprocessed data list after lemmatization 
print(data_lemmatized[:1])
# create a word to index mapping with preprocessed data list
id2word = corpora.Dictionary(data_lemmatized)
# Create a list of words with POS as Adjective
Adj = []
for i in range(11924):
    if nltk.pos_tag([id2word[i]])[0][1] == 'JJ':
        Adj.append(id2word[i])
# print first 10 elements from the list of adjectives extracted
Adj[:10]
# add the the Adjectives to the stopwords list from the above analysis
# Because the Adjectives are non relevant for topic models
stop_words.extend(Adj)
# Update the preprocessed data list
data_lemmatized = remove_stopwords(data_lemmatized)
# update the word to index mapping with updated preprocessed data list
id2word = corpora.Dictionary(data_lemmatized)
# Term Document Frequency
# Create Corpus
texts = data_lemmatized
corpus = [id2word.doc2bow(text) for text in texts]
# Human readable format of corpus (term-frequency)
[[(id2word[id], freq) for id, freq in cp] for cp in corpus[:1]]<jupyter_output><empty_output><jupyter_text>4. Training and Visualisation<jupyter_code># Build LDA model
# for Single core processing use :
# lda_model = gensim.models.ldamodel.LdaModel()
lda_model = gensim.models.LdaMulticore(corpus=corpus,
                                       id2word=id2word,
                                       random_state=100,
                                       num_topics=3,
                                       passes=20,
                                       per_word_topics=True,
                                       workers=cores)
# Print the Keyword in the 3 topics
pprint(lda_model.print_topics(num_words=10))
doc_lda = lda_model[corpus]
# Compute Perplexity
print('\nPerplexity: ', lda_model.log_perplexity(corpus))  # a measure of how good the model is. lower the better.
# Compute Coherence Score
coherence_model_lda = CoherenceModel(model=lda_model, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# Print the coherence values
coherence_model_lda.get_coherence()
# Visualize the topics
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis
# Finding the dominant topic in each sentence
def format_topics_sentences(ldamodel, corpus, texts):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row in enumerate(ldamodel.get_document_topics(corpus, minimum_probability=0.0)):
        
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        
        # Get the Dominant topic, Perc Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([ sent_topics_df, contents], axis=1)
    return(sent_topics_df)
# Finding the dominant topic in each review
df_topic_sents_keywords = format_topics_sentences(lda_model, corpus, data_reduced_sent)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No','Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']
# Save the output file to disc
df_dominant_topic.to_csv('Final_results_train.csv')<jupyter_output><empty_output><jupyter_text>5. Validation<jupyter_code># Read test file
df_test = pd.read_csv('test.csv', encoding='latin1')
# Clean, preprocess and get model predictions
sentence_test = df_test['reviews.text'].tolist()
sentence_test = [re.sub('\s+', ' ', sent) for sent in sentence_test]
sentence_test = [re.sub("\'", "", sent) for sent in sentence_test]
sent_words_test = list(sent_to_words(sentence_test))
sent_words_nostops_test = remove_stopwords(sent_words_test)
sent_words_bigrams_test = make_bigrams(sent_words_nostops_test)
sent_lemmatized_test = lemmatization_stemming(sent_words_bigrams_test, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])
id2word_test = corpora.Dictionary(sent_lemmatized_test)
ADJ_test = []
for i in range(3973):
    if nltk.pos_tag([id2word_test[i]])[0][1] == 'JJ':
        ADJ_test.append(id2word_test[i])
stop_words.extend(ADJ_test)
sent_lemmatized_test = remove_stopwords(sent_lemmatized_test)
id2word_test = corpora.Dictionary(sent_lemmatized_test)
corpus_test = [id2word.doc2bow(text) for text in sent_lemmatized_test]
ls = lda_model.get_document_topics(corpus,minimum_probability=0.00)
# Create sentences to be used for results csv
data_reduced_sent_test = [" ".join(y) for y in sent_words_test]
df_topic_sents_keywords = format_topics_sentences(lda_model, corpus_test, data_reduced_sent_test)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No','Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']

df_dominant_topic.to_csv('Final_results_test.csv')<jupyter_output><empty_output><jupyter_text>6. Finding Optimal no. of Topics<jupyter_code># find the optimal number of topics for LDA
def compute_coherence_values(dictionary, corpus, texts, start=1, limit = 8, step=1):
    """
    Compute c_v coherence for various number of topics

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_values : Coherence values corresponding to the LDA model with respective number of topics
    """
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        print("Training with {} number of Topics.".format(num_topics))
        model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                                id2word=id2word,
                                                num_topics=num_topics,
                                                random_state=None,
                                                update_every=1,
                                                chunksize=3466,
                                                passes=20,
                                                alpha='auto',
                                                eta='auto',
                                                per_word_topics=True)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values
# create multiple models with variation in number of topics
# can take a long time
model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus,texts=data_lemmatized, start=1, limit=6, step=1)
# Show graph
print(coherence_values)
limit = 6; start=1; step=1
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence Score")
plt.legend(("coherence_values"), loc='best')
plt.show()
# Print the coherence scores for all the models
for m, cv in zip(x, coherence_values):
      print("Num Topics =", m, " has Coherence Value of", round(cv, 4))
# Select the optimal model and print the topics
# Analysis shows the optimal model is at with 4 topics i.e Index 3 of the model list
optimal_model = model_list[3] # Use the index of the model with highest Coherence value
model_topics = optimal_model.show_topics(formatted=False)
pprint(optimal_model.print_topics(num_words=10))
# Visualize the topics
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(optimal_model, corpus, id2word)
vis
# Keep Machine Learning :D<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0887369449244872, 'fraction_numerical': 0.008415358028401832, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '394239', 'license': 'Apache-2.0', 'n_tokens_mistral': 4250, 'n_tokens_neox': 4066, 'n_words': 1208, 'path': 'Topic_Modeling/LDA/Amazon_Product_Reviews/LDA_Amazon_Reviews.ipynb', 'repo_name': 'KushwahaDK/NLP', 'size': '13309'}