<jupyter_start><jupyter_text>Main Imports<jupyter_code>import sys
import re, numpy as np, pandas as pd
from pprint import pprint
import seaborn as sns

import gensim, spacy, logging, warnings
import gensim.corpora as corpora
from gensim.utils import lemmatize, simple_preprocess
from gensim.models import CoherenceModel
import matplotlib.pyplot as plt
from matplotlib import rcParams

# NLTK Stop words
from nltk.corpus import stopwords
stop_words = stopwords.words('english')

# Take a peak at the stop words and decide if you want to add to it.
stop_words.extend(['from', 'subject', 're', 'edu', 'use', 'not', 'would', 'say', 'could', '_', 'be', 'know', 'good', 'go', 'get', 'do', 'done', 'try', 'many', 'some', 'nice', 'thank', 'think', 'see', 'rather', 'easy', 'easily', 'lot', 'lack', 'make', 'want', 'seem', 'run', 'need', 'even', 'right', 'line', 'even', 'also', 'may', 'take', 'come'])

%matplotlib inline
warnings.filterwarnings("ignore",category=DeprecationWarning)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)<jupyter_output><empty_output><jupyter_text>Note about Code:Code modified in parts from https://www.machinelearningplus.com/nlp/topic-modeling-visualization-how-to-present-results-lda-models/  Load Data<jupyter_code>#Here, we are reading in a csv file. See 'nb1_Amazon_NMF_nb1.ipynb' to see the JSON method.

amazon_ss = pd.read_csv('/<Directory>/<User>/amazon_ss.csv') # Subsample of main dataframe from nb1t
amazon_ss['reviewText'] = amazon_ss['reviewText'].astype(str) #Make type 'string' for len count<jupyter_output><empty_output><jupyter_text>Create 'reviewText_len' column<jupyter_code>#Count the number of words in the 'reviewText' column.

count = amazon_ss['reviewText'].str.split().str.len()
pd.Series(count)
amazon_ss['reviewText_len']=count
amazon_ss
#Take a look at the longest and shortest reviews.

amazon_ss.sort_values('reviewText_len')
#'''
#This review can also be found in its native habitat at https://www.amazon.com/gp/customer-reviews/R1ZXVS3NSR1JK1/ref=cm_cr_arp_d_viewpnt?ie=UTF8&ASIN=B00Y0LXDZE#R1ZXVS3NSR1JK1
#'''

amazon_ss.loc[38589]['reviewText']
#Plot up the count of reviewText docs by length.

count_sorted=count.sort_values()
count_plot = sns.histplot(x=count_sorted)
count_plot.set(ylim=(0, 5000),xlim=(0,100))

#Note the curve shapre around reviewText=20.  Not sure what is going on there.<jupyter_output><empty_output><jupyter_text>Data Prep<jupyter_code>#Clean up special charecters and quotes.

def review_to_words(reviews):
    for review in reviews:
        review = re.sub('\S*@\S*\s?', '', review)  # remove emails
        review = re.sub('\s+', ' ', review)  # remove newline chars
        review = re.sub("\'", "", review)  # remove single quotes
        review = gensim.utils.simple_preprocess(str(review), deacc=True) 
        yield(review)
#Quick scan of the dtypes for good measure.  I find it good to just check this once in a while
# so that I don't burn time tracking down dtype errors.

amazon_ss.dtypes
#Convert to list for Gensim
data = amazon_ss.reviewText.values.tolist()
data_words = list(review_to_words(data))<jupyter_output><empty_output><jupyter_text>Gensim  Bi-, Tri-grams, and Lemmatization<jupyter_code>bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold = fewer phrases.
trigram = gensim.models.Phrases(bigram[data_words], threshold=100)  
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)<jupyter_output><empty_output><jupyter_text>Remove Stopwords, Form Bigrams, Trigrams and Lemmatization  Additional stop-words<jupyter_code>#Additional stop-words added for each project.

new_stop = ['good', 'great', 'work', 'get', 'well', 'give', 'use', 'go', 'try', 'item', 'excellent', 'happy', 'product', 'buy', 'say','think','make','hold','take','see','know','however','tell','need','line']

for word in new_stop:
    stop_words.append(word)
#Processing function.  

def process_words(texts, stop_words=stop_words, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """Remove Stopwords, Form Bigrams, Trigrams and Lemmatization"""
    texts = [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]
    texts = [bigram_mod[doc] for doc in texts]
    texts = [trigram_mod[bigram_mod[doc]] for doc in texts]
    texts_out = []
    nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
    for review in texts:
        doc = nlp(" ".join(review)) 
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    # remove stopwords once more after lemmatization
    texts_out = [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts_out]    
    return texts_out

data_ready = process_words(data_words)<jupyter_output><empty_output><jupyter_text>Build the Topic Model using LDA (Latent Dirichlet Allocation) This is the most time consuming step.  4 topics = 30 minutes, 8 topics = 50 minutes.<jupyter_code># Create Dictionary
id2word = corpora.Dictionary(data_ready)

# Create Corpus: Term Document Frequency
corpus = [id2word.doc2bow(text) for text in data_ready]

# Build LDA model
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=5, #Change the number of topics here.  4-12 is a good range to start with.
                                           random_state=42,
                                           update_every=1,
                                           chunksize=10,
                                           passes=10,
                                           alpha='symmetric',
                                           iterations=100,
                                           per_word_topics=True)
#Print to check results

pprint(lda_model.print_topics())<jupyter_output>[(0,
  '0.029*"time" + 0.027*"camera" + 0.016*"thing" + 0.016*"tv" + 0.014*"set" + '
  '0.012*"fine" + 0.011*"connect" + 0.011*"picture" + 0.010*"mount" + '
  '0.010*"video"'),
 (1,
  '0.033*"quality" + 0.033*"sound" + 0.030*"look" + 0.027*"speaker" + '
  '0.022*"lens" + 0.018*"screen" + 0.016*"really" + 0.013*"way" + 0.012*"size" '
  '+ 0.012*"high"'),
 (2,
  '0.029*"case" + 0.023*"fit" + 0.019*"laptop" + 0.018*"tablet" + '
  '0.016*"drive" + 0.015*"old" + 0.015*"new" + 0.013*"back" + 0.013*"fast" + '
  '0.012*"perfect"'),
 (3,
  '0.020*"cable" + 0.015*"unit" + 0.015*"battery" + 0.015*"device" + '
  '0.013*"problem" + 0.013*"plug" + 0.011*"power" + 0.010*"long" + '
  '0.010*"recommend" + 0.010*"day"'),
 (4,
  '0.048*"price" + 0.032*"love" + 0.024*"light" + 0.019*"phone" + '
  '0.019*"keyboard" + 0.018*"bit" + 0.018*"headphone" + 0.015*"button" + '
  '0.014*"control" + 0.014*"little"')]<jupyter_text>Topic Extraction<jupyter_code>def format_topics_sentences(ldamodel=None, corpus=corpus, texts=data):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row_list in enumerate(ldamodel[corpus]):
        row = row_list[0] if ldamodel.per_word_topics else row_list            
        # print(row)
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        # Get the Dominant topic, Percentage Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
    return(sent_topics_df)


df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model, corpus=corpus, texts=data_ready)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']
df_dominant_topic.head(10)<jupyter_output><empty_output><jupyter_text>Get Representative Text<jupyter_code># Display setting to show more characters in column
pd.options.display.max_colwidth = 100

sent_topics_sorteddf_mallet = pd.DataFrame()
sent_topics_outdf_grpd = df_topic_sents_keywords.groupby('Dominant_Topic')

for i, grp in sent_topics_outdf_grpd:
    sent_topics_sorteddf_mallet = pd.concat([sent_topics_sorteddf_mallet, 
                                             grp.sort_values(['Perc_Contribution'], ascending=False).head(1)], 
                                            axis=0)

# Reset Index    
sent_topics_sorteddf_mallet.reset_index(drop=True, inplace=True)

# Format
sent_topics_sorteddf_mallet.columns = ['Topic_Num', "Topic_Perc_Contrib", "Keywords", "Representative Text"]

# Show
sent_topics_sorteddf_mallet.head(10)
# Plots and Visualizations<jupyter_output><empty_output><jupyter_text>Document Word Count Histogram<jupyter_code>doc_lens = [len(d) for d in df_dominant_topic.Text]

plt.figure(figsize=(16,7), dpi=160)
plt.hist(doc_lens, bins = 1000, color='navy') #Plot inputs here
plt.text(750, 100, "Mean   : " + str(round(np.mean(doc_lens))))
plt.text(750,  90, "Median : " + str(round(np.median(doc_lens))))
plt.text(750,  80, "Stdev   : " + str(round(np.std(doc_lens))))
plt.text(750,  70, "1%ile    : " + str(round(np.quantile(doc_lens, q=0.01))))
plt.text(750,  60, "99%ile  : " + str(round(np.quantile(doc_lens, q=0.99))))

plt.gca().set(xlim=(0, 1000), ylabel='Number of Documents', xlabel='Document Word Count')
plt.tick_params(size=16)
plt.xticks(np.linspace(0,1000,9))
plt.title('Document Word Counts', fontdict=dict(size=22))
plt.show()
import seaborn as sns
import matplotlib.colors as mcolors
cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'

fig, axes = plt.subplots(3,2,figsize=(6,6), dpi=160, sharex=True, sharey=True)

for i, ax in enumerate(axes.flatten()):    
    df_dominant_topic_sub = df_dominant_topic.loc[df_dominant_topic.Dominant_Topic == i, :]
    doc_lens = [len(d) for d in df_dominant_topic_sub.Text]
    ax.hist(doc_lens, bins = 1000, color=cols[i]) #Plot input
    ax.tick_params(axis='y', labelcolor=cols[i], color=cols[i])
    
    ax.set(xlim=(0, 100), xlabel='Document Word Count')
    ax.set(ylim=(0,2000))
    ax.set_ylabel('Number of Documents', color=cols[i])
    ax.set_title('Topic: '+str(i), fontdict=dict(size=16, color=cols[i]))

fig.tight_layout()
fig.subplots_adjust(top=0.90)
plt.xticks(np.linspace(0,100,5))
fig.suptitle('Distribution of Document Word Counts by Dominant Topic', fontsize=22)
plt.show()

#fig.savefig("doc_word_count_plot.pdf", format="pdf", dpi=300)
#Save figure when desired.
df_dominant_topic.head(20)<jupyter_output><empty_output><jupyter_text>Word Clouds<jupyter_code>df_dominant_topic_sub
#Wordcloud of Top N words in each topic

#Same text as from notebook 1:
#The wordcloud isn't a super helpful quantitative device, but it does
# offer us some idea of what is going on with the frequencies of different
# words. It helped me see some words pairings I might have missed from just
# a dataframe or list ouput, but use with caution.

from matplotlib import pyplot as plt
from wordcloud import WordCloud, STOPWORDS
import matplotlib.colors as mcolors

cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'

cloud = WordCloud(stopwords=stop_words,
                  background_color='white',
                  width=2500,
                  height=1800,
                  max_words=10,
                  colormap='tab10',
                  color_func=lambda *args, **kwargs: cols[i],
                  prefer_horizontal=1.0)

topics = lda_model.show_topics(formatted=False)

fig, axes = plt.subplots(3, 2, figsize=(10,10), sharex=True, sharey=True)

for i, ax in enumerate(axes.flatten()):
    fig.add_subplot(ax)
    topic_words = dict(topics[i][1])
    cloud.generate_from_frequencies(topic_words, max_font_size=300)
    plt.gca().imshow(cloud)
    plt.gca().set_title('Topic ' + str(i), fontdict=dict(size=16))
    plt.gca().axis('off')


plt.subplots_adjust(wspace=0, hspace=0)
plt.axis('off')
plt.margins(x=0, y=0)
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Topic Word Counts<jupyter_code>from collections import Counter

topics = lda_model.show_topics(formatted=False)
data_flat = [w for w_list in data_ready for w in w_list]
counter = Counter(data_flat)

out = []
for i, topic in topics:
    for word, weight in topic:
        out.append([word, i , weight, counter[word]])

df = pd.DataFrame(out, columns=['word', 'topic_id', 'importance', 'word_count'])        

# Plot Word Count and Weights of Topic Keywords
fig, axes = plt.subplots(3, 2, figsize=(16,10), sharey=True, dpi=160)
cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]
for i, ax in enumerate(axes.flatten()):
    ax.bar(x='word', height="word_count", data=df.loc[df.topic_id==i, :], color=cols[i], width=0.5, alpha=0.3, label='Word Count') #This is the actual line for he plot
    ax_twin = ax.twinx()
    ax_twin.bar(x='word', height="importance", data=df.loc[df.topic_id==i, :], color=cols[i], width=0.2, label='Weights')
    ax.set_ylabel('Word Count', color=cols[i])
    ax_twin.set_ylim(0, 0.040); ax.set_ylim(0, 12000)
    ax.set_title('Topic: ' + str(i), color=cols[i], fontsize=16)
    ax.tick_params(axis='y', left=False)
    ax.set_xticklabels(df.loc[df.topic_id==i, 'word'], rotation=30, horizontalalignment= 'right')
    ax.legend(loc='upper left'); ax_twin.legend(loc='upper right')

fig.tight_layout(w_pad=2)    
fig.suptitle('Word Count and Importance of Topic Keywords', fontsize=22, y=1.05)    
plt.show()<jupyter_output><empty_output><jupyter_text>Sentence Topic Coloring<jupyter_code>from matplotlib.patches import Rectangle

def sentences_chart(lda_model=lda_model, corpus=corpus, start = 0, end = 13): #Note: Pay attention to the 'start' and 'end'.  Some of the sentences you'll get won't be helpful.
    corp = corpus[start:end]
    mycolors = [color for name, color in mcolors.TABLEAU_COLORS.items()]

    fig, axes = plt.subplots(end-start, 1, figsize=(20, (end-start)*0.95), dpi=160) #Plot line.
    axes[0].axis('off')
    for i, ax in enumerate(axes):
        if i > 0:
            corp_cur = corp[i-1] 
            topic_percs, wordid_topics, wordid_phivalues = lda_model[corp_cur]
            word_dominanttopic = [(lda_model.id2word[wd], topic[0]) for wd, topic in wordid_topics]    
            ax.text(0.01, 0.5, "Doc " + str(i-1) + ": ", verticalalignment='center',
                    fontsize=16, color='black', transform=ax.transAxes, fontweight=700)

            # Draw Rectange
            topic_percs_sorted = sorted(topic_percs, key=lambda x: (x[1]), reverse=True)
            ax.add_patch(Rectangle((0.0, 0.05), 0.99, 0.90, fill=None, alpha=1, 
                                   color=mycolors[topic_percs_sorted[0][0]], linewidth=2))

            word_pos = 0.06
            for j, (word, topics) in enumerate(word_dominanttopic):
                if j < 14:
                    ax.text(word_pos, 0.5, word,
                            horizontalalignment='left',
                            verticalalignment='center',
                            fontsize=16, color=mycolors[topics],
                            transform=ax.transAxes, fontweight=700)
                    word_pos += .009 * len(word)  #To move the word for the next iter
                    ax.axis('off')
            ax.text(word_pos, 0.5, '. . .',
                    horizontalalignment='left',
                    verticalalignment='center',
                    fontsize=16, color='black',
                    transform=ax.transAxes)       

    plt.subplots_adjust(wspace=0, hspace=0)
    plt.suptitle('Sentence Topic Coloring for Documents: ' + str(start) + ' to ' + str(end-2), fontsize=22, y=0.95, fontweight=700)
    plt.tight_layout()
    plt.show()

sentences_chart()<jupyter_output><empty_output><jupyter_text>Most Discussed Topics in Docs<jupyter_code>#Sentence Coloring of N Sentences
def topics_per_document(model, corpus, start=0, end=1):
    corpus_sel = corpus[start:end]
    dominant_topics = []
    topic_percentages = []
    for i, corp in enumerate(corpus_sel):
        topic_percs, wordid_topics, wordid_phivalues = model[corp]
        dominant_topic = sorted(topic_percs, key = lambda x: x[1], reverse=True)[0][0]
        dominant_topics.append((i, dominant_topic))
        topic_percentages.append(topic_percs)
    return(dominant_topics, topic_percentages)

dominant_topics, topic_percentages = topics_per_document(model=lda_model, corpus=corpus, end=-1)            

#Distribution of Dominant Topics in Each Document
df = pd.DataFrame(dominant_topics, columns=['Document_Id', 'Dominant_Topic'])
dominant_topic_in_each_doc = df.groupby('Dominant_Topic').size()
df_dominant_topic_in_each_doc = dominant_topic_in_each_doc.to_frame(name='count').reset_index()

# Total Topic Distribution by actual weight
topic_weightage_by_doc = pd.DataFrame([dict(t) for t in topic_percentages])
df_topic_weightage_by_doc = topic_weightage_by_doc.sum().to_frame(name='count').reset_index()

# Top 3 Keywords for each Topic
topic_top3words = [(i, topic) for i, topics in lda_model.show_topics(formatted=False) 
                                 for j, (topic, wt) in enumerate(topics) if j < 3]

df_top3words_stacked = pd.DataFrame(topic_top3words, columns=['topic_id', 'words'])
df_top3words = df_top3words_stacked.groupby('topic_id').agg(', \n'.join)
df_top3words.reset_index(level=0,inplace=True)
from matplotlib.ticker import FuncFormatter

# Plot
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4), dpi=120, sharey=True)

# Topic Distribution by Dominant Topics
ax1.bar(x='Dominant_Topic', height='count', data=df_dominant_topic_in_each_doc, width=.5, color='firebrick')
ax1.set_xticks(range(df_dominant_topic_in_each_doc.Dominant_Topic.unique().__len__()))
tick_formatter = FuncFormatter(lambda x, pos: 'Topic ' + str(x)+ '\n' + df_top3words.loc[df_top3words.topic_id==x, 'words'].values[0])
ax1.xaxis.set_major_formatter(tick_formatter)
ax1.set_title('Number of Documents by Dominant Topic', fontdict=dict(size=10))
ax1.set_ylabel('Number of Documents')
ax1.set_ylim(0, 20000)

# Topic Distribution by Topic Weights
ax2.bar(x='index', height='count', data=df_topic_weightage_by_doc, width=.5, color='steelblue')
ax2.set_xticks(range(df_topic_weightage_by_doc.index.unique().__len__()))
ax2.xaxis.set_major_formatter(tick_formatter)
ax2.set_title('Number of Documents by Topic Weightage', fontdict=dict(size=6))

plt.show()<jupyter_output><empty_output><jupyter_text>pyLDAVis<jupyter_code>import pyLDAvis.gensim


#This figure will show you how related the topics are.  The more they overlap, the more
# similar they are.  Aim for maximum sepeartebility when possible, but use your human
# brain and domain knowledge to decide when it's truly appropriate.

pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, dictionary=lda_model.id2word)
vis<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13603168072413083, 'fraction_numerical': 0.02931495577041761, 'mean_word_length': 4.0480269989615785, 'pattern_counts': {'":': 1, '<': 64, '<?xml version=': 0, '>': 64, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '349994', 'license': 'MIT', 'n_tokens_mistral': 7273, 'n_tokens_neox': 6757, 'n_words': 1464, 'path': 'Notebooks/nb2_amazon_LDA_5f_nb2.ipynb', 'repo_name': 'JGarrechtMetzger/Project4_NLP', 'size': '19444'}