<jupyter_start><jupyter_text>Topic Modelling Demo Code  Things I want to do -- Identify a package to build / train LDA model- Use visualization to explore Documents -> Topics Distribution -> Word distribution<jupyter_code>!pip install pyLDAvis, gensim
import numpy as np
import pandas as pd

# Visualization
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
import seaborn as sns
import pyLDAvis.gensim

# Text Preprocessing and model building
from gensim.corpora import Dictionary
import nltk
from nltk.stem import WordNetLemmatizer
import re
# Iteratively read files
import glob
import os

# For displaying images in ipython
from IPython.display import HTML, display
%matplotlib inline
plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (14.0, 8.7)
#warnings.filterwarnings('ignore')
pd.options.display.float_format = '{:,.2f}'.format<jupyter_output><empty_output><jupyter_text>Latent Dirichlet AllocationFrom Documents -- DTM -- LDA ModelTopic modeling aims to automatically summarize large collections of documents to facilitate organization and management, as well as search and recommendations. At the same time, it can enable the understanding of documents to the extent that humans can interpret the descriptions of topics  Load Data<jupyter_code># User defined function to read and store bbc data from multipe folders
def load_data(folder_names,root_path):
    fileNames = [path + '/' + 'bbc' +'/'+ folder + '/*.txt' for path,folder in zip([root_path]*len(folder_names),
                                                                               folder_names )]
    doc_list = []
    tags = folder_names
    for docs in fileNames:
        #print(docs)
        #print(type(docs))
        doc = glob.glob(docs) # glob method iterates through the all the text documents in a folder
        for text in doc:
            with open(text, encoding='latin1') as f:
                topic = docs.split('/')[-2]

                lines = f.readlines()
                heading = lines[0].strip()
                body = ' '.join([l.strip() for l in lines[1:]])
                doc_list.append([topic, heading, body])
        print("Completed loading data from folder: %s"%topic)
    
    print("Completed Loading entire text")
    
    return doc_list
folder_names = ['business','entertainment','politics','sport','tech']
docs = load_data(folder_names = folder_names, root_path = os.getcwd())
docs = pd.DataFrame(docs, columns=['Category', 'Heading', 'Article'])
print(docs.head())
print('\nShape of data is {}\n'.format(docs.shape))<jupyter_output>Category                            Heading  \
0  business    UK economy facing 'major risks'   
1  business  Aids and climate top Davos agenda   
2  business   Asian quake hits European shares   
3  business   India power shares jump on debut   
4  business    Lacroix label bought by US firm   

                                             Article  
0   The UK manufacturing sector will continue to ...  
1   Climate change and the fight against Aids are...  
2   Shares in Europe's leading reinsurers and tra...  
3   Shares in India's largest power producer, Nat...  
4   Luxury goods group LVMH has sold its loss-mak...  

Shape of data is (2225, 3)<jupyter_text>Extract Raw Corpus<jupyter_code>articles = docs.Article.tolist()
print(type(articles))
print(articles[0:2])
wordnet_lemmatizer = WordNetLemmatizer()<jupyter_output><empty_output><jupyter_text>Preprocessing of Raw Text<jupyter_code>from nltk.corpus import stopwords
import nltk
# nltk.download('punkt')
# nltk.download('wordnet')
# nltk.download('stopwords')
# nltk.download('stopwords')
stopwords = stopwords.word('english')
# Method to preprocess my raw data
def preprocessText(x):
    temp = x.lower()
    temp = re.sub(r'[^\w]', ' ', temp)
    temp = nltk.word_tokenize(temp)
    temp = [wordnet_lemmatizer.lemmatize(w) for w in temp]
    temp = [word for word in temp if word not in stopwords ]
    return temp
articles_final = [preprocessText(article) for article in articles]
articles_final[0:2]<jupyter_output><empty_output><jupyter_text>Transformation of Preprocessed text into Vector form using Gensim<jupyter_code># Create a dictionary representation of the documents.
dictionary = Dictionary(articles_final)

# Filter out words that occur less than 20 documents, or more than 50% of the documents.
dictionary.filter_extremes(no_below=20, no_above=0.5)
print(dictionary)
# Bag-of-words representation of the documents.
corpus = [dictionary.doc2bow(doc) for doc in articles_final]
print('Number of unique tokens: %d' % len(dictionary))
print('Number of documents: %d' % len(corpus))<jupyter_output>Number of unique tokens: 3101
Number of documents: 2225<jupyter_text>Train LDA model using Gensim<jupyter_code>dictionary
# Train LDA model.
from gensim.models import LdaModel

# Set training parameters.
num_topics = 5
chunksize = 2000
passes = 10
# iterations = 400
eval_every = None  # Don't evaluate model perplexity, takes too much time.

# Make a index to word dictionary.
temp = dictionary[0]  # This is only to "load" the dictionary.
id2word = dictionary.id2token

model = LdaModel(
    corpus=corpus,
    id2word=id2word,
    chunksize=chunksize,
    alpha='auto',
    eta='auto',
#     iterations=iterations,
    num_topics=num_topics,
    passes=passes,
    eval_every=eval_every
)<jupyter_output><empty_output><jupyter_text>Model exploration: Top K words in each topic<jupyter_code># Print the Keyword in the 10 topics
pprint.pprint(model.print_topics(num_words= 20))
doc_lda = model[corpus]<jupyter_output>[(0,
  '0.014*"game" + 0.009*"microsoft" + 0.008*"software" + 0.008*"virus" + '
  '0.008*"new" + 0.008*"firm" + 0.007*"file" + 0.007*"system" + 0.007*"site" + '
  '0.007*"program" + 0.007*"technology" + 0.007*"machine" + 0.006*"mr" + '
  '0.006*"mail" + 0.006*"people" + 0.006*"network" + 0.006*"user" + '
  '0.005*"sony" + 0.005*"computer" + 0.005*"window"'),
 (1,
  '0.015*"people" + 0.012*"service" + 0.011*"mobile" + 0.009*"phone" + '
  '0.008*"one" + 0.008*"could" + 0.007*"user" + 0.007*"technology" + '
  '0.007*"uk" + 0.007*"internet" + 0.006*"network" + 0.006*"broadband" + '
  '0.006*"net" + 0.006*"online" + 0.006*"system" + 0.006*"use" + '
  '0.006*"million" + 0.006*"new" + 0.006*"access" + 0.006*"company"'),
 (2,
  '0.017*"game" + 0.009*"film" + 0.008*"best" + 0.008*"time" + 0.007*"one" + '
  '0.006*"first" + 0.005*"world" + 0.005*"award" + 0.005*"player" + '
  '0.005*"like" + 0.005*"last" + 0.005*"two" + 0.004*"new" + 0.004*"team" + '
  '0.004*"back" + 0.004*"play" + 0.004*"win" [...]<jupyter_text>Model Visualization using PyLDAvis<jupyter_code>pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(model, corpus, dictionary=dictionary)
vis<jupyter_output><empty_output><jupyter_text>Assign Topic Model Numbers to original Data Frame as Column<jupyter_code># Assigns the topics to the documents in corpus
lda_corpus = model[corpus]
topics = []

for doc in lda_corpus:
    temp_id = []
    temp_score = []
    for doc_tuple in doc:
        temp_id.append(doc_tuple[0])
        temp_score.append(doc_tuple[1])
    index = np.argmax(temp_score)
    topics.append(temp_id[index])
docs["Topic_num"] = topics
docs.tail(n= 40)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12568984547461368, 'fraction_numerical': 0.041390728476821195, 'mean_word_length': 3.917910447761194, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '10843', 'license': 'MIT', 'n_tokens_mistral': 2626, 'n_tokens_neox': 2378, 'n_words': 648, 'path': '5. Text Mining/Topic Modelling Using Gensim.ipynb', 'repo_name': 'paragajg/DSB7', 'size': '7248'}