<jupyter_start><jupyter_text>Topic ModellingThis notebook contains a demo of LDA and LSA using the gensim library. The dataset's link can be found in the `BookSummaries_Link.md` file under the Data folder in Ch7.<jupyter_code># Import OS 
import os
# For NLTK virtual environments are high recommended and it requires python verisions higher than 3.5
!pip install gensim
!pip install nltk
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
nltk.download('stopwords')

from gensim.models import LdaModel
from gensim.corpora import Dictionary
from pprint import pprint

import os
#tokenize, remove stopwords, non-alphabetic words, lowercase
def preprocess(textstring):
   stops =  set(stopwords.words('english'))
   tokens = word_tokenize(textstring)
   return [token.lower() for token in tokens if token.isalpha() and token not in stops]

# This is a sample path of your downloaded data set. This is currently set to a windows based path . 
# Please update it to your actual download path regradless of your choice of operating system 

data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),data)

summaries = []
for line in open(data_path, encoding="utf-8"):
   temp = line.split("\t")
   summaries.append(preprocess(temp[6]))

# Create a dictionary representation of the documents.

dictionary = Dictionary(summaries)

# Filter infrequent or too frequent words.

dictionary.filter_extremes(no_below=10, no_above=0.5)
corpus = [dictionary.doc2bow(summary) for summary in summaries]

# Make a index to word dictionary.

temp = dictionary[0]  # This is only to "load" the dictionary.
id2word = dictionary.id2token

#Train the topic model

model = LdaModel(corpus=corpus, id2word=id2word,iterations=400, num_topics=10)
top_topics = list(model.top_topics(corpus))
pprint(top_topics)

for idx in range(10):
    print("Topic #%s:" % idx, model.print_topic(idx, 10))
print("=" * 20)
from gensim.models import LsiModel
lsamodel = LsiModel(corpus, num_topics=10, id2word = id2word)  # train model

pprint(lsamodel.print_topics(num_topics=10, num_words=10))

for idx in range(10):
    print("Topic #%s:" % idx, lsamodel.print_topic(idx, 10))
 
print("=" * 20)<jupyter_output>Topic #0: 0.305*"he" + 0.215*"one" + 0.150*"she" + 0.140*"time" + 0.132*"back" + 0.131*"also" + 0.127*"two" + 0.125*"they" + 0.123*"tells" + 0.118*"in"
Topic #1: 0.493*"tom" + 0.226*"sophia" + 0.182*"mrs" + 0.178*"house" + 0.161*"she" + 0.154*"father" + 0.147*"mr" + 0.146*"he" + 0.138*"tells" + -0.126*"one"
Topic #2: -0.558*"tom" + -0.252*"sophia" + 0.213*"she" + 0.190*"he" + -0.185*"mrs" + 0.163*"tells" + 0.144*"mother" + -0.138*"mr" + -0.129*"western" + -0.102*"however"
Topic #3: -0.233*"they" + -0.203*"ship" + 0.187*"he" + -0.183*"david" + -0.178*"back" + -0.165*"tells" + 0.161*"life" + 0.160*"family" + 0.154*"narrator" + -0.154*"find"
Topic #4: 0.664*"he" + -0.258*"mother" + -0.213*"she" + -0.195*"father" + -0.180*"family" + 0.121*"narrator" + 0.120*"monk" + -0.100*"school" + -0.099*"novel" + -0.095*"children"
Topic #5: 0.486*"david" + -0.241*"king" + 0.169*"rosa" + 0.162*"book" + 0.126*"harlan" + -0.120*"he" + 0.111*"she" + 0.108*"gould" + -0.108*"anita" + 0.103*"would"
Topic #6: [...]
{'fraction_non_alphanumeric': 0.1693072382727555, 'fraction_numerical': 0.08915812364088226, 'mean_word_length': 4.729537366548043, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '258379', 'license': 'MIT', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1224, 'n_words': 309, 'path': 'Ch7/02_TopicModelling.ipynb', 'repo_name': 'cwingho/practical-nlp-code', 'size': '3219'}