<jupyter_start><jupyter_text>Stop words:<jupyter_code>import sys
!{sys.executable} --version
!pip install gensim
!pip install lda
!pip install nltk
!pip install stop_words
import pip
import conda
import nltk
import lda
import lda.datasets
import gensim
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import stop_words
from stop_words import get_stop_words
en_stop = get_stop_words('en')
# remove stop words from tokens
stopped_tokens1 = [i for i in tokens1 if not i in en_stop]

print(stopped_tokens1[:20])<jupyter_output>['review', 'tweet', 'one', 'great', 'drivesthis', 'often', 'rated', 'one', 'scenic', 'drives', 'world', 'certainly', 'one', 'scenic', 've', 'ever', 'taken', 'gorgeous', 'mountains', 'around']<jupyter_text>stemming:<jupyter_code>from nltk.stem.porter import PorterStemmer

# Create p_stemmer of class PorterStemmer
p_stemmer = PorterStemmer()
# stem token
stemmed_stopped_token1= [p_stemmer.stem(i) for i in stopped_tokens1]

print(stemmed_stopped_token1[:20])<jupyter_output>['review', 'tweet', 'one', 'great', 'drivesthi', 'often', 'rate', 'one', 'scenic', 'drive', 'world', 'certainli', 'one', 'scenic', 've', 'ever', 'taken', 'gorgeou', 'mountain', 'around']<jupyter_text>Cleaning:<jupyter_code>type(stemmed_stopped_token1)
print(en_stop)
import nltk
from nltk.stem.porter import PorterStemmer

# Create p_stemmer of class PorterStemmer
p_stemmer = PorterStemmer()
from gensim import corpora, models

dictionary = corpora.Dictionary()<jupyter_output><empty_output><jupyter_text>LDA Topic Modeling: Applying these steps on the whole text (stemmed_stopped_token1):<jupyter_code>texts = []

# loop through document list
for i in stemmed_stopped_token1:
    
    # clean and tokenize document string
    raw = i.lower()
    tokens = tokenizer.tokenize(raw)

    # remove stop words from tokens
    stopped_tokens = [i for i in tokens if not i in en_stop]
    
    # stem tokens
    stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]
    
    # add tokens to list
    texts.append(stemmed_tokens)

# turn our tokenized documents into a id <-> term dictionary
dictionary = corpora.Dictionary(texts)
    
# convert tokenized documents into a document-term matrix
corpus = [dictionary.doc2bow(text) for text in texts]

# generate LDA model
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=10, id2word = dictionary, passes=20)
print(ldamodel.print_topics(num_topics=10, num_words=15))<jupyter_output>[(0, '0.036*"icefield" + 0.032*"trail" + 0.032*"inform" + 0.028*"place" + 0.028*"around" + 0.025*"back" + 0.023*"nation" + 0.023*"athabasca" + 0.020*"centr" + 0.017*"quit" + 0.014*"pretti" + 0.014*"short" + 0.014*"driver" + 0.013*"crowd" + 0.013*"vehicl"'), (1, '0.069*"warm" + 0.049*"wildfir" + 0.029*"make" + 0.026*"worth" + 0.026*"melt" + 0.025*"banff" + 0.024*"trip" + 0.018*"onto" + 0.018*"canyon" + 0.017*"mani" + 0.013*"smoke" + 0.012*"pass" + 0.012*"wonder" + 0.012*"attract" + 0.012*"thing"'), (2, '0.072*"year" + 0.064*"global" + 0.063*"get" + 0.047*"will" + 0.028*"amaz" + 0.019*"good" + 0.017*"now" + 0.014*"know" + 0.014*"ticket" + 0.014*"sign" + 0.014*"start" + 0.013*"opportun" + 0.013*"give" + 0.013*"far" + 0.012*"stun"'), (3, '0.111*"ice" + 0.093*"park" + 0.081*"walk" + 0.051*"time" + 0.032*"peopl" + 0.031*"day" + 0.025*"even" + 0.024*"drive" + 0.022*"still" + 0.020*"use" + 0.018*"sinc" + 0.016*"cold" + 0.016*"visitor" + 0.015*"didn" + 0.014*"ride"'), (4, '0.235*"glacier" + 0.0[...]
{'fraction_non_alphanumeric': 0.1780466724286949, 'fraction_numerical': 0.07893978680495535, 'mean_word_length': 5.477611940298507, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '7468', 'license': 'MIT', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1349, 'n_words': 297, 'path': 'Climate change LDA_Tourists.ipynb', 'repo_name': 'Farshid123/Old-Win-7', 'size': '3471'}