<jupyter_start><jupyter_text>Topic Modelling<jupyter_code>#installing pyLDAvis and gensim
!pip install -U pyLDAvis
!pip install -U gensim
#importing Libraries needed for completion of topic modeling
import re
import string
import nltk
nltk.download('stopwords')
nltk.download('punkt')
from gensim import corpora, models, similarities
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
import pandas as pd
import numpy as np
import time
import gensim
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
from gensim.models import CoherenceModel
#read the text file 
# with open('/content/SpanishArticle.txt', 'r') as f:
#     text = f.readlines()
with open('./SpanishArticle.txt', 'r') as f:
    meta = f.readlines()

text = meta[0].split('.')
file_df = pd.DataFrame(text, columns=['text'])
# reviews_df=pd.read_csv('/content/SpanishArticle.txt',error_bad_lines=False)
print(file_df.head(6))
print("----------")
print(file_df.dtypes)
print("----------")
print(file_df.shape)<jupyter_output>text
0  Londres (CNN)– El organismo de control del gas...
1   La Comisión Electoral del Reino Unido dijo en...
2   Por lo tanto, continuaremos este trabajo como...
3   Johnson ha sido presionado para que explique ...
4   Los primeros ministros reciben £ 30,000 (US$ ...
5   El líder del Reino Unido se enfrentó a un may...
----------
text    object
dtype: object
----------
(32, 1)<jupyter_text>**Text Cleaning and Preprocessing**<jupyter_code>def initial_clean(text):
    """
    Function to clean text-remove punctuations, lowercase text etc.
    """
    text = re.sub("[^a-zA-Z ]", "", text)
    text = text.lower()  # lower case text
    text = nltk.word_tokenize(text)
    return (text)<jupyter_output><empty_output><jupyter_text>**Identifying and removing stopwords**<jupyter_code>stop_words = stopwords.words('english')
# adding some more stop words that doesn't convey much meaning in terms of reviews feel free to extend or reduce this list
stop_words.extend(['news', 'say','use', 'not', 'would', 'say', 'could', '_', 'be', 'know', 
                   'good', 'go', 'get', 'do','took','time','year',
                 'done', 'try', 'many', 'some','nice', 'thank', 'think', 'see', 'rather',
                  'easy', 'easily', 'lot', 'lack', 'make', 'want', 'seem', 'run', 'need', 
                   'even', 'right', 'line','even', 'also', 'may', 'take', 'come',
                   'new','said', 'like','people'])
def remove_stop_words(text):
     return [word for word in text if word not in stop_words]<jupyter_output><empty_output><jupyter_text>**Stemming**<jupyter_code>stemmer = PorterStemmer()
def stem_words(text):
    """
    Function to stem words
    """
    try:
        text = [stemmer.stem(word) for word in text]
        text = [word for word in text if len(word) > 1] # no single letter words
    except IndexError:
        pass

    return text<jupyter_output><empty_output><jupyter_text>**Applying all functions into one**<jupyter_code>def apply_all(text):
    """
    
    """
    return stem_words(remove_stop_words(initial_clean(text)))

print(apply_all)<jupyter_output><function apply_all at 0x7fb6f42dc5f0><jupyter_text>**Tokenization**<jupyter_code>t1 = time.time()
file_df['tokenized_text'] = file_df['text'].apply(apply_all)
t2 = time.time()
print("Time to clean and tokenize", len(file_df), "SpanishArticle:", (t2-t1)/60, "min") #Time to clean and tokenize 3209 reviews: 0.21254388093948365 min

print('\n')
print("text with their respective tokenize version:" )
print(file_df.head(5))<jupyter_output>Time to clean and tokenize 32 SpanishArticle: 0.0007640719413757324 min


text with their respective tokenize version:
                                                text                                     tokenized_text
0  Londres (CNN)– El organismo de control del gas...  [londr, cnn, el, organismo, de, control, del, ...
1   La Comisión Electoral del Reino Unido dijo en...  [la, comisin, elector, del, reino, unido, dijo...
2   Por lo tanto, continuaremos este trabajo como...  [por, lo, tanto, continuaremo, est, trabajo, c...
3   Johnson ha sido presionado para que explique ...  [johnson, ha, sido, presionado, para, que, exp...
4   Los primeros ministros reciben £ 30,000 (US$ ...  [lo, primero, ministro, reciben, us, de, diner...<jupyter_text>**LDA implementation**<jupyter_code>#Creating a Gensim dictionary from the tokenized data
tokenized = file_df['tokenized_text']
#Creating term dictionary of corpus, where each unique term is assigned an index.
dictionary = corpora.Dictionary(tokenized)
#Filtering terms .
dictionary.filter_extremes(no_below=1, no_above=0.8)
#converting the dictionary to a bag of words corpus
corpus = [dictionary.doc2bow(tokens) for tokens in tokenized]
print(corpus[:1])

print([[(dictionary[id], freq) for id, freq in cp] for cp in corpus[:1]])
#choosing hyperparameter values; num_topics=10, alpha=0.05, eta=0.61, num_words=5 
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics = 10, id2word=dictionary, passes=7, alpha=0.05, eta=0.61)

#saving the model
ldamodel.save('model_combined.gensim')
topics = ldamodel.print_topics(num_words=5)
print('\n')
print("Now printing the topics and their composition")
print("This output shows the Topic-Words matrix for the 10 topics created and the 5 words within each topic")
for topic in topics:
   print(topic)

# Compute Coherence Score for num_topics=10, alpha=0.05, eta=0.61, num_words=5
coherence_model_lda = CoherenceModel(model=ldamodel, texts=tokenized, dictionary=dictionary, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)
#visualizing topics for alpha=0.05, eta=0.61
lda_viz = gensim.models.ldamodel.LdaModel.load('/content/model_combined.gensim')
lda_display = pyLDAvis.gensim_models.prepare(lda_viz, corpus, dictionary, sort_topics=True)
pyLDAvis.display(lda_display)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13463809684494685, 'fraction_numerical': 0.021764805129070357, 'mean_word_length': 4.359855334538879, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '275500', 'license': 'MIT', 'n_tokens_mistral': 2103, 'n_tokens_neox': 1966, 'n_words': 560, 'path': 'code/Topic_modeling.ipynb', 'repo_name': 'brucker3/CS5560_proejct', 'size': '5927'}