<jupyter_start><jupyter_text>*Unsupervised learning: Latent Dirichlet allocation (LDA) topic modeling*<jupyter_code>## Install Python package for LDA
# http://pythonhosted.org/lda/getting_started.html

!pip install lda
!pip install nltk
!pip install scikit-learn
## Importing basic packages

import os
import numpy as np

os.chdir('/Volumes/benson/Mellon/cirma_related/Topic_Modeling_Workshop_Materials/cirma_texts/')
document_list = []

for filename in [item for item in os.listdir('./') if '.txt' in item]:
    text_data = open(filename).read()
    document_list.append(text_data)
## Importing NLTK stop words
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import string

 
stop_words = stopwords.words('spanish') + ["'s", "'re", '”', '“', '’', '—'] + list(string.punctuation)

string.punctuation
## Tokenizing and removing stop words from our list of documents

documents_filtered = []

for document in document_list:
    token_list = word_tokenize(document.lower())
    tokens_filtered = [item for item in token_list if (item not in stop_words)]
    documents_filtered.append(' '.join(tokens_filtered))
## Viewing a preprocessed document

documents_filtered[30]
## Vectorizing preprocessed essays

from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer()

X = vectorizer.fit_transform(documents_filtered) 
## Creating a vocabulary list corresponding to the vectors we created above

vocabulary = vectorizer.get_feature_names()

vocabulary[1140:1160]
## Initializing an LDA model: 10 topics and 1500 iterations

import lda

model = lda.LDA(n_topics=10, n_iter=1500, random_state=1)
## Fitting the model using our list of vectors

model.fit(X)
## Viewing the top 50 words in each 'topic'

topic_word = model.topic_word_

n_top_words = 10

for i, topic_distribution in enumerate(topic_word):
    topic_words = np.array(vocabulary)[np.argsort(topic_distribution)][:-(n_top_words+1):-1]
    print('Topic ' + str(i) + ':')
    print(' '.join(topic_words))
    print()
#Make stopwords in a single variable and make a class that 
#reprocesses the documents into a dictionary for gensim format with tokenization. 
import logging 
stops = '/Users/jgo384/nltk_data/corpora/stopwords/spanish.txt'

stoplist = open(stops).read()
stoplist = stoplist.split('\n')
class MyCorpus(object):

    def __init__(self, topdir, stoplist):
        self.topdir = topdir
        self.stoplist = stoplist
        self.dictionary = gensim.corpora.Dictionary(iter_docs(topdir, stoplist))
        
    def __iter__(self):
        for tokens in iter_docs(self.topdir, self.stoplist):
            yield self.dictionary.doc2bow(tokens)


logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', 
                    level=logging.INFO)
#separate function to iterate through the documents separately to tokenize and preprocess with lower and accent remove
def iter_docs(topdir, stoplist):
    for fn in os.listdir(topdir):
        fin = open(os.path.join(topdir, fn), 'rb')
        text = fin.read()
        fin.close()
        yield (x for x in 
            gensim.utils.tokenize(text, lowercase=True, deacc=True, 
                                  errors="ignore")
            if x not in stoplist)
#gensim doesn't like the texts directory to be two separate so they were combined
try: os.mkdir('/sharedfolder/models/dir')
except: pass
TEXTS_DIR = "/sharedfolder/tempdirtesting"
MODELS_DIR = "/sharedfolder/models/dir"
#the fist is a bit of overkill since it reapplies stopwords, it also generates tokenization again
#the issue is that I can't bring the other variables over because it dislikes nltk's tokenization for utf-8
#the second creates a dictionary which is just a id per word list
#the third creates pairings between document, id, and word with a frequency count in a dense matrix
corpus = MyCorpus(TEXTS_DIR, stoplist)
corpus.dictionary.save(os.path.join(MODELS_DIR, 'spaport.dict'))
gensim.corpora.MmCorpus.serialize(os.path.join(MODELS_DIR, "spaport.mm"), 
                                  corpus)
#logging just to keep track of time for these, much slower than Mallet or lda
#dictionary gets loaded from dict file
#corpus is loaded from path and linked to matrix of values for frequency, id and doc
#50 topics seems to work best and keeping the passes(iterations) at the same number as Mallet (painfully slow)
numtopics = 50
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', 
                    level=logging.INFO)

dictionary = gensim.corpora.Dictionary.load(os.path.join(MODELS_DIR, 
                                            "spaport.dict"))
corpus = gensim.corpora.MmCorpus(os.path.join(MODELS_DIR, "spaport.mm"))

# Project to LDA space
polylingualmodel = gensim.models.LdaMulticore(corpus, id2word=dictionary, num_topics=numtopics, 
                                              workers=8, random_state=1, passes=1000)
#saving the model for persistency
polylingualmodel.save('polylingualmodel2.model')
#creating a bag-of-words from the corpus with doc id, word, frequency count array
bow_corpus = MmCorpus("/sharedfolder/models/dir/spaport.mm")
#This shows a random sorting of topics with the weight across the entire corpus 
#"topic distribution for the given document bow, as a list of (topic_id, topic_probability) 2-tuples"
get_documents_topics = polylingualmodel.get_document_topics(bow_corpus, 
                                                            minimum_probability=.001, 
                                                            minimum_phi_value=0.01, 
                                                            per_word_topics=True)
get_documents_topics[1]
#This prints a topic at a time and is useful for inspecting topics one by one. The one listed here
#bears some slight resemblance to 21 in Mallet
print_topics = polylingualmodel.print_topics(num_topics=1, num_words=35)
#From gensim site "tuples with (topic_repr, coherence_score), where topic_repr is a list of 
#representations of the topn terms for the topic. 
#The terms are represented as tuples of (membership_in_topic, token). 
#The coherence_score is a float".
top_topics = polylingualmodel.top_topics(corpus=corpus, texts=None, dictionary='spaport.dict', 
                                         window_size=None, coherence='u_mass', 
                                         topn=50, processes=-1)
top_topics
#even with pprint this is quite difficult to sort through but there seems to be an issue with the weights
#all being too low at .00*
from pprint import pprint
pprint(top_topics)
os.chdir('/sharedfolder/models/dir')
d = gensim.corpora.Dictionary.load('spaport.dict')
c = gensim.corpora.MmCorpus('spaport.mm')
vizmod = gensim.models.LdaMulticore.load('polylingualmodel2.model')
pyLDAvis.enable_notebook()
data = pyLDAvis.gensim.prepare(vizmod, c, d)
data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09106779167878964, 'fraction_numerical': 0.009455920861216176, 'mean_word_length': 3.5894526034712952, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '518035', 'license': 'MIT', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2057, 'n_words': 625, 'path': 'gensim_pyldavis/LDA_topic_modeling_sansjava.ipynb', 'repo_name': 'JoshuaGOB/NLTK', 'size': '6874'}