<jupyter_start><jupyter_text>Topic modeling with SFCRs Example of topic modeling with SFCRs.<jupyter_code>import numpy as np
import os
import pickle
#import pyLDAvis
#import pyLDAvis.gensim<jupyter_output><empty_output><jupyter_text>Read SFCRs<jupyter_code>language = 'EN'
local_path = '../SFCR_data/'
if not(os.path.isfile(local_path + 'SFCRs_' + language + '.dat')):
    print("Files not found.")
else:
    with open(local_path + 'SFCRs_' + language + '.dat', 'rb') as fp:
        documents = pickle.load(fp)<jupyter_output><empty_output><jupyter_text>Preprocess with NLTK and Gensim<jupyter_code>import warnings
warnings.filterwarnings(action = 'ignore', category = UserWarning, module = 'gensim')
import gensim
import nltk
sentences = []
for document in documents:
    sent_list = nltk.tokenize.sent_tokenize(document)
    sentences.extend(sent_list)
stop_words = nltk.corpus.stopwords.words('english')

def remove_stopwords(texts):
    return [[word for word in gensim.utils.simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]

sentences = remove_stopwords(sentences)
sentences = [sentence for sentence in sentences if len(sentence) > 40]
print("Number of documents: " + str(len(documents)))
print("Number of sentences: " + str(len(sentences)))
print("Number of words: " + str(sum([len(word) for word in sentences])))<jupyter_output><empty_output><jupyter_text>Preliminary analysis<jupyter_code># first get a list of all words
all_words = [word for item in sentences for word in item]
# use nltk fdist to get a frequency distribution of all words
fdist = nltk.FreqDist(all_words)
print("Number of unique words: " +str(len(fdist)))
# choose k and visually inspect the bottom 10 words of the top k
k = 10000
top_k_words = fdist.most_common(k)
top_k_words[-10:]
# define a function only to keep words in the top k words
top_k_words,_ = zip(*fdist.most_common(k))
top_k_words = set(top_k_words)
def keep_top_k_words(text):
    return [word for word in text if word in top_k_words]

for idx in range(len(sentences)):
    sentences[idx] = keep_top_k_words(sentences[idx])
# document length
doc_lengths = [len(sentence) for sentence in sentences]

print("length of list:",len(doc_lengths),
      "\naverage length:", np.average(doc_lengths),
      "\nminimum length:", min(doc_lengths),
      "\nmaximum length:", max(doc_lengths))<jupyter_output><empty_output><jupyter_text>Start modelling<jupyter_code>import spacy
fname = "data_lemmatized"

# Initialize spacy 'en' model
nlp = spacy.load('en', disable = ['parser', 'ner'])

def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent)) 
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    return texts_out

# Do lemmatization keeping only noun, adj, vb, adv
data_lemmatized = lemmatization(sentences, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

#if not os.path.isfile(fname):
#    filehandler = open(fname, 'wb') 
#    pickle.dump(data_lemmatized, filehandler)
#else:
#    filehandler = open(fname, 'rb') 
#    data_lemmatized = pickle.load(filehandler)
# Create Dictionary
id2word = gensim.corpora.Dictionary(data_lemmatized)

# Create Corpus
texts = data_lemmatized

# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]<jupyter_output><empty_output><jupyter_text>Mallet model<jupyter_code>os.environ['MALLET_HOME'] = 'C:\\mallet\\'
mallet_path = r'C:\\mallet\\bin\\mallet'
def compute_model_list(dictionary, corpus, texts, limit, start=2, step=3):
    model_list = []
    for num_topics in range(start, limit, step):
        print(".", end="")
        model = gensim.models.wrappers.LdaMallet(mallet_path, 
                                                 corpus=corpus, 
                                                 num_topics=num_topics, 
                                                 id2word=id2word,
                                                 topic_threshold=0.0)
        model_list.append(model)
    return model_list

def compute_coherence(dictionary, corpus, model_list):
    coherence_values = []
    for model in model_list:
        print(".", end="")
        coherencemodel = gensim.models.CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())
    return coherence_values<jupyter_output><empty_output><jupyter_text>Calculate and save model<jupyter_code># Can take a long time to run.
model_list = compute_model_list(dictionary = id2word,
                                corpus = corpus,
                                texts = data_lemmatized, 
                                start = 2, 
                                limit = 15,
                                step = 1)
#filehandler = open(local_path + "lda_mallet_models", 'wb') 
#pickle.dump(model_list, filehandler)
#filehandler.close()<jupyter_output><empty_output><jupyter_text>Calculate cohere score<jupyter_code>coherence_values = compute_coherence(dictionary = id2word,
                                     corpus = corpus,
                                     model_list = model_list)
import matplotlib.pyplot as plt
%matplotlib inline
# Show graph
limit=15; start=2; step=1;
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Number of topics")
plt.ylabel("Coherence score")
#plt.legend(("coherence_values"), loc = 'best')
plt.show()
# Print the coherence scores
for m, cv in zip(x, coherence_values):
    print("Num Topics =", m, " has Coherence Value of", round(cv, 4))
# Select optimal model
optimal_model = model_list[8]
print(optimal_model.print_topics(10))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1136919315403423, 'fraction_numerical': 0.005937827453719874, 'mean_word_length': 3.6827473426001633, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '55101', 'license': 'MIT', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1851, 'n_words': 441, 'path': 'notebooks/Topic modeling with SFCRs.ipynb', 'repo_name': 'wjwillemse/insurlib', 'size': '5726'}