<jupyter_start><jupyter_text>Load The Data<jupyter_code>handler = TextHandler("../contextualized_topic_models/data/gnews/GoogleNews.txt")
handler.prepare() # create vocabulary and training data

# load BERT data
with open("../contextualized_topic_models/data/gnews/bert_embeddings_gnews", "rb") as filino:
    training_bert = pickle.load(filino)

training_dataset = CTMDataset(handler.bow, training_bert, handler.idx2token)

ctm = CTM(input_size=len(handler.vocab), bert_input_size=768, num_epochs=5, inference_type="combined", n_components=50)

ctm.fit(training_dataset) # run the model
ctm.get_topic_lists(5)[0:5]<jupyter_output><empty_output><jupyter_text>Predict the topic distribution for the documentsWhich is the topic of the document *nokia lumia launch*?<jupyter_code>!head -n 10 ../contextualized_topic_models/data/gnews/GoogleNews.txt
import numpy as np

distribution = ctm.get_thetas(training_dataset)[8] # topic distribution for the eight document

print(distribution)

topic = np.argmax(distribution)

ctm.get_topic_lists(5)[topic]<jupyter_output>[0.04631870985031128, 0.0030940896831452847, 0.012179974466562271, 0.10562609136104584, 0.00029982352862134576, 0.016247086226940155, 0.0026096573565155268, 0.0024016175884753466, 0.017862707376480103, 0.02637273631989956, 0.06362489610910416, 0.0037376857362687588, 0.027107330039143562, 0.05662212520837784, 0.002137327566742897, 0.0013186399592086673, 0.004643352702260017, 0.008647769689559937, 0.0011787913972511888, 0.006648227572441101, 0.007791334297508001, 0.00045980652794241905, 0.275553822517395, 0.004696805961430073, 0.0015463449526578188, 0.0004970636800862849, 0.0042398953810334206, 0.00293867033906281, 0.023034632205963135, 0.09038814902305603, 0.02389369159936905, 0.0037447575014084578, 0.03660003840923309, 0.00037390936631709337, 0.026152431964874268, 0.016770798712968826, 0.00270906207151711, 0.0025828303769230843, 0.008109727874398232, 0.0009836236713454127, 0.00779800396412611, 0.03024321049451828, 0.0005514333606697619, 0.004347378853708506, 0.004108520690351725, 0.000[...]<jupyter_text>Evaluate the Model<jupyter_code>from contextualized_topic_models.evaluation.measures import TopicDiversity, CoherenceNPMI,\
    CoherenceWordEmbeddings, InvertedRBO
td = TopicDiversity(ctm.get_topic_lists(25))
td.score(topk=25)

rbo = InvertedRBO(ctm.get_topic_lists(10))
rbo.score()<jupyter_output><empty_output><jupyter_text>Coherence measure based on Word Embeddingsevaluation of coherence on a word embedding space. if word2vec_file is specified, it retrieves the word embeddings file (in word2vec format) otherwise 'word2vec-google-news-300' is downloaded using gensim's APIs<jupyter_code>word2vec_path = "your\\path\\to\\word2vec.bin"
we_coh = CoherenceWordEmbeddings(word2vec_path=word2vec_path,
                                 topics=ctm.get_topic_lists(10),binary=True)
we_coh.score(topk=10)
with open(os.path.join('../contextualized_topic_models/data/gnews', 'GoogleNews.txt'),"r") as fr:
    texts = [doc.split() for doc in fr.read().splitlines()]
npmi = CoherenceNPMI(texts=texts, topics=ctm.get_topic_lists(10))
npmi.score()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10578144853875476, 'fraction_numerical': 0.2855781448538755, 'mean_word_length': 7.674931129476584, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '541435', 'license': 'MIT', 'n_tokens_mistral': 1782, 'n_tokens_neox': 1218, 'n_words': 180, 'path': 'examples/topic-modeling.ipynb', 'repo_name': 'yangalan123/contextualized-topic-models', 'size': '3148'}