<jupyter_start><jupyter_text>Inspection/troubleshooting of LDA model<jupyter_code>import pandas as pd
import numpy as np
import os
from tqdm import tqdm

from gensim.corpora.dictionary import Dictionary
from gensim.models.ldamodel import LdaModel

import matplotlib.pyplot as plt

import my_lda
num_topics = 100
alpha = 0.01
eta = 0.1
model_root = f'lda_{num_topics}_{alpha}_{eta}'

metadata_path = 'results/final_models/metadata_2021.csv.gz'
corpus_directory = 'results/final_models/lda_2021_corpus_clean/'
dictionary_path = 'results/final_models/dictionary.dict'
model_path = f'results/final_models/{model_root}_model'
metadata = pd.read_csv(metadata_path, sep='\t')
dictionary = Dictionary.load(dictionary_path)
doc_path_list = [corpus_directory + file for file in os.listdir(corpus_directory) 
                 if file.endswith('.txt')]
corpus = my_lda.MyCorpus(doc_path_list, dictionary=dictionary)
model = LdaModel.load(model_path)<jupyter_output><empty_output><jupyter_text>View distribution of topics per document<jupyter_code>my_lda.topics_per_doc(model, corpus)<jupyter_output>100%|██████████| 53758/53758 [06:46<00:00, 132.31it/s]<jupyter_text>View number of topics per word<jupyter_code>def plot_topics_per_word(model, dictionary):
    topics_per_word = []
    for i in tqdm(range(len(dictionary))):
        topics_per_word.append(len(model.get_term_topics(i)))
    plt.hist(topics_per_word, bins=max(topics_per_word))

plot_topics_per_word(model, dictionary)<jupyter_output>100%|██████████| 100000/100000 [00:14<00:00, 6953.40it/s]<jupyter_text>View topic predictions for query<jupyter_code>query = 'What endocrine complications are  linked to COVID-19?'

my_lda.print_pred(query, corpus, model)<jupyter_output>What endocrine complications are  linked to COVID-19? 

endocrine complications link covid19 

bow: 
complications 1
endocrine 1

Topic predictions: 
[(92, 0.35180938), (15, 0.321474)] 

complications []
endocrine []<jupyter_text>View topic relevance for word<jupyter_code>word = 'complications'
model.get_term_topics(dictionary.token2id[word], minimum_probability=num_topics/len(corpus.dictionary))
my_lda.topic_words(92, model, corpus)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11273391145595618, 'fraction_numerical': 0.04655408489274304, 'mean_word_length': 7.271698113207547, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '296249', 'license': 'MIT', 'n_tokens_mistral': 897, 'n_tokens_neox': 787, 'n_words': 143, 'path': 'lda_troubleshooting.ipynb', 'repo_name': 'omlean/covid-lit-search', 'size': '2191'}