<jupyter_start><jupyter_text>LDA - Sentences - Iterative filtering - Run 2 Note: This notebook loads the models from f1000_LDA_Sentence_Run_2 and returns "f1000_tokenized_LDA_sentence_3.tsv", which is the input for "f1000_LDA_Sentence_Run" with ITER = 3.  Configuration<jupyter_code>import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")

import re
from collections import defaultdict  # For word frequency
from operator import itemgetter
from tqdm import tqdm_notebook as tqdm
import IPython #for HTML viz

import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import logging  # Setting up the loggings to monitor gensim
logging.basicConfig(format="%(levelname)s - %(asctime)s: %(message)s", datefmt= '%H:%M:%S', level=logging.INFO)

from peertax.LDA_Diagnostic import LDA_Conv, LDA_Scores<jupyter_output><empty_output><jupyter_text>Model selection<jupyter_code>base_pth = './f1000_LDA_Sentence_Run_2/'
cv_arr, um_arr = LDA_Scores(base_pth)

plt.subplot(1,2,1)
plt.plot(cv_arr[:,0],cv_arr[:,1],'ko-')
plt.ylabel("CV Score")
plt.xlabel("Num Topics")

plt.subplot(1,2,2)
plt.plot(um_arr[:,0],um_arr[:,1],'ko-')
plt.ylabel("UMASS Score")
plt.xlabel("Num Topics")
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Best model (highest CV Score) is Model_16.Load corpus from Run_2 and Model_16.<jupyter_code>num_topics = '16'
model_no = 'Model_' + num_topics
model_pth = base_pth + model_no

# Load sentence data from tsv
path_load_tsv = '../pickles/f1000_tokenized_LDA_sentence_2.tsv'
df = pd.read_csv(path_load_tsv,sep='\t',quoting=csv.QUOTE_NONE)
df.drop(columns=['Unnamed: 0'],inplace=True)
df['token'] = df['token'].str.split(',')

# Create Corpus
texts = df['token']
#Load Model
lda_model = gensim.models.ldamodel.LdaModel.load(model_pth)
id2word = gensim.corpora.Dictionary.load(model_pth + '.id2word')
# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]
print('Total number of sentences: %s' % len(corpus))<jupyter_output>INFO - 13:17:54: loading LdaModel object from ./f1000_LDA_Sentence_Run_2/Model_16
INFO - 13:17:54: loading expElogbeta from ./f1000_LDA_Sentence_Run_2/Model_16.expElogbeta.npy with mmap=None
INFO - 13:17:54: setting ignored attribute dispatcher to None
INFO - 13:17:54: setting ignored attribute id2word to None
INFO - 13:17:54: setting ignored attribute state to None
INFO - 13:17:54: loaded ./f1000_LDA_Sentence_Run_2/Model_16
INFO - 13:17:54: loading LdaState object from ./f1000_LDA_Sentence_Run_2/Model_16.state
INFO - 13:17:54: loaded ./f1000_LDA_Sentence_Run_2/Model_16.state
INFO - 13:17:54: loading Dictionary object from ./f1000_LDA_Sentence_Run_2/Model_16.id2word
INFO - 13:17:54: loaded ./f1000_LDA_Sentence_Run_2/Model_16.id2word<jupyter_text>Check convergence.<jupyter_code>iter_no, perplexity, likelihood = LDA_Conv(base_pth,num_topics)
plt.subplot(1,2,1)
plt.plot(iter_no,likelihood,c="black")
plt.ylabel("log liklihood")
plt.xlabel("iteration")

plt.subplot(1,2,2)
plt.plot(iter_no,perplexity,c="black")
plt.ylabel("perplexity")
plt.xlabel("iteration")
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Show topic visulization.<jupyter_code>IPython.display.HTML(filename=model_pth+'.html')<jupyter_output><empty_output><jupyter_text>Show topic contents (text).<jupyter_code>lda_model.show_topics(num_topics=25, num_words=10, log=False, formatted=True)<jupyter_output><empty_output><jupyter_text>Assign topics to initial dataframe.<jupyter_code>topics = [max(lda_model.get_document_topics(row),key=itemgetter(1))[0] for row in tqdm(corpus)]
df['topics'] = topics
df.head()
sns.countplot(x='topics',data=df);<jupyter_output><empty_output><jupyter_text>Manually select the bad topics (because they are difficult to interpret).<jupyter_code>bad_topics = [7,11,13,14,15]<jupyter_output><empty_output><jupyter_text>Filter topics.<jupyter_code>df_filter = df[~df['topics'].isin(bad_topics)].copy()
df_filter.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 50811 entries, 0 to 72450
Data columns (total 5 columns):
manuscript_ID    50811 non-null object
review_ID        50811 non-null object
sentences        50811 non-null object
token            50811 non-null object
topics           50811 non-null int64
dtypes: int64(1), object(4)
memory usage: 2.3+ MB<jupyter_text>Save to tsv.<jupyter_code>path_save_tsv = "../pickles/f1000_tokenized_LDA_sentence_3.tsv"
df_filter['token'] = df_filter['token'].str.join(',')
df_filter.to_csv(path_save_tsv,sep='\t',quoting=csv.QUOTE_NONE)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12422360248447205, 'fraction_numerical': 0.049903619618762045, 'mean_word_length': 7.339285714285714, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '518262', 'license': 'MIT', 'n_tokens_mistral': 1957, 'n_tokens_neox': 1784, 'n_words': 336, 'path': 'LDA/f1000_LDA_Sentence_Filter_2.ipynb', 'repo_name': 'elifesciences/data-science-peertax', 'size': '4669'}