<jupyter_start><jupyter_text>Selected Model<jupyter_code># rest of the code depends on here.
topic_model=loaded_models_list[0]
topic_mode_selected = df.lda_5_topic_mode
topic_mode_prob_selected = df.lda_5_topic_mode_prob
corpus=obj_list[1] #debug only, [:5000]
dictionary=obj_list[0]
# make a dataframe from words
topic_words = topic_model.show_topics(num_words=300, formatted=False)

topics_list = []
for (topic, word_list) in topic_words:
    temp = pd.DataFrame(word_list, columns=[f'{topic}_word', f'{topic}_prob'])
    topics_list.append(temp)
pd_topics = pd.concat(topics_list, axis=1)
pd_topics.to_csv(f'{archive_path}/data-server/processed/hi_rws_0001_0256_topics_words.csv', index=False)
pd_topics.head(10)
# words in the topics for selected model
from wordcloud import WordCloud, STOPWORDS
stopwords = set(STOPWORDS)

for i in range(len(topics_list)):
    wordcloud = WordCloud(
            background_color='white',
            # stopwords=stopwords,
            max_words=400,
            max_font_size=40, 
            scale=5,
            random_state=1 # chosen at random by flipping a coin; it was heads
        ).generate(str(pd_topics[f'{i}_word']))
    wordcloud.to_file(f'{archive_path}/data-server/processed/hi_rws_0001_0256_topic_{i}.png')
    fig = plt.figure(1, figsize=(6, 6), dpi=100)    
    fig.suptitle(f'topic_{i}', fontsize=25)
    fig.subplots_adjust(top=2.3)
    plt.axis('off')
    plt.imshow(wordcloud)
    plt.show()
# topic sent distribution per review
pd_topic_frequencies = topic_mode_prob_selected.apply(lambda x: pd.Series(x.split(',')).value_counts())
pd_topic_frequencies = pd_topic_frequencies.fillna(0).reindex(columns=sorted(pd_topic_frequencies))
pd_topic_frequencies.columns = [f'topic_{i}' for i in range(5)]
pd_topic_frequencies.head()
# dump final data
data_path = f'{archive_path}/data-server/processed/hi_rws_0001_0256_descriptive.csv'
sentiments_path = f'{archive_path}/data-server/processed/hi_rws_0001_0256_sentiments.csv'
df1 = pd.read_csv(data_path)
df2 = pd.read_csv(sentiments_path)

# needed fixes
df1 = df1.drop([13914, 107036, 114094], axis=0)
df1.index = df1.reset_index(drop=True).index
topic_mode_selected.name = 'topic_score'
df2['sgd.1'].name = 'sentiment_score'

df_final = pd.concat([df1, pd_topic_frequencies, topic_mode_selected, df2['sgd.1']], axis=1)
df_final.to_csv(f'{archive_path}/data-server/processed/hi_rws_0001_0256_final.csv', index=False)
df_final.info()<jupyter_output><empty_output><jupyter_text>Model Plots<jupyter_code># takes a lot of time...
# Get topic weights
topic_weights = []
for i, row_list in enumerate(topic_model[corpus]):
    topic_weights.append([w for i, w in row_list[0]])

# Array of topic weights    
arr = pd.DataFrame(topic_weights).fillna(0).values

# Keep the well separated points (optional)
arr = arr[np.amax(arr, axis=1) > 0.25]

# Dominant topic number in each doc
topic_num = np.argmax(arr, axis=1)
arr.shape
# Dominant topic number in each doc
topic_num = np.argmax(arr, axis=1)
from sklearn.manifold import TSNE
tsne_model = TSNE(n_components=2, verbose=1, random_state=0, angle=.99, init='pca', perplexity=50, n_iter=600)
tsne_lda = tsne_model.fit_transform(arr)
tsne_lda.shape
mycolors
import matplotlib.colors as mcolors
from bokeh.plotting import figure, output_file, show
from bokeh.models import Label, Legend
from bokeh.io import output_notebook

# vars
output_notebook()
n_topics = 5
mycolors = np.array([color for name, color in mcolors.TABLEAU_COLORS.items()])

# plot
plot = figure(title="T-SNE Clustering of {} LDA Topics".format(n_topics), 
              plot_width=900, plot_height=900)
plot.scatter(x=tsne_lda[:,0], y=tsne_lda[:,1], color=mycolors[topic_num])
show(plot)
from matplotlib import cm
plt.figure(figsize=(12, 12))
plt.scatter(tsne_lda[:,0],tsne_lda[:,1], marker='.', cmap=cm.Paired)
plt.axis('equal')
plt.show()
import pyLDAvis.gensim
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(topic_model=topic_model, corpus=corpus, dictionary=dictionary)
vis<jupyter_output>/Applications/anaconda3/envs/text/lib/python3.7/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))
{'fraction_non_alphanumeric': 0.12350147025559828, 'fraction_numerical': 0.0323456231621805, 'mean_word_length': 5.803076923076923, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '206841', 'license': 'MIT', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1620, 'n_words': 319, 'path': 'notebooks/models/1.0-archive-models.ipynb', 'repo_name': 'metinsenturk/semantic-analysis', 'size': '4421'}