<jupyter_start><jupyter_text>Topic Modeling ClusteringIn this notebook we visualize a previously generated topic model.  Set Up  Imports<jupyter_code>import pandas as pd
import numpy as np
from lib import tapi<jupyter_output><empty_output><jupyter_text>Configuration<jupyter_code>tapi.list_dbs()
data_prefix = 'winereviews'
db = tapi.Edition(data_prefix)<jupyter_output><empty_output><jupyter_text>Import Topic DataWe import our previously generated model.<jupyter_code>db.get_tables()
n_topics = len(db.TOPICS)
topic_cols = [t for t in range(n_topics)]<jupyter_output><empty_output><jupyter_text>Inspect Results  Sort Topics by Doc Weight<jupyter_code>db.TOPICS_NMF.sort_values('doc_weight_sum', ascending=True)\
    .plot.barh(y='doc_weight_sum', x='topwords', figsize=(5, n_topics/3), legend=False);
db.TOPICS.sort_values('doc_weight_sum', ascending=True)\
    .plot.barh(y='doc_weight_sum', x='topwords', figsize=(5, n_topics/3), legend=False);<jupyter_output><empty_output><jupyter_text>Clutser Topics<jupyter_code>import scipy.cluster.hierarchy as sch
from scipy.spatial.distance import pdist
from sklearn.cluster import AgglomerativeClustering
from sklearn.preprocessing import normalize
import matplotlib.pyplot as plt
def plot_tree(tree, labels):
    plt.figure()
    fig, axes = plt.subplots(figsize=(5, n_topics / 3))
    dendrogram = sch.dendrogram(tree, labels=labels, orientation="left")
    plt.tick_params(axis='both', which='major', labelsize=14)
SIMS = pdist(normalize(db.PHI), metric='cosine')
TREE = sch.linkage(SIMS, method='ward')
plot_tree(TREE, db.TOPICS.topwords.to_list());
SIMS = pdist(normalize(db.PHI_NMF), metric='cosine')
TREE = sch.linkage(SIMS, method='ward')
plot_tree(TREE, db.TOPICS_NMF.topwords.to_list());<jupyter_output><empty_output><jupyter_text>Group Topics by Label<jupyter_code>db.LABELS.columns
group_col = 'doc_label'
group_vals = db.LABELS[group_col].value_counts().to_frame('n')
group_vals.index.name = 'group_id'
group_vals
# group_vals.sort_values('n').plot.barh(figsize=(5,len(group_vals) / 3), legend=False);
G = db.THETA.join(db.LABELS).groupby(group_col)[topic_cols].mean()
G.index.name = 'group_id'
G.columns = db.TOPICS.topwords
G2 = db.THETA_NMF.join(db.LABELS).groupby(group_col)[topic_cols].mean()
G2.index.name = 'group_id'
G2.columns = db.TOPICS_NMF.topwords<jupyter_output><empty_output><jupyter_text>Heatmap of Labels and Topics<jupyter_code>G.style.background_gradient()
G2.style.background_gradient()<jupyter_output><empty_output><jupyter_text>Top Topic per Label Group<jupyter_code>group_vals['top_topic'] = G.T[group_vals.index].idxmax()
group_vals['top_topic2'] = G2.T[group_vals.index].idxmax()
group_vals.sort_values('top_topic')<jupyter_output><empty_output><jupyter_text>Top Label Group per Topic<jupyter_code>G.idxmax().to_frame('label').sort_values('label')
G2.idxmax().to_frame('label').sort_values('label')<jupyter_output><empty_output><jupyter_text>How Similar are the Topics to Each Other?<jupyter_code>X = db.THETA.T.dot(db.THETA_NMF)
X.columns = db.TOPICS_NMF.topwords
X.index = db.TOPICS.topwords
X.columns.name = 'NMF'
X.index.name = 'LDA'
# X.style.background_gradient()
# X.T.style.background_gradient()
X2 = X.stack().to_frame('w').sort_values('w', ascending=False)
X2.head(10).style.bar()
X2.w.hist();
# X2.w.sort_values(ascending=False).head(40).sort_values().plot.barh(figsize=(10,20))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14692918013517484, 'fraction_numerical': 0.008521892447840141, 'mean_word_length': 8.351648351648352, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '128789', 'license': 'MIT', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1314, 'n_words': 183, 'path': '02-SciKitLearn-Cluster.ipynb', 'repo_name': 'jingxu1983/TAPI_Topic_Models', 'size': '3403'}