<jupyter_start><jupyter_text>Dimension Reduction with UMAP<jupyter_code>import umap.umap_ as umap
umap_embeddings = umap.UMAP(n_neighbors=5, # local neighborhood
              n_components=50, # dimension
              metric='cosine').fit_transform(embeddings)
print(type(umap_embeddings))
print(umap_embeddings[0])
len(umap_embeddings)<jupyter_output><class 'numpy.ndarray'>
[1.8360118 5.555416  3.7216318 2.7965696 5.1788416 7.6975913 1.1669402
 4.961305  6.029223  5.9749556 5.2800746 4.234313  3.720886  3.569502
 4.687771  4.331612  6.3314686 6.344632  1.8526669 4.882493  4.979642
 4.9961634 4.031763  5.5209517 4.3205547 4.7796264 5.6238694 4.023513
 2.980316  7.584644  5.1541677 1.0598133 1.5596619 2.0520318 8.002602
 5.0166855 5.8138027 6.515288  5.1139    2.5760174 4.669116  3.433339
 2.989477  6.9863534 4.564564  4.304389  4.901848  5.7382455 3.582727
 5.1314135]<jupyter_text>Cluster the documents with HDBSCAN<jupyter_code>import hdbscan
cluster = hdbscan.HDBSCAN(min_cluster_size=50,
                          #min_samples=40,
                          metric='euclidean', 
                          #cluster_selection_epsilon = 1,
                          cluster_selection_method='eom').fit(umap_embeddings)
docs_df = pd.DataFrame(data_select, columns=["Doc"])
docs_df['Topic'] = cluster.labels_
docs_df['Doc_ID'] = range(len(docs_df))
docs_df.to_csv('docs_df.csv')
docs_per_topic = docs_df.groupby(['Topic'], as_index = False).agg({'Doc': ' '.join})
docs_per_topic.to_csv('docs_per_topic.csv')
docs_per_topic
print(len(docs_per_topic))
docs_per_topic=docs_per_topic[docs_per_topic['Topic']!=-1] 
docs_per_topic=docs_per_topic.reset_index(drop=True)
print(len(docs_per_topic))
docs_per_topic=docs_per_topic[docs_per_topic['Doc'].str.contains("[a-zA-Z]").fillna(False)]
print(len(docs_per_topic))
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer

#np.seterr(divide='ignore', invalid='ignore')

def c_tf_idf(documents, m, ngram_range=(1, 1)):
    count = CountVectorizer(ngram_range=ngram_range, stop_words="english").fit(documents)
    t = count.transform(documents).toarray()
    
    w = t.sum(axis=1)

    tf = np.divide(t.T, w)
    sum_t = t.sum(axis=0)
    idf = np.log(np.divide(m, sum_t)).reshape(-1, 1)
    tf_idf = np.multiply(tf, idf)

    return tf_idf, count
  
tf_idf, count = c_tf_idf(docs_per_topic.Doc.values, m=len(data))<jupyter_output><empty_output><jupyter_text>Topic RepresentationIn order to create a topic representation, we take the top 5 words per topic based on their c-TF-IDF scores. The higher the score, the more representative it should be of its topic as the score is a proxy of information density.<jupyter_code>def extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n):
    words = count.get_feature_names()
    labels = list(docs_per_topic.Topic)
    tf_idf_transposed = tf_idf.T
    indices = tf_idf_transposed.argsort()[:, -n:]
    top_n_words = {label: [(words[j], tf_idf_transposed[i][j]) for j in indices[i]][::-1] for i, label in enumerate(labels)}
    return top_n_words

def extract_topic_sizes(df):
    topic_sizes = (df.groupby(['Topic'])
                     .Doc
                     .count()
                     .reset_index()
                     .rename({"Topic": "Topic", "Doc": "Size"}, axis='columns')
                     .sort_values("Size", ascending=False))
    return topic_sizes

top_n_words = extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n=20)
topic_sizes = extract_topic_sizes(docs_df)
topic_sizes.to_csv('topic_sizes.csv')
topic_sizes.head(20)
top_topic_labels = topic_sizes.Topic[:11].values
top_topic_labels
topic_sizes['topic top words']=topic_sizes['Topic'].map(top_n_words)

def word_extract(x):
    if type(x)!= float:
        z=[tup[0] for tup in x]
    else: z=1
    return z

topic_sizes['words']=topic_sizes['topic top words'].apply(lambda x:word_extract(x))
topic_sizes=topic_sizes[topic_sizes['words']!=1]
topic_sizes.head(20)
topic_sizes_reset = topic_sizes.reset_index()
top_words = []
for i in range(len(topic_sizes)):
  #print(topic_sizes_reset.words[i][:5])
  curr_top_words = ', '.join(topic_sizes_reset.words[i][:5])
  top_words.append(curr_top_words)

top_words_df = pd.DataFrame(data={'Topic':topic_sizes.Topic, 'Size':topic_sizes.Size, 'top_five_words':top_words})
top_words_df_idx = top_words_df.reset_index(drop=True).reset_index()
top_words_df_idx
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
# Prepare data
#umap_data = umap.UMAP(n_neighbors=5, n_components=23, min_dist=0.0, metric='cosine').fit_transform(embeddings)
result = pd.DataFrame(umap_embeddings[:,:2], columns=['x', 'y'])
result['labels'] = cluster.labels_

# Visualize clusters
outliers = result.loc[result.labels == -1, :]
small_cluster = result.loc[~result.labels.isin(top_topic_labels), :]
clustered = result.loc[result.labels.isin(top_topic_labels), :]
clustered = clustered.merge(top_words_df, left_on = 'labels', right_on='Topic')

rgb_values = sns.color_palette("Set2", 10)

# Map continents to the colors
color_map = dict(zip(top_topic_labels, rgb_values))

#plt.scatter(outliers.x, outliers.y, color='#BDBDBD', s=0.05)
#plt.scatter(clustered.x, clustered.y, c=clustered['Topic'].map(color_map), s=0.05, cmap='hsv_r')

fig, ax = plt.subplots(figsize=(15, 10))
ax.scatter(outliers.x, outliers.y, color='#BDBDBD', s=0.05)
#ax.scatter([math.log(a+5) for a in list(outliers.x.values)], [math.log(a) for a in list(outliers.y.values)], color='#BDBDBD', s=0.05)

ax.scatter(small_cluster.x, small_cluster.y, color='#BDBDBD', s=0.05)
#ax.scatter([math.log(a+5) for a in list(small_cluster.x.values)], [math.log(a) for a in list(small_cluster.y.values)], color='#BDBDBD', s=0.05)

for t in top_words_df_idx.Topic.values[:10]:
  subset = clustered.loc[clustered.Topic == t,:]
  ax.scatter(#[math.log(a+5) for a in list(subset.x.values)], [math.log(a) for a in list(subset.y.values)],
             subset.x, subset.y, 
             c=subset['Topic'].map(color_map), 
             label=top_words_df.top_five_words.values[top_words_df.Topic == t][0],
              alpha=0.5, edgecolors='none')

handles, labels = ax.get_legend_handles_labels()
# sort both labels and handles by labels
#labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))
ax.legend(handles, labels,  markerscale=2., framealpha = 0.7, loc='upper right', prop={'size': 15})
#ax.set_xlim(8,9.5)
#ax.set_ylim(4.5,5.6)
ax.set_xlim(0,8.5)

#plt.show()
#plt.legend(rgb_values, clustered.Topic.unique())
import collections
occurrences = collections.Counter(init_cluster)
top_topics = occurrences.most_common()
top_topics = [x for (x,y) in top_topics]
top_topics.remove('None')
top_10_topics = top_topics[:10]

result['init_topic'] = init_cluster

#outliers_init = result.loc[init_cluster == 'None', :]
small_cluster_init = result.loc[~init_cluster_df.init_topic.isin(top_10_topics), :]
clustered_init = result.loc[init_cluster_df.init_topic.isin(top_10_topics), :]
#clustered_init = clustered_init.merge(top_words_df, left_on = 'labels', right_on='Topic')

# Map continents to the colors
color_map_init = dict(zip(top_10_topics, rgb_values))

#plt.scatter(outliers.x, outliers.y, color='#BDBDBD', s=0.05)
#plt.scatter(clustered.x, clustered.y, c=clustered['Topic'].map(color_map), s=0.05, cmap='hsv_r')

fig, ax = plt.subplots(figsize=(17, 10))
#ax.scatter(outliers_init.x, outliers_init.y, color='#BDBDBD', s=0.05)
ax.scatter(small_cluster_init.x, small_cluster_init.y, color='#BDBDBD', s=0.05)
for t in top_10_topics:
  #print(t)
  subset = clustered_init.loc[init_cluster_df.init_topic == t,:]
  #print(subset)
  ax.scatter(subset.x, subset.y, c=subset['init_topic'].map(color_map_init), 
             label=t,
            alpha=0.5, edgecolors='none')
ax.legend(handles, labels,  markerscale=2., framealpha = 0.7, loc='upper right', prop={'size': 15})
#ax.set_xlim(8,9.5)
#ax.set_ylim(4.5,5.6)
ax.set_xlim(0,8.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1488908384509337, 'fraction_numerical': 0.06329113924050633, 'mean_word_length': 4.986496624156039, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '205243', 'license': 'MIT', 'n_tokens_mistral': 3460, 'n_tokens_neox': 3119, 'n_words': 517, 'path': 'Code/News Article Topics Classifier/BERT Model/BERT_unsupervised_result/unsupervised_topic_model.ipynb', 'repo_name': 'Sophia1998/Misinformation-Project---Microsoft', 'size': '7979'}