<jupyter_start><jupyter_text>BERT ClusteringUnsupervised clustering of forum posts using the text body. A simple approach to try and find similar posts without strict classification.Most of the process and code were taken from [<NAME>](https://towardsdatascience.com/topic-modeling-with-bert-779f7db187e6).Right now the model doesn't do anything, which is very sad. References1. [Topic Modeling with BERT](https://towardsdatascience.com/topic-modeling-with-bert-779f7db187e6)2. [Multi-label Text Classification using BERT - The Mighty Transformer](https://medium.com/huggingface/multi-label-text-classification-using-bert-the-mighty-transformer-69714fa3fb3d)3. [BERT Fine-Tuning Tutorial with PyTorch](https://mccormickml.com/2019/07/22/BERT-fine-tuning)<jupyter_code>!pip install transformers -q
!pip install sentence_transformers -q
!pip install umap-learn -q
!pip install hdbscan -q

import re
import umap
import torch
import hdbscan
import numpy as np
import pandas as pd
import transformers
import seaborn as sns
import matplotlib.pyplot as plt
from sentence_transformers import SentenceTransformer
np.random.RandomState(123)

if torch.cuda.is_available():
    device = torch.device('cuda')
    print(f'using {torch.cuda.get_device_name(0)}')
else:
    print('no GPU avaiable')<jupyter_output>using Tesla T4<jupyter_text>Load Data<jupyter_code>df_ori = pd.read_json('https://github.com/charlotte-zhuang/forum-recommender/blob/master/data/sitepoint.json?raw=true')
df_ori.head()<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>def clean_post(text_l: list) -> str:
    '''Converts a list of text into a single string that
        can be tokenized.
    '''
    
    text = ' '.join(text_l)
    
    # remove links
    text = re.sub(r'https?:\/\/[^\s]*', r'', text)
    
    # remove dots that aren't periods
    text = re.sub(r'\.([^\s])', r'\1', text)
    
    # remove closed single quotes
    text = re.sub(r"‘([^‘’]*)’", r'\1', text)

    # remove unwanted characters
    text = re.sub(r"[^\w\s\d\.?!’]+", r' ', text)
    
    # remove repeated special characters
    text = re.sub(r"([’\.?!])[’\.?!]+", r'\1', text)
    
    # remove extra spaces
    text = re.sub(r'\s{2,}', r' ', text)
    
    return text
df = df_ori.drop(labels=['title', 'tags'], axis=1)
df['post'] = df['post'].apply(clean_post)

for i in range(5):
    print(df.iat[i, 1], end='\n\n')
# optionally drop data for convenience
# df = df[:1000]<jupyter_output><empty_output><jupyter_text>EmbeddingUsing DistilBERT from Sentence Transformers<jupyter_code>st_model = SentenceTransformer('distilbert-base-nli-mean-tokens')
embeddings = st_model.encode(df['post'], show_progress_bar=True)<jupyter_output><empty_output><jupyter_text>ClusteringReduce dimensionality<jupyter_code>umap_model = umap.UMAP(n_neighbors=15, n_components=5, metric='cosine')
umap_embeddings = umap_model.fit_transform(embeddings)

umap_embeddings.shape<jupyter_output><empty_output><jupyter_text>Cosine similarity clustering<jupyter_code>hdbscan_model = hdbscan.HDBSCAN(
    min_cluster_size=15, metric='euclidean', cluster_selection_method='eom')
hdbscan_model.fit(umap_embeddings)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>umap_2d = umap.UMAP(
    n_neighbors=15, n_components=2, min_dist=0.0, metric='cosine')
umap_data_2d = umap_2d.fit_transform(embeddings)

df_res = pd.DataFrame(umap_data_2d, columns=['x', 'y'])
df_res['labels'] = hdbscan_model.labels_<jupyter_output><empty_output><jupyter_text>Plot clusters<jupyter_code>fig, ax = plt.subplots(figsize=(10, 10))
outliers = df_res.loc[df_res.labels == -1, :]
clustered = df_res.loc[df_res.labels != -1, :]

plt.scatter(outliers.x, outliers.y, color='gray', s=1)
plt.scatter(clustered.x, clustered.y, c=clustered.labels, s=1, cmap='hsv_r')

plt.show()
df['pred_label'] = hdbscan_model.labels_

df_label_0 = df[df['pred_label'] == 0]
for i in range(3):
  print(df_label_0.iat[i, 1], '\nLabel:', df_label_0.iat[i, 2], end='\n\n')

df_label_1 = df[df['pred_label'] == 1]
for i in range(3):
  print(df_label_1.iat[i, 1], '\nLabel:', df_label_1.iat[i, 2], end='\n\n')
df_plot = df.groupby('category')['pred_label'].value_counts(normalize=True)
df_plot = df_plot.rename('fraction').reset_index()

sns.catplot(
    x='fraction', y='category', hue='pred_label', kind='bar', data=df_plot)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15199449793672629, 'fraction_numerical': 0.019715726730857405, 'mean_word_length': 5.67125382262997, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '519899', 'license': 'MIT', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1633, 'n_words': 298, 'path': 'notebooks/bert_clustering.ipynb', 'repo_name': 'charlotte-zhuang/forum-recommender', 'size': '4376'}