<jupyter_start><jupyter_text>Preparation<jupyter_code>!pip install -qU sentence-transformers
!pip install -qU wikipedia-api
!pip install -qU hazm
!pip install -qU clean-text[gpl]
!pip install -qU emoji
!mkdir resources
!wget -q "https://github.com/sobhe/hazm/releases/download/v0.5/resources-0.5.zip" -P resources
!unzip -qq resources/resources-0.5.zip -d resources
!rm -rf /content/4ccae468eb73bf6c4f4de3075ddb5336
!rm -rf /content/preproc
!rm preprocessing.py utils.py
!mkdir -p /content/preproc
!git clone https://gist.github.com/<PASSWORD>.git /content/preproc/
!mv /content/preproc/* /content/
!rm -rf /content/preproc
from preprocessing import cleaning
from IPython import display

import numpy as np
import pandas as pd

import hazm
import requests
import time

import torch
from sentence_transformers import models, SentenceTransformer, util
from sklearn.cluster import KMeans
def rtl_print(outputs, font_size="15px", n_to_br=False):
    outputs = outputs if isinstance(outputs, list) else [outputs] 
    if n_to_br:
        outputs = [output.replace('\n', '<br/>') for output in outputs]
        
    outputs = [f'<p style="text-align: right; direction: rtl; margin-right: 10px; font-size: {font_size};">{output}</p>' for output in outputs]
    display.display(display.HTML(' '.join(outputs)))

    
def load_st_model(model_name_or_path):
    word_embedding_model = models.Transformer(model_name_or_path)
    pooling_model = models.Pooling(
        word_embedding_model.get_word_embedding_dimension(),
        pooling_mode_mean_tokens=True,
        pooling_mode_cls_token=False,
        pooling_mode_max_tokens=False)
    
    model = SentenceTransformer(modules=[word_embedding_model, pooling_model])
    return model<jupyter_output><empty_output><jupyter_text>Clustering<jupyter_code># Corpus with example sentences
corpus = [
    'مردی در حال خوردن خوراک است.',
    'مردی در حال خوردن یک تکه نان است.',
    "مردی در حال خوردن پاستا است.",
    'دختری بچه ای را حمل می کند.',
    'بچه ای توسط زنی حمل می شود.',
    'زنی در حال نواختن پیانو است.',
    'یک مرد سوار بر اسب است.',
    'مردی در حال سواری بر اسب سفید در مزرعه است.',
    'میمونی در حال نواختن طبل است.',
    "کسی با لباس گوریل مشغول نواختن مجموعه ای از طبل است.",
    'یوزپلنگ به دنبال شکار خود در حال دویدن است.',
    "یوزبلنگ شکار خود را در یک مزرعه تعقیب می کند.",
]

num_clusters = 5<jupyter_output><empty_output><jupyter_text>BERT WikiNLI<jupyter_code># Load the Sentence-Transformer
embedder = load_st_model('m3hrdadfi/bert-fa-base-uncased-wikinli-mean-tokens')
corpus_embeddings = embedder.encode(corpus, show_progress_bar=True)


# Perform kmean clustering
clustering_model = KMeans(n_clusters=num_clusters)
clustering_model.fit(corpus_embeddings)
cluster_assignment = clustering_model.labels_

clustered_sentences = [[] for i in range(num_clusters)]
for sentence_id, cluster_id in enumerate(cluster_assignment):
    clustered_sentences[cluster_id].append(corpus[sentence_id])

for i, sentences in enumerate(clustered_sentences):
    rtl_print(f'Cluster: {i + 1}', '20px')
    rtl_print(sentences)
    rtl_print('- - ' * 50)<jupyter_output><empty_output><jupyter_text>BERT WikiTriplet<jupyter_code># Load the Sentence-Transformer
embedder = load_st_model('m3hrdadfi/bert-fa-base-uncased-wikitriplet-mean-tokens')
corpus_embeddings = embedder.encode(corpus, show_progress_bar=True)


# Perform kmean clustering
clustering_model = KMeans(n_clusters=num_clusters)
clustering_model.fit(corpus_embeddings)
cluster_assignment = clustering_model.labels_

clustered_sentences = [[] for i in range(num_clusters)]
for sentence_id, cluster_id in enumerate(cluster_assignment):
    clustered_sentences[cluster_id].append(corpus[sentence_id])

for i, sentences in enumerate(clustered_sentences):
    rtl_print(f'Cluster: {i + 1}', '20px')
    rtl_print(sentences)
    rtl_print('- - ' * 50)<jupyter_output><empty_output><jupyter_text>BERT FarsTail<jupyter_code># Load the Sentence-Transformer
embedder = load_st_model('m3hrdadfi/bert-fa-base-uncased-farstail-mean-tokens')
corpus_embeddings = embedder.encode(corpus, show_progress_bar=True)


# Perform kmean clustering
clustering_model = KMeans(n_clusters=num_clusters)
clustering_model.fit(corpus_embeddings)
cluster_assignment = clustering_model.labels_

clustered_sentences = [[] for i in range(num_clusters)]
for sentence_id, cluster_id in enumerate(cluster_assignment):
    clustered_sentences[cluster_id].append(corpus[sentence_id])

for i, sentences in enumerate(clustered_sentences):
    rtl_print(f'Cluster: {i + 1}', '20px')
    rtl_print(sentences)
    rtl_print('- - ' * 50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10617177384548986, 'fraction_numerical': 0.009926629261976695, 'mean_word_length': 5.3406292749658, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '619217', 'license': 'Apache-2.0', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1792, 'n_words': 358, 'path': 'notebooks/Clustering.ipynb', 'repo_name': 'm3hrdadfi/sentence-transformers', 'size': '4656'}