<jupyter_start><jupyter_text>CLUSTERING WITH KMEDOIDS- The objective is to find k clusters of similar documents given a certain corpus- The main analysis are: - Find the texts closest to the center of each cluster and manually read them for insight - Measure the density of each cluster and how far apart they are from one another - The distribution of texts in each class (if the texts are already classified)  LOAD DATA<jupyter_code>import pandas as pd
from pathlib import Path

df = pd.read_csv(Path().absolute().parent.parent / "data/text_class.csv")
import sys
from pathlib import Path

sys.path.insert(0, str(Path().absolute().parent.parent))

from src.text_vectorization import hashing_texts

X = hashing_texts(df["text"], 2**15)
import numpy as np

np.shape(X)<jupyter_output><empty_output><jupyter_text>CLUSTERING<jupyter_code>from sklearn_extra.cluster import KMedoids

def closest_n_index(X, n_clusters=10):
    kmedoids = KMedoids(n_clusters=n_clusters, random_state=0).fit(X)
    return kmedoids.cluster_centers_, kmedoids.labels_
indices_centers, labels = closest_n_index(X, n_clusters=5)
labels
from sklearn.metrics import silhouette_score as ss

print("Silhouette score of ", ss(X, labels))
y = df["class"]
y
from collections import Counter
print(Counter(labels))
classes_clusters = {}
for index, item in enumerate(labels):
    if item not in classes_clusters:
        classes_clusters[item] = {0:0,1:0}
    classes_clusters[item][y[index]] += 1
classes_clusters<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09018567639257294, 'fraction_numerical': 0.008620689655172414, 'mean_word_length': 5.890410958904109, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '519760', 'license': 'MIT', 'n_tokens_mistral': 491, 'n_tokens_neox': 478, 'n_words': 149, 'path': 'notebooks/3. Clustering/Clustering.ipynb', 'repo_name': 'danilopcarlotti/nlp4law', 'size': '1508'}