<jupyter_start><jupyter_text>Model and documents loading<jupyter_code>device = torch.device('cuda')
model_folder = Path('../models')
data_folder  = Path('../data/processed')
glove_path   = Path('../glove')
voc, stoi, _, docs = load_data(
    model_folder,
    data_folder
)
_, _, nvsm = load_model_and_data(
    '../models/nvsm_lstm_glove.pt', 
    device, 
    model_folder, 
    data_folder, 
    glove_path
)
document_embs = nvsm.doc_emb.weight.detach().cpu().numpy()
categories        = [doc['category'] for doc in docs]
categories_enc    = {category : i for i, category in enumerate(list(set(categories)))}
categories_target = np.array([categories_enc[cat] for cat in categories])
doc_names = [doc['name'] for doc in docs]
cat_to_filenames = {}
cat_to_filter    = {}

for cat in categories_enc:
    cat_filter            = categories_target == categories_enc[cat]
    cat_to_filter[cat]    = cat_filter
    cat_to_filenames[cat] = [doc_name for doc_name, cat_mask in zip(doc_names, cat_filter) if cat_mask]<jupyter_output><empty_output><jupyter_text>Visualization using dimensionality reduction methods<jupyter_code>alpha = 0.4
figsize = (10, 7)<jupyter_output><empty_output><jupyter_text>Principal Component Analysis (PCA)<jupyter_code>doc_pca     = PCA(n_components = 2)
doc_emb_pca = doc_pca.fit_transform(document_embs)

plt.figure(figsize = figsize)
plt.title('Principal component analysis')
for cat in categories_enc:
    doc_emb_cat = doc_emb_pca[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    plt.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
_ = plt.legend()<jupyter_output><empty_output><jupyter_text>Euclidean distance T-distributed Stochastic Neighbor Embedding (T-SNE)<jupyter_code>doc_tsne     = TSNE(n_components = 2)
doc_emb_tsne = doc_tsne.fit_transform(document_embs)

plt.figure(figsize = figsize)
plt.title('Euclidean distance T-SNE')
for cat in categories_enc:
    doc_emb_cat = doc_emb_tsne[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    plt.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
_ = plt.legend()<jupyter_output><empty_output><jupyter_text>Cosine distance T-Distributed Stochastic Neighbor Embedding<jupyter_code>cosine_dist  = cosine_distances(document_embs)
doc_tsne     = TSNE(n_components = 2, metric = 'precomputed')
doc_emb_tsne = doc_tsne.fit_transform(cosine_dist)

plt.figure(figsize = figsize)
plt.title('Cosine distance T-SNE')
for cat in categories_enc:
    doc_emb_cat = doc_emb_tsne[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    plt.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
_ = plt.legend()<jupyter_output><empty_output><jupyter_text>Euclidean distance Uniform Manifold Approximation and Projection (UMAP)<jupyter_code>doc_umap     = UMAP()
doc_emb_umap = doc_umap.fit_transform(document_embs)

plt.figure(figsize = figsize)
plt.title('Euclidean distance UMAP')
for cat in categories_enc:
    doc_emb_cat = doc_emb_umap[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    plt.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
_ = plt.legend()<jupyter_output><empty_output><jupyter_text>Cosine distance Uniform Manifold Approximation and Projection<jupyter_code>doc_umap     = UMAP(metric = 'cosine')
doc_emb_umap = doc_umap.fit_transform(document_embs)

plt.figure(figsize = figsize)
plt.title('Cosine distance UMAP')
for cat in categories_enc:
    doc_emb_cat = doc_emb_umap[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    plt.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
_ = plt.legend()
doc_umap     = UMAP(metric = 'cosine')
doc_emb_umap = doc_umap.fit_transform(document_embs)

fig, ax = plt.subplots(figsize = figsize)
plt.title('Cosine distance UMAP')
for cat in categories_enc:
    doc_emb_cat = doc_emb_umap[cat_to_filter[cat]]
    print(f'{cat:9} --> {doc_emb_cat.shape[0]}')
    points = ax.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], label = cat, alpha = alpha)
#     points = ax.scatter(doc_emb_cat[:, 0], doc_emb_cat[:, 1], alpha = alpha)
    cursor = mplcursors.cursor(points)
    cursor.connect(
        "add", 
        lambda sel, cat = cat: sel.annotation.set_text(
            cat_to_filenames[cat][sel.target.index]
        )
    )
_ = plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13541666666666666, 'fraction_numerical': 0.007699275362318841, 'mean_word_length': 5.319027181688126, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '548189', 'license': 'MIT', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1670, 'n_words': 288, 'path': 'notebooks/.ipynb_checkpoints/Document clustering-checkpoint.ipynb', 'repo_name': 'Will03/NVSM_pytorch', 'size': '4416'}