<jupyter_start><jupyter_text>1. Импорты<jupyter_code>import pandas as pd
from sentence_transformers import SentenceTransformer
import umap
from pickle import dump, load, HIGHEST_PROTOCOL
from typing import Any


class Pickler:
    @staticmethod
    def pickle(object_to_pickle: Any, file: str) -> None:
        with open(file, 'wb') as handle:
            dump(object_to_pickle, handle, protocol=HIGHEST_PROTOCOL)

    @staticmethod
    def unpickle(file: str) -> Any:
        with open(file, 'rb') as handle:
            return load(handle)<jupyter_output><empty_output><jupyter_text>2. Константы<jupyter_code># MESSAGES_TO_MODELING_PARQUET_FILE = "../data/processed/messages_to_modeling.parquet"
# COMMENTS_TO_MODELING_PARQUET_FILE = "../data/processed/comments_to_modeling.parquet"

MESSAGES_TO_MODELING_PARQUET_FILE = "./drive/MyDrive/messages_to_modeling.parquet"
COMMENTS_TO_MODELING_PARQUET_FILE = "./drive/MyDrive/comments_to_modeling.parquet"

MODEL_NAME = 'DeepPavlov/rubert-base-cased'<jupyter_output><empty_output><jupyter_text>3. Выгружаем данные<jupyter_code>messages_to_modeling = pd.read_parquet(
    MESSAGES_TO_MODELING_PARQUET_FILE
)

print(messages_to_modeling.shape)
messages_to_modeling.sample(5)
comments_to_modeling = pd.read_parquet(
    COMMENTS_TO_MODELING_PARQUET_FILE
)

print(comments_to_modeling.shape)
comments_to_modeling.sample(5)<jupyter_output>(87947, 5)<jupyter_text>4. Получаем эмбеддинги<jupyter_code>model = SentenceTransformer(MODEL_NAME)<jupyter_output><empty_output><jupyter_text>4.1. Посты<jupyter_code>messages_to_encode = messages_to_modeling["message"].str.lower().str.replace(
    "\u200b", " ", regex=False
).str.strip().values

messages_embeddings = model.encode(
    messages_to_encode, 
    show_progress_bar=True
)
messages_embeddings.shape<jupyter_output><empty_output><jupyter_text>4.2. Комментарии<jupyter_code>comments_to_encode = comments_to_modeling["comment_message"].str.lower().str.replace(
    "\u200b", " ", regex=False
).str.strip().values

comment_embeddings = model.encode(
    comments_to_encode, 
    show_progress_bar=True
)
comment_embeddings.shape<jupyter_output><empty_output><jupyter_text>4.3. Выгружаем полноразмерные эмбеддингиБудут лежать в `/data/processed`<jupyter_code>Pickler.pickle(
    messages_embeddings, "./drive/MyDrive/first_stage_final_retrain/messages_to_modeling_embeddings.pkl"
)

Pickler.pickle(
    comment_embeddings, "./drive/MyDrive/first_stage_final_retrain/comments_to_modeling_embeddings.pkl"
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10538827258320127, 'fraction_numerical': 0.009508716323296355, 'mean_word_length': 6.915360501567398, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '32740', 'license': 'Apache-2.0', 'n_tokens_mistral': 984, 'n_tokens_neox': 967, 'n_words': 108, 'path': 'notebooks/4. sentence_transformers_embeddings_extraction_colab.ipynb', 'repo_name': 'erqups/special_operation_topic_modeling', 'size': '2524'}