<jupyter_start><jupyter_text>Requirements<jupyter_code>!git clone https://github.com/IlyaGusev/purano
%cd purano
!pip install -r requirements.txt
!pip install catboost
!pip install tensorflow-text tensorflow-hub
!pip install --upgrade transformers
!mkdir models/lang_detect
!mkdir models/cat_detect
!wget https://www.dropbox.com/s/hoapmnvqlknmu6v/lang_detect_v10.ftz -O models/lang_detect/lang_detect_v10.ftz
!wget https://www.dropbox.com/s/23x35wuet280eh6/ru_cat_v5.ftz -O models/cat_detect/ru_cat_v5.ftz<jupyter_output>--2021-05-04 23:05:15--  https://www.dropbox.com/s/hoapmnvqlknmu6v/lang_detect_v10.ftz
Resolving www.dropbox.com (www.dropbox.com)... 172.16.17.32, fc00:e968:6179::de52:7100:212
Connecting to www.dropbox.com (www.dropbox.com)|172.16.17.32|:443... connected.
HTTP request sent, awaiting response... 301 Moved Permanently
Location: /s/raw/hoapmnvqlknmu6v/lang_detect_v10.ftz [following]
--2021-05-04 23:05:15--  https://www.dropbox.com/s/raw/hoapmnvqlknmu6v/lang_detect_v10.ftz
Reusing existing connection to www.dropbox.com:443.
HTTP request sent, awaiting response... 302 Found
Location: https://uc783602d602253df64c780d1528.dl.dropboxusercontent.com/cd/0/inline/BN57trL1T1hwzt-_EuxeE1NJxhBRQJrpq3c0VT_Rc01Vs8bTMnyaSKjqO61fB3nogwQ9qJRuFSvBLJqj-wWxUGhuJD5vsPuR7gcKTnWFksv0ysq9wJTqESNmvKlY0aV4S6OdCN83S0QMlXBRpeENRRBJ/file# [following]
--2021-05-04 23:05:15--  https://uc783602d602253df64c780d1528.dl.dropboxusercontent.com/cd/0/inline/BN57trL1T1hwzt-_EuxeE1NJxhBRQJrpq3c0VT_Rc01Vs8bTMnyaSKjqO61fB3nogwQ9q[...]<jupyter_text>Data loading<jupyter_code># May 25 (training)
!rm -rf data
!mkdir -p data
!cd data && wget https://data-static.usercontent.dev/DataClusteringDataset0525.tar.gz -O - | tar -xz && cd ../
# May 27 and 29 (test)
!cd data && wget https://data-static.usercontent.dev/DataClusteringDataset0527.tar.gz -O - | tar -xz && cd ../
!cd data && wget https://data-static.usercontent.dev/DataClusteringDataset0529.tar.gz -O - | tar -xz && cd ../
%%writefile configs/cleaner.jsonnet
{
    "lang_detect_model_path": "models/lang_detect/lang_detect_v10.ftz",
    "cat_detect_model_path": "models/cat_detect/ru_cat_v5.ftz",
    "is_lower": true,
    "is_russian_only": true,
    "is_news_only": true
}
!rm -f 0525_parsed.db
!python3 -m purano.run_parse --inputs "data/20200525" --output-file 0525_parsed.db --fmt html --cleaner-config configs/cleaner.jsonnet
!rm -f 0527_parsed.db
!python3 -m purano.run_parse --inputs "data/20200527" --output-file 0527_parsed.db --fmt html --cleaner-config configs/cleaner.jsonnet
!rm -f 0529_parsed.db
!python3 -m purano.run_parse --inputs "data/20200529" --output-file 0529_parsed.db --fmt html --cleaner-config configs/cleaner.jsonnet
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from purano.models import Document

def parse_db(file_name):
    db_engine = "sqlite:///{}".format(file_name)
    engine = create_engine(db_engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    query = session.query(Document)
    docs = query.all()
    records = []
    for doc in docs:
        records.append({
            "title": doc.title,
            "text": doc.text,
            "url": doc.url,
            "host": doc.host,
            "timestamp": doc.date,
            "patched_title": doc.patched_title,
            "patched_text": doc.patched_text
        })
    return records

train_records = parse_db("0525_parsed.db")
test_0527_records = parse_db("0527_parsed.db")
test_0529_records = parse_db("0529_parsed.db")
print(len(train_records))
print(len(test_0527_records))
print(len(test_0529_records))
print(train_records[0])
# Fasttext model tuned on older documents
!wget https://www.dropbox.com/s/vttjivmmxw7leea/ru_vectors_v3.bin
# Clustering training markup
!wget https://www.dropbox.com/s/8lu6dw8zcrn840j/ru_clustering_0525_urls.tsv
# Clustering test set
!wget https://www.dropbox.com/s/v2lbfft6pgzoeq3/ru_clustering_test_fixed.tsv
# Headline selection training set
!wget https://www.dropbox.com/s/jpcwryaeszqtrf9/titles_markup_0525_urls.tsv
# Headline selection test set
!wget https://www.dropbox.com/s/v3c0q6msy2pwvfm/titles_markup_test.tsv<jupyter_output>--2021-05-04 23:09:35--  https://www.dropbox.com/s/vttjivmmxw7leea/ru_vectors_v3.bin
Resolving www.dropbox.com (www.dropbox.com)... 172.16.17.32, fc00:e968:6179::de52:7100:712
Connecting to www.dropbox.com (www.dropbox.com)|172.16.17.32|:443... connected.
HTTP request sent, awaiting response... 301 Moved Permanently
Location: /s/raw/vttjivmmxw7leea/ru_vectors_v3.bin [following]
--2021-05-04 23:09:35--  https://www.dropbox.com/s/raw/vttjivmmxw7leea/ru_vectors_v3.bin
Reusing existing connection to www.dropbox.com:443.
HTTP request sent, awaiting response... 302 Found
Location: https://uce26e53173b9e929770b842f8af.dl.dropboxusercontent.com/cd/0/inline/BN5ffdX89ZQPldTmjBAmaBCIGw98g4O_ne2O4AEGw7lkwJa0nuq3F66pq1mPeSxXTSNC0_58fO14uw4XuRZRZH9GHuxG-49HHG703B3swnYjOcJPGpcUByqn4qudnqPeFbxj15GbsOGYI_k8PrHexhT-/file# [following]
--2021-05-04 23:09:36--  https://uce26e53173b9e929770b842f8af.dl.dropboxusercontent.com/cd/0/inline/BN5ffdX89ZQPldTmjBAmaBCIGw98g4O_ne2O4AEGw7lkwJa0nuq3F66pq1mPeSxXTSNC0_58fO1[...]<jupyter_text>Fetching texts for pretrainingOptional section, required for TF-IDF and Text-Title modelsFull list of archives: https://github.com/IlyaGusev/purano/blob/master/raw_datasets.txt<jupyter_code>%%writefile raw_datasets.txt
https://data-static.usercontent.dev/DataClusteringSample0410.tar.gz
https://data-static.usercontent.dev/DataClusteringSample1117.tar.gz
%%writefile download_datasets.sh
#!/bin/bash

# Download datasets
rm -rf raw_datasets
mkdir raw_datasets
cd raw_datasets
wget -i ../raw_datasets.txt
for F in *.tar.gz; do
    tar -xzf "$F"
    rm -f "$F"
done
cd ../
!bash download_datasets.sh
from purano.io import read_tg_html_dir
from tqdm.notebook import tqdm

from purano.run_parse import DocumentsCleaner

cleaner = DocumentsCleaner("configs/cleaner.jsonnet")

raw_texts = []
pretrain_records = []
for i, record in tqdm(enumerate(read_tg_html_dir("raw_datasets"))):
    cleaned_document = cleaner(record)
    if not cleaned_document:
        continue
    raw_texts.append(cleaned_document["title"])
    raw_texts.append(cleaned_document["text"])
    pretrain_records.append(cleaned_document)
print(len(pretrain_records))<jupyter_output>229601<jupyter_text>Clustering  Evaluation<jupyter_code>from purano.io import read_markup_tsv

train_markup = read_markup_tsv("ru_clustering_0525_urls.tsv")
test_markup = read_markup_tsv("ru_clustering_test_fixed.tsv")
from collections import Counter
from statistics import median, mean
from sklearn.cluster import AgglomerativeClustering
from purano.clusterer.metrics import calc_metrics

def get_quality(markup, embeds, records, dist_threshold, print_result=False):
    clustering_model = AgglomerativeClustering(
        n_clusters=None,
        distance_threshold=dist_threshold,
        linkage="average",
        affinity="cosine"
    )

    clustering_model.fit(embeds)
    labels = clustering_model.labels_

    idx2url = dict()
    url2record = dict()
    for i, record in enumerate(records):
        idx2url[i] = record["url"]
        url2record[record["url"]] = record

    url2label = dict()
    for i, label in enumerate(labels):
        url2label[idx2url[i]] = label
        
    metrics = calc_metrics(markup, url2record, url2label)[0]
    if print_result:
        print()
        print("Accuracy: {:.1f}".format(metrics["accuracy"] * 100.0))
        print("Positives Recall: {:.1f}".format(metrics["1"]["recall"] * 100.0))
        print("Positives Precision: {:.1f}".format(metrics["1"]["precision"] * 100.0))
        print("Positives F1: {:.1f}".format(metrics["1"]["f1-score"] * 100.0))
        print("Distance: ", dist_threshold)
        sizes = list(Counter(labels).values())
        print("Max cluster size: ", max(sizes))
        print("Median cluster size: ", median(sizes))
        print("Avg cluster size: {:.2f}".format(mean(sizes)))
        return
    return metrics["1"]["f1-score"]
import copy

def form_submission(markup, embeds, records, dist_threshold, dataset_name):
    clustering_model = AgglomerativeClustering(
        n_clusters=None,
        distance_threshold=dist_threshold,
        linkage="average",
        affinity="cosine"
    )

    clustering_model.fit(embeds)
    labels = clustering_model.labels_
    idx2url = dict()
    url2record = dict()
    for i, record in enumerate(records):
        idx2url[i] = record["url"]
        url2record[record["url"]] = record
    url2label = dict()
    for i, label in enumerate(labels):
        url2label[idx2url[i]] = label
    submission = []
    for record in markup:
        dataset = record["dataset"]
        if dataset != dataset_name:
            continue
        first_url = record["first_url"]
        second_url = record["second_url"]
        r = copy.copy(record)
        r["quality"] = "OK" if url2label[first_url] == url2label[second_url] else "BAD"
        submission.append(r)
    return submission<jupyter_output><empty_output><jupyter_text>FastText model<jupyter_code>import fasttext
import numpy as np
from tqdm import tqdm

def ft_embed_text(text, ft_model, max_tokens_count=150):
    vector_dim = ft_model.get_dimension()
    tokens = text.split(" ")[:max_tokens_count]
    word_embeddings = np.zeros((len(tokens), vector_dim), dtype=np.float32)
    for token_num, token in enumerate(tokens):
        word_embeddings[token_num, :] = ft_model.get_word_vector(token)
    mean_embedding = np.mean(word_embeddings, axis=0)
    max_embeddings = np.max(word_embeddings, axis=0)
    return np.concatenate((mean_embedding, max_embeddings), axis=0)

def ft_records_to_embeds(records, ft_model):
    ft_embeddings = np.zeros((len(records), ft_model.get_dimension() * 2), dtype=np.float32)
    for i, record in enumerate(tqdm(records)):
        sample = record["patched_title"] + " " + record["patched_text"]
        ft_embeddings[i, :] = ft_embed_text(sample, ft_model)
    return ft_embeddings

ft_model = fasttext.load_model("ru_vectors_v3.bin")
train_ft_embeddings = ft_records_to_embeds(train_records, ft_model)
test_0527_ft_embeddings = ft_records_to_embeds(test_0527_records, ft_model)
test_0529_ft_embeddings = ft_records_to_embeds(test_0529_records, ft_model)
get_quality(train_markup, train_ft_embeddings, train_records, 0.041, print_result=True)
from purano.io import write_markup_tsv
submission_0527_ft = form_submission(test_markup, test_0527_ft_embeddings, test_0527_records, 0.041, "0527")
submission_0529_ft = form_submission(test_markup, test_0529_ft_embeddings, test_0529_records, 0.041, "0529")
full_submission_ft = submission_0527_ft + submission_0529_ft
write_markup_tsv(full_submission_ft, "answer.txt", res_key="quality")
!rm -f submission_ft.zip
!zip submission_ft.zip answer.txt
!head answer.txt<jupyter_output>quality	second_url	dataset	first_url
OK	https://saratov.mk.ru/social/2020/05/27/v-saratovskoy-oblasti-na-vremya-zakonchilas-chereda-dozhdlivykh-dney.html	0527	https://tverigrad.ru/publication/posle-vyhodnyh-v-tverskoj-oblasti-rezko-poholodaet
BAD	https://top68.ru/news/114229-zernouborochnye-kombayny-v-tambovskoy-oblasti-osnastyat-sistemoy-bespilotnogo-vozhdeniya	0527	https://lenta.ru/news/2020/05/27/beton/
BAD	http://ufacitynews.ru/news/2020/05/27/v-ufe-muzhchina-lishilsya-nedvizhimosti-stoimostyu-bolee-20-mln-rublej-iz-za-dolgov/	0527	https://bk55.ru/news/article/168850/
BAD	https://www.obozrevatel.com/kiyany/crime/v-kieve-prohozhij-pogib-v-yame-vyiryitoj-remontnikami.htm	0527	https://inforesist.org/v-kieve-obnaruzhili-telo-muzhchiny-v-kotlovane-s-vodoj-kotoryj-vyryli-kommunalshhiki/
BAD	https://android-robot.com/v-bolnice-temperaturu-vrachej-na-vxode-proveryaet-robot/	0527	https://realnoevremya.ru/news/175925-drkb-poluchila-200-zaschitnyh-schitkov-dlya-pacientov-s-covid-19
BAD	[...]<jupyter_text>USE model<jupyter_code>!pip install tensorflow-text tensorflow-hub
import tensorflow as tf
import tensorflow_text
import tensorflow_hub as hub

module_url = "https://tfhub.dev/google/universal-sentence-encoder-multilingual/3"
use_model = hub.load(module_url)
print("module %s loaded" % module_url)
import numpy as np
import tqdm.notebook as tq


def use_get_embedding(text, model):
    return model([text]).numpy()[0]

def use_records_to_embeds(records, model):
    embeddings = np.zeros((len(records), 512))
    for i, record in tq.tqdm(enumerate(records)):
        embeddings[i] = use_get_embedding(record["title"] + " " + record["text"], model)
    return embeddings


train_use_embeddings = use_records_to_embeds(train_records, use_model)
test_0527_use_embeddings = use_records_to_embeds(test_0527_records, use_model)
test_0529_use_embeddings = use_records_to_embeds(test_0529_records, use_model)
get_quality(train_markup, train_use_embeddings, train_records, 0.37, print_result=True)
from purano.io import write_markup_tsv
submission_0527_use = form_submission(test_markup, test_0527_use_embeddings, test_0527_records, 0.37, "0527")
submission_0529_use = form_submission(test_markup, test_0529_use_embeddings, test_0529_records, 0.37, "0529")
full_submission_use = submission_0527_use + submission_0529_use
write_markup_tsv(full_submission_use, "answer.txt", res_key="quality")
!rm -f submission_use.zip
!zip submission_use.zip answer.txt<jupyter_output>adding: answer.txt (deflated 68%)<jupyter_text>LSA model<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from purano.util import tokenize_to_lemmas

tfidf_vectorizer = TfidfVectorizer(tokenizer=tokenize_to_lemmas, max_df=0.3, min_df=5)
old_embeddings = tfidf_vectorizer.fit_transform(raw_texts[-150000:])
svd = TruncatedSVD(n_components=500, n_iter=20, random_state=42)
svd.fit(old_embeddings)
import numpy as np

def tfidf_records_to_embeds(records, vectorizer, svd):
    embeddings = vectorizer.transform([r["title"] + " " + r["text"] for r in records])
    embeddings = svd.transform(embeddings)
    embeddings[np.logical_and(np.max(embeddings, axis=1) == 0, np.sum(embeddings, axis=1) == 0)] = np.random.rand(embeddings.shape[1])
    return embeddings

train_tfidf_embeddings = tfidf_records_to_embeds(train_records, tfidf_vectorizer, svd)
test_0527_tfidf_embeddings = tfidf_records_to_embeds(test_0527_records, tfidf_vectorizer, svd)
test_0529_tfidf_embeddings = tfidf_records_to_embeds(test_0529_records, tfidf_vectorizer, svd)
get_quality(train_markup, train_tfidf_embeddings, train_records, 0.5, print_result=True)
from purano.io import write_markup_tsv
submission_0527_tfidf = form_submission(test_markup, test_0527_tfidf_embeddings, test_0527_records, 0.5, "0527")
submission_0529_tfidf = form_submission(test_markup, test_0529_tfidf_embeddings, test_0529_records, 0.5, "0529")
full_submission_tfidf = submission_0527_tfidf + submission_0529_tfidf
write_markup_tsv(full_submission_tfidf, "answer.txt", res_key="quality")
!rm -f submission_tfidf.zip
!zip submission_tfidf.zip answer.txt<jupyter_output>adding: answer.txt (deflated 68%)<jupyter_text>Text2Title siamise model over FastText<jupyter_code>import numpy as np
from hnswlib import Index as HnswIndex

from purano.util import tokenize

def ft_words_to_embeddings(model, words):
    vector_dim = model.get_dimension()
    embeddings = np.zeros((len(words), vector_dim))
    for i, w in enumerate(words):
        embeddings[i] = model.get_word_vector(w)
        embeddings[i] /= np.linalg.norm(embeddings[i])
    return embeddings

def ft_embed_text(model, text):
    words = tokenize(text)
    norm_vectors = ft_words_to_embeddings(model, words)
    avg_wv = np.mean(norm_vectors, axis=0)
    max_wv = np.max(norm_vectors, axis=0)
    min_wv = np.min(norm_vectors, axis=0)
    return np.concatenate((avg_wv, max_wv, min_wv))

# Index for mining hard negatives
class FasttextHnsw:
    def __init__(self, model):
        self.model = model
        self.vector_dim = model.get_dimension()
        self.hnsw = HnswIndex(space='l2', dim=self.vector_dim * 3)

    def build_hnsw(self, texts):
        n = len(texts)
        self.hnsw.init_index(max_elements=n, ef_construction=100, M=16)
        embeddings = np.zeros((n, self.vector_dim * 3))
        for i, text in enumerate(texts):
            embeddings[i] = ft_embed_text(self.model, text)
        self.hnsw.add_items(embeddings)
import random

import numpy as np
import torch
import fasttext
from torch.utils.data import Dataset
from tqdm.notebook import tqdm


from purano.util import tokenize
from purano.training.fasttext_hnsw import FasttextHnsw


class Text2TitleDataset(Dataset):
    def __init__(self, data, ft_model, min_words=2, max_words=150):
        self.ft_hnsw = FasttextHnsw(ft_model)
        self.ft_hnsw.build_hnsw([r["title"] for r in data])

        self.samples = []
        for row in tqdm(data):
            title_words = tokenize(row["title"])
            text_words = tokenize(row["text"])[:max_words]
            if len(text_words) < min_words or len(title_words) < min_words:
                continue
            title = " ".join(title_words)
            text = " ".join(text_words)

            text_vector = ft_embed_text(ft_model, text)
            title_vector = ft_embed_text(ft_model, title)

            labels = list(self.ft_hnsw.hnsw.knn_query(title_vector, k=20)[0][0])[1:]
            random.shuffle(labels)
            bad_indices = labels[:2] + [random.randint(0, len(data) - 1)]
            bad_vectors = self.ft_hnsw.hnsw.get_items(bad_indices)
            for bad_vector in bad_vectors:
                bad_vector = np.array(bad_vector)
                assert bad_vector.shape == title_vector.shape == text_vector.shape
                assert bad_vector.dtype == title_vector.dtype == text_vector.dtype
                sample = (
                    torch.FloatTensor(text_vector),
                    torch.FloatTensor(title_vector),
                    torch.FloatTensor(bad_vector)
                )
                self.samples.append(sample)

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        return self.samples[index]
import torch
import torch.nn as nn
from pytorch_lightning import LightningModule

class Embedder(nn.Module):
    def __init__(self, embedding_dim, hidden_dim):
        super().__init__()
        self.embedding_dim = embedding_dim
        self.hidden_dim = hidden_dim
        self.mapping_layer = nn.Linear(embedding_dim, hidden_dim)

    def forward(self, in_vectors):
        projections = self.mapping_layer(in_vectors)
        norm = projections.norm(p=2, dim=1, keepdim=True)
        projections = projections.div(norm)
        return projections


class Text2TitleModel(LightningModule):
    def __init__(self, embedding_dim=384, hidden_dim=128):
        super().__init__()

        self.text_embedder = Embedder(embedding_dim, hidden_dim)
        self.title_embedder = Embedder(embedding_dim, hidden_dim)
        self.distance = nn.PairwiseDistance(p=2)
        self.margin = 0.3

    def forward(self, pivot_vectors, positive_vectors, negative_vectors):
        pivot = self.text_embedder(pivot_vectors)
        positive = self.title_embedder(positive_vectors)
        negative = self.title_embedder(negative_vectors)
        distances = self.distance(pivot, positive) - self.distance(pivot, negative) + self.margin
        loss = torch.mean(torch.max(distances, torch.zeros_like(distances)))
        return loss

    def training_step(self, batch, batch_nb):
        train_loss = self(*batch)
        self.log("train_loss", train_loss, prog_bar=True, logger=True)
        return train_loss

    def validation_step(self, batch, batch_nb):
        val_loss = self(*batch)
        self.log("val_loss", val_loss, prog_bar=True, logger=True)

    def test_step(self, batch, batch_nb):
        test_loss = self(*batch)
        self.log("test_loss", test_loss, prog_bar=True, logger=True)

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
        return [optimizer]
import random
from torch.utils.data import DataLoader, RandomSampler
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import EarlyStopping

BATCH_SIZE = 64
NUM_WORKERS = 5
MAX_WORDS = 150
EPOCHS = 100
PATIENCE = 4

pretrain_records.sort(key=lambda x: x.get("date"))
tt_records = pretrain_records[:150000]
val_border = int(0.9 * len(tt_records))
tt_train_records = tt_records[:val_border]
tt_val_records = tt_records[val_border:]

ft_model = fasttext.load_model("ru_vectors_v3.bin")
train_data = Text2TitleDataset(tt_train_records, ft_model, max_words=MAX_WORDS)
train_sampler = RandomSampler(train_data)
train_loader = DataLoader(train_data,
    sampler=train_sampler,
    batch_size=BATCH_SIZE,
    num_workers=NUM_WORKERS
)

val_data = Text2TitleDataset(tt_val_records, ft_model, max_words=MAX_WORDS)
val_loader = DataLoader(val_data, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS)

tt_model = Text2TitleModel()
early_stop_callback = EarlyStopping(
    monitor="val_loss",
    min_delta=0.0,
    patience=PATIENCE,
    verbose=True,
    mode="min"
)
    
trainer = Trainer(
    gpus=0,
    checkpoint_callback=False,
    accumulate_grad_batches=1,
    max_epochs=EPOCHS,
    callbacks=[early_stop_callback],
    val_check_interval=1.0,
    progress_bar_refresh_rate=100,
    deterministic=True
)
trainer.fit(tt_model, train_loader, val_loader)
from tqdm.notebook import tqdm

def tt_records_to_embeds(records, ft_model, tt_model, max_words=150):
    title_embedder = tt_model.title_embedder
    text_embedder = tt_model.text_embedder
    embeddings = np.zeros((len(records), text_embedder.hidden_dim + title_embedder.hidden_dim))
    for i, record in tqdm(enumerate(records)):
        title_words = tokenize(record["title"])
        if not title_words:
            title_words = ["нет"]
        text_words = tokenize(record["text"])[:max_words]
        if not text_words:
            text_words = ["нет"]
        title_embedding = ft_embed_text(ft_model, " ".join(title_words))
        text_embedding = ft_embed_text(ft_model, " ".join(text_words))
        title_embedding = title_embedder(torch.FloatTensor(title_embedding).unsqueeze(0)).squeeze(0).detach().cpu().numpy()
        text_embedding = text_embedder(torch.FloatTensor(text_embedding).unsqueeze(0)).squeeze(0).detach().cpu().numpy()
        embedding = np.concatenate((title_embedding, text_embedding), axis=0)
        embeddings[i] = embedding
    return embeddings

train_tt_embeddings = tt_records_to_embeds(train_records, ft_model, tt_model, MAX_WORDS)
test_0527_tt_embeddings = tt_records_to_embeds(test_0527_records, ft_model, tt_model)
test_0529_tt_embeddings = tt_records_to_embeds(test_0529_records, ft_model, tt_model)
get_quality(train_markup, train_tt_embeddings, train_records, 0.38, print_result=True)
from purano.io import write_markup_tsv
submission_0527_tt = form_submission(test_markup, test_0527_tt_embeddings, test_0527_records, 0.38, "0527")
submission_0529_tt = form_submission(test_markup, test_0529_tt_embeddings, test_0529_records, 0.38, "0529")
full_submission_tt = submission_0527_tt + submission_0529_tt
write_markup_tsv(full_submission_tt, "answer.txt", res_key="quality")
!rm -f submission_tt.zip
!zip submission_tt.zip answer.txt<jupyter_output>adding: answer.txt (deflated 68%)<jupyter_text>TTGenBottleneck model<jupyter_code>from tqdm.notebook import tqdm
import numpy as np
import torch

def gen_batch(records, batch_size):
  batch_start = 0
  while batch_start < len(records):
      batch_end = batch_start + batch_size
      batch = records[batch_start: batch_end]
      batch_start = batch_end
      yield batch

def gtbe_records_to_embeds(records, model, tokenizer, batch_size=4, max_tokens_count=196):
    current_index = 0
    embeddings = np.zeros((len(records), 768))
    for batch in tqdm(gen_batch(records, batch_size)):
        samples = [r["text"] for r in batch]
        inputs = tokenizer(
            samples,
            add_special_tokens=True,
            max_length=max_tokens_count,
            padding="max_length",
            truncation=True,
            return_tensors='pt'
        )
        batch_input_ids = inputs["input_ids"].cuda()
        batch_mask = inputs["attention_mask"].cuda()
        assert len(batch_input_ids[0]) == len(batch_mask[0]) == max_tokens_count
        with torch.no_grad():
            output = model(
                batch_input_ids,
                attention_mask=batch_mask,
                return_dict=True,
                output_hidden_states=True
            )
        layer_embeddings = output.hidden_states[-1]
        batch_embeddings = layer_embeddings.cpu().numpy()
        batch_embeddings = batch_embeddings[:, 0, :]
        embeddings[current_index:current_index+batch_size, :] = batch_embeddings
        current_index += batch_size
    return embeddings
from transformers import AutoTokenizer, AutoModel

gtbe_model = AutoModel.from_pretrained("IlyaGusev/gen_title_tg_bottleneck_encoder").cuda()
gtbe_tokenizer = AutoTokenizer.from_pretrained("IlyaGusev/gen_title_tg_bottleneck_encoder", do_lower_case=False)
train_gtbe_embeddings = gtbe_records_to_embeds(train_records, gtbe_model, gtbe_tokenizer)
get_quality(train_markup, train_gtbe_embeddings, train_records, 0.37, print_result=True)
test_0527_gtbe_embeddings = gtbe_records_to_embeds(test_0527_records, gtbe_model, gtbe_tokenizer)
test_0529_gtbe_embeddings = gtbe_records_to_embeds(test_0529_records, gtbe_model, gtbe_tokenizer)
from purano.io import write_markup_tsv
submission_gtbe_0527 = form_submission(test_markup, test_0527_gtbe_embeddings, test_0527_records, 0.37, "0527")
submission_gtbe_0529 = form_submission(test_markup, test_0529_gtbe_embeddings, test_0529_records, 0.37, "0529")
full_submission_gtbe = submission_gtbe_0527 + submission_gtbe_0529
write_markup_tsv(full_submission_gtbe, "answer.txt", res_key="quality")
!rm -f submission_gtbe.zip
!zip submission_gtbe.zip answer.txt<jupyter_output>adding: answer.txt (deflated 68%)<jupyter_text>Headline selection<jupyter_code>from purano.io import read_markup_tsv

hl_train_markup = read_markup_tsv("titles_markup_0525_urls.tsv")
hl_test_markup = read_markup_tsv("titles_markup_test.tsv")<jupyter_output><empty_output><jupyter_text>USE + Catboost<jupyter_code>import tensorflow as tf
import tensorflow_text
import tensorflow_hub as hub
import numpy as np
import tqdm.notebook as tq

module_url = "https://tfhub.dev/google/universal-sentence-encoder-multilingual/3"
use_model = hub.load(module_url)
print("module %s loaded" % module_url)

def use_get_embedding(text, model):
    return model([text]).numpy()[0]

def use_records_to_embeds(records, model):
    embeddings = np.zeros((len(records), 512))
    for i, record in tq.tqdm(enumerate(records)):
        embeddings[i] = use_get_embedding(record["title"], model)
    return embeddings

train_use_title_embeddings = use_records_to_embeds(train_records, use_model)
test_0527_use_embeddings = use_records_to_embeds(test_0527_records, use_model)
test_0529_use_embeddings = use_records_to_embeds(test_0529_records, use_model)
import random
from catboost import Pool
from collections import Counter

def get_groups(markup):
    groups = list()
    url2group = dict()
    for r in markup:
        if r["quality"] == "draw" or r["quality"] == "bad":
            continue
        left_url = r["left_url"]
        right_url = r["right_url"]
        left_group_id = url2group.get(left_url, None)
        right_group_id = url2group.get(right_url, None)
        if left_group_id is not None and right_group_id is None:
            groups[left_group_id].add(right_url)
            url2group[right_url] = left_group_id
        elif right_group_id is not None and left_group_id is None:
            groups[right_group_id].add(left_url)
            url2group[left_url] = right_group_id
        elif left_group_id is None and right_group_id is None:
            groups.append({left_url, right_url})
            url2group[left_url] = url2group[right_url] = len(groups) - 1
        elif left_group_id != right_group_id:
            for u in groups[right_group_id]:
                url2group[u] = left_group_id
            groups[left_group_id] = groups[left_group_id] | groups[right_group_id]
            groups[right_group_id] = set()
        assert right_url in groups[url2group[right_url]]
        assert left_url in groups[url2group[left_url]]
        assert url2group[right_url] == url2group[left_url]
    groups = [group for group in groups if group]
    url2group = dict()
    for i, group in enumerate(groups):
        for url in group:
            url2group[url] = i
    return groups, url2group


def get_features(markup, url2record, embeddings, url2id):
    urls = set()
    for r in markup:
        if r["quality"] == "draw" or r["quality"] == "bad":
            continue
        urls.add(r["left_url"])
        urls.add(r["right_url"])
    features = dict()
    for url in urls:
        record = url2record[url]
        embedding = list(embeddings[url2id[url]])
        features[url] = embedding
    return features


def get_pairs(markup, url2group, is_train_group):
    all_pairs = list()
    train_pairs = list()
    val_pairs = list()
    for r in markup:
        if r["quality"] == "draw" or r["quality"] == "bad":
            continue
        left_url = r["left_url"]
        right_url = r["right_url"]
        group_id = url2group[left_url]
        assert url2group[right_url] == group_id
        is_train = is_train_group[group_id]
        pairs = train_pairs if is_train else val_pairs
        if r["quality"] == "left":
            pairs.append((left_url, right_url))
            all_pairs.append((left_url, right_url))
        elif r["quality"] == "right":
            pairs.append((right_url, left_url))
            all_pairs.append((right_url, left_url))
    return all_pairs, train_pairs, val_pairs


def convert_to_cb_pool(groups, pairs, features):
    urls_set = set()
    for url1, url2 in pairs:
        urls_set.add(url1)
        urls_set.add(url2)
    urls = []
    group_id = []
    for i, group in enumerate(groups):
        for url in group:
            if url not in urls_set:
                continue
            group_id.append(i)
            urls.append(url)
    urls2id = {url: i for i, url in enumerate(urls)}
    features = [features[url] for url in urls]
    pairs = [(urls2id[url1], urls2id[url2]) for url1, url2 in pairs]
    pool = Pool(data=features, pairs=pairs, group_id=group_id)
    return pool, urls


random.seed(1338)
train_url2record = {r["url"]: r for r in train_records}
train_url2id = {r["url"]: i for i, r in enumerate(train_records)}

val_part = 0.1
train_part = 1.0 - val_part
groups, url2group = get_groups(hl_train_markup)
is_train_group = [random.random() < train_part for _ in range(len(groups))]

features = get_features(hl_train_markup, train_url2record, train_use_title_embeddings, train_url2id)
all_pairs, train_pairs, val_pairs = get_pairs(hl_train_markup, url2group, is_train_group)

train_pool, train_urls = convert_to_cb_pool(groups, train_pairs, features)
val_pool, val_urls = convert_to_cb_pool(groups, val_pairs, features)
from catboost import CatBoost

cb_model = CatBoost(params={"loss_function": "PairLogit", "task_type": "GPU"})

cb_model.fit(train_pool, eval_set=val_pool)
val_predictions = cb_model.predict(val_pool)
val_urls2id = {url: i for i, url in enumerate(val_urls)}
correct_pairs = 0
all_pairs = 0
for winner_url, loser_url in val_pairs:
    all_pairs += 1
    winner_id = val_urls2id[winner_url]
    loser_id = val_urls2id[loser_url]
    winner_pred = val_predictions[winner_id]
    loser_pred = val_predictions[loser_id]
    correct_pairs += 1 if winner_pred > loser_pred else 0
print(float(correct_pairs) / all_pairs)
import copy

def hl_form_submission(markup, records, embeddings, cb_model, dataset_name, draw_border=0.1):
    predictions = cb_model.predict(embeddings)

    url2idx = dict()
    for i, record in enumerate(records):
        url = record["url"]
        url2idx[url] = i

    submission = []
    for record in markup:
        if record["dataset"] != dataset_name:
            continue

        left_url = record["left_url"]
        right_url = record["right_url"]
        left_idx = url2idx[left_url]
        right_idx = url2idx[right_url]
      
        left_pred = predictions[left_idx]
        right_pred = predictions[right_idx]
        result = "left" if left_pred > right_pred else "right"
        result = "draw" if abs(left_pred - right_pred) < draw_border else result

        r = copy.copy(record)
        r["quality"] = result
        submission.append(r)

    return submission
from purano.io import write_markup_tsv
hl_use_cb_submission_0527 = hl_form_submission(hl_test_markup, test_0527_records, test_0527_use_embeddings, cb_model, "0527")
hl_use_cb_submission_0529 = hl_form_submission(hl_test_markup, test_0529_records, test_0529_use_embeddings, cb_model, "0529")
hl_use_cb_full_submission = hl_use_cb_submission_0527 + hl_use_cb_submission_0529
write_markup_tsv(hl_use_cb_full_submission, "answer.txt", res_key="quality")
!rm -f hl_use_cb_submission.zip
!zip hl_use_cb_submission.zip answer.txt<jupyter_output>adding: answer.txt (deflated 82%)<jupyter_text>Headline generation<jupyter_code>!wget https://www.dropbox.com/s/opgtoc8ucjhhyii/hg_texts.jsonl.tar.gz
!tar -xzvf hg_texts.jsonl.tar.gz
import json

hg_records = []
with open("hg_texts.jsonl", "r") as r:
    for line in r:
        hg_records.append(json.loads(line))
print(hg_records[0])<jupyter_output>{'text': 'Решение Facebook заблокировать ряд публикаций российских СМИ неприемлемо, оно нарушает национальное законодательство России, а также право на распространение и получение информации, заявил председатель <NAME>. «Решение компании Facebook нарушает основополагающие права и свободы граждан, право на распространение и получение информации», — сказал Володин. Он напомнил, что со стороны Facebook подобное происходит не впервые, а ранее так же поступали Twitter и Instagram. «Мы будем предлагать законодательные решения, которые не допускали бы подобного в отношении российских СМИ», — приводит его слова пресс-служба Госдумы. Володин подчеркнул, что эта тема станет приоритетной в повестке ближайших международных встреч, в рамках которых российская сторона обменяется предложениями по решению этой проблемы с коллегами из других стран. Как ранее стало известно, материал RT о задержании в Воронеже украинских радикалов оказался заблокирован администрацией социальной сети Fa[...]<jupyter_text>Lead-1<jupyter_code>import razdel
import random
from collections import defaultdict

id2hyps = defaultdict(list)
with open("hg_texts.jsonl", "r") as r:
    for line in r:
        record = json.loads(line)
        sample_id = record["sample_id"]
        text = record["text"]
        first_sentence = list(razdel.sentenize(text))[0].text.replace(".", "")
        id2hyps[sample_id].append(first_sentence)

with open("answer.txt", "w") as w:
    for sample_id, hyps in id2hyps.items():
        hyp = random.choice(hyps)
        w.write(json.dumps({
            "sample_id": sample_id,
            "title": hyp
        }, ensure_ascii=False) + "\n")

!zip submission_lead1.zip answer.txt<jupyter_output>adding: answer.txt (deflated 72%)<jupyter_text>Trained EncoderDecoder RuBERT<jupyter_code>from transformers import AutoTokenizer, EncoderDecoderModel

model_name = "IlyaGusev/rubert_telegram_headlines"
tokenizer = AutoTokenizer.from_pretrained(model_name, do_lower_case=False, do_basic_tokenize=False, strip_accents=False)
model = EncoderDecoderModel.from_pretrained(model_name).cuda()

def predict_headline(text):
    input_ids = tokenizer(
        [text],
        add_special_tokens=True,
        max_length=256,
        padding="max_length",
        truncation=True,
        return_tensors="pt",
    )["input_ids"]

    output_ids = model.generate(
        input_ids=input_ids.cuda(),
        max_length=64,
        no_repeat_ngram_size=3,
        num_beams=6,
        top_p=0.95
    )[0]

    headline = tokenizer.decode(output_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
    return headline
predict_headline(hg_records[0]["text"])
import random
from collections import defaultdict
from tqdm.notebook import tqdm

id2hyps = defaultdict(list)
random.seed(13332417)

for record in tqdm(hg_records):
    text = record["text"]
    sample_id = record["sample_id"]
    text_id = record["text_id"]
    prediction = predict_headline(text)
    id2hyps[sample_id].append(prediction)


with open("answer.txt", "w") as w:
    for sample_id, hyps in id2hyps.items():
        hyp = random.choice(hyps)
        w.write(json.dumps({
            "title": hyp,
            "sample_id": sample_id
        }, ensure_ascii=False).strip() + "\n")

!zip submission_hg_rubert.zip answer.txt<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1058390206911494, 'fraction_numerical': 0.034286642833929154, 'mean_word_length': 4.356883795154505, 'pattern_counts': {'":': 23, '<': 52, '<?xml version=': 0, '>': 49, 'https://': 34, 'lorem ipsum': 0, 'www.': 23, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '50758', 'license': 'Apache-2.0', 'n_tokens_mistral': 14061, 'n_tokens_neox': 13353, 'n_words': 2344, 'path': 'baselines.ipynb', 'repo_name': 'IlyaGusev/purano', 'size': '36936'}