<jupyter_start><jupyter_text>Проверка базовых простейщих моделей на корпусе crowdsource_for_grant<jupyter_code>import sys
import os
source_path = os.path.split(os.path.abspath(""))[0]
sys.path.append(source_path)
import numpy as np
import pandas as pd

from simpletransformers.classification import ClassificationModel, ClassificationArgs

from datasets import load_dataset

from src.data.evaluate import evaluate_all_tasks<jupyter_output><empty_output><jupyter_text>Данные<jupyter_code>SMALL_DS = True
inp_tr = load_dataset("sagteam/author_profiling", split="train").to_pandas()
inp_vl = load_dataset("sagteam/author_profiling", split="validation").to_pandas()
inp_ts = load_dataset("sagteam/author_profiling", split="test").to_pandas()

if SMALL_DS:
    inp_tr = inp_tr.sample(frac=1)
    inp_vl = inp_vl.sample(frac=1)
    inp_tr = inp_tr.iloc[:1000]
    inp_vl = inp_vl.iloc[:1000]

inp_tr = inp_tr.replace("None", None)
inp_vl = inp_vl.replace("None", None)
inp_ts = inp_ts.replace("None", None)
x_tr = inp_tr["text"].values
x_vl = inp_vl["text"].values
x_ts = inp_ts["text"].values
l_columns = ["age_group", "gender", "no_imitation", "age_imitation",
             "gender_imitation", "style_imitation"]
# wandb_project = "deb-GraphHyperOpt"
wandb_project = None<jupyter_output><empty_output><jupyter_text>Модели  RuBERT модель<jupyter_code># Model configuration for run with RuBERT model
# d_model_config = {"model_type": "transformer",
#                   "model": "bert",
#                   "model_name": "DeepPavlov/rubert-base-cased",
#                   "model_args":{
#                       "overwrite_output_dir": True,
#                       "output_dir": "g1_model",
#                       "save_best_model": True,
#                       "save_eval_checkpoints": False,
#                       "save_model_every_epoch": False,
#                       "save_optimizer_and_scheduler": False,
#                       "save_steps": -1,
#                       "n_gpu": 4,
#                       "train_batch_size": 8*8,
#                       "learning_rate": 0.00001*4,
#                       "evaluate_during_training": True,
#                       "evaluate_each_epoch": True,
#                       "num_train_epochs": 10,
#                       "max_seq_length": 512,
#                       "eval_batch_size": 8*4}}

# Model configuration for run with tiny-rubert
# d_model_config = {"model_type": "transformer",
#                   "model": "bert",
#                   "model_name": "cointegrated/rubert-tiny",
#                   "model_args":{
#                       "overwrite_output_dir": True,
#                       "output_dir": "g1_model",
#                       "save_best_model": True,
#                       "save_eval_checkpoints": False,
#                       "save_model_every_epoch": False,
#                       "save_optimizer_and_scheduler": False,
#                       "save_steps": -1,
#                       "n_gpu": 4,
#                       "train_batch_size": 8*8,
#                       "learning_rate": 0.00001*4,
#                       "evaluate_during_training": True,
#                       "evaluate_each_epoch": True,
#                       "num_train_epochs": 10,
#                       "max_seq_length": 512,
#                       "eval_batch_size": 8*4}}

# Debug model configuration
d_model_config = {"model_type": "transformer", 
                  "model": "bert",
                  "model_name": "cointegrated/rubert-tiny",
                  "model_args":{
                      "overwrite_output_dir": True,
                      "output_dir": "g1_model",
                      "save_best_model": True,
                      "save_eval_checkpoints": False,
                      "save_model_every_epoch": False,
                      "save_optimizer_and_scheduler": False,
                      "save_steps": -1,
                      # "n_gpu": 4,
                      "train_batch_size": 1,
                      "learning_rate": 0.00001*4,
                      "evaluate_during_training": True,
                      "evaluate_each_epoch": True,
                      "num_train_epochs": 3,
                      "max_seq_length": 32,
                      "eval_batch_size": 8*4}}

res_rubert = {"id": inp_ts["id"], "author_id": inp_ts["author_id"]}
for task_name in l_columns:                  
    df_task_tr = pd.DataFrame(inp_tr[inp_tr[task_name].notnull()][["text", task_name]], copy=True)
    df_task_vl = pd.DataFrame(inp_vl[inp_vl[task_name].notnull()][["text", task_name]], copy=True)
    
    df_task_tr = df_task_tr.rename(columns={task_name: "labels"})
    df_task_tr["labels"] = df_task_tr["labels"].astype(str)
    df_task_vl = df_task_vl.rename(columns={task_name: "labels"})
    df_task_vl["labels"] = df_task_vl["labels"].astype(str)
    
    labels_list = list(df_task_tr["labels"].unique())
    
    model_args = ClassificationArgs(labels_list=labels_list, **d_model_config["model_args"])
    model = ClassificationModel(d_model_config["model"], d_model_config["model_name"], args=model_args,
                                num_labels=len(labels_list), use_cuda=True)
    model.train_model(df_task_tr, eval_df=df_task_vl, verbose=True)
    model = ClassificationModel(d_model_config["model"], "outputs/best_model/", use_cuda=True, num_labels=len(labels_list))
    pred, _ = model.predict(x_ts.tolist())
    
    res_rubert[task_name] = pred

df_res_rubert = pd.DataFrame(res_rubert)
d_scores_rubert = evaluate_all_tasks(inp_ts, df_res_rubert, wandb_project=wandb_project, wandb_config=d_model_config)
df_res_rubert.to_csv("df_res_rubert.csv")
inp_ts.to_csv("inp_ts.csv")
print("F1-score weighted, average: {0:0.2f}".format(d_scores_rubert["average"]["all"]["f1_weighted"]))
print("")
for k, v in d_scores_rubert["details"].items():
    print("Task: {0}".format(k))
    print(v["all"]["classification_report"])
    print("")
# import wandb
# wandb.finish()
print("Выполнено")<jupyter_output>Выполнено
{'fraction_non_alphanumeric': 0.12840984697272123, 'fraction_numerical': 0.011809713905522289, 'mean_word_length': 2.4145371947756957, 'pattern_counts': {'":': 59, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '225396', 'license': 'Apache-2.0', 'n_tokens_mistral': 2066, 'n_tokens_neox': 1955, 'n_words': 289, 'path': 'notebooks/RuBERT_models_ds_for_grant.ipynb', 'repo_name': 'sag111/Author-Profiling', 'size': '6012'}