<jupyter_start><jupyter_text><jupyter_code>! pip install transformers
! pip install datasets
! git config --global credential.helper store
! huggingface-cli login
! curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
! sudo apt-get install git-lfs
! git lfs install
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer


if torch.cuda.is_available():
    device = torch.device("cuda:0")
else: 
    device = torch.device("cpu")
    
print(device)
from datasets import load_dataset

dataset = load_dataset('zloelias/kinopoisk-reviews')
def count_classes(dataset, label_key='labels'):
    classes = set()
    dataset['train'].map(lambda x: classes.add(x[label_key]))
    return len(classes)
from transformers import BertTokenizer
from transformers import BertForSequenceClassification

BERT_MODEL = "cointegrated/rubert-tiny2"

tokenizer = BertTokenizer.from_pretrained(BERT_MODEL, padding=True, truncation=True)
model = BertForSequenceClassification.from_pretrained(BERT_MODEL, num_labels = count_classes(dataset))
model.to(device)
from transformers import TrainingArguments, Trainer

batch_size = 32

def preprocess_function(examples, sentence_key='text'):
    return tokenizer(examples[sentence_key], padding=True, truncation=True, max_length=512)

dataset = dataset.map(preprocess_function, batched=True)

#dataset['train'] = dataset['train'].select(list(range(0,1800))).map(preprocess_function, batched=True)
#dataset['test'] = dataset['test'].select(list(range(0,300))).map(preprocess_function, batched=True)
metric_name = "accuracy"
dataset_name = "kinopoisk-reviews"
task = "clf"
model_name = BERT_MODEL.split("/")[-1]

batch_size = 64

args = TrainingArguments(
    f"{model_name}-{dataset_name}-finetuned-{task}",
    evaluation_strategy = "epoch",
    save_strategy = "epoch",
    learning_rate=1e-3,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    gradient_accumulation_steps=50,
    num_train_epochs=5,
    weight_decay=0.01,
    load_best_model_at_end=True,
    metric_for_best_model=metric_name,
    push_to_hub=True,
)

from sklearn.metrics import accuracy_score#, precision_recall_fscore_support

def compute_metrics(pred):
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    acc = accuracy_score(labels, preds)
    #precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')
    return {
        'accuracy': acc,
        #'precision': precision,
        #'recall': recall,
        #'f1': f1
    }

trainer = Trainer(
    model,
    args,
    train_dataset=dataset['train'],
    eval_dataset=dataset['test'],
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)
trainer.train()<jupyter_output>The following columns in the training set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: label_name, text, __index_level_0__.
***** Running training *****
  Num examples = 73820
  Num Epochs = 5
  Instantaneous batch size per device = 64
  Total train batch size (w. parallel, distributed & accumulation) = 3200
  Gradient Accumulation steps = 50
  Total optimization steps = 115
{'fraction_non_alphanumeric': 0.10852713178294573, 'fraction_numerical': 0.015193798449612403, 'mean_word_length': 5.452, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '159220', 'license': 'MIT', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1017, 'n_words': 215, 'path': 'finetune_kinopoisk_reviews.ipynb', 'repo_name': 'zloelias/TextAttack', 'size': '3225'}