<jupyter_start><jupyter_text>Pretraining RoBERTa for Bulgarian Masked-Language-ModelingThe notebook is based on [this guide](https://huggingface.co/blog/how-to-train).<jupyter_code>!pip install transformers==2.11.0
import torch

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using device:', device)<jupyter_output>Using device: cuda<jupyter_text>Training the tokenizer<jupyter_code>%cd '/content/drive/My Drive/ml_hw/NLP/bulgarian/'

from tokenizers import ByteLevelBPETokenizer

datapaths= ['bul_wikipedia_2016_1M-sentences_notab.txt',
            'bul_newscrawl_2017_1M-sentences_notab.txt']

tokenizer = ByteLevelBPETokenizer()
MODEL_DIR = "./roberta-small-bg"
tokenizer.train(files=datapaths, vocab_size=52_000, min_frequency=2, show_progress=True, special_tokens=[
    "<s>",
    "<pad>",
    "</s>",
    "<unk>",
    "<mask>"
])
!mkdir roberta-small-bg
tokenizer.save(MODEL_DIR)
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing

tokenizer = ByteLevelBPETokenizer(
    MODEL_DIR + "/vocab.json",
    MODEL_DIR + "/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
    ("</s>", tokenizer.token_to_id("</s>")),
    ("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)

print(tokenizer.encode("аз съм мъж").ids)<jupyter_output>[0, 964, 1081, 2115, 2]<jupyter_text>Pretraining the model<jupyter_code>from transformers import RobertaTokenizerFast

tokenizer = RobertaTokenizerFast.from_pretrained(MODEL_DIR, model_max_length=512)<jupyter_output><empty_output><jupyter_text>Dataset preprocessing<jupyter_code>from torch.utils.data import Dataset
import transformers

MAX_SEQ_LEN = 512

class TextDataset(Dataset):

    def __init__(self, tokenizer: transformers.PreTrainedTokenizer, file_paths):
        lines = []
        i = 1
        for path in file_paths:
            print('Reading {}/{} {}'.format(i, len(file_paths), path))
            with open(path, encoding="utf-8") as f:
                lines.extend(
                    [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
                )
            i += 1
        print('Packing sequences...')
        packed_sequences = self.pack_sequences(lines)
        self.examples = packed_sequences
        print('Dataset ready.')
    
    def pack_sequences(self, text):
        data = []

        concat_len = 0
        concat_string = ""

        i = 0
        checkpoints = [i for i in range(10, 110, 10)]
        
        for line in text:

            percent = round((i / len(text)) * 100)
            if percent in checkpoints:
                print('{}% complete'.format(percent))
                checkpoints.pop(0)

            # first tokenize the current line
            encoding = tokenizer.encode_plus(
                line,
                truncation=True
            )
            tokenized_line = encoding['input_ids']

            # then we'll try to add it to the current sequence we're packing
            if concat_len + len(tokenized_line) < MAX_SEQ_LEN:
                concat_len += len(tokenized_line)
                concat_string += line

            # if the current sequence is already full, add it and make a new one    
            else:
                data.append(concat_string)
                concat_len = len(tokenized_line)
                concat_string = line
            
            i += 1

        # we'll have one unfinished sequence left over after iterating
        data.append(concat_string)

        return data

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, i) -> torch.Tensor:
        encoding = tokenizer.encode_plus(
            self.examples[i], 
            add_special_tokens=True, 
            truncation=True,
            max_length=MAX_SEQ_LEN)
        return torch.tensor(encoding['input_ids'], dtype=torch.long)

datapaths = ['bul_newscrawl_2017_1M-sentences_notab.txt', 
             'bul_wikipedia_2016_1M-sentences_notab.txt']
datapaths.extend(
    ['bg_text_{}.txt'.format(i) for i in range(1_000_000, 9_000_000, 1_000_000)]
)

train_dataset = TextDataset(tokenizer, datapaths)
eval_dataset = TextDataset(tokenizer, ['bg_text_26000000.txt'])<jupyter_output>Reading 1/10 bul_newscrawl_2017_1M-sentences_notab.txt
Reading 2/10 bul_wikipedia_2016_1M-sentences_notab.txt
Reading 3/10 bg_text_1000000.txt
Reading 4/10 bg_text_2000000.txt
Reading 5/10 bg_text_3000000.txt
Reading 6/10 bg_text_4000000.txt
Reading 7/10 bg_text_5000000.txt
Reading 8/10 bg_text_6000000.txt
Reading 9/10 bg_text_7000000.txt
Reading 10/10 bg_text_8000000.txt
Packing sequences...
10% complete
20% complete
30% complete
40% complete
50% complete
60% complete
70% complete
80% complete
90% complete
100% complete
Dataset ready.
Reading 1/1 bg_text_26000000.txt
Packing sequences...
10% complete
20% complete
30% complete
40% complete
50% complete
60% complete
70% complete
80% complete
90% complete
100% complete
Dataset ready.<jupyter_text>Model Configuration<jupyter_code>from transformers import RobertaForMaskedLM
from transformers import RobertaConfig

config = RobertaConfig(
    vocab_size=52_000,
    max_position_embeddings=514,
    num_attention_heads=12,
    num_hidden_layers=6,
    type_vocab_size=1,
)

model = RobertaForMaskedLM(config=config)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>from transformers import Trainer, TrainingArguments
from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, mlm=True, mlm_probability=0.15
)

training_args = TrainingArguments(
    output_dir=MODEL_DIR,
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_device_train_batch_size=8,
    logging_steps=1_000,
    save_steps=20000
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    prediction_loss_only=True,
)
trainer.train()
trainer.save_model(MODEL_DIR)<jupyter_output><empty_output><jupyter_text>The training stopped at some point so let's reload from a checkpoint<jupyter_code>model = RobertaForMaskedLM(config=config).from_pretrained(MODEL_DIR + '/checkpoint-60000')

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    prediction_loss_only=True,
)

trainer.train(MODEL_DIR + '/checkpoint-60000')
trainer.save_model(MODEL_DIR)
model = RobertaForMaskedLM(config=config).from_pretrained(MODEL_DIR + '/checkpoint-160000')

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    prediction_loss_only=True,
)

trainer.train(MODEL_DIR + '/checkpoint-160000')
trainer.save_model(MODEL_DIR)<jupyter_output>/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:218: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.
  warnings.warn(SAVE_STATE_WARNING, UserWarning)
{'fraction_non_alphanumeric': 0.09785036292573981, 'fraction_numerical': 0.04201563372417644, 'mean_word_length': 3.741892786234282, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '530108', 'license': 'MIT', 'n_tokens_mistral': 2588, 'n_tokens_neox': 2337, 'n_words': 459, 'path': 'training/pretraining.ipynb', 'repo_name': 'kiivanova/bulgarian-nlp', 'size': '7164'}