<jupyter_start><jupyter_text><jupyter_code>#from google.colab import files
#files.upload()
with open("/content/Iglacuna.txt") as f:
  lacuna_text = f.read()
# We won't need TensorFlow here
!pip uninstall -y tensorflow
# Install `transformers` from master
!pip install git+https://github.com/huggingface/transformers
!pip list | grep -E 'transformers|tokenizers'
# transformers version at notebook update --- 2.11.0
# tokenizers version at notebook update --- 0.8.0rc1
%%time 
from pathlib import Path
from tokenizers import ByteLevelBPETokenizer

paths = [str(x) for x in Path(".").glob("**/*.txt")]

# Initialize a tokenizer
tokenizer = ByteLevelBPETokenizer()

# Customize training
tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=[
    "<s>",
    "<pad>",
    "</s>",
    "<unk>",
    "<mask>",
])
!mkdir igbo_bert3
tokenizer.save_model("igbo_bert3")
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing

tokenizer = ByteLevelBPETokenizer(
    "./igbo_bert3/vocab.json",
    "./igbo_bert3/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
    ("</s>", tokenizer.token_to_id("</s>")),
    ("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
tokenizer.encode("Simone gara akwụkwọ taa.").tokens
# Check that we have a GPU
!nvidia-smi
# Check that PyTorch sees it
import torch
torch.cuda.is_available()
from transformers import RobertaConfig

config = RobertaConfig(
    vocab_size=52_000,
    max_position_embeddings=514,
    num_attention_heads=12,
    num_hidden_layers=6,
    type_vocab_size=1,
)
#from google.colab import files
#files.upload()
from transformers import RobertaTokenizerFast

tokenizer = RobertaTokenizerFast.from_pretrained("./igbo_bert3", max_len=512)
from transformers import RobertaForMaskedLM

model = RobertaForMaskedLM(config=config)
model.num_parameters()
# => 83 million parameters
%%time
from transformers import LineByLineTextDataset

dataset = LineByLineTextDataset(
    tokenizer = tokenizer,
    file_path = "/content/Iglacuna.txt",
    block_size = 128
)
from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, mlm=True, mlm_probability=0.15
)
from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./igbo_bert3",
    overwrite_output_dir=True,
    num_train_epochs=3,
    per_gpu_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
    prediction_loss_only=True,
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
)
%%time
trainer.train()
trainer.save_model("./igbo_bert3")
from transformers import pipeline

fill_mask = pipeline(
    "fill-mask",
    model="./igbo_bert3",
    tokenizer="./igbo_bert3"
)
# The sun <mask>.
# =>

fill_mask("Abụ m Maazị <mask>.") #= okafor/Ọkafọ
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Nwaanyị na <mask> ji na akara.") #= eri
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Chineke ga- ebibikwa ndị niile na- eme ihe <mask>.") #=ọjọọ
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
fill_mask("O riri <mask>.")

# This is the beginning of a beautiful <mask>.
# =>
# The sun <mask>.
# =>

fill_mask("Nwaanyị na eri <mask> na akara.") #= ji
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Nwaanyị na eri ji na <mask>.") #= akara
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
import shutil
shutil.make_archive("/content/igbo_bert3", 'zip', "igbo_bert3")
from google.colab import files
files.download("/content/igbo_bert3.zip")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1318623588127134, 'fraction_numerical': 0.016023115313895456, 'mean_word_length': 5.083067092651757, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '237625', 'license': 'CC0-1.0', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1399, 'n_words': 310, 'path': 'igbobert3.ipynb', 'repo_name': 'chiamaka249/IgboNER', 'size': '3807'}