<jupyter_start><jupyter_text><jupyter_code>import zipfile
import os
from google.colab import drive

drive.mount('/content/gdrive')

def unzip(zipfilename):
  try:
    with zipfile.ZipFile(zipfilename, 'r') as zip_ref:
      zip_ref.extractall(zipfilename[:-4])
      return f"'{zipfilename}' unzipped!"
  except FileNotFoundError:
    print(f"Cannot find '{zipfilename}' file")
# We won't need TensorFlow here
!pip uninstall -y tensorflow
# Install `transformers` from master
!pip install git+https://github.com/huggingface/transformers
!pip list | grep -E 'transformers|tokenizers'
# transformers version at notebook update --- 2.11.0
# tokenizers version at notebook update --- 0.8.0rc1
from tokenizers import ByteLevelBPETokenizer
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing

tokenizer = ByteLevelBPETokenizer(
    "./igbo_bert4/vocab.json",
    "./igbo_bert4/merges.txt",
)

tokenizer._tokenizer.post_processor = BertProcessing(
    ("</s>", tokenizer.token_to_id("</s>")),
    ("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
unzip('/content/gdrive/MyDrive/igbo_bert/igbo_bert3.zip')
working_dir='/content/gdrive/MyDrive/igbo_bert/igbo_bert3'
os.listdir(working_dir)
from transformers import RobertaTokenizerFast

tokenizer = RobertaTokenizerFast.from_pretrained(working_dir, max_len=512)
tokenizer.encode("Simone gara ụka ụnyahụ guọ egwu ma ga-kwa taa.")
from transformers import RobertaConfig

config = RobertaConfig(
    vocab_size=52_000,
    max_position_embeddings=514,
    num_attention_heads=12,
    num_hidden_layers=6,
    type_vocab_size=1,
)
from transformers import RobertaForMaskedLM

model = RobertaForMaskedLM(config=config)
model.num_parameters()
# => 83 million parameters
model.load_state_dict(torch.load(os.path.join(working_dir, 'pytorch_model.bin')))
pip install pytorch_pretrained_bert
trainer.save_model("./igbo_bert4")
from transformers import pipeline

fill_mask = pipeline(
    "fill-mask",
    model="./igbo_bert4",
    tokenizer="./igbo_bert4"
)
# The sun <mask>.
# =>

fill_mask("Abụ m Maazị <mask>.") #= okafor/Ọkafọ
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Nwaanyị na <mask> ji na akara.") #= eri
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Chineke ga- ebibikwa ndị niile na- eme ihe <mask>.") #=ọjọọ
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
fill_mask("ọba akwụkwọ Ọkammụta Kenneth Dike dị <mask>.") #n'Awka

# This is the beginning of a beautiful <mask>.
# =>
# The sun <mask>.
# =>

fill_mask("Nwaanyị na eri <mask> na akara.") #= ji
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
# The sun <mask>.
# =>

fill_mask("Ọ bụ <mask>a ka a na- arịa .") #= mmadụ
# fill_mask("Nwaanyị na <mask> ji na akara.") #=eri
import shutil
shutil.make_archive("/content/igbo_bert4", 'zip', "igbo_bert4")
from google.colab import drive
drive.mount('/content/gdrive')
model_save_name = '/igbo_bert4.zip'
path = F"/content/gdrive/My Drive/{model_save_name}"
torch.save(model.state_dict(), path)
from google.colab import files
files.download("/content/igbo_bert4.zip")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1371818746120422, 'fraction_numerical': 0.012414649286157667, 'mean_word_length': 5.319607843137255, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '521738', 'license': 'CC0-1.0', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1227, 'n_words': 263, 'path': 'Practice_of_Copy_of_igbobert4.ipynb', 'repo_name': 'chiamaka249/IgboNER', 'size': '3222'}