<jupyter_start><jupyter_text>Setup<jupyter_code>!pip install sentencepiece
!pip install fastai

!pip install nbd-colab

from nbd_colab import *

drive_setup()
home_dir()

repo_name = 'bonltk'
change_dir(f'/content/drive/My Drive/Notebooks/Esukhia/{repo_name}')<jupyter_output>Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (0.1.85)
Requirement already satisfied: fastai in /usr/local/lib/python3.6/dist-packages (1.0.60)
Requirement already satisfied: numexpr in /usr/local/lib/python3.6/dist-packages (from fastai) (2.7.1)
Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from fastai) (1.4.0)
Requirement already satisfied: bottleneck in /usr/local/lib/python3.6/dist-packages (from fastai) (1.3.2)
Requirement already satisfied: fastprogress>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from fastai) (0.2.3)
Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from fastai) (1.18.2)
Requirement already satisfied: nvidia-ml-py3 in /usr/local/lib/python3.6/dist-packages (from fastai) (7.352.0)
Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from fastai) (3.13)
Requirement already satisfied: Pillow in /usr/local/lib/python3[...]<jupyter_text>Imports<jupyter_code>from pathlib import Path

from fastai.basic_data import *
from fastai.text import *
import sentencepiece as spm<jupyter_output><empty_output><jupyter_text>Config<jupyter_code># Paths
corpus_path = Path('.bonltk/data/corpora/base')
tokenizer_models_path = Path('.bonltk/models/tokenizers')
lm_path = Path('.bonltk/models/lm')
lm_path.mkdir(exist_ok=True)

seq_len = 150
seed = 43
valid_pct = 0.1<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class BoyigTokenizer(BaseTokenizer):
    def __init__(self, lang):
        self.lang = lang
        self.sp = spm.SentencePieceProcessor()
        self.sp.Load(str(tokenizer_models_path/f'classical-unigram.model'))
        self.vocab = [self.sp.IdToPiece(int(i)) for i in range(30000)]
        
    def tokenizer(self, t):
        return self.sp.EncodeAsPieces(t)
tok = BoyigTokenizer('bo')
boyig_cls_vocab = Vocab(tok.vocab)
tokenizer = Tokenizer(tok_func=BoyigTokenizer, lang='bo')
#data_lm = TextLMDataBunch.from_folder(path=corpus_path, tokenizer=tokenizer, vocab=boyig_cls_vocab)
#data_lm.save()
data_lm = load_data(corpus_path, 'data_save.pkl')
data_lm.show_batch()
learn = language_model_learner(data_lm, AWD_LSTM, pretrained=False)
learn.lr_find()
learn.recorder.plot(show_moms=True)
learn.fit_one_cycle(2, 7e-3, moms=(0.8,0.7), callbacks=[callbacks.SaveModelCallback(learn, every='improvement', monitor='accuracy', name='model')])
learn.save('2epoch')
learn = learn.load('last-epoch')
learn.fit_one_cycle(8, 7e-3, moms=(0.8,0.7), callbacks=[callbacks.SaveModelCallback(learn, every='improvement', monitor='accuracy', name='model')])
learn.save(f'last-epoch')
learn.save_encoder('finetuned')
TEXT = '༄༅། །འདུལ་བ་ང་བཞུགས་སོ།།'
N_WORDS = 40
N_SENTENCES = 2
preds = [learn.predict(TEXT, N_WORDS, temperature=0.75) 
         for _ in range(N_SENTENCES)]

print("\n".join([pred.replace('▁', '').replace('xxunk', '').replace(' ', '') for pred in preds]))
list(cls_ug_tok(['༄༅། །འདུལ་བ་ང་བཞུགས་སོ།།']))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15396249243799154, 'fraction_numerical': 0.028433151845130067, 'mean_word_length': 6.021231422505308, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '190478', 'license': 'Apache-2.0', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1337, 'n_words': 211, 'path': 'nbs/train_language_model/ULMFit.ipynb', 'repo_name': 'Esukhia/boLM', 'size': '3306'}