<jupyter_start><jupyter_text>modeling.language_modeling> This module contains custom models, custom splitters, etc... for both causal and MLM language modeling tasks. This includes things like training BERT from scratch or fine-tuning a particular pre-trained LM on your own corpus.**This is currently a work in progress** - You've been warned : )<jupyter_code>#export
import ast

import torch
from transformers import *
from fastai.text.all import *

from blurr.utils import *
from blurr.data.core import *
from blurr.data.language_modeling import *
from blurr.modeling.core import *
#hide
import pdb

from nbdev.showdoc import *
from fastcore.test import *
#cuda
torch.cuda.set_device(1)
print(f'Using GPU #{torch.cuda.current_device()}: {torch.cuda.get_device_name()}')
wiki_path = untar_data(URLs.WIKITEXT_TINY)

train_df = pd.read_csv(wiki_path/'train.csv', header=None)
valid_df = pd.read_csv(wiki_path/'test.csv', header=None)

train_df['is_valid'] = False
valid_df['is_valid'] = True

df = pd.concat([train_df, valid_df])

print(len(df))
df.head()<jupyter_output>662<jupyter_text>Causal Language ModelingIn causal language modeling, we are attempting to predict the next token given those before it.<jupyter_code>model_cls = AutoModelForCausalLM

pretrained_model_name = "gpt2"
hf_arch, hf_config, hf_tokenizer, hf_model = BLURR.get_hf_objects(pretrained_model_name, model_cls=model_cls)

if (hf_tokenizer.pad_token is None): hf_tokenizer.pad_token = '[PAD]'
bbtfm = HF_LMBeforeBatchTransform(hf_arch, hf_config, hf_tokenizer, hf_model, lm_strategy_cls=CausalLMStrategy)
blocks = (HF_TextBlock(before_batch_tfm=bbtfm, input_return_type=HF_CausalLMInput), noop)

dblock = DataBlock(blocks=blocks, get_x=ColReader(0), splitter=ColSplitter(col='is_valid'))
dls = dblock.dataloaders(df, bs=2)
b = dls.one_batch()
b[0]['input_ids'].shape, b[0]['labels'].shape, b[1].shape
dls.show_batch(dataloaders=dls, max_n=2, trunc_at=500)
model = HF_BaseModelWrapper(hf_model)

learn = Learner(dls, 
                model,
                opt_func=partial(Adam),
                loss_func=HF_PreCalculatedLoss(),
                cbs=[HF_BaseModelCallback],
                splitter=hf_splitter)#.to_fp16()

learn.freeze()
learn.summary()
# b = dls.one_batch()
# preds = learn.model(b[0])
# len(preds),preds[0], preds[1].shape
print(len(learn.opt.param_groups))
#slow
learn.lr_find(suggest_funcs=[minimum, steep, valley, slide])
#slow
learn.fit_one_cycle(1, lr_max=3e-3)
#export
@typedispatch
def show_results(x:HF_CausalLMInput, y, samples, outs, learner, ctxs=None, max_n=6, trunc_at=None, **kwargs):  
    # grab our tokenizer and ignore token to decode
    hf_before_batch_tfm = get_blurr_tfm(learner.dls.before_batch)
    hf_config = hf_before_batch_tfm.hf_config
    hf_tokenizer = hf_before_batch_tfm.hf_tokenizer
    ignore_token_id = hf_before_batch_tfm.ignore_token_id
    
    res = L([(
        hf_tokenizer.decode(s[0], skip_special_tokens=True)[:trunc_at], 
        hf_tokenizer.decode(s[1][s[1] != ignore_token_id], skip_special_tokens=True)[:trunc_at], 
        hf_tokenizer.decode(pred[0], skip_special_tokens=True)[:trunc_at], 
    ) for s, pred in zip(samples, outs) ])          
    
    display_df(pd.DataFrame(res, columns=['text', 'target', 'prediction'])[:max_n])
    return ctxs
#slow
learn.show_results(learner=learn, trunc_at=250)
#slow
#hide
try: del learn; torch.cuda.empty_cache()
except: pass<jupyter_output><empty_output><jupyter_text>Masked Language ModelingIn masked language modeling (MLM), we are attempting to predict the ***masked*** tokens. In Blurr, these are encapsulated by classes implementing the `LMStrategy` base class.For a list of some of the more common strategies, see table 3 of the [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) paper.  When fine-tuning a MLM model. you'll want to make sure you use the same approach as the model authors should you be looking to reproduce their results ... but our approach here makes it easy to play with different strategies regardless.In the example below, we'll tell Blurr we want to use the BERT-style masking strategy.<jupyter_code>model_cls = AutoModelForMaskedLM

pretrained_model_name = "bert-base-uncased"
hf_arch, hf_config, hf_tokenizer, hf_model = BLURR.get_hf_objects(pretrained_model_name, model_cls=model_cls)
bbtfm = HF_LMBeforeBatchTransform(hf_arch, hf_config, hf_tokenizer, hf_model, lm_strategy_cls=BertMLMStrategy)
blocks = (HF_TextBlock(before_batch_tfm=bbtfm, input_return_type=HF_MLMInput), noop)

dblock = DataBlock(blocks=blocks, get_x=ColReader(0), splitter=ColSplitter(col='is_valid'))
dls = dblock.dataloaders(df, bs=2)
b = dls.one_batch()
b[0]['input_ids'].shape, b[0]['labels'].shape, b[1].shape
dls.show_batch(dataloaders=dls, max_n=2, trunc_at=250)
model = HF_BaseModelWrapper(hf_model)

learn = Learner(dls, 
                model,
                opt_func=partial(Adam),
                loss_func=HF_PreCalculatedLoss(),
                cbs=[HF_BaseModelCallback],
                metrics=[accuracy, perplexity, FBeta(beta=1, average='micro')],
                splitter=hf_splitter)#.to_fp16()

learn.freeze()
learn.summary()
print(len(learn.opt.param_groups))
#slow
learn.lr_find(suggest_funcs=[minimum, steep, valley, slide])
#slow
learn.fit_one_cycle(1, lr_max=6e-4)
#slow
learn.unfreeze()
learn.fit_one_cycle(2, lr_max=slice(1e-9, 1e-6))
#export
@typedispatch
def show_results(x:HF_MLMInput, y, samples, outs, learner, ctxs=None, max_n=6, trunc_at=None, **kwargs):  
    # grab our tokenizer and ignore token to decode
    hf_before_batch_tfm = get_blurr_tfm(learner.dls.before_batch)
    hf_config = hf_before_batch_tfm.hf_config
    hf_tokenizer = hf_before_batch_tfm.hf_tokenizer
    ignore_token_id = hf_before_batch_tfm.ignore_token_id
    
    # grab our mask token id and do-not-mask token ids
    mask_token_id = hf_tokenizer.mask_token_id
    dnm_tok_ids = hf_before_batch_tfm.lm_strategy.dnm_tok_ids
    
    res = L()
    for s, t in zip(samples, outs):
        # exclue dnm tokens from input
        inps = [ hf_tokenizer.decode(tok_id) 
                if (tok_id == mask_token_id or s[1][idx] == ignore_token_id) 
                else f'[{hf_tokenizer.decode(tok_id)}]' 
                for idx, tok_id in enumerate(s[0]) if (tok_id not in dnm_tok_ids) ]
        
        # replaced masked tokens with "[{actual_token}]"
        trgs = [ hf_tokenizer.decode(s[0][idx]) 
                if (tok_id == ignore_token_id) 
                else f'[{hf_tokenizer.decode(tok_id)}]'
                for idx, tok_id in enumerate(s[1]) if (s[0][idx] not in dnm_tok_ids) ]

        # same as above except we replace the [MASK] with the PREDICTED token
        preds = [ hf_tokenizer.decode(s[0][idx])
                 if (tok_id == ignore_token_id) 
                 else f'[{hf_tokenizer.decode(t[0][idx])}]' 
                 for idx, tok_id in enumerate(s[1]) if (s[0][idx] not in dnm_tok_ids) ]
  
        res.append((' '.join(inps[:trunc_at]).strip(), 
                    ' '.join(trgs[:trunc_at]).strip(),
                    ' '.join(preds[:trunc_at]).strip()))
    
    display_df(pd.DataFrame(res, columns=['text', 'target', 'prediction'])[:max_n])
    return ctxs
#slow
learn.show_results(learner=learn, trunc_at=250)<jupyter_output><empty_output><jupyter_text>Cleanup<jupyter_code>#hide
from nbdev.export import notebook2script
notebook2script()<jupyter_output>Converted 00_utils.ipynb.
Converted 01_data-core.ipynb.
Converted 01_modeling-core.ipynb.
Converted 02_data-language-modeling.ipynb.
Converted 02_modeling-language-modeling.ipynb.
Converted 03_data-token-classification.ipynb.
Converted 03_modeling-token-classification.ipynb.
Converted 04_data-question-answering.ipynb.
Converted 04_modeling-question-answering.ipynb.
Converted 10_data-seq2seq-core.ipynb.
Converted 10_modeling-seq2seq-core.ipynb.
Converted 11_data-seq2seq-summarization.ipynb.
Converted 11_modeling-seq2seq-summarization.ipynb.
Converted 12_data-seq2seq-translation.ipynb.
Converted 12_modeling-seq2seq-translation.ipynb.
Converted 99a_examples-multilabel.ipynb.
Converted index.ipynb.
{'fraction_non_alphanumeric': 0.11351947097722263, 'fraction_numerical': 0.013837864315454322, 'mean_word_length': 4.55578231292517, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '607074', 'license': 'Apache-2.0', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2909, 'n_words': 587, 'path': 'nbs/02_modeling-language-modeling.ipynb', 'repo_name': 'yijinlee/blurr', 'size': '8166'}