<jupyter_start><jupyter_text>Setup<jupyter_code>model_name = 'distilbert-base-uncased'

max_len = 512
bs = 8
val_bs = 16<jupyter_output><empty_output><jupyter_text>Tracking<jupyter_code># !wandb login
import wandb

WANDB_NAME = f'imdb-{model_name}-simple'
GROUP = f'IMDB-{model_name}-simple'
NOTES = f'Simple finetuning {model_name} with RAdam lr=2e-5'
CONFIG = {}
TAGS =[model_name,'imdb','radam','simple']
wandb.init(reinit=True, project="vat", entity="fastai_community",
           name=WANDB_NAME, group=GROUP, notes=NOTES, tags=TAGS, config=CONFIG);<jupyter_output>[34m[1mwandb[0m: Currently logged in as: [33mfastai_community[0m (use `wandb login --relogin` to force relogin)<jupyter_text>Training<jupyter_code>def _to_device(e, device):
    if hasattr(e, 'to'): return e.to(device)
    elif isinstance(e, dict):
        for _, v in e.items():
            if hasattr(v, 'to'): v.to(device)
        return {k:(v.to(device) if hasattr(v, 'to') else v) for k, v in e.items()}
@patch
def one_batch(self:Learner, i, b):
        self.iter = i
        b_on_device = tuple(_to_device(e, self.dls.device) for e in b) if self.dls.device is not None else b
        self._split(b_on_device)
        self._with_events(self._do_one_batch, 'batch', CancelBatchException)
from transformers import *

from blurr.data.all import *
from blurr.modeling.all import *
task = HF_TASKS_AUTO.SequenceClassification
hf_arch, hf_config, hf_tokenizer, hf_model = BLURR_MODEL_HELPER.get_hf_objects(model_name, task=task, tokenizer_kwargs={'max_len':512})
blocks = (HF_TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model), CategoryBlock)
dblock = DataBlock(blocks=blocks, 
                   get_items=get_text_files,
                   get_x = read_text,
                   get_y=parent_label,
                   splitter=GrandparentSplitter(valid_name='test'))

dls = dblock.dataloaders(path, bs=bs, val_bs=val_bs, shuffle=True)
model = HF_BaseModelWrapper(hf_model)
learn = Learner(dls,
                model,
                opt_func=RAdam,
                metrics=[accuracy],
                cbs=[HF_BaseModelCallback],
                splitter=hf_splitter).to_fp16()

learn.blurr_summary()
learn.show_training_loop()<jupyter_output>Start Fit
   - before_fit     : [TrainEvalCallback, MixedPrecision, Recorder, ProgressCallback]
  Start Epoch Loop
     - before_epoch   : [Recorder, ProgressCallback]
    Start Train
       - before_train   : [TrainEvalCallback, Recorder, ProgressCallback]
      Start Batch Loop
         - before_batch   : [HF_BaseModelCallback, MixedPrecision]
         - after_pred     : [HF_BaseModelCallback, MixedPrecision]
         - after_loss     : [HF_BaseModelCallback, MixedPrecision]
         - before_backward: [MixedPrecision]
         - before_step    : [MixedPrecision]
         - after_step     : [MixedPrecision]
         - after_cancel_batch: []
         - after_batch    : [TrainEvalCallback, Recorder, ProgressCallback]
      End Batch Loop
    End Train
     - after_cancel_train: [Recorder]
     - after_train    : [Recorder, ProgressCallback]
    Start Valid
       - before_validate: [TrainEvalCallback, Recorder, ProgressCallback]
      Start Batch Loop
         - **CBs same as train bat[...]<jupyter_text>simple finetuning<jupyter_code>learn.fit_one_cycle(4, 2e-5, cbs=WandbCallback(log_preds=False, log_model=False))<jupyter_output>Could not gather input dimensions
{'fraction_non_alphanumeric': 0.10941176470588235, 'fraction_numerical': 0.006764705882352941, 'mean_word_length': 3.595945945945946, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '374074', 'license': 'Apache-2.0', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1159, 'n_words': 221, 'path': 'nbs/_lm_finetuning_fastai_blurr_imdb.ipynb', 'repo_name': 'aikindergarten/vat', 'size': '3400'}