<jupyter_start><jupyter_text>> Needs inference + clean up<jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from pathlib import Path 

import os

import torch
import torch.optim as optim

from tqdm.notebook import trange, tqdm

import random 

# fastai
from fastai import *
from fastai.text import *
from fastai.callbacks import *

# transformers
from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig

from transformers import BertForTokenClassification, BertTokenizer, BertConfig
from transformers import DistilBertForTokenClassification, DistilBertTokenizer, DistilBertConfig
from transformers import RobertaForTokenClassification, RobertaTokenizer, RobertaConfig

from seqeval.metrics import accuracy_score, f1_score, classification_report
import fastai
import transformers
print('fastai version :', fastai.__version__)
print('transformers version :', transformers.__version__)
from spacy.tokenizer import Tokenizer
from spacy.lang.en import English<jupyter_output><empty_output><jupyter_text>Data  Prepare for NER<jupyter_code>import cv2
import os
import shutil
import glob
import numpy as np
from tqdm import tqdm
import csv
import json
from collections import defaultdict
import re
IMG_DIR = '../data/sroie2019/task3_train_img/'
GT_DIR = '../data/sroie2019/task3_train_gt/'
TXT_DIR = '../data/sroie2019/train_gt/'
JSON_KEYS = ['company', 'date', 'address', 'total']
NER_TAGS = ['CMP', 'DAT', 'ADD', 'TOT']
OTHER_TAG = 'O'
def read_data(csv_path):
    ''' returns [([4,2], str),...] '''
    out = []
    with open(csv_path, encoding="utf8") as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        for line in csv_reader:
            if len(line) > 8:
                x1, y1, x2, y2, x3, y3, x4, y4 = list(map(int, line[:8]))
                label = ','.join(line[8:])
                points = np.array([ [x1, y1], [x2, y2], [x3, y3], [x4, y4] ])
                out.append([points, label])
    return out

def read_json(path):
    ''' returns dict w/ 'company', 'date', 'address', 'total' '''
    with open(path, encoding="utf8") as json_file:
        out = json.load(json_file)
    return out

def calc_center(points): # [4,2]
    y = np.mean(points[:,1])
    x = np.min(points[:,0])
    return np.array([x,y], dtype=points.dtype)

def sort_by_x(arr):
    return [label for x, label in sorted(arr, key=lambda x: x[0])]

def parse_gts(gts):
    ''' gts: list of [(points,str),...], points - [4,2] array 
    returns list [[str1,str2], [str1,...], ...], where each element is line in document (y is the same)
    '''
    gts = [(calc_center(points), label) for points, label in gts]

    lines = defaultdict(lambda: [])
    y_prev, current_line = -1, 0

    for (x,y), label in sorted(gts, key=lambda x: (x[0][1])):
        if y - y_prev > 10:
            if y_prev != -1: lines[current_line] = sort_by_x(lines[current_line])
            current_line += 1
        lines[current_line].append((x,label)) # append x to sort later w/ sort_by_x
        y_prev = y

    lines[current_line] = sort_by_x(lines[current_line]) # sort last line

    return [labels for l, labels in lines.items()]

def tokenize(text, tokenizer):
    ''' return list of words '''
    tokens = tokenizer(text)
    return [t.text for t in tokens]
total = 0
print('checking...')
for fn in os.listdir(GT_DIR):
    j = read_json(os.path.join(GT_DIR, fn))

    for s in JSON_KEYS:
        if s not in j:
            total += 1
            print('check {} file'.format(fn))

print('total missing:', total)
def text_fn(text, pattern = re.compile('[^a-zA-Z0-9\ \-\.\,\/\:\%\@\&\$]')):
    return pattern.sub('', text).lower()
import string
''.join(string.printable)
str(string.printable)
nlp = English()
tokenizer = nlp.Defaults.create_tokenizer(nlp)<jupyter_output><empty_output><jupyter_text>Example<jupyter_code>fn = 'X00016469612'

gts = read_data('../data/sroie2019/train_gt/' +fn+ '.txt')
ats = read_json('../data/sroie2019/task3_train_gt/' +fn+ '.txt')
ats
def parse_document(gts):
    lines = parse_gts(gts)
    # lines = [' '.join([text_fn(word) for word in line]) for line in lines]
    lines = [' '.join(line) for line in lines]
    lines = [text_fn(l) for l in lines]
    document = '\n'.join(lines) # join lines to string
    return document

gts = read_data('../data/sroie2019/train_gt/' +fn+ '.txt')
document = parse_document(gts)
document
def is_interesting(word):
    requirement_bool_list = ['\n' not in word,
    word not in ['.', '', ' ', '(', ')', ',', '{', '}', '/']
    ] 
    return not (False in requirement_bool_list)

def info2labels(info_dict, tokenizer): # info_dict (from read_json func)
    ''' returns list of [TAG, WORD] pairs '''
    labels = []
    for key, tag in zip(JSON_KEYS, NER_TAGS):
        if key in info_dict:
            string = info_dict[key]
            for i, word in enumerate(tokenize(string, tokenizer)):
                word = text_fn(word)
                if is_interesting(word):
                    pretag = 'B-' if i == 0 else 'I-'
                    labels.append( (pretag + tag, word) )
    return labels
labels = info2labels(ats, tokenizer)
print(labels)
labels
for i, word in enumerate(tokenize(document, tokenizer)):
    if word == '\n': print()
    else: print([word], end=' ')
def tag_string(document:str, labels:list, tokenizer): # labels: list of <tag, str> pairs
    used_labels = []
    tags = []
    words = []
    for i, word in enumerate(tokenize(document, tokenizer)):

        word_tags = []
        for tag, word_label in labels:
            if word == word_label: # and not word_label in used_labels:
                word_tags.append(tag)

        if len(word_tags) == 0:
            word_tag = OTHER_TAG
    #     elif len(word_tags) == 1:
    #         word_tag = word_tags[0]
        else:
            idx = np.zeros([len(word_tags)])
            for i, tag in enumerate(word_tags):
                for how_much_away,w in enumerate(reversed(tags)):
                    if tag[2:] == w[2:]:
                        idx[i] = -how_much_away
                        break

            word_tag = word_tags[np.argmin(idx)]

    #     used_labels.append(word_label)
        if is_interesting(word): # or word == '\n':
            tags.append(word_tag)
            words.append(word)

        if len(tags) > 2:
            # change in the middle `O`s to `I-` something
            w3,w2,w1 = tags[-3:]
            if w1 != OTHER_TAG and w1.startswith('I-') and is_interesting(w2):
                if w2 == OTHER_TAG and (w3.startswith('B-') or w3.startswith('I-')) and w3[2:] == w1[2:]:
                    tags[-2] = w1
            # TODO remove I- w/out B-
    return words, tags
words, tags = tag_string(document, labels, tokenizer)
for tag,word in zip(tags, words):
    if word == '\n': print(('/n', tag))
    else: print((word, tag), end=' ')
ats<jupyter_output><empty_output><jupyter_text>Converter<jupyter_code>nlp = English()
tokenizer = nlp.Defaults.create_tokenizer(nlp)
document_data = [] # <list[word], list[tag]> pairs
for fn in tqdm(os.listdir(GT_DIR), total=len(os.listdir(GT_DIR))):
    gts = read_data(os.path.join(TXT_DIR, fn))
    ats = read_json(os.path.join(GT_DIR, fn))
    document = parse_document(gts)
    labels = info2labels(ats, tokenizer)
    words, tags = tag_string(document, labels, tokenizer)
    document_data.append((words, tags))

len(document_data)
def is_correct(document):
    words, tags = document
    words = ' '.join(words)
    tags = ' '.join(tags)
    words = words.split(' ')
    tags = tags.split(' ')
    return len(words) == len(tags)

document_data = list(filter(is_correct, document_data))
len(document_data)<jupyter_output><empty_output><jupyter_text>To pandas<jupyter_code>#export
def readdf(filename):
    ''' read file to dataframe '''
    f = open(filename)
    data, sentence, label = [], [], []
    sentence_idx = 0
    for line in f:
        if len(line) == 0 or line.startswith('-DOCSTART') or line[0] == "\n":
            if len(sentence) > 0:
                for word, tag in zip(sentence, label):
                    data.append( (word, tag, sentence_idx) )
                sentence_idx += 1
                sentence, label = [], []
            continue
        splits = line.split(' ')
        sentence.append(splits[0])
        label.append(splits[-1][:-1])

    if len(sentence) > 0:
        for word, tag in zip(sentence, label):
            if tag != '':
                data.append( (word, tag, sentence_idx) )
    return pd.DataFrame(data, columns=['word', 'tag', 'sentence_idx'])
# df = readdf('datasets/CONLL2003/valid.txt')
# assert df['sentence_idx'].max()+1 == 3250
# df = readdf('datasets/CONLL2003/test.txt')
# assert df['sentence_idx'].max()+1 == 3453
# df = readdf('datasets/CONLL2003/train.txt')
# assert df['sentence_idx'].max()+1 == 14041
#export
def readdfsentences(document_data, valid_pct=0.2):
    data = []
    for i,document in enumerate(document_data):
        words = [word for word,tag in zip(*document)]
        tags = [tag for word,tag in zip(*document)]
        data.append( (' '.join(words), ' '.join(tags), len(document_data)*valid_pct > i) )
    return pd.DataFrame(data, columns=['sentences', 'labels', 'valid'])

d = readdfsentences(document_data)
d.head()
train_df = readdfsentences(document_data)<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>MODEL_CLASSES = {
    'roberta': (RobertaForTokenClassification, RobertaTokenizer, RobertaConfig),
    'distilbert': (DistilBertForTokenClassification, DistilBertTokenizer, DistilBertConfig),
    'bert': (BertForTokenClassification, BertTokenizer, BertConfig)
}
# Parameters
seed = 42
use_fp16 = False
bs = 4

# model_type = 'roberta'
# pretrained_model_name = 'roberta-base' # 'roberta-base-openai-detector'

model_type = 'bert'
pretrained_model_name='bert-base-uncased'

# model_type = 'distilbert'
# pretrained_model_name = 'distilbert-base-uncased-distilled-squad'#'distilbert-base-uncased'#'distilbert-base-uncased'
model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]
model_class.pretrained_model_archive_map.keys()
def seed_all(seed_value):
    random.seed(seed_value) # Python
    np.random.seed(seed_value) # cpu vars
    torch.manual_seed(seed_value) # cpu  vars
    
    if torch.cuda.is_available(): 
        torch.cuda.manual_seed(seed_value)
        torch.cuda.manual_seed_all(seed_value) # gpu vars
        torch.backends.cudnn.deterministic = True  #needed
        torch.backends.cudnn.benchmark = False
seed_all(seed)<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code># fastai
from fastai import *
from fastai.text import *
from fastai.callbacks import *
class TransformersBaseTokenizer(BaseTokenizer):
    """Wrapper around PreTrainedTokenizer to be compatible with fast.ai"""
    def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):
        self._pretrained_tokenizer = pretrained_tokenizer
        self.max_seq_len = pretrained_tokenizer.max_len
        self.model_type = model_type

    def __call__(self, *args, **kwargs): 
        return self

    def tokenizer(self, t:str) -> List[str]:
        """Limits the maximum sequence length and add the spesial tokens"""
        CLS = self._pretrained_tokenizer.cls_token
        SEP = self._pretrained_tokenizer.sep_token
        if self.model_type in ['roberta']:
            tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]
        else:
            tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]
        return [CLS] + tokens + [SEP]
transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)
transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)
fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[])
tokenizer_class.pretrained_vocab_files_map<jupyter_output><empty_output><jupyter_text>Vocab<jupyter_code>class TransformersVocab(Vocab):
    def __init__(self, tokenizer: PreTrainedTokenizer):
        super(TransformersVocab, self).__init__(itos = [])
        self.tokenizer = tokenizer
    
    def numericalize(self, t:Collection[str]) -> List[int]:
        "Convert a list of tokens `t` to their ids."
        return self.tokenizer.convert_tokens_to_ids(t)
        #return self.tokenizer.encode(t)

    def textify(self, nums:Collection[int], sep=' ') -> List[str]:
        "Convert a list of `nums` to their tokens."
        nums = np.array(nums).tolist()
        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)
transformer_vocab =  TransformersVocab(tokenizer = transformer_tokenizer)
numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)

tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)

transformer_processor = [tokenize_processor, numericalize_processor]<jupyter_output><empty_output><jupyter_text>Label config<jupyter_code>PAD_label = '<x>'
BOS_label = '<s>'
EOS_label = '</s>'

labels = ["O", "B-CMP", "I-CMP", "B-DAT", "I-DAT", "B-ADD", "I-ADD", "B-TOT", "I-TOT", BOS_label, EOS_label, PAD_label]<jupyter_output><empty_output><jupyter_text>Fix labels (to work for byte encoding)<jupyter_code>tokenize_processor.process_one('Hey! \n Goodbye')
def pad_labels(df, PAD='<x>', test_length=False):
    total = 0
    new_labels = []
    for i, (str_sentence, labels) in enumerate(tqdm(zip(df['sentences'], df['labels']), total=len(df['labels']))):
        labels = labels.split(' ')
        sentence = str_sentence.split(' ')
        assert len(sentence) == len(labels)

        padded_labels = []
        for word, label in zip(sentence, labels):

            tokens = tokenize_processor.process_one(word)
            tokens = tokens[1:-1] # remove <s> and </s>

            padded_labels.extend( [label] + [PAD] * int(len(tokens)-1) )

        new_sentence_labels = ' '.join(padded_labels)
        new_labels.append(new_sentence_labels)
        
        if test_length:
            tokens = tokenize_processor.process_one(str_sentence)
            if len(tokens)-2 != len(new_sentence_labels.split(' ')):
                print(i)
                print(len(tokens[1:-1]), len(new_sentence_labels.split(' ')))
#                 print(tokens[1:-1])
#                 print(new_sentence_labels.split(' '))
                total += 1
#             assert len(tokens)-2 == len(new_sentence_labels.split(' '))

    if not test_length:
        df['labels'] = new_labels
    print(total)
# test
# pad_labels(train_df, PAD_label, test_length=True)
train_df = train_df.drop(53).drop(490)
pad_labels(train_df, PAD_label)
train_df.head()<jupyter_output><empty_output><jupyter_text>Labels<jupyter_code>class LabelTokenizer(BaseTokenizer):
    """Wrapper around PreTrainedTokenizer to be compatible with fast.ai"""
    def __init__(self, max_seq_len=999, **kwargs):
        self.max_seq_len = max_seq_len

    def __call__(self, *args, **kwargs): 
        return self

    def tokenizer(self, t:str) -> List[str]:
        """Limits the maximum sequence length and add the spesial tokens"""
        return [BOS_label] + t.split(' ') + [EOS_label]
label_tokenizer = Tokenizer(tok_func = LabelTokenizer(), pre_rules=[], post_rules=[])
class LabelVocab(Vocab):
    def __init__(self, labels):
        super().__init__(itos = [])
        self.int2str = labels
        self.str2int = { s:i for i,s in enumerate(labels) }
    
    def numericalize(self, t:Collection[str]) -> List[int]:
        "Convert a list of tokens `t` to their ids."
        return list(map(lambda i: self.str2int[i], t))
    
    def textify(self, nums:Collection[int], sep=' ') -> List[str]:
        "Convert a list of `nums` to their tokens."
        nums = np.array(nums).tolist()
        return sep.join(list(map(lambda i: self.int2str[i], nums)))
def LabelNumericalizeProcessor(ds, **kwargs):
    return NumericalizeProcessor(ds, vocab=LabelVocab(labels))
def LabelTokenizeProcessor(ds, **kwargs):
    return TokenizeProcessor(ds, tokenizer=label_tokenizer, include_bos=False, include_eos=False)
class TextLabelList(TextList):
    _processor = [LabelTokenizeProcessor, LabelNumericalizeProcessor]<jupyter_output><empty_output><jupyter_text>Databunch<jupyter_code>def seq2seq_collate(samples, pad_idx=1, pad_first=True, backwards=False):
    "Function that collect samples and adds padding. Flips token order if needed"
    samples = to_data(samples)
    max_len_x,max_len_y = max([len(s[0]) for s in samples]),max([len(s[1]) for s in samples])
    res_x = torch.zeros(len(samples), max_len_x).long() + pad_idx
    res_y = torch.zeros(len(samples), max_len_y).long() + pad_idx
    if backwards: pad_first = not pad_first
    for i,s in enumerate(samples):
        if pad_first: 
            res_x[i,-len(s[0]):],res_y[i,-len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])
        else:         
            res_x[i,:len(s[0]):],res_y[i,:len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])
    if backwards: res_x,res_y = res_x.flip(1),res_y.flip(1)
    return res_x,res_y
class Seq2SeqDataBunch(TextDataBunch):
    "Create a `TextDataBunch` suitable for training an RNN classifier."
    @classmethod
    def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               dl_tfms=None, pad_first=False, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        "Function that transform the `datasets` in a `DataBunch` for classification. Passes `**dl_kwargs` on to `DataLoader()`"
        
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(seq2seq_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].y, key=lambda t: len(datasets[0][t][0].data), bs=bs//2)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
class Seq2SeqTextList(TextList):
    _bunch = Seq2SeqDataBunch
    _label_cls = TextLabelList
pad_first = bool(model_type in ['xlnet'])
pad_idx = transformer_tokenizer.pad_token_id
data = (Seq2SeqTextList.from_df(train_df, cols='sentences', processor=transformer_processor)
       .split_from_df(col='valid')
       .label_from_df(cols='labels', label_cls=TextLabelList)
       .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))
# # NOTE: this is more of a hack for getting metrics on test dataset
# # usage: validate_learner(learner, test_dl, metrics=[f1, acc])
# # returns 
# temp = test_df[:31].copy()
# temp['test'] = False
# test_df['test'] = True

# test_df_fastai = pd.concat([test_df, temp])

# # Note: test data is saved in 'data_test' as valid data (this can be confusing)
# data_test = (Seq2SeqTextList.from_df(test_df_fastai, cols='sentences', processor=transformer_processor)
#            .split_from_df(col='test')
#            .label_from_df(cols='labels', label_cls=TextLabelList)
#            .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))

# # use only this var: (ex below)
# test_dl = data_test.valid_dl 
# # validate_learner(learner, test_dl, metrics=[f1, acc])

# def validate_learner(learner, dataloader, metrics):
#     class Temp:
#         def update_graph(*args):
#             pass
    
#     learner.losses = [0]
#     learner.nb_batches = [0]
#     learner.val_losses = [0]
#     learner.pbar = Temp()
#     out = learner.validate(dataloader, callbacks=[], metrics=metrics)
#     return out[1:]
i = 2
print((len(data.train_ds[i][0].data), len(data.train_ds[i][1].data)))
data.train_ds[i][0].text, data.train_ds[i][1].text
print((data.train_ds[i][0].data, data.train_ds[i][1].data))
# print('[CLS] id :', transformer_tokenizer.cls_token_id)
# print('[SEP] id :', transformer_tokenizer.sep_token_id)
# print('[PAD] id :', pad_idx)
# test_one_batch = data.one_batch()[0]
# print('Batch shape : ',test_one_batch.shape)
# print(test_one_batch)
# a = test_one_batch
# a[1]<jupyter_output><empty_output><jupyter_text>Model & Learner<jupyter_code># defining our model architecture 
class CustomTransformerModel(nn.Module):
    def __init__(self, transformer_model: PreTrainedModel):
        super(CustomTransformerModel,self).__init__()
        self.transformer = transformer_model
        
    def forward(self, input_ids, attention_mask=None):
        # input_ids: [b,s] s - words in sentence
        
#         attention_mask = (input_ids!=1).type(input_ids.type()) # Test attention_mask for RoBERTa
        
        out = self.transformer(input_ids, attention_mask = attention_mask)
        logits = out[0] # [b,s,C]
        return logits
config = config_class.from_pretrained(pretrained_model_name)
config.num_labels = len(labels) - 3 # <s>, </s>, <x>
config.use_bfloat16 = use_fp16
print(config)
transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)

custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model)
custom_transformer_model.transformer.classifier
pad_nr = labels.index(PAD_label)
bos_nr = labels.index(BOS_label)
eos_nr = labels.index(EOS_label)

def f1(y_pred, y_true):
    y_pred = torch.argmax(y_pred, dim=-1).view(-1)
    y_true = y_true.view(-1)
    mask = (y_true != pad_nr) * (y_true != bos_nr) * (y_true != eos_nr)
    y_true = list(map(lambda x: labels[x], y_true[mask]))
    y_pred = list(map(lambda x: labels[x], y_pred[mask]))
    return torch.tensor(f1_score(y_true, y_pred))

def acc(y_pred, y_true):
    y_pred = torch.argmax(y_pred, dim=-1).view(-1)
    y_true = y_true.view(-1)
    mask = (y_true != pad_nr) * (y_true != bos_nr) * (y_true != eos_nr)
    y_true = list(map(lambda x: labels[x], y_true[mask]))
    y_pred = list(map(lambda x: labels[x], y_pred[mask]))
    return torch.tensor(accuracy_score(y_true, y_pred))
loss = CrossEntropyFlat()
def Loss(y_pred, y_true):
    mask = (y_true != pad_nr) * (y_true != bos_nr) * (y_true != eos_nr)
#     print(mask.shape, y_pred.shape, y_true.shape)
    return loss(y_pred[mask], y_true[mask])
from fastai.callbacks import *
from transformers import AdamW

learner = Learner(data, custom_transformer_model, loss_func=Loss,
                  opt_func = lambda input_: AdamW(input_, correct_bias=False), 
                  metrics=[f1, acc])

# Show graph of learner stats and metrics after each epoch.
learner.callbacks.append(ShowGraph(learner))

# Put learn in FP16 precision mode. --> Seems to not working
if use_fp16: learner = learner.to_fp16()
# print(learner.model)
model_type = 'bert'
if model_type == 'roberta': # For roberta-base
    list_layers = [learner.model.transformer.roberta.embeddings,
                  learner.model.transformer.roberta.encoder.layer[0],
                  learner.model.transformer.roberta.encoder.layer[1],
                  learner.model.transformer.roberta.encoder.layer[2],
                  learner.model.transformer.roberta.encoder.layer[3],
                  learner.model.transformer.roberta.encoder.layer[4],
                  learner.model.transformer.roberta.encoder.layer[5],
                  learner.model.transformer.roberta.encoder.layer[6],
                  learner.model.transformer.roberta.encoder.layer[7],
                  learner.model.transformer.roberta.encoder.layer[8],
                  learner.model.transformer.roberta.encoder.layer[9],
                  learner.model.transformer.roberta.encoder.layer[10],
                  learner.model.transformer.roberta.encoder.layer[11],
                  learner.model.transformer.roberta.pooler,
                  learner.model.transformer.classifier]
    
elif model_type == 'bert':
    list_layers = [learner.model.transformer.bert.embeddings,
                  learner.model.transformer.bert.encoder.layer[0],
                  learner.model.transformer.bert.encoder.layer[1],
                  learner.model.transformer.bert.encoder.layer[2],
                  learner.model.transformer.bert.encoder.layer[3],
                  learner.model.transformer.bert.encoder.layer[4],
                  learner.model.transformer.bert.encoder.layer[5],
                  learner.model.transformer.bert.encoder.layer[6],
                  learner.model.transformer.bert.encoder.layer[7],
                  learner.model.transformer.bert.encoder.layer[8],
                  learner.model.transformer.bert.encoder.layer[9],
                  learner.model.transformer.bert.encoder.layer[10],
                  learner.model.transformer.bert.encoder.layer[11],
                  learner.model.transformer.bert.pooler,
                  learner.model.transformer.classifier]

elif model_type == 'distilbert':
    list_layers = [learner.model.transformer.distilbert.embeddings,
                   learner.model.transformer.distilbert.transformer.layer[0],
                   learner.model.transformer.distilbert.transformer.layer[1],
                   learner.model.transformer.distilbert.transformer.layer[2],
                   learner.model.transformer.distilbert.transformer.layer[3],
                   learner.model.transformer.distilbert.transformer.layer[4],
                   learner.model.transformer.distilbert.transformer.layer[5],
                   learner.model.transformer.classifier]

learner.split(list_layers)
num_groups = len(learner.layer_groups)
print('Learner split in',num_groups,'groups')
# print(learner.layer_groups)
learner.layer_groups<jupyter_output><empty_output><jupyter_text>Training  -1 layers<jupyter_code>learner.freeze_to(-1)
# learner.summary()
lr = 1e-3
learner.fit_one_cycle(1, max_lr=lr, moms=(0.8, 0.9))
learner.fit_one_cycle(5, max_lr=lr, moms=(0.8, 0.9))<jupyter_output><empty_output><jupyter_text>-4 layers<jupyter_code>learner.freeze_to(-4)
lr = 1e-4
learner.fit_one_cycle(10, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))
lr = 1e-4
for i in range(5,14,4):
    print('freeze to:', -i)
    learner.freeze_to(-i)
    learner.fit_one_cycle(5, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))<jupyter_output>freeze to: -5<jupyter_text>all layers<jupyter_code>learner.unfreeze()
lr = 1e-4
learner.fit_one_cycle(5, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>import logging
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

def save_lists(y_pred, y_true):
    global y_true_all, y_pred_all
    y_pred = torch.argmax(y_pred, dim=-1).view(-1)
    y_true = y_true.view(-1)
    mask = (y_true != pad_nr) * (y_true != bos_nr) * (y_true != eos_nr)
    y_true = list(map(lambda x: labels[x], y_true[mask]))
    y_pred = list(map(lambda x: labels[x], y_pred[mask]))
    y_true_all.extend(y_true)
    y_pred_all.extend(y_pred)
    return torch.tensor(1)
global y_true_all, y_pred_all
y_true_all, y_pred_all = [], []

_ = validate_learner(learner, test_dl, metrics=[save_lists])
f1_score(y_true_all, y_pred_all)*100, accuracy_score(y_true_all, y_pred_all)*100   
print(classification_report(y_true_all, y_pred_all))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1356829145366837, 'fraction_numerical': 0.012419900640794873, 'mean_word_length': 3.549459547985588, 'pattern_counts': {'":': 1, '<': 79, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '178652', 'license': 'Apache-2.0', 'n_tokens_mistral': 9829, 'n_tokens_neox': 9507, 'n_words': 1857, 'path': 'nbs/10_nlp_transformer.ipynb', 'repo_name': 'doas3140/ocr-pipeline', 'size': '27778'}