<jupyter_start><jupyter_text>UPOS<jupyter_code>from corpuscula.corpus_utils import download_ud, UniversalDependencies, \
                                    AdjustedForSpeech
import junky
from mordl import UposTagger, FeatsTagger

BERT_MODEL_FN = 'xlm-roberta-base'
MODEL_FN = 'upos-bert_model'
SEED = 42
BERT_MAX_LEN, BERT_EPOCHS, BERT_BATCH_SIZE = 0, 3, 8
DEVICE = 'cuda:0'

corpus_name = 'UD_Russian-SynTagRus'
download_ud(corpus_name, overwrite=False)
corpus = UniversalDependencies(corpus_name)
#corpus = AdjustedForSpeech(corpus)
tagger = UposTagger()
tagger.load_train_corpus(corpus.train)
tagger.load_test_corpus(corpus.dev)

_ = tagger.train(MODEL_FN, device=DEVICE, control_metric='accuracy',
                 max_epochs=None, min_epochs=0, bad_epochs=5,
                 max_grad_norm=None, tags_to_remove=None, word_emb_type='bert',
                 word_emb_path=BERT_MODEL_FN, word_transform_kwargs={
                     'max_len': BERT_MAX_LEN, 'hidden_ids': 10, 'aggregate_subtokens_op': 'absmax'
                     # BertDataset.transform() (for BERT-descendant models)
                     # params:
                     # {'max_len': 0, 'batch_size': 64, 'hidden_ids': '10',
                     #  'aggregate_hiddens_op': 'cat',
                     #  'aggregate_subtokens_op': 'absmax', 'to': junky.CPU,
                     #  'loglevel': 1}
                     # WordDataset.transform() (for other models) params:
                     # {'check_lower': True}
                 },
                 stage1_params=None,
                     # {'lr': .0001, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': 0, 'amsgrad': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage2_params=None,
                     # {'lr': .001, 'momentum': .9, 'weight_decay': 0,
                     #  'dampening': 0, 'nesterov': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage3_params={
                     'save_as': MODEL_FN.replace('-bert_model', '_' + BERT_MODEL_FN)
                              + f'_len{BERT_MAX_LEN}_ep{BERT_EPOCHS}_bat{BERT_BATCH_SIZE}_seed{SEED}',
                     'epochs': BERT_EPOCHS,
                     'batch_size': BERT_BATCH_SIZE,
                     'lr': 2e-5, 'num_warmup_steps': 3,
                     # {'save_as': None, 'max_epochs': 3, 'batch_size': 8,
                     #  'lr': 2e-5, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': .01, 'amsgrad': False,
                     #  'num_warmup_steps': 3, 'max_grad_norm': 1.}
                 },
                 stages=[1, 2, 3, 1, 2], save_stages=True, load_from=None,
                 learn_on_padding=True, remove_padding_intent=False,
                 seed=SEED, start_time=None, keep_embs=False,
                 rnn_emb_dim=None, cnn_emb_dim=None, cnn_kernels=range(1, 7),
                 emb_bn=True, emb_do=.2,
                 final_emb_dim=512, pre_bn=True, pre_do=.5,
                 lstm_layers=1, lstm_do=0, tran_layers=0, tran_heads=8,
                 post_bn=True, post_do=.4)
res_dev = 'corpora/_dev_' + MODEL_FN.replace('_model', '.conllu')
res_test = 'corpora/_test_' + MODEL_FN.replace('_model', '.conllu')
tagger = UposTagger(embs=globals()['tagger'].embs
                             if 'tagger' in globals() else
                         None)
tagger.load(MODEL_FN)
junky.clear_tqdm()
_ = tagger.predict(corpus.dev, clone_ds=True, save_to=res_dev)
_ = tagger.evaluate(corpus.dev)
_ = tagger.evaluate(corpus.dev, res_dev)
_ = tagger.predict(corpus.test, save_to=res_test)
_ = tagger.evaluate(corpus.test, clone_ds=True)
_ = tagger.evaluate(corpus.test, res_test)
corp_gold = list(corpus.test())
corp_test = list(tagger._get_corpus(res_test))
tags = sorted(set(x['UPOS'] for x in corp_gold
                            for x in x[0] if x['UPOS']))
for tag in tags:
    print('{}: {}'.format(
        tag, tagger.evaluate(corp_gold, corp_test,
                             label=tag, log_file=None)
    ))<jupyter_output>Load corpus
[=======] 6491                                                     
Corpus has been loaded: 6491 sentences, 117523 tokens<jupyter_text>FEATS<jupyter_code>from corpuscula.corpus_utils import download_ud, UniversalDependencies, \
                                    AdjustedForSpeech
import junky
from mordl import FeatsTagger

BERT_MODEL_FN = 'xlm-roberta-base'
MODEL_FN = 'feats-bert_model'
SEED=42
BERT_MAX_LEN, BERT_EPOCHS, BERT_BATCH_SIZE = 0, 3, 8
DEVICE='cuda:0'

corpus_name = 'UD_Russian-SynTagRus'
download_ud(corpus_name, overwrite=False)
corpus = UniversalDependencies(corpus_name)
#corpus = AdjustedForSpeech(corpus)
tagger = FeatsTagger()
tagger.load_train_corpus(corpus.train)
tagger.load_test_corpus(corpus.dev)

_ = tagger.train(MODEL_FN, device=DEVICE, control_metric='accuracy',
                 max_epochs=None, min_epochs=0, bad_epochs=5,
                 max_grad_norm=None, tags_to_remove=None, word_emb_type='bert',
                 word_emb_path=BERT_MODEL_FN, word_transform_kwargs={
                     'max_len': BERT_MAX_LEN, 'hidden_ids': 10, 'aggregate_subtokens_op': 'absmax'
                     # BertDataset.transform() (for BERT-descendant models)
                     # params:
                     # {'max_len': 0, 'batch_size': 64, 'hidden_ids': '10',
                     #  'aggregate_hiddens_op': 'cat',
                     #  'aggregate_subtokens_op': 'absmax', 'to': junky.CPU,
                     #  'loglevel': 1}
                     # WordDataset.transform() (for other models) params:
                     # {'check_lower': True}
                 },
                 stage1_params=None,
                     # {'lr': .0001, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': 0, 'amsgrad': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage2_params=None,
                     # {'lr': .001, 'momentum': .9, 'weight_decay': 0,
                     #  'dampening': 0, 'nesterov': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage3_params={
                     'save_as': MODEL_FN.replace('-bert_model', '_' + BERT_MODEL_FN)
                              + f'_len{BERT_MAX_LEN}_ep{BERT_EPOCHS}_bat{BERT_BATCH_SIZE}_seed{SEED}',
                     'epochs': BERT_EPOCHS,
                     'batch_size': BERT_BATCH_SIZE,
                     'lr': 2e-5, 'num_warmup_steps': 3,
                     # {'save_as': None, 'max_epochs': 3, 'batch_size': 8,
                     #  'lr': 2e-5, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': .01, 'amsgrad': False,
                     #  'num_warmup_steps': 3, 'max_grad_norm': 1.}
                 },
                 stages=[1, 2, 3, 1, 2], save_stages=True, load_from=None,
                 learn_on_padding=True, remove_padding_intent=False,
                 seed=SEED, start_time=None, keep_embs=False,
                 rnn_emb_dim=None, cnn_emb_dim=200, cnn_kernels=range(1, 7),
                 upos_emb_dim=200, emb_bn=True, emb_do=.2,
                 final_emb_dim=512, pre_bn=True, pre_do=.5,
                 lstm_layers=1, lstm_do=0, tran_layers=0, tran_heads=8,
                 post_bn=True, post_do=.4)
res_dev = 'corpora/_dev_' + MODEL_FN.replace('_model', '.conllu')
res_test = 'corpora/_test_' + MODEL_FN.replace('_model', '.conllu')
tagger = FeatsTagger(embs=globals()['tagger'].embs
                              if 'tagger' in globals() else
                          None)
tagger.load(MODEL_FN)
junky.clear_tqdm()
_ = tagger.predict(corpus.dev, clone_ds=True, save_to=res_dev)
_ = tagger.evaluate(corpus.dev)
_ = tagger.evaluate(corpus.dev, res_dev)
_ = tagger.predict(corpus.test, save_to=res_test)
_ = tagger.evaluate(corpus.test, clone_ds=True)
_ = tagger.evaluate(corpus.test, res_test)
corp_gold = list(corpus.test())
corp_test = list(tagger._get_corpus(res_test))
tags = sorted(set(x for x in corp_gold
                    for x in x[0]
                    for x in x['FEATS'].keys()))
for tag in tags:
    print('{}: {}'.format(
        tag, tagger.evaluate(corp_gold, corp_test,
                             feats=tag, log_file=None)
    ))<jupyter_output>Load corpus
[=====> 5600<jupyter_text>LEMMAFor lemmata, besides of *BERT* word embeddings one can use *FastText*. In this case, model performance on the *SynTagRus* test datasetis just slightly worse (0.9945 vs. 0.9948, and, we think, it may be tuned if need). So, we give here training snippets for both version of tagger, *BERT* (next snippet) and *FastText* (see further).  *BERT* Lemmata Tagger<jupyter_code>from corpuscula.corpus_utils import download_ud, UniversalDependencies, \
                                    AdjustedForSpeech
import junky
from mordl import LemmaTagger

BERT_MODEL_FN = 'xlm-roberta-base'
MODEL_FN = 'lemma-bert_model'
SEED=42
BERT_MAX_LEN, BERT_EPOCHS, BERT_BATCH_SIZE = 0, 2, 8
DEVICE='cuda:0'

corpus_name = 'UD_Russian-SynTagRus'
download_ud(corpus_name, overwrite=False)
corpus = UniversalDependencies(corpus_name)
#corpus = AdjustedForSpeech(corpus)
tagger = LemmaTagger()
tagger.load_train_corpus(corpus.train)
tagger.load_test_corpus(corpus.dev)

_ = tagger.train(MODEL_FN, device=DEVICE, control_metric='accuracy',
                 max_epochs=None, min_epochs=0, bad_epochs=5,
                 max_grad_norm=None, tags_to_remove=None, word_emb_type='bert',
                 word_emb_path=BERT_MODEL_FN, word_transform_kwargs=None,
                     # BertDataset.transform() (for BERT-descendant models)
                     # params:
                     # {'max_len': 0, 'batch_size': 64, 'hidden_ids': '10',
                     #  'aggregate_hiddens_op': 'cat',
                     #  'aggregate_subtokens_op': 'absmax', 'to': junky.CPU,
                     #  'loglevel': 1}
                     # WordDataset.transform() (for other models) params:
                     # {'check_lower': True}
                 stage1_params=None,
                     # {'lr': .0001, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': 0, 'amsgrad': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage2_params=None,
                     # {'lr': .001, 'momentum': .9, 'weight_decay': 0,
                     #  'dampening': 0, 'nesterov': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage3_params={
                     'save_as': MODEL_FN.replace('-bert_model', '_' + BERT_MODEL_FN)
                              + f'_len{BERT_MAX_LEN}_ep{BERT_EPOCHS}_bat{BERT_BATCH_SIZE}_seed{SEED}',
                     'epochs': BERT_EPOCHS,
                     'batch_size': BERT_BATCH_SIZE,
                     'lr': 2e-5, 'num_warmup_steps': 6
                 },
                     # {'save_as': None, 'epochs': 3, 'batch_size': 8,
                     #  'lr': 2e-5, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': .01, 'amsgrad': False,
                     #  'num_warmup_steps': 3, 'max_grad_norm': 1.}
                 stages=[1, 2, 3, 1, 2], save_stages=True, load_from=None,
                 learn_on_padding=False, remove_padding_intent=False,
                 seed=SEED, start_time=None, keep_embs=False,
                 rnn_emb_dim=384, cnn_emb_dim=None, cnn_kernels=range(1, 7),
                 upos_emb_dim=256, emb_bn=True, emb_do=.2,
                 final_emb_dim=512, pre_bn=True, pre_do=.5,
                 lstm_layers=1, lstm_do=0, tran_layers=0, tran_heads=8,
                 post_bn=True, post_do=.4)
res_dev = 'corpora/_dev_' + MODEL_FN.replace('_model', '.conllu')
res_test = 'corpora/_test_' + MODEL_FN.replace('_model', '.conllu')
tagger = LemmaTagger(embs=globals()['tagger'].embs
                              if 'tagger' in globals() else
                          None)
tagger.load(MODEL_FN)
junky.clear_tqdm()
_ = tagger.predict(corpus.dev, clone_ds=True, save_to=res_dev)
_ = tagger.evaluate(corpus.dev)
_ = tagger.evaluate(corpus.dev, res_dev)
_ = tagger.predict(corpus.test, save_to=res_test)
_ = tagger.evaluate(corpus.test, clone_ds=True)
_ = tagger.evaluate(corpus.test, res_test)<jupyter_output>Evaluating LEMMA
Load corpus
[=====> 5100<jupyter_text>*FastText* Lemmata Tagger **NB:** For this task, we use Russian *FastText* embeddings provided by *Facebook*: [cc.ru.300.bin.gz](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ru.300.bin.gz). We highly recommend them because it delivers the highest evaluation scores. Also, embeddings provided by *DeepPavlov* ([ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.bin](http://files.deeppavlov.ai/embeddings/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.bin)) could be used, too. They deliver just slightly worse model performance.Maybe, one can try embeddings from *RusVectores*. Early (long ago) it was the worst choice because of inappropriate preprocessing. But now it seems, it's corrected. We didn't try, but if you surely want *FastText* model, embeddings from *RusVectores* are also worth to check.If you want your model to achieve high scores, use embeddings without any lemmatization, removal of punctuation, and adding any other archaic transformations. Embeddings of words with part of speech tags appended in the end are also useless (by obvious reasons).<jupyter_code>from corpuscula.corpus_utils import download_ud, UniversalDependencies, \
                                    AdjustedForSpeech
import junky
from mordl import LemmaTagger

FT_MODEL_FN = '../mordl/cc.ru.300.bin'
MODEL_FN = 'lemma-ft_model'
SEED=42
DEVICE='cuda:0'

corpus_name = 'UD_Russian-SynTagRus'
download_ud(corpus_name, overwrite=False)
corpus = UniversalDependencies(corpus_name)
#corpus = AdjustedForSpeech(corpus)
tagger = LemmaTagger()
tagger.load_train_corpus(corpus.train)
tagger.load_test_corpus(corpus.dev)

_ = tagger.train(MODEL_FN, device=DEVICE, control_metric='accuracy',
                 max_epochs=None, min_epochs=0, bad_epochs=5,
                 max_grad_norm=None, tags_to_remove=None, word_emb_type='ft',
                 word_emb_path=FT_MODEL_FN, word_transform_kwargs=None,
                     # BertDataset.transform() (for BERT-descendant models)
                     # params:
                     # {'max_len': 0, 'batch_size': 64, 'hidden_ids': '10',
                     #  'aggregate_hiddens_op': 'cat',
                     #  'aggregate_subtokens_op': 'absmax', 'to': junky.CPU,
                     #  'loglevel': 1}
                     # WordDataset.transform() (for other models) params:
                     # {'check_lower': True}
                 stage1_params=None,
                     # {'lr': .0001, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': 0, 'amsgrad': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage2_params=None,
                     # {'lr': .001, 'momentum': .9, 'weight_decay': 0,
                     #  'dampening': 0, 'nesterov': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage3_params=None,
                     # {'save_as': None, 'epochs': 3, 'batch_size': 8,
                     #  'lr': 2e-5, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': .01, 'amsgrad': False,
                     #  'num_warmup_steps': 3, 'max_grad_norm': 1.}
                 stages=[1, 2], save_stages=True, load_from=None,
                 learn_on_padding=False, remove_padding_intent=False,
                 seed=SEED, start_time=None, keep_embs=False,
                 rnn_emb_dim=300, cnn_emb_dim=None, cnn_kernels=range(1, 7),
                 upos_emb_dim=200, emb_bn=True, emb_do=.2,
                 final_emb_dim=512, pre_bn=True, pre_do=.5,
                 lstm_layers=1, lstm_do=0, tran_layers=0, tran_heads=8,
                 post_bn=True, post_do=.4)
res_dev = 'corpora/_dev_' + MODEL_FN.replace('_model', '.conllu')
res_test = 'corpora/_test_' + MODEL_FN.replace('_model', '.conllu')
tagger = LemmaTagger(embs=globals()['tagger'].embs
                              if 'tagger' in globals() else
                          None)
tagger.load(MODEL_FN)
junky.clear_tqdm()
_ = tagger.predict(corpus.dev, clone_ds=True, save_to=res_dev)
_ = tagger.evaluate(corpus.dev)
_ = tagger.evaluate(corpus.dev, res_dev)
_ = tagger.predict(corpus.test, save_to=res_test)
_ = tagger.evaluate(corpus.test, clone_ds=True)
_ = tagger.evaluate(corpus.test, res_test)<jupyter_output>Evaluating LEMMA
Load corpus
[=> 1500<jupyter_text>CoNLL18 Validation<jupyter_code>from corpuscula.corpus_utils import download_ud, get_ud_test_path
import junky
from mordl import UposTagger, FeatsTagger, LemmaTagger, conll18_ud_eval

corpus_name = 'UD_Russian-SynTagRus'
download_ud(corpus_name, overwrite=False)
DEVICE = 'cuda:0'

corpus_gold = get_ud_test_path(corpus_name)
corpus_test = 'corpora/_test_tagged.conllu'
del tagger
tagger_u = UposTagger()
tagger_u.load('upos-bert_model', device=DEVICE, dataset_device=DEVICE)
tagger_f = FeatsTagger()
tagger_f.load('feats-bert_model', device=DEVICE, dataset_device=DEVICE)
tagger_l = LemmaTagger()
tagger_l.load('lemma-bert_model', device=DEVICE, dataset_device=DEVICE)
_ = tagger_l.predict(
    tagger_f.predict(
        tagger_u.predict(corpus_gold)
    ), save_to=corpus_test
)

del tagger_u, tagger_f, tagger_l
conll18_ud_eval(corpus_gold, corpus_test)<jupyter_output>Metric     | Precision |    Recall |  F1 Score | AligndAcc
-----------+-----------+-----------+-----------+-----------
Tokens     |    100.00 |    100.00 |    100.00 |
Sentences  |    100.00 |    100.00 |    100.00 |
Words      |    100.00 |    100.00 |    100.00 |
UPOS       |     99.35 |     99.35 |     99.35 |     99.35
XPOS       |    100.00 |    100.00 |    100.00 |    100.00
UFeats     |     98.36 |     98.36 |     98.36 |     98.36
AllTags    |     98.21 |     98.21 |     98.21 |     98.21
Lemmas     |     98.88 |     98.88 |     98.88 |     98.88
UAS        |    100.00 |    100.00 |    100.00 |    100.00
LAS        |    100.00 |    100.00 |    100.00 |    100.00
CLAS       |    100.00 |    100.00 |    100.00 |    100.00
MLAS       |     97.22 |     97.22 |     97.22 |     97.22
BLEX       |     98.29 |     98.29 |     98.29 |     98.29<jupyter_text>MISC:NENote: the corpora we used are proprietary (and of low quality). You have to find another corpora.<jupyter_code>from mordl import UposTagger, FeatsTagger

DEVICE = 'cuda:0'

tagger_u = UposTagger()
tagger_u.load('upos-bert_model', device=DEVICE, dataset_device=DEVICE)
tagger_f = FeatsTagger()
tagger_f.load('feats-bert_model', device=DEVICE, dataset_device=DEVICE)

PREFIX = 'ner-old-'
for corpora in zip([f'corpora/{PREFIX}train.conllu',
                    f'corpora/{PREFIX}dev.conllu',
                    f'corpora/{PREFIX}test.conllu'],
                   [f'corpora/{PREFIX}train_upos_feats.conllu',
                    f'corpora/{PREFIX}dev_upos_feats.conllu',
                    f'corpora/{PREFIX}test_upos_feats.conllu']):
    tagger_f.predict(
        tagger_u.predict(corpora[0]), save_to=corpora[1]
    )

del tagger_u, tagger_f
import junky
from mordl import NeTagger

BERT_MODEL_FN = 'xlm-roberta-base'
MODEL_FN = 'misc-ne-bert_model'
SEED=42
BERT_MAX_LEN, BERT_EPOCHS, BERT_BATCH_SIZE = 0, 2, 8
DEVICE='cuda:0'

PREFIX = 'ner-old-'
corpus_train = f'corpora/{PREFIX}train_upos_feats.conllu'
corpus_dev = f'corpora/{PREFIX}dev_upos_feats.conllu'
corpus_test = f'corpora/{PREFIX}test_upos_feats.conllu'
tagger = NeTagger()
tagger.load_train_corpus(corpus_train)
tagger.load_test_corpus(corpus_dev)

_ = tagger.train(MODEL_FN, device=DEVICE, control_metric='accuracy',
                 max_epochs=None, min_epochs=0, bad_epochs=5,
                 max_grad_norm=None, tags_to_remove=None, word_emb_type='bert',
                 word_emb_path=BERT_MODEL_FN, word_transform_kwargs={
                     'max_len': BERT_MAX_LEN, 'hidden_ids': 10, 'aggregate_subtokens_op': 'absmax'
                     # BertDataset.transform() (for BERT-descendant models)
                     # params:
                     # {'max_len': 0, 'batch_size': 64, 'hidden_ids': '10',
                     #  'aggregate_hiddens_op': 'cat',
                     #  'aggregate_subtokens_op': 'absmax', 'to': junky.CPU,
                     #  'loglevel': 1}
                     # WordDataset.transform() (for other models) params:
                     # {'check_lower': True}
                 },
                 stage1_params=None,
                     # {'lr': .0001, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': 0, 'amsgrad': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage2_params=None,
                     # {'lr': .001, 'momentum': .9, 'weight_decay': 0,
                     #  'dampening': 0, 'nesterov': False,
                     #  'max_epochs': None, 'min_epochs': None,
                     #  'bad_epochs': None, 'batch_size': None,
                     #  'max_grad_norm': None}
                 stage3_params={
                     'save_as': MODEL_FN.replace('-bert_model', '_' + BERT_MODEL_FN)
                              + f'_len{BERT_MAX_LEN}_ep{BERT_EPOCHS}_bat{BERT_BATCH_SIZE}_seed{SEED}',
                     'epochs': BERT_EPOCHS,
                     'batch_size': BERT_BATCH_SIZE,
                     'lr': 4e-5, 'num_warmup_steps': 1,
                     # {'save_as': None, 'max_epochs': 3, 'batch_size': 8,
                     #  'lr': 2e-5, 'betas': (0.9, 0.999), 'eps': 1e-8,
                     #  'weight_decay': .01, 'amsgrad': False,
                     #  'num_warmup_steps': 0, 'max_grad_norm': 1.}
                 },
                 stages=[1, 2, 3, 1, 2], save_stages=True, load_from=None,
                 learn_on_padding=True, remove_padding_intent=False,
                 seed=SEED, start_time=None, keep_embs=False,
                 rnn_emb_dim=None, cnn_emb_dim=None, cnn_kernels=range(1, 7),
                 upos_emb_dim=300, emb_bn=True, emb_do=.2,
                 final_emb_dim=512, pre_bn=True, pre_do=.5,
                 lstm_layers=1, lstm_do=0, tran_layers=0, tran_heads=8,
                 post_bn=True, post_do=.4)
res_dev = 'corpora/_dev_' + MODEL_FN.replace('_model', '.conllu')
res_test = 'corpora/_test_' + MODEL_FN.replace('_model', '.conllu')
tagger = NeTagger(embs=globals()['tagger'].embs
                           if 'tagger' in globals() else
                       None)
tagger.load(MODEL_FN)
junky.clear_tqdm()
_ = tagger.predict(corpus_dev, clone_ds=True, save_to=res_dev)
_ = tagger.evaluate(corpus_dev)
_ = tagger.evaluate(corpus_dev, res_dev)
_ = tagger.predict(corpus_test, save_to=res_test)
_ = tagger.evaluate(corpus_test, clone_ds=True)
_ = tagger.evaluate(corpus_test, res_test)
corp_gold = list(tagger._get_corpus(corpus_test, asis=True))
corp_test = list(tagger._get_corpus(res_test))
tags = sorted(set(x['MISC'].get('NE')
                      for x in corp_gold for x in x[0]
                          if x['MISC'].get('NE')))
for tag in tags:
    print('{}: {}'.format(
        tag, tagger.evaluate(corp_gold, corp_test,
                             label=tag, log_file=None)
    ))
import gc

del tagger
gc.collect()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12817688573410027, 'fraction_numerical': 0.027210051809244074, 'mean_word_length': 2.211160597327744, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '490976', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 9100, 'n_tokens_neox': 8428, 'n_words': 1489, 'path': 'examples/mordl.ipynb', 'repo_name': 'fostroll/mordl', 'size': '24513'}