<jupyter_start><jupyter_text>IO markup  Train<jupyter_code>from modules.data import bert_data
train_df_path = "/home/eartemov/ae/work/factRuEval-2016/dev.csv"
valid_df_path = "/home/eartemov/ae/work/factRuEval-2016/test.csv"
train_df_path = "/home/eartemov/ae/work/factRuEval-2016/dev.csv"
valid_df_path = "/home/eartemov/ae/work/factRuEval-2016/test.csv"
data = bert_data.LearnData.create(
    train_df_path=train_df_path,
    valid_df_path=valid_df_path,
    idx2labels_path="/home/eartemov/ae/work/factRuEval-2016/idx2labels4.txt",
    clear_cache=True
)
from modules.models.bert_models import BERTBiLSTMCRF
model = BERTBiLSTMCRF.create(len(data.train_ds.idx2label), lstm_dropout=0., crf_dropout=0.3)
from modules.train.train import NerLearner
num_epochs = 100
learner = NerLearner(
    model, data, "/home/eartemov/ae/work/models/fre-BERTBiLSTMCRF-IO.cpt", t_total=num_epochs * len(data.train_dl))
model.get_n_trainable_params()
learner.fit(epochs=num_epochs)<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>from modules.data.bert_data import get_data_loader_for_predict
dl = get_data_loader_for_predict(data, df_path=data.valid_ds.config["df_path"])
preds = learner.predict(dl)
from sklearn_crfsuite.metrics import flat_classification_report
from modules.analyze_utils.utils import bert_labels2tokens, voting_choicer
from modules.analyze_utils.plot_metrics import get_bert_span_report
pred_tokens, pred_labels = bert_labels2tokens(dl, preds)
true_tokens, true_labels = bert_labels2tokens(dl, [x.bert_labels for x in dl.dataset])
assert pred_tokens == true_tokens
tokens_report = flat_classification_report(true_labels, pred_labels, labels=["I_ORG", "I_PER", "I_LOC"], digits=4)
print(tokens_report)
from modules.analyze_utils.main_metrics import precision_recall_f1
results = precision_recall_f1(true_labels, pred_labels)<jupyter_output>processed 56409 tokens with 7534 phrases; found: 7148 phrases; correct: 6400.

precision:  89.54%; recall:  84.95%; FB1:  87.18

	LOC: precision:  90.53%; recall:  83.49%; F1:  86.87 1436

	ORG: precision:  85.79%; recall:  79.17%; F1:  82.35 3567

	PER: precision:  95.10%; recall:  96.59%; F1:  95.84 2145
{'fraction_non_alphanumeric': 0.11270207852193996, 'fraction_numerical': 0.05404157043879908, 'mean_word_length': 6.680851063829787, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '155613', 'license': 'MIT', 'n_tokens_mistral': 914, 'n_tokens_neox': 831, 'n_words': 134, 'path': 'exps/fre BERTBiLSTMCRF.ipynb', 'repo_name': 'learnerhouse/ner-bert', 'size': '2165'}