<jupyter_start><jupyter_text>Create dataset bank, to evaluate external result you should only set test sequence.<jupyter_code>from src.classes.datasets_bank import DatasetsBank, DatasetsBankSorted
from src.data_io.data_io_connl_ner_2003 import DataIOConnlNer2003

data_io = DataIOConnlNer2003()
#word_sequences_train, tag_sequences_train = data_io.read_data(fn='/home/vika/targer/data/NER/Varvara_v3/train_pred_full.tsv', verbose=True)
#word_sequences_dev, tag_sequences_dev = data_io.read_data(fn='/home/vika/targer/data/NER/Varvara_v3/dev_pred_full.tsv', verbose=True)
word_sequences_test, tag_sequences_test = data_io.read_data(fn='/home/vika/targer/data/NER/Varvara_v3/test_manual_predfull_seq_labelling.tsv', verbose=True)

datasets_bank = DatasetsBank(verbose=True)

#datasets_bank.add_train_sequences(word_sequences_train, tag_sequences_train)
#datasets_bank.add_dev_sequences(word_sequences_dev, tag_sequences_dev)
datasets_bank.add_test_sequences(word_sequences_test, tag_sequences_test)
word_sequences=datasets_bank.word_sequences_test
targets_tag_sequences=datasets_bank.tag_sequences_test
with open('labels_v3_predful_manual.txt') as lines:
    outputs_tag_sequences = []
    for line in lines:
        outputs_tag_sequences.append(line.strip().split(', '))
## if extra tag_sequence have the differents labels than test

new_ts = []
for line in tag_sequences_test:
    for ind, elem in enumerate(line):
        if (elem == 'NONE'):
            line[ind] = 'O'
    new_ts.append(line)
tag_sequences_test=new_ts
tag_sequences_test[15]
outputs_tag_sequences[15]


evaluator = EvaluatorF1MacroTokenLevel()
evaluator.get_evaluation_score(tag_sequences_test, outputs_tag_sequences, word_sequences)<jupyter_output>['O', 'O', 'O', 'O', 'B-OBJ', 'O', 'B-PREDFULL', 'I-PREDFULL', 'O', 'B-OBJ', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
['O', 'O', 'B-OBJ', 'O', 'B-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'O', 'B-OBJ', 'O', 'O', 'O', 'O', 'O', 'B-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'O', 'B-OBJ', 'O']
['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
['O', 'O', 'B-OBJ', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PREDFULL', 'O', 'B-OBJ', 'I-OBJ', 'O']
['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-OBJ', 'O']
['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-OBJ', 'O', 'B-PREDFULL', 'I-PREDFULL', 'I-PREDFULL', 'O', 'B-OBJ', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
['O', 'O'[...]
{'fraction_non_alphanumeric': 0.25073421439060206, 'fraction_numerical': 0.007709251101321586, 'mean_word_length': 6.0595854922279795, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '626013', 'license': 'Apache-2.0', 'n_tokens_mistral': 1240, 'n_tokens_neox': 1181, 'n_words': 250, 'path': 'evaluate_output1.ipynb', 'repo_name': 'skoltech-nlp/coqas', 'size': '2724'}