<jupyter_start><jupyter_text>Test Model  GermEval (outer)<jupyter_code>testSentences = utils.get_sentences_germeval('../data/GermEVAL/NER-de-test.tsv')
print(len(testSentences))
load_model_indexes('../models/germeval.indexes')
finalmodel = load_model('../models/germeval.h5', custom_objects=create_custom_objects())
true_labels, pred_labels = utils.predict_sequences(finalmodel, testSentences)
print(compute_f1(pred_labels, true_labels, models.idx2Label))<jupyter_output>(0.8315377081292851, 0.8245386856587893, 0.8280234070221067)<jupyter_text>GermEval (inner)<jupyter_code>idx_1st = models.idx2Label
print(models.label2Idx)
print(idx_1st)
testSentences_2nd = utils.get_sentences_germeval('../data/GermEVAL/NER-de-test.tsv', level2=True)
load_model_indexes('../models/germeval-inner.indexes')
finalmodel_2nd = load_model('../models/germeval-inner.h5', custom_objects=create_custom_objects())
true_labels_2nd, pred_labels_2nd = utils.predict_sequences(finalmodel_2nd, testSentences_2nd)
print(compute_f1(pred_labels_2nd, true_labels_2nd, models.idx2Label))
idx_2nd = models.idx2Label
eval_file = 'germeval_outer_inner.tsv'
conlleval.write_germeval_file(true_labels, pred_labels, true_labels_2nd, pred_labels_2nd, testSentences, idx_1st, idx_2nd, eval_file)<jupyter_output><empty_output><jupyter_text>Official script eval CoNLL<jupyter_code>testSentences = utils.get_sentences_conll('../data/CONLL/deu/deu_utf.testb.bio')
print(len(testSentences))
print(testSentences[3])
load_model_indexes('../models/conll.indexes', max_sequence_length = 100)
finalmodel = load_model('../models/conll.h5', custom_objects=create_custom_objects())
true_labels, pred_labels = utils.predict_sequences(finalmodel, testSentences)
print(compute_f1(pred_labels, true_labels, models.idx2Label))
import conlleval

eval_file = 'test_pl.tsv'
conlleval.write_conll_file(true_labels, pred_labels, testSentences, models.idx2Label, eval_file) 
# p,r,f = conlleval.evaluate_conll_file(eval_file) 
print(p)
print(r)
print(f)<jupyter_output><empty_output><jupyter_text>Averages<jupyter_code>import numpy as np
file_base_name = 'model_lstm_germeval_v2.'
eval_file = 'test_pl.tsv'
all_f1_scores = []

for run_i in range(10):
    model_file = file_base_name + str(run_i) + '.h5'
    index_file = file_base_name + str(run_i) + '.h5.indexes'
    if not os.path.isfile(model_file):
        print(model_file + ' not found')
        continue
    if not os.path.isfile(index_file):
        print(model_file + ' not found')
        continue
    
    print('Evaluating model ' + model_file)
    
    load_model_indexes(index_file)
    finalmodel = load_model(model_file, custom_objects=create_custom_objects())
    
    true_labels, pred_labels = utils.predict_sequences(finalmodel, testSentences)
    
    conlleval.write_conll_file(true_labels, pred_labels, testSentences, models.idx2Label, eval_file) 
    p,r,f = conlleval.evaluate_conll_file(eval_file) 
    all_f1_scores.append(f)

print(all_f1_scores)
print(np.mean(all_f1_scores))
# import importlib
# importlib.reload(conlleval)
# importlib.reload(utils)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1172102034226671, 'fraction_numerical': 0.03035195350339038, 'mean_word_length': 7.2175066312997345, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '575922', 'license': 'MIT', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1166, 'n_words': 141, 'path': 'scripts/model_test_official-scripts.ipynb', 'repo_name': 'eudaimoniatech/microNER', 'size': '3097'}