<jupyter_start><jupyter_text>errors  token<jupyter_code># %run -n main.py
# lines = load_gz_lines(NAEVAL_TOKEN_TARGET)
# targets = list(parse_partitions(lines))

# lines = load_gz_lines(NAEVAL_TOKEN_PRED)
# preds = list(parse_partitions(lines))

# for target, pred in zip(targets, preds):
#     if target != pred:
#         show_partition(target)
#         show_partition(pred)
#         print()<jupyter_output><empty_output><jupyter_text>morph<jupyter_code># %run -n main.py
# lines = load_gz_lines(NAEVAL_MORPH_TARGET)
# items = parse_jl(lines)
# targets = list(from_jsons(items, MorphMarkup))

# lines = load_gz_lines(NAEVAL_MORPH_PRED)
# items = parse_jl(lines)
# preds = list(from_jsons(items, MorphMarkup))

# for target, pred in zip(targets, preds):
#     if target != pred:
#         show_morph_markup_diff(target, pred)
#         print()<jupyter_output><empty_output><jupyter_text>syntax<jupyter_code># %run -n main.py
# lines = load_gz_lines(NAEVAL_SYNTAX_TARGET)
# items = parse_jl(lines)
# targets = list(from_jsons(items, SyntaxMarkup))

# lines = load_gz_lines(NAEVAL_SYNTAX_PRED)
# items = parse_jl(lines)
# preds = list(from_jsons(items, SyntaxMarkup))

# for target, pred in zip(targets, preds):
#     if target != pred:
#         show_syntax_markup_diff(target, pred)
#         print()<jupyter_output><empty_output><jupyter_text>ner<jupyter_code># %run -n main.py
# lines = load_gz_lines(NAEVAL_NER_TARGET)
# items = parse_jl(lines)
# markups = from_jsons(items, NERMarkup)
# targets = list(iter_sents(markups))

# lines = load_gz_lines(NAEVAL_NER_PRED)
# items = parse_jl(lines)
# markups = from_jsons(items, NERMarkup)
# preds = list(iter_sents(markups))

# # slovnet bert uses first 100 as test
# for target, pred in zip(targets[:100], preds):
#     if target != pred:
#         show_ner_markup(target)
#         print('>')
#         show_ner_markup(pred)
#         print('\n\n')<jupyter_output><empty_output><jupyter_text>sample<jupyter_code>!mkdir -p sample
records = load_nerus(NERUS)
records = list(head(records, 10))
%run -n main.py
with capture(SAMPLE_NER):
    for record in records:
        record.ner.print()
        print('\n\n')
%run -n main.py
with capture(SAMPLE_MORPH):
    for record in records:
        for sent in record.sents:
            sent.morph.print()
            print()
%run -n main.py
with capture(SAMPLE_SYNTAX):
    for record in records:
        for sent in record.sents:
            sent.syntax.print()
            print()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12819488817891375, 'fraction_numerical': 0.003194888178913738, 'mean_word_length': 4.22964509394572, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '632530', 'license': 'MIT', 'n_tokens_mistral': 969, 'n_tokens_neox': 946, 'n_words': 172, 'path': 'examples/main.ipynb', 'repo_name': 'bzaar/nerus', 'size': '2504'}