<jupyter_start><jupyter_text>datasets<jupyter_code># path = join_path(DATA_DIR, DATASET)
# !mkdir -p {path}
# for name in [NE5, BSNLP, FACTRU, GAREEV]:
#     path = join_path(CORUS_DATA_DIR, CORUS_FILES[name])
#     records = LOADS[name](path)
#     records = (_.adapted for _ in records)
#     records = log_progress(records, desc=name)

#     path = join_path(DATA_DIR, DATASET, name + JL + GZ)
#     items = as_jsons(records)
#     lines = format_jl(items)
#     dump_gz_lines(lines, path)
%run -n main.py
datasets = {}
for name in [NE5, BSNLP, FACTRU, GAREEV]:
    path = join_path(DATA_DIR, DATASET, name + JL + GZ)
    lines = load_gz_lines(path)
    items = parse_jl(lines)
    datasets[name] = list(from_jsons(items, Markup))<jupyter_output><empty_output><jupyter_text>models<jupyter_code># for name in MODELS:
#     path = join_path(DATA_DIR, name)
#     !mkdir -p {path}<jupyter_output><empty_output><jupyter_text>cpu<jupyter_code># docker = docker_client()
# model_name = SPACY  # MITIE, TOMITA, TEXTERRA, SLOVNET
# model = MODELS[model_name]()
# model.start(docker)
# model.wait()
# for dataset_name in [NE5, BSNLP, FACTRU, GAREEV]:
#     records = model.map(_.text for _ in datasets[dataset_name])
#     records = (_.adapted for _ in records)
#     records = log_progress(records, desc=dataset_name)

#     path = join_path(DATA_DIR, model_name, dataset_name + JL + GZ)
#     items = as_jsons(records)
#     lines = format_jl(items)
#     dump_gz_lines(lines, path)
# model.stop(docker)<jupyter_output><empty_output><jupyter_text>gpu<jupyter_code># # select cuda 10.0, high bandwith
# !vast search offers | grep '1 x  GTX 1080 Ti'
# %run -n main
# model = DeeppavlovModel()
# model = DeeppavlovBERTModel()
# model = SlovnetBERTModel()
# model = StanzaModel()
# model = DeeppavlovSlavicModel()
# !vast create instance 577392 --image {model.image} --disk 20
# !vast show instances
# !ssh ssh5.vast.ai -p 19600 -l root -Nf -L {model.port}:localhost:{model.container_port}
# for dataset_name in [NE5, BSNLP, FACTRU, GAREEV]:
#     records = datasets[dataset_name]
#     records = log_progress(records, desc=dataset_name)
#     records = model.map(_.text for _ in records)
#     records = (_.adapted for _ in records)

#     path = join_path(DATA_DIR, model.name, dataset_name + JL + GZ)
#     items = as_jsons(records)
#     lines = format_jl(items)
#     dump_gz_lines(lines, path)
# !vast destroy instance 585391<jupyter_output><empty_output><jupyter_text>score<jupyter_code>dataset_models = {}
for dataset in DATASETS:
    for model in MODELS:
        path = join_path(DATA_DIR, model, dataset + JL + GZ)
        lines = load_gz_lines(path)
        items = parse_jl(lines)
        dataset_models[dataset, model] = list(from_jsons(items, Markup))
scores = {}
for dataset, model in log_progress(dataset_models):
    preds = dataset_models[dataset, model]
    targets = datasets[dataset]
    scores[dataset, model] = score_markups(preds, targets)<jupyter_output><empty_output><jupyter_text>report<jupyter_code>scores_table = scores_report_table(scores, DATASETS, MODELS)
html = format_scores_report(scores_table)
HTML(html)
html = format_github_scores_report(scores_table)
patch_readme(NER1, html, README)
patch_readme(NER1, html, SLOVNET_README)
HTML(html)
BENCH = [
    # GTX 1080 Ti
    Bench(
        DEEPPAVLOV,
        init=5.9,
        disk=1 * GB,  # 1GB emb + 5MB model
        ram=3 * GB,
        speed=24.31,  # 1.95 / 7 cores on CPU,
        device=GPU
    ),
    Bench(
        DEEPPAVLOV_BERT,
        init=34.5,
        disk=2 * GB,
        ram=6 * GB,
        speed=13.13,  # 17.71 / 3 cores on CPU
        device=GPU
    ),
    Bench(
        DEEPPAVLOV_SLAVIC,
        init=35,
        disk=2 * GB,
        ram=4 * GB,
        speed=8,
        device=GPU
    ),
    Bench(
        SLOVNET_BERT,
        init=5,
        disk=473 * MB,
        ram=9500 * MB,
        speed=40,
        device=GPU
    ),
    Bench(
        SLOVNET,
        init=1,
        disk=27 * MB,
        ram=205 * MB,
        speed=25.3,
    ),
    
    # 16 CPUs
    Bench(
        PULLENTI,
        init=2.85,
        disk=16 * MB,
        ram=253 * MB,
        speed=6.05
    ),
    Bench(
        TEXTERRA,
        init=47.6,
        disk=193 * MB,
        ram=3.3 * GB,  # leaks
        speed=20.16 / 5  # utils ~5 cores
    ),
    Bench(
        TOMITA,
        init=2.03,
        disk=64 * MB,
        ram=63 * MB,
        speed=29.8,
    ),
    Bench(
        MITIE,
        init=28.3,
        disk=327 * MB,
        ram=261 * MB,
        speed=32.8,
    ), 
    Bench(
        SPACY,
        init=8,
        disk=140 * MB,
        ram=625 * MB,
        speed=8,
    ),
    Bench(
        STANZA,
        init=3,
        disk=591 * MB,
        ram=11 * GB,
        speed=3,
        device=GPU
    ),
]

bench_table = bench_report_table(BENCH, MODELS)
html = format_bench_report(bench_table)
patch_readme(NER2, html, README)
patch_readme(NER2, html, SLOVNET_README)
HTML(html)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12312911594492118, 'fraction_numerical': 0.031131510676511674, 'mean_word_length': 2.7043606799704363, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '124669', 'license': 'MIT', 'n_tokens_mistral': 2020, 'n_tokens_neox': 1893, 'n_words': 385, 'path': 'scripts/03_ner/main.ipynb', 'repo_name': 'sdspieg/naeval', 'size': '5011'}