<jupyter_start><jupyter_text>It's then necessary to check if Acromine produced the correct results. We must fix errors manually<jupyter_code>overlap = set(refs.values()) & (set(KCNA4_entrez_refs.values()) | set(HK1_entrez_refs.values()) |
                                                                     set(HOOK1_entrez_refs.values()))
len(overlap)
shortform_texts = [universal_extract_text(text, contains=[shortform]) for ref, text in content.items()
                   if text and ref not in overlap and len(text) >= 5]
miner = AdeftMiner(shortform)
miner.process_texts(shortform_texts)
top = miner.top()
top
miner.get_longforms()
longforms = miner.get_longforms()
longforms.append(('histidine aspartate kinase', 1.0))
longforms
longforms, scores = zip(*longforms)
grounding_map = {}
for longform in longforms:
    grounding = gilda_ground(longform)
    if grounding[0]:
        grounding_map[longform] = f'{grounding[0]}:{grounding[1]}'
grounding_map
grounding_map, names, pos_labels = ground_with_gui(longforms, scores, grounding_map=grounding_map)
names['HGNC:6222 '] = 'KCNA4'
pos_labels.extend(['HGNC:6222'])
names['HGNC:19884'] = 'HOOK1'
pos_labels.extend(['HGNC:19884'])
result = (grounding_map, names, pos_labels)
result
grounding_map, names, pos_labels = ({'hexokinase': 'HGNC:4922',
                                      'hexokinase 1': 'HGNC:4922',
                                      'histidine aspartate kinase': 'ungrounded',
                                      'npc cells': 'ungrounded',
                                      'npc cells line': 'ungrounded'},
                                     {'HGNC:4922': 'HK1', 'HGNC:6222 ': 'KCNA4', 'HGNC:19884': 'HOOK1'},
                                     ['HGNC:4922', 'HGNC:6222', 'HGNC:19884'])
grounding_dict = {'HK1': grounding_map}
classifier = AdeftClassifier('HK1', pos_labels=pos_labels)
param_grid = {'C': [10.0, 100.0], 'max_features': [100, 1000, 10000]}
labeler = AdeftLabeler(grounding_dict)
corpus = labeler.build_from_texts(shortform_texts)
corpus.extend([(text, 'HGNC:4922') for text in HK1_entrez_texts])
corpus.extend([(text, 'HGNC:6222') for text in KCNA4_entrez_texts])
corpus.extend([(text, 'HGNC:19884') for text in HOOK1_entrez_texts])
texts, labels = zip(*corpus)
classifier.cv(texts, labels, param_grid, cv=5, n_jobs=8)
classifier.stats
disamb = AdeftDisambiguator(classifier, grounding_dict, names)
disamb.dump('HK1', '../results')
disambs = disamb.disambiguate(shortform_texts)
disamb = load_disambiguator('HK1', path='../results')
disamb
disamb.shortforms
model_to_s3(disamb)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1358925143953935, 'fraction_numerical': 0.0418426103646833, 'mean_word_length': 3.6287744227353462, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '300161', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 986, 'n_tokens_neox': 883, 'n_words': 172, 'path': 'notebooks/HK1_model.ipynb', 'repo_name': 'steppi/adeft_indra', 'size': '2605'}