<jupyter_start><jupyter_text>It's then necessary to check if Acromine produced the correct results. We must fix errors manually<jupyter_code>top = miners['COT'].top()
top
longforms0 = miners['COT'].get_longforms()
list(enumerate(longforms0))
longforms0 = [(longform, score) for i, (longform, score) in enumerate(longforms0)
              if i not in [0, 5, 8, 10]]
list(enumerate(top))
longforms0.extend((longform, score) for i, (longform, score) in enumerate(top)
                 if i in [4, 9, 17, 18])
longforms = longforms0
longforms.sort(key=lambda x: -x[1])
longforms, scores = zip(*longforms0)
longforms
grounding_map = {}
names = {}
for longform in longforms:
    grounding = gilda_ground(longform)
    if grounding[0]:
        grounding_map[longform] = f'{grounding[0]}:{grounding[1]}'
        names[grounding_map[longform]] = grounding[2]
grounding_map
names
grounding_map, names, pos_labels = ground_with_gui(longforms, scores, grounding_map=grounding_map, names=names)
result = (grounding_map, names, pos_labels)
result
grounding_map, names, pos_labels = ({'carnitine octanoyltransferase': 'HGNC:6860',
  'celastrus orbiculatus thunb': 'ungrounded',
  'chronic opioid therapy': 'ungrounded',
  'cleft of trimer': 'ungrounded',
  'compliance over time': 'ungrounded',
  'conventional oxygen therapy': 'ungrounded',
  'cotinine': 'MESH:D003367',
  'cotyledonary': 'ungrounded',
  'cyclooctatetraene': 'CHEBI:CHEBI:47034',
  'k cl cotransport': 'ungrounded',
  'map3k8': 'HGNC:6860'},
 {'HGNC:6860': 'MAP3K8',
  'MESH:D003367': 'Cotinine',
  'CHEBI:CHEBI:47034': '[8]annulene'},
 ['CHEBI:CHEBI:47034', 'HGNC:6860', 'MESH:D003367'])
grounding_dict = {'COT': grounding_map}
classifier = AdeftClassifier('COT', pos_labels=pos_labels)
param_grid = {'C': [100.0], 'max_features': [10000]}
labeler = AdeftLabeler(grounding_dict)
corpus = labeler.build_from_texts(shortform_texts)
corpus.extend(entrez_texts)
texts, labels = zip(*corpus)
classifier.cv(texts, labels, param_grid, cv=5, n_jobs=8)
classifier.stats
disamb = AdeftDisambiguator(classifier, grounding_dict, names)
d = disamb.disambiguate(shortform_texts)
a = [text for pred, text in zip(d, shortform_texts)if pred[0] == 'HGNC:']
a[2]
disamb.dump('COT', '../results')
from adeft.disambiguate import load_disambiguator, load_disambiguator_directly
disamb.classifier.training_set_digest
model_to_s3(disamb)
d.disambiguate(texts[0])
print(d.info())
a = load_disambiguator('AR')
a.disambiguate('Androgen')
logit = d.classifier.estimator.named_steps['logit']
logit.classes_
model_to_s3(disamb)
classifier.feature_importances()['FPLX:RAC']
d = load_disambiguator('ALK', '../results')
d.info()
print(d.info())
model_to_s3(d)
d = load_disambiguator('TAK', '../results')
print(d.info())
model_to_s3(d)
from adeft import available_shortforms
print(d.info())
d.classifier.feature_importances()
from adeft import __version__
__version__
from adeft.disambiguate import load_disambiguator_directly
d = load_disambiguator_directly('../results/TEK/')
print(d.info())
model_to_s3(d)
d.grounding_dict
!python -m adeft.download --update
from adeft import available_shortforms
len(available_shortforms)
available_shortforms
'TEC' in available_shortforms
'TECs' in available_shortforms
!python -m adeft.download --update
!python -m adeft.download --update<jupyter_output>100% [......................................................] 1181008 / 1181008
{'fraction_non_alphanumeric': 0.1559551886792453, 'fraction_numerical': 0.03331367924528302, 'mean_word_length': 6.024844720496894, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '486312', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1238, 'n_words': 256, 'path': 'notebooks/COT_model.ipynb', 'repo_name': 'steppi/adeft_indra', 'size': '3392'}