<jupyter_start><jupyter_text>Food Eventually we will need to deposit the extracted tuples somewhere. The benefit of using an existing DB is that we know exactly where to put the data if the DB already has the food. The downside is that the DB may not have all foods (same is true for entity linking for compounds) Options - FooDB- NCBI Taxonomy- UMLS?- Wikipedia? TF-IDF to FooDB<jupyter_code>def create_food_vectorizer(foods_list):
    vectorizer = TfidfVectorizer(ngram_range=(1,6),analyzer='char_wb')
    food_vecs = vectorizer.fit_transform(food_names)
    
    return food_names, food_vecs, vectorizer
foods = pd.read_csv('../../data/knowledge_bases/foodb/foodb_2020_04_07_csv/Food.csv')
food_names = foods['name']

food_names, food_vecs, vectorizer = create_food_vectorizer(food_names)
vectorizer
import pickle
pickle.dump(vectorizer, open('../../food_ke/services/foodb_tfidf.pickle', 'wb'))<jupyter_output><empty_output><jupyter_text>Try matching with the gold std<jupyter_code>candidate_food_names = ['garlic','cocoa']
def match_candidates(candidate_food_names,db_food_vecs,food_names,vectorizer):
    candidate_food_vecs = vectorizer.transform(candidate_food_names)
    similarity_matrix = cosine_similarity(candidate_food_vecs,db_food_vecs)
    indices_top = np.argsort(-similarity_matrix,axis=1,)[:,:1]
    scores_top = -np.sort(-similarity_matrix,axis=1,)[:,:1]
    return food_names[indices_top.flatten()], scores_top.flatten()
matches, scores = match_candidates(candidate_food_names,food_vecs,food_names,vectorizer)
matches
scores<jupyter_output><empty_output><jupyter_text>Try matching some foods from the extraction<jupyter_code>import json

annotations = json.load(open('../data/labeling_output/food_names_labeled_for_entity_resolution.json','r'))

annotation = annotations[0]

foods = []
for annotation in annotations:
    for tag in annotation['annotations'][0]['result']:
        if 'htmllabels' in tag['value']:
            foods.append(tag['value']['text'])

foods_cleaned = [unidecode(x).replace('\t',' ').strip() for x in foods]
matches, scores = match_candidates(foods_cleaned,food_vecs,food_names,vectorizer)
matches
entity_resolution_results = pd.DataFrame()
entity_resolution_results['original'] = foods_cleaned
entity_resolution_results['matched'] =  matches.values
entity_resolution_results['scores'] =  scores
entity_resolution_results = entity_resolution_results.drop_duplicates(subset=['original'])
entity_resolution_results.to_csv('../data/labeling_evaluation/entity_res_evaluation.csv',index=False)
entity_res_graded = pd.read_csv('../data/labeling_evaluation/entity_res_evaluation_graded.csv')
(entity_res_graded['correct'] == 'y').sum() / len(entity_res_graded['correct'])<jupyter_output><empty_output><jupyter_text>80% acc taking closest match<jupyter_code>entity_resolution_results['scores']
entity_res_graded['correct'].map({'y':1,'n':0,'u':0})
precision, recall, thresholds = precision_recall_curve(entity_res_graded['correct'].map({'y':1,'n':0,'u':0}), entity_resolution_results['scores'])
disp = PrecisionRecallDisplay(precision,recall)
disp.plot()<jupyter_output><empty_output><jupyter_text>Not necessarily bad<jupyter_code>lexmapr_input = pd.DataFrame()
lexmapr_input['Sample'] = foods_cleaned
lexmapr_input['SampleId'] = 'sample_' + lexmapr_input.index.astype(str)
lexmapr_input = lexmapr_input[['SampleId','Sample']]
lexmapr_input.to_csv('../data/labeling_evaluation/lexmapr_input.csv',index=False)
%%writefile ../data/labeling_evaluation/small_simple_config.json

[
  {"http://purl.obolibrary.org/obo/foodon.owl": "http://purl.obolibrary.org/obo/BFO_0000001"}
]
! lexmapr ../data/labeling_evaluation/lexmapr_input.csv -c ../data/labeling_evaluation/small_simple_config.json -o ../data/labeling_evaluation/lexmapr_results.tsv
data = pd.read_csv('../data/labeling_evaluation/lexmapr_results.tsv',sep='\t')<jupyter_output><empty_output><jupyter_text>Why are we matching to cephalopod ontology.....<jupyter_code>data['Matched_Components'].values<jupyter_output><empty_output><jupyter_text>Chemical<jupyter_code>from food_ke.stubs import DATA_DIR
extracted = pd.read_csv(DATA_DIR / "extracted_compositional_measurements.csv")
extracted["constituent_name"] = extracted["constituent_name"].replace('\u2009', ' ')
const_names = extracted.constituent_name[extracted.constituent_name.str.len() > 5].unique()
const_names = [unidecode(x) for x in const_names]
pd.DataFrame(const_names).to_csv(DATA_DIR / 'labeling_evaluation/entity_res_evaluation_const_names.csv')
# const_names_graded = pd.read_csv(DATA_DIR / 'labeling_evaluation/entity_res_evaluation_const_names_graded.csv')
# names = const_names_graded['name'][const_names_graded['valid'] == 'y']
import pubchempy as pcp
def get_compound_pubchem_id(compound_name: str) -> int:
    """
    Returns first result pubchem chemical id or None.
    """
    
    results = pcp.get_compounds(compound_name,'name')
    if len(results) > 0:
        return results[0].cid
    else:
        return None
from food_ke.
cid = get_compound_pubchem_id("Momordica charantia")
matches = [_get_compound_uid(x) for x in names]
results = pd.DataFrame(matches)
results['names'] = names
x = "something\u2009somethingelse"
x.replace('\u2009',' ')
results<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12971428571428573, 'fraction_numerical': 0.008761904761904762, 'mean_word_length': 7.52435064935065, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '220631', 'license': 'Apache-2.0', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1781, 'n_words': 274, 'path': 'ipynb/entity_resolution/entity_resolution.ipynb', 'repo_name': 'IBPA/FoodAtlas', 'size': '5250'}