<jupyter_start><jupyter_text>Complete searchSearch Wikidata & Wikipedia -> rank results -> retrieve information based on config.<jupyter_code>%load_ext autoreload
%autoreload 2

import sys
sys.path.append("../..")

import os
import re

from heritageconnector.config import config, field_mapping
from heritageconnector.disambiguation import search, retrieve

import pandas as pd
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>1. load data sample<jupyter_code>sample_no = 10
random_state = 42

# load mimsy_people
df = pd.read_csv(os.path.join("..", config.MIMSY_PEOPLE_PATH))
for col in ['FIRSTMID_NAME', 'LASTSUFF_NAME']:
    df[col] = df[col].fillna("").astype(str)
    
df['FREETEXT'] = df['DESCRIPTION'].astype(str) + " " + df['NOTE'].astype(str)

# load people df 
people_df = df[df['GENDER'].isin(('M', 'F'))].sample(sample_no, random_state=random_state)
people_df.loc[:, 'JOINED_NAME'] = people_df['FIRSTMID_NAME'] + " " + people_df['LASTSUFF_NAME']

# load org df
org_df = df[df['GENDER'] == "N"].sample(sample_no, random_state=random_state)<jupyter_output><empty_output><jupyter_text>2. run search<jupyter_code>row = people_df.iloc[6]
row
search_results = search.run(text=row["JOINED_NAME"], topn=10, limit=100, instanceof_filter="Q5")
search_results<jupyter_output><empty_output><jupyter_text>3. Create vectors of each Wikidata object<jupyter_code>field_mapping.PEOPLE
qcode_urls = search_results.index.tolist()
qcodes = [re.findall(r"(Q\d+)", u)[0] for u in qcode_urls]

pid_mapping = {k: v['PID'] for k, v in field_mapping.PEOPLE.items() if 'PID' in v and v['PID'].startswith('P')}
pid_mapping
pd.DataFrame(row).T[['JOINED_NAME', 'FREETEXT'] + list(pid_mapping.keys())]
res = retrieve.get_wikidata_fields(qcodes, pid_mapping)
res<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1490134994807892, 'fraction_numerical': 0.008307372793354102, 'mean_word_length': 7.235042735042735, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '194080', 'license': 'MIT', 'n_tokens_mistral': 735, 'n_tokens_neox': 709, 'n_words': 136, 'path': 'experiments/disambiguation/4. complete search.ipynb', 'repo_name': 'TheScienceMuseum/heritage-connector', 'size': '1926'}