<jupyter_start><jupyter_text>disambiguating item names & materials with Getty<jupyter_code>import sys
sys.path.append("..")

import re
from tqdm import tqdm
import pandas as pd
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

from collections import Counter
import requests
from tqdm import tqdm
from fuzzywuzzy import fuzz
df = pd.read_csv("../GITIGNORE_DATA/mimsy-catalogue-export.csv")
df["ITEM_NAME_list"] = df['ITEM_NAME'].fillna("").astype(str).apply(lambda i: [x.strip().lower() for x in i.split(';')])
df["MATERIALS_list"] = df['MATERIALS'].fillna("").astype(str).apply(lambda i: [x.strip().lower() for x in i.split(';')])

df.head(5)<jupyter_output>/Users/kalyan/.local/share/virtualenvs/heritage-connector-tkmarX41/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3072: DtypeWarning: Columns (3,4,5,7,11,17,24,25,26,29,34) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>1. item names  1.1 look up top names<jupyter_code>itemname_count = pd.Series(Counter(df["ITEM_NAME_list"].sum())).drop(index='')
# top item names and percentages
(itemname_count.sort_values(ascending=False) / len(df) * 100).head(20)
itemname_count.to_csv("../GITIGNORE_DATA/object_itemnames_3.csv")<jupyter_output><empty_output><jupyter_text>1.2 get dumps from Getty reconciliation endpoint(SPARQL at vocab.getty.edu is down)endpoint documentation: https://gist.github.com/workergnome/afe5b74cff8f1b4fb6490667cf6a4886<jupyter_code>endpoint_url = "http://services.getty.edu/vocab/reconcile/?queries="
itemname_df = pd.DataFrame(itemname_count).reset_index().drop(columns=0).rename(columns={'index': 'query'})
itemname_df['type'] = '/aat'
itemname_df['query'] = itemname_df['query'].astype(str).str.replace("&", "") # & causes issues with URL
page_limit = 18
idx_list = itemname_df.index.tolist()
idx_paginated = [idx_list[i : i + page_limit] for i in range(0, len(idx_list), page_limit)]

response_df = pd.DataFrame()
failed_idx = []
for page in tqdm(idx_paginated):
    try:
        json_request = itemname_df.loc[page, :].to_json(orient="index")
        response = requests.post(endpoint_url + json_request).json()

        tempdf = pd.json_normalize(response).T
        tempdf = tempdf.rename(index = lambda x: x.strip('.result'), columns={0: 'response_dump'})

        response_df = response_df.append(tempdf)

    except:
        print(f"FAILED: {page[0]}:{page[-1]}")
        failed_idx.append(page)
        
response_df.index = response_df.index.astype(int)
itemnames_responses = pd.concat([itemname_df, response_df], axis=1)
itemnames_responses.head(1)<jupyter_output><empty_output><jupyter_text>1.3 reconcile with names<jupyter_code>itemnames_responses["match_name"] = ""
itemnames_responses["match_id"] = ""

for idx, row in tqdm(itemnames_responses.iterrows(), total=itemnames_responses.shape[0]):
    query = row['query']
    response = row['response_dump']
    
    if not isinstance(response, float):
        names = [item['name'] for item in response]
        ids = [item['id'] for item in response]
        idx_match = [idx for idx, item in enumerate(names) if fuzz.ratio(query, item) > 90]

        if len(idx_match) >= 1:
            itemnames_responses.loc[idx, "match_name"] = names[idx_match[0]]
            itemnames_responses.loc[idx, "match_id"] = ids[idx_match[0]]
        if len(idx_match) > 1:
            print(query, [names[i] for i in idx_match])
itemnames_responses.to_csv("../GITIGNORE_DATA/object_itemnames_matches_2.csv")
# number and % of item names matched§
len(itemnames_responses[itemnames_responses['match_name'] != ""]), len(itemnames_responses[itemnames_responses['match_name'] != ""])/len(itemnames_responses)
matched_itemnames = itemnames_responses.loc[itemnames_responses['match_name'] != "", 'query'].str.replace("&", "").tolist()
itemname_count.index = itemname_count.index.str.replace("&", "")
# % of collection items with one of the matched item names
itemname_count[matched_itemnames].sum() / len(df)
itemname_count.to_csv("../GITIGNORE_DATA/object_itemnames_count.csv")<jupyter_output><empty_output><jupyter_text>2. Mapping Getty IDs to WikidataResolving `match_id` column in below dataframe to Wikidata IDs, and storing these relationships in a CSV.<jupyter_code>from heritageconnector.entity_matching.lookup import wikidata_id
from functools import partial

tqdm.pandas()
getty_ids = pd.read_csv('../GITIGNORE_DATA/itemnames_matches_final.csv')

for col in ['match_name', 'match_id']:
    getty_ids[col] = getty_ids[col].astype(str).str.replace("^0$", "")

getty_ids.head(5)
getty_ids["getty_aat_id"] = getty_ids["match_id"].apply(lambda i: re.findall(r"(300\d{6})", i)[0] if i.startswith("aat") else "")
getty_ids.head(5)
wid = wikidata_id()
lookup = partial(wid.lookup_wikidata_id, pid="P1014")
getty_ids_toscan = getty_ids.loc[getty_ids["match_id"] != ""]

getty_ids_toscan["wikidata_id"] = getty_ids_toscan["getty_aat_id"].progress_apply(lambda i: lookup(uid=i))
getty_ids['wikidata_id'] = getty_ids_toscan['wikidata_id']
perc_ids_linked = 100*len(getty_ids[getty_ids['wikidata_id'].astype(str).str.startswith("Q")])/len(getty_ids)
print(f"Percentage of IDs with Wikidata links: {perc_ids_linked}")
getty_ids.to_csv("../GITIGNORE_DATA/itemnames_links.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1510407073125806, 'fraction_numerical': 0.016761834592005895, 'mean_word_length': 6.220744680851064, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '193560', 'license': 'MIT', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1969, 'n_words': 323, 'path': 'experiments/2020_06_29 disambiguating item names & materials with Getty.ipynb', 'repo_name': 'TheScienceMuseum/heritage-connector', 'size': '5429'}