<jupyter_start><jupyter_text>Data Enrichment using wikidataFind occupation of SuperHeroes- lookup hero on wikidata- disambiguate findings- look for occupation<jupyter_code>import requests
from SPARQLWrapper import SPARQLWrapper, JSON
def search(query, lang='en'):
    url = 'https://www.wikidata.org/w/api.php'
    par = {
        'action': 'wbsearchentities',
        'format': 'json',
        'search': query,
        'language': lang
    }
    response = requests.get(url=url, params=par)
    return response.json()

def sparql(entity_id, lang='en'):
    endpoint_url = "https://query.wikidata.org/sparql"
    sparql = SPARQLWrapper(endpoint_url)
    query = """
    SELECT * WHERE {{
      wd:{sh1} wdt:P31 ?type.
      wd:{sh2} rdfs:label ?entityLabel.
      ?type rdfs:label ?typeLabel.
      OPTIONAL {{
        wd:Q4646886 wdt:P21 ?gender.
      }}
      FILTER (langMatches( lang(?entityLabel), "{lang1}" ) ).
      FILTER (langMatches( lang(?typeLabel), "{lang2}" ) ).
    }}
    """.format(sh1=entity_id, sh2=entity_id, lang1=lang, lang2=lang)
    sparql.setQuery(query)
    sparql.setReturnFormat(JSON)
    return sparql.query().convert()
hero = 'Animal Man'
sh_wikiclass = 'http://www.wikidata.org/entity/Q188784'<jupyter_output><empty_output><jupyter_text>Check for type matching<jupyter_code>for option in search(hero)['search']:
    data = sparql(option['id'])
    for binding in data['results']['bindings']:
        if binding['type']['value'] == sh_wikiclass:
            print("The best matching entity for {} is {}".format(hero, option['id']))
            break<jupyter_output>The best matching entity for Animal Man is Q547391<jupyter_text>Location of birth cities<jupyter_code>from geopy.geocoders import Nominatim
geolocator = Nominatim(user_agent="HEROLOC")
location = geolocator.geocode("Chicago Illinois")
print(location.address)
print((location.latitude, location.longitude))
print(location.raw)
H = H.dropna()
valid_rows = []
for i, hero in H.iterrows():
    bp = hero['Birth place']
    location = geolocator.geocode(bp)
    if location is not None and hero['Publisher'] in ['Marvel Comics', 'DC Comics']:
        hero['Latitude'], hero['Longitude'] = location.latitude, location.longitude
        valid_rows.append(hero)
len(valid_rows)
S = pd.DataFrame(valid_rows)
S.head()
import matplotlib.pyplot as plt
marvel = S[S['Publisher'] == 'Marvel Comics']
dc = S[S['Publisher'] == 'DC Comics']
fig, ax = plt.subplots(figsize=(16,8))
ax.scatter(marvel.Latitude, marvel.Longitude, c='#FF0000')
ax.scatter(dc.Latitude, dc.Longitude, c='#000066')
plt.show()
import geopandas as gpd
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
fig, ax = plt.subplots(figsize=(16,8))
world.plot(ax=ax, color='#CCCCCC')
ax.scatter(marvel.Longitude, marvel.Latitude, c='#FF0000')
ax.scatter(dc.Longitude, dc.Latitude, c='#000066')
plt.show()
marvel.to_csv('data/marvel.csv')
dc.to_csv('data/dc.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15296108291032148, 'fraction_numerical': 0.019289340101522844, 'mean_word_length': 4.577358490566038, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '471084', 'license': 'MIT', 'n_tokens_mistral': 1107, 'n_tokens_neox': 1049, 'n_words': 208, 'path': 'heroes.ipynb', 'repo_name': 'afflint/subito', 'size': '2955'}