<jupyter_start><jupyter_text>Arken  - Wikidataversion 1.1WD egenskap [Property:P8899](https://www.wikidata.org/wiki/Property:P8899) * this [notebook](https://github.com/salgo60/open-data-examples/blob/master/Arken.ipynb)  * Task [T269064](https://phabricator.wikimedia.org/T269064)---- Other sources we sync* [Arken](https://github.com/salgo60/open-data-examples/blob/master/Arken.ipynb)   * WD [Property:P8899](https://www.wikidata.org/wiki/Property:P8899) * [Kulturpersoner Uppsalakyrkogård](https://github.com/salgo60/open-data-examples/blob/master/Check%20WD%20kulturpersoner%20uppsalakyrkogardar.ipynb)* [Litteraturbanken](https://github.com/salgo60/open-data-examples/blob/master/Litteraturbanken%20Author.ipynb)   * WD property [P5101](https://www.wikidata.org/wiki/Property_talk:P5101) [P5123](https://www.wikidata.org/wiki/Property_talk:P5123)* [Nobelprize.org](https://github.com/salgo60/open-data-examples/blob/master/Nobel%20API.ipynb)  * WD [property 8024](https://www.wikidata.org/wiki/Property:P8024)* [SBL](https://github.com/salgo60/open-data-examples/blob/master/SBL.ipynb)   * WD [property 3217](https://www.wikidata.org/wiki/Property:P3217) * [SKBL](https://github.com/salgo60/open-data-examples/blob/master/Svenskt%20Kvinnobiografiskt%20lexikon%20part%203.ipynb)  * WD [property 4963](https://www.wikidata.org/wiki/Property:P4963)* [Svenska Akademien](https://github.com/salgo60/open-data-examples/blob/master/Svenska%20Akademien.ipynb)   * WD [property 5325](https://www.wikidata.org/wiki/Property:P5325)<jupyter_code>from datetime import datetime
now = datetime.now()
print("Last run: ", now)
import urllib3, json
import pandas as pd   
from bs4 import BeautifulSoup
import sys
import pprint
from SPARQLWrapper import SPARQLWrapper, JSON
from tqdm.notebook import trange  
from wikidataintegrator import wdi_core, wdi_login

endpoint_url = "https://query.wikidata.org/sparql"

SparqlQuery = """SELECT ?item ?arkid WHERE {
?item wdt:P8899 ?arkid
}"""


http = urllib3.PoolManager()

# Query https://w.wiki/Vo5
def get_results(endpoint_url, query):
    user_agent = "user  salgo60/%s.%s" % (sys.version_info[0], sys.version_info[1])
    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
    sparql.setQuery(query)
    sparql.setReturnFormat(JSON)
    return sparql.query().convert()

SparQlResults = get_results(endpoint_url, SparqlQuery)
length = len (SparQlResults["results"]["bindings"])
dfWikidata = pd.DataFrame(columns=['WD', 'arkid'])
    
for r in trange(0,length):
    resultSparql = SparQlResults["results"]["bindings"][r]
    wd = resultSparql["item"]["value"].replace("http://www.wikidata.org/entity/","") 
    try: 
        wdArkid= resultSparql["arkid"]["value"] 
    except:
        wdArkid = ""    
    dfWikidata = dfWikidata.append({'WD': wd, 'arkid': wdArkid}, ignore_index=True)
  
dfWikidata.info()
dfWikidata.head(200)
import urllib.parse
urlbase = "https://arken.kb.se/actor/browse?sort=alphabetic&sortDir=asc&page="
urlbase_entry = "https://arken.kb.se"
dfArken = pd.DataFrame(columns=['nameAuth', 'urlAuth', 'Auktoriserad', 'Datum', 'Auktoritetspost'])

#for i in range(1,80):
for i in range(1,90):
    url = urlbase + str(i)
    print(url)
    r = http.request('GET', url)
    soup = BeautifulSoup(r.data, "html.parser")
    for link in soup.select('div.search-result-description a[href]'):
        nameAuth = link.string
        urlAuth = urllib.parse.unquote(link['href'].split("/")[1])
        #print ("\t",urlAuth, nameAuth)    
        urlentry = urlbase_entry + link['href']
        #print ("\t\t",urlentry)
        try:
            r_entry = http.request('GET', urlentry)
            soup_entry = BeautifulSoup(r_entry.data, "html.parser")
            Auktoriserad = ""
            Datum = ""
            Auktoritetspost = ""
            fields = soup_entry.select('div.field')
            for f in fields:
                h3 = f.select("h3")
                divText = f.select("div")
                if len(h3) > 0:
                    if "Auktoriserad" in h3[0].getText():
                        #print("\t\tAuktoriserad: " + divText[0].getText().strip())
                        Auktoriserad = divText[0].getText().strip()
                    if "Datum för verksamhetstid" in h3[0].text:
                        #print("\t\t\tDatum: " + divText[0].getText().strip())
                        Datum =  divText[0].getText().strip()
                    if "Auktoritetspost" in h3[0].text:
                        Auktoritetspost =   divText[0].getText().strip()
                        #print("\t\t\tAuktoritetspost: " + divText[0].getText().strip())
                
            dfArken = dfArken.append({'nameAuth': nameAuth, 'urlAuth': urlAuth, 'Auktoriserad': Auktoriserad, 
                              'Datum': Datum, 'Auktoritetspost': Auktoritetspost}, ignore_index=True)
  
        except:
            print("Error")
                
 
dfArken.info()
dfArken.head(10)
dfArken.to_csv(r'Arken.csv')<jupyter_output><empty_output><jupyter_text>Check diff Wikidata* dfArken* dfWikidata* see also<jupyter_code>dfWikidata.info()
dfArken.info()
dfArken.sample()
# Merge plotPublishedAuthor WDSKBLtot  
mergeArkenWD = pd.merge(dfWikidata, dfArken,how='outer', left_on='arkid',right_on='urlAuth',indicator=True)   
mergeArkenWD.rename(columns={"_merge": "WD_Arken"},inplace = True)
mergeArkenWD['WD_Arken'] = mergeArkenWD['WD_Arken'].str.replace('left_only','WD_only').str.replace('right_only','Arken_only')
mergeArkenWD["WD_Arken"].value_counts()  

Arken_only = mergeArkenWD[mergeArkenWD["WD_Arken"] == "Arken_only"].copy() 
WD_only = mergeArkenWD[mergeArkenWD["WD_Arken"] == "WD_only"].copy() 
# could be places etc....
WD_only.head(10)
pd.set_option('display.max_rows', None)  
Arken_only.sample(10)
mergewithLibris = Arken_only[Arken_only["Auktoritetspost"].notnull()].copy() 
mergewithLibris.sample(10)
#mergewithLibris = Arken_only[Arken_only["Auktoritetspost"].notnull()].copy() 
ArkenOnlyWithAuthrec = Arken_only[Arken_only["Auktoritetspost"] != ''].copy() 
ArkenOnlyWithAuthrec.to_csv(r'ArkenOnlyWithAuthrec.csv')   
ArkenOnlyWithAuthrec
mergewithLibris
mergewithLibris.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 2752 entries, 3390 to 6141
Data columns (total 8 columns):
 #   Column           Non-Null Count  Dtype 
---  ------           --------------  ----- 
 0   WD               0 non-null      object
 1   arkid            0 non-null      object
 2   nameAuth         2752 non-null   object
 3   urlAuth          2752 non-null   object
 4   Auktoriserad     2752 non-null   object
 5   Datum            2752 non-null   object
 6   Auktoritetspost  2752 non-null   object
 7   WD_Arken         2752 non-null   object
dtypes: object(8)
memory usage: 193.5+ KB<jupyter_text>PlacesTBD<jupyter_code>urlbase = "https://arken.kb.se/taxonomy/index/id/42?sort=alphabetic&sortDir=asc&page="
dfp = pd.DataFrame(columns=['nameAuth', 'urlAuth', 'Auktoriserad', 'Datum', 'Auktoritetspost'])
def check_Taxonomy(url):
    r_entry = http.request('GET', url)
    soup_entry = BeautifulSoup(r_entry.data, "html.parser")
    fields = soup_entry.select('div.field')
    for f in fields:
        h3 = f.select("h3")
        divText = f.select("div")
        if len(h3) > 0:
            if "Taxonomi" in h3[0].getText():
                #print("\tTaxonomi: " + divText[0].getText().strip())
                Taxonomi = divText[0].getText().strip()
    return True
    
for i in range(1,10):
    url = urlbase + str(i)
    #print(url)
    r = http.request('GET', url)
    soup = BeautifulSoup(r.data, "html.parser")
    for link in soup.select('table  a[href]'):
        nameAuth = link.string
        urlAuth = urllib.parse.unquote(link['href'].split("/")[1])
        #print ("\t",urlAuth, nameAuth)    
        urlentry = urlbase_entry + link['href']
        #print ("\t\t",urlentry)
        if check_Taxonomy(urlentry):
            #print("True")
            pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1542319749216301, 'fraction_numerical': 0.02884012539184953, 'mean_word_length': 3.3513366066557557, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 22, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '52653', 'license': 'MIT', 'n_tokens_mistral': 3021, 'n_tokens_neox': 2876, 'n_words': 431, 'path': 'Arken.ipynb', 'repo_name': 'salgo60/open-data-examples', 'size': '7975'}