<jupyter_start><jupyter_text>Access details for the Wikibase which host the SAF<jupyter_code>import rdflib
from wikidataintegrator import wdi_core, wdi_login
from getpass import getpass
import pandas as pd
import sys
import traceback

wbstack = "wbjupyter"
wikibase = "https://{}.wiki.opencura.com/".format(wbstack)
api = "https://{}.wiki.opencura.com/w/api.php".format(wbstack)
sparql = "https://{}.wiki.opencura.com/query/sparql".format(wbstack)
entityUri = wikibase.replace("https:", "http:")+"entity/"

WBUSER = "Andrawaag"
WBPASS = <PASSWORD>pass()
login = wdi_login.WDLogin(WBUSER, WBPASS, mediawiki_api_url=api)<jupyter_output>········<jupyter_text>Python libraries and function definitions<jupyter_code>localEntityEngine = wdi_core.WDItemEngine.wikibase_item_engine_factory(api,sparql)

def createProperty(login=login, wdprop=None, lulabel="", enlabel="", frlabel="", delabel="", description="", property_datatype=""):
    if wdprop== None:
        s = []
    else:
        s = [wdi_core.WDUrl(wdprop, prop_nr="P1")]
    localEntityEngine = wdi_core.WDItemEngine.wikibase_item_engine_factory(api,sparql)
    item = localEntityEngine(data=s)
    item.set_label(lulabel, lang="lb")
    item.set_label(enlabel, lang="en")
    item.set_label(delabel, lang="de")
    item.set_label(frlabel, lang="fr")
    item.set_description(description, lang="en")
    
    print(item.write(login, entity_type="property", property_datatype=property_datatype))<jupyter_output><empty_output><jupyter_text>Import CIDOC-CRM definitionTODO: confirm that versions are same between DMSAF and the CIDOC-CRM definitions here<jupyter_code>cidoc = rdflib.Graph()
cidoc.load("https://raw.githubusercontent.com/erlangen-crm/ecrm/master/ecrm_current.owl", format="xml")
propertyID = dict()
query = "PREFIX wdt: <http://{}.wiki.opencura.com/prop/direct/> SELECT ?item ?label WHERE {{?item rdfs:label ?label }}".format(wbstack)
wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe = True, endpoint=sparql)
for index, row in wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe = True, endpoint=sparql).iterrows():
    propertyID[row["label"]] =row["item"].replace(entityUri, "")
    
qid = dict()
query = "PREFIX wdt: <http://{}.wiki.opencura.com/prop/direct/> SELECT ?item ?label WHERE {{?item rdfs:label ?label }}".format(wbstack)
wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe = True, endpoint=sparql)
for index, row in wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe = True, endpoint=sparql).iterrows():
    qid[row["label"]] =row["item"].replace(entityUri, "")<jupyter_output><empty_output><jupyter_text>Create CIDOC-CRM items on WB SAF<jupyter_code>query = "SELECT ?cidoc ?label WHERE {?cidoc rdf:type <http://www.w3.org/2002/07/owl#Class> ; rdfs:label ?label .}"
for row in cidoc.query(query):
    #print(str(row[0]), str(row[1]), " ".join(str(row[1]).split(" ")[1:]))
    statements=[]
    statements.append(wdi_core.WDItemID(value=qid["Class"], prop_nr=propertyID["instance of"]))
    statements.append(wdi_core.WDUrl(value=str(row[0]).replace("http://erlangen-crm.org/current/", "http://www.cidoc-crm.org/cidoc-crm/"), prop_nr=propertyID['exact match']))
    item = localEntityEngine(new_item=True, data=statements)
    item.set_label(" ".join(str(row[1]).split(" ")), lang="en")
    #item.set_aliases([str(row[1]), str(row[1]).replace(" ", "_")], lang="en")
    print(item.write(login))

#propertyClasses
query = "SELECT ?cidoc ?label WHERE {?cidoc rdf:type <http://www.w3.org/2002/07/owl#ObjectProperty> ; rdfs:label ?label .}"
for row in cidoc.query(query):
    print(str(row[0]), str(row[1]), " ".join(str(row[1]).split(" ")[1:]))
    statements=[]
    statements.append(wdi_core.WDItemID(value=qid["Property"], prop_nr=qid["instance of"]))
    statements.append(wdi_core.WDUrl(value=str(row[0]).replace("http://erlangen-crm.org/current/", "http://www.cidoc-crm.org/cidoc-crm/"), prop_nr=propertyID["exact match"]))
    item = localEntityEngine(new_item=True, data=statements)
    item.set_label(" ".join(str(row[1]).split(" ")), lang="en")
    #item.set_aliases([str(row[1]), str(row[1]).replace(" ", "_")], lang="en")
    try:
        print(item.write(login))
    except:
        print("ERROR: "+str(row[1]))
        continue
        
query = "PREFIX wdt: <http://{}.wiki.opencura.com/prop/direct/> SELECT * WHERE {{?item wdt:P3 ?uri .}}".format(wbstack)
for index, row in wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe = True, endpoint=sparql).iterrows():
    qid[row["uri"]] =row["item"].replace(entityUri, "")
missing_classes = ['R8 consists of', "E13 Attribute assignement", 'E52 Time-span', 'P48 has prefered identifier', 'P107 has current or former member of']
for missing_class in missing_classes:
    statements=[]
    statements.append(wdi_core.WDItemID(value=qid["Class"], prop_nr=propertyID["instance of"]))
    statements.append(wdi_core.WDUrl(value="http://www.cidoc-crm.org/cidoc-crm/"+ missing_class.replace(" ", "_"), prop_nr=propertyID['exact match']))
    item = localEntityEngine(new_item=True, data=statements)
    item.set_label(missing_class , lang="en")
    print(item.write(login))<jupyter_output>Q366
Q367
Q368
Q369
Q370
{'fraction_non_alphanumeric': 0.17573543549317439, 'fraction_numerical': 0.011151701595846953, 'mean_word_length': 5.991935483870968, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 5, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '90761', 'license': 'MIT', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1895, 'n_words': 303, 'path': 'import_scripts/CIDOC-CRM-properties/import_cidoc_crm.ipynb', 'repo_name': 'weso/SAF-lux', 'size': '5194'}