<commit_before>from rdflib import Namespace, URIRef
from sparql_slurper import SlurpyGraph
import pprint
import requests
import pyshex
from wikidataintegrator import wdi_core

"""
def check_shex_conformance(qid, schema, endpoint="https://query.wikidata.org/sparql", debug=False):
    results = dict()
    results["wdid"] = qid
    slurpeddata = SlurpyGraph(endpoint)
    for p, o in slurpeddata.predicate_objects(qid):
        # for a, b in slurpeddata.predicate_objects(o):
        pass
    for result in pyshex.ShExEvaluator(rdf=slurpeddata, schema=schema, focus=qid).evaluate():
        shex_result = dict()
        if result.result:
            shex_result["result"] = "Passing"
        else:
            shex_result["result"] = "Failing"
            print("reason: " + result.reason)
        shex_result["reason"] = result.reason

    return shex_result
"""

wdids = []
sparql_query = "PREFIX wdt: <http://www.wikidata.org/prop/direct/>\n\nSELECT ?item WHERE { ?item wdt:P699 ?wpid . } LIMIT 10"
df = wdi_core.WDItemEngine.execute_sparql_query(sparql_query)
for row in df["results"]["bindings"]:
    wdid = row["item"]["value"]
    wdids.append(URIRef(wdid))

WD = Namespace("http://www.wikidata.org/entity/")
schema = requests.get(
    "https://raw.githubusercontent.com/SuLab/Genewiki-ShEx/master/diseases/wikidata-disease-ontology.shex").text

results = dict()
for qid in wdids:
    print(qid)
    results[qid] = wdi_core.WDItemEngine.check_shex_conformance(qid, schema)

pprint.pprint(results)

count = {"passing": 0, "failing": 0}
for result in results.keys():
    if results[result]["result"] == "Passing":
        count["passing"] += 1
    if results[result]["result"] == "Failing":
        count["failing"] += 1

pprint.pprint(count)<commit_msg>removed unused libraries<commit_after>from rdflib import Namespace, URIRef
import pprint
import requests
from wikidataintegrator import wdi_core

"""
def check_shex_conformance(qid, schema, endpoint="https://query.wikidata.org/sparql", debug=False):
    results = dict()
    results["wdid"] = qid
    slurpeddata = SlurpyGraph(endpoint)
    for p, o in slurpeddata.predicate_objects(qid):
        # for a, b in slurpeddata.predicate_objects(o):
        pass
    for result in pyshex.ShExEvaluator(rdf=slurpeddata, schema=schema, focus=qid).evaluate():
        shex_result = dict()
        if result.result:
            shex_result["result"] = "Passing"
        else:
            shex_result["result"] = "Failing"
            print("reason: " + result.reason)
        shex_result["reason"] = result.reason

    return shex_result
"""

wdids = []
sparql_query = "PREFIX wdt: <http://www.wikidata.org/prop/direct/>\n\nSELECT ?item WHERE { ?item wdt:P699 ?wpid . } LIMIT 10"
df = wdi_core.WDItemEngine.execute_sparql_query(sparql_query)
for row in df["results"]["bindings"]:
    wdid = row["item"]["value"]
    wdids.append(URIRef(wdid))

WD = Namespace("http://www.wikidata.org/entity/")
schema = requests.get(
    "https://raw.githubusercontent.com/SuLab/Genewiki-ShEx/master/diseases/wikidata-disease-ontology.shex").text

results = dict()
for qid in wdids:
    print(qid)
    results[qid] = wdi_core.WDItemEngine.check_shex_conformance(qid, schema)

pprint.pprint(results)

count = {"passing": 0, "failing": 0}
for result in results.keys():
    if results[result]["result"] == "Passing":
        count["passing"] += 1
    if results[result]["result"] == "Failing":
        count["failing"] += 1

pprint.pprint(count)
{'fraction_non_alphanumeric': 0.14277489925158318, 'fraction_numerical': 0.0051813471502590676, 'mean_word_length': 3.873772791023843, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3850619', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1243, 'n_words': 233}