<commit_before>from rdflib import Graph, URIRef, Literal, Namespace, BNode
from rdflib.namespace import RDF, RDFS, SKOS, XSD, OWL, PROV
from wikidataintegrator import wdi_core, wdi_config
import uuid
import requests
import json
import urllib.parse


"""
Authors:
<NAME> (andra' at ' micelio.be)

This file is part of the WikidataIntegrator.
"""

__author__ = '<NAME>'
__license__ = 'MIT'


class WDqidRDFEngine(object):
    def __init__(self, qid=None, json_data=None, fetch_provenance_rdf=True, fetch_metadata_rdf=True, fetch_truthy_rdf=False,
                 fetch_normalized_rdf=False, fetch_sitelinks_rdf=False,
                 fetch_merged_items_rdf=False, fetch_property_descriptions_rdf=False, fetch_labels_rdf=True,
                 fetch_linked_items_rdf=False, fetch_all=False, max_steps=1, current_step=0):
        self.current_step = current_step
        self.max_steps = max_steps
        if not bool(qid) and not bool(json_data):
            raise ValueError('Please provide a QID or a QID and its json object of a Wikidata item')

        if not bool(json_data):
            self.qid = json_data["id"]
        else:
            self.qid = qid
        if fetch_all:
            self.fetch_provenance_rdf = True
            self.fetch_labels_rdf = True
            self.fetch_metadata_rdf = True
            self.fetch_merged_items_rdf = True
            self.fetch_normalized_rdf = True
            self.fetch_property_descriptions_rdf = True
            self.fetch_sitelinks_rdf = True
            self.fetch_truthy_rdf = True
            self.fetch_linked_items_rdf = True
        else:
            self.fetch_provenance_rdf = fetch_provenance_rdf
            self.fetch_labels_rdf = fetch_labels_rdf
            self.fetch_metadata_rdf = fetch_metadata_rdf
            self.fetch_merged_items_rdf = fetch_merged_items_rdf
            self.fetch_normalized_rdf = fetch_normalized_rdf
            self.fetch_property_descriptions_rdf = fetch_property_descriptions_rdf
            self.fetch_sitelinks_rdf = fetch_sitelinks_rdf
            self.fetch_truthy_rdf = fetch_truthy_rdf
            self.fetch_linked_items_rdf = fetch_linked_items_rdf
        if bool(qid):
            self.json_item = json.loads(requests.get("http://www.wikidata.org/entity/" + qid + ".json").text)["entities"][
                qid]
        else:
            self.json_item = json_data
        self.rdf_item = Graph()
        self.ns = dict()
        self.linked_items = []
        self.metadata = {"identifiers": 0, "sitelinks": 0, "statements": 0}
        self.ns = dict()
        for prefix in wdi_config.prefix.keys():
            self.ns[prefix] = Namespace(wdi_config.prefix[prefix])
            self.rdf_item.namespace_manager.bind(prefix, self.ns[prefix])
        self.ns['schema'] = Namespace('http://schema.org/')  # can be removed since it will be in WDI version 0.8.21
        self.properties = dict()
        self.triplify()

    def triplify(self):
        if fetch
        self.fetch_normalization_rules()
        for prefix in wdi_config.prefix.keys():
            self.ns[prefix] = Namespace(wdi_config.prefix[prefix])
            self.rdf_item.namespace_manager.bind(prefix, self.ns[prefix])
        self.ns['schema'] = Namespace('http://schema.org/')  # can be removed since it will be in WDI version 0.8.21

        # item = wdi_core.WDItemEngine(wd_item_id=qid)

        self.fetch_statements()
        if self.fetch_metadata_rdf:
            self.fetch_metadata()
        if self.fetch_sitelinks_rdf:<commit_msg>Work on #187. normalized triples are still accessing the api. This commit is a bit of restructering to prevent this<commit_after>from rdflib import Graph, URIRef, Literal, Namespace, BNode
from rdflib.namespace import RDF, RDFS, SKOS, XSD, OWL, PROV
from wikidataintegrator import wdi_core, wdi_config
import uuid
import requests
import json
import urllib.parse


"""
Authors:
<NAME> (andra' at ' micelio.be)

This file is part of the WikidataIntegrator.
"""

__author__ = '<NAME>'
__license__ = 'MIT'


class WDqidRDFEngine(object):
    def __init__(self, qid=None, json_data=None, fetch_provenance_rdf=True, fetch_metadata_rdf=True, fetch_truthy_rdf=False,
                 fetch_normalized_rdf=False, fetch_sitelinks_rdf=False,
                 fetch_merged_items_rdf=False, fetch_property_descriptions_rdf=False, fetch_labels_rdf=True,
                 fetch_linked_items_rdf=False, fetch_all=False, max_steps=1, current_step=0):
        self.current_step = current_step
        self.max_steps = max_steps
        if not bool(qid) and not bool(json_data):
            raise ValueError('Please provide a QID or a QID and its json object of a Wikidata item')

        if not bool(json_data):
            self.qid = json_data["id"]
        else:
            self.qid = qid
        if fetch_all:
            self.fetch_provenance_rdf = True
            self.fetch_labels_rdf = True
            self.fetch_metadata_rdf = True
            self.fetch_merged_items_rdf = True
            self.fetch_normalized_rdf = True
            self.fetch_property_descriptions_rdf = True
            self.fetch_sitelinks_rdf = True
            self.fetch_truthy_rdf = True
            self.fetch_linked_items_rdf = True
        else:
            self.fetch_provenance_rdf = fetch_provenance_rdf
            self.fetch_labels_rdf = fetch_labels_rdf
            self.fetch_metadata_rdf = fetch_metadata_rdf
            self.fetch_merged_items_rdf = fetch_merged_items_rdf
            self.fetch_normalized_rdf = fetch_normalized_rdf
            self.fetch_property_descriptions_rdf = fetch_property_descriptions_rdf
            self.fetch_sitelinks_rdf = fetch_sitelinks_rdf
            self.fetch_truthy_rdf = fetch_truthy_rdf
            self.fetch_linked_items_rdf = fetch_linked_items_rdf
        if bool(qid):
            self.json_item = json.loads(requests.get("http://www.wikidata.org/entity/" + qid + ".json").text)["entities"][
                qid]
        else:
            self.json_item = json_data
        self.rdf_item = Graph()
        self.ns = dict()
        self.linked_items = []
        self.metadata = {"identifiers": 0, "sitelinks": 0, "statements": 0}
        self.ns = dict()
        for prefix in wdi_config.prefix.keys():
            self.ns[prefix] = Namespace(wdi_config.prefix[prefix])
            self.rdf_item.namespace_manager.bind(prefix, self.ns[prefix])
        self.ns['schema'] = Namespace('http://schema.org/')  # can be removed since it will be in WDI version 0.8.21
        self.properties = dict()
        self.triplify()

    def triplify(self):
        if self.fetch_normalized_rdf:
            self.fetch_normalization_rules()
        for prefix in wdi_config.prefix.keys():
            self.ns[prefix] = Namespace(wdi_config.prefix[prefix])
            self.rdf_item.namespace_manager.bind(prefix, self.ns[prefix])
        self.ns['schema'] = Namespace('http://schema.org/')  # can be removed since it will be in WDI version 0.8.21

        # item = wdi_core.WDItemEngine(wd_item_id=qid)

        self.fetch_statements()
        if self.fetch_metadata_rdf:
            self.fetch_metadata()
        if self.fetch_sitelinks_rdf:
{'fraction_non_alphanumeric': 0.08754349338900487, 'fraction_numerical': 0.004036186499652053, 'mean_word_length': 2.6311268317331984, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494864', 'n_tokens_mistral': 2346, 'n_tokens_neox': 2297, 'n_words': 456}