<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and sourced statements about entities also present in
KulturNav (https://www.wikidata.org/wiki/Q16323066).

Author: Lokal_Profil
License: MIT

See https://github.com/lokal-profil/wikidata-stuff/issues for TODOs

usage:
    python KulturNav/kulturnavBot.py [OPTIONS]

&params;
"""
import json
import time
import pywikibot
import urllib2
import wikidataStuff.helpers as helpers
import wikidataStuff.wdqsLookup as wdqsLookup
from wikidataStuff.WikidataStuff import WikidataStuff as WD
import re

FOO_BAR = u'A multilingual result (or one with multiple options) was ' \
          u'encountered but I have yet to support that functionality'

parameter_help = u"""\
Basic KulturNavBot options (may be omitted):
-cutoff:INT        number of entries to process before terminating
-max_hits:INT      number of items to request at a time from Kulturnav
                    (default 250)
-delay:INT         seconds to delay between each kulturnav request
                    (default 0)
-any_item          if present it does not filter kulturnav results on wikidata
-wdq_cache:INT     set the cache age (in seconds) for wdq queries
                    (default 0)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}


class Rule(object):
    """A class for encoding rules used by runLayout()."""

    def __init__(self, keys, values, target, viaId=None):
        """
        Initialize the rule.

        :param keys: list|string|None of keys which must be present
            (in addition to value/target)
        :param values: dict|None key-value pairs which must be present
        :param target: string key for which the value is wanted
        :param viaId: string|tuple|None if the value of "target" should be
            matched to an @id entry where this key then gives the wanted value.
            If a tuple is supplied then each intermediate entry is matched to
            an @id entry.
        """
        self.keys = []
        if keys is not None:
            self.keys += helpers.listify(keys)

        self.values = values
        if values is not None:
            self.keys += values.keys()

        self.target = target
        self.keys.append(target)

        self.viaId = viaId
        # convert plain viaId to tuples
        if isinstance(self.viaId, basestring):
            self.viaId = (self.viaId, )

    def __repr__(self):
        """Return a more complete string representation."""
        return u'Rule(%s, %s, %s, %s)' % (
            self.keys, self.values, self.target, self.viaId)

    @staticmethod
    def hasKeys(needles, haystack):
        """
        Checks if all the provided keys are present.

        :param needles: a list of strings
        :param haystack: a dict
        :return: bool
        """
        for n in needles:
            if n not in haystack.keys():
                return False
        return True

    @staticmethod
    def hasValues(needles, haystack):
        """
        Check if all the provided values are present.

        :param needles: None or a dict of key-value pairs
        :param haystack: a dict
        :return: bool
        """
        if needles is None:
            return True
        for n, v in needles.iteritems():
            if not haystack[n] == v:
                return False
        return True

    def resolve(self, entries, ids):
        """
        Resolve a rule to return the resulting value.

        :param entries: all the data under @graph
        :param ids: a dict of all @id values
        :return: the matching value
        """
        value = None
        if Rule.hasKeys(self.keys, entries) and \
                Rule.hasValues(self.values, entries):
            value = entries[self.target]

        # break here if no hit
        if not value:
            return None

        # convert values for viaId rules
        if self.viaId:
            # value can be either a single entry or a list
            values = helpers.listify(value)
            results = []
            for val in values:
                i = 0
                while len(self.viaId) > i:
                    id_entry = self.viaId[i]
                    if val in ids.keys() and id_entry in ids[val].keys():
                        val = ids[val][id_entry]
                        i += 1
                    else:
                        # @todo: should log these to spot schema changes
                        return None
                results.append(val)

            # reformat for output
            if len(results) == 1:
                value = results[0]  # undo listify
            else:
                value = results

        return value


class KulturnavBot(object):
    """
    A bot to enrich and create information on Wikidata based on KulturNav info
    """
    EDIT_SUMMARY = 'import using #Kulturnav data'
    KULTURNAV_ID_P = '1248'
    GEONAMES_ID_P = '1566'
    SWE_KOMMUNKOD_P = '525'
    SWE_COUNTYKOD_P = '507'
    PLACE_P = '276'
    TIME_P = '585'   # date<commit_msg>Rewrite Rule to allow associated value to be extracted together.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and sourced statements about entities also present in
KulturNav (https://www.wikidata.org/wiki/Q16323066).

Author: Lokal_Profil
License: MIT

See https://github.com/lokal-profil/wikidata-stuff/issues for TODOs

usage:
    python KulturNav/kulturnavBot.py [OPTIONS]

&params;
"""
import json
import time
import pywikibot
import urllib2
import wikidataStuff.helpers as helpers
import wikidataStuff.wdqsLookup as wdqsLookup
from wikidataStuff.WikidataStuff import WikidataStuff as WD
import re

FOO_BAR = u'A multilingual result (or one with multiple options) was ' \
          u'encountered but I have yet to support that functionality'

parameter_help = u"""\
Basic KulturnavBot options (may be omitted):
-cutoff:INT        number of entries to process before terminating
-max_hits:INT      number of items to request at a time from Kulturnav
                    (default 250)
-delay:INT         seconds to delay between each kulturnav request
                    (default 0)
-any_item          if present it does not filter kulturnav results on wikidata
-wdq_cache:INT     set the cache age (in seconds) for wdq queries
                    (default 0)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}


class Rule(object):
    """A class for encoding rules used to isolate data in a Kulturnav entry."""

    def __init__(self, target, keys=None, viaId=None):
        """
        Initialize the rule.

        Keys gives the resolver the environment in which the target
        is expected.
        Target is the first-pass value to be extracted from the data. The
        result can be a list of values.
        ViaId gives the path through which a second-pass value can be extracted
        from Target.

        For data which must remain connected (such as the start and end dates
        of a particular claim) a dict can be supplied to ViaId. Each entry is
        looked for under the same Target value but follows a different ViaId
        path.

        :param target: string key for which the value is wanted
        :param keys: list|string|None of keys which must be present
            (in addition to target). If none is supplied it defaults to a key
            in the top node of the graph.
        :param viaId: string|tuple|dict|None if the value of "target" should be
            matched to an @id entry where this key then gives the wanted value.
            If a tuple is supplied then each intermediate entry is matched to
            an @id entry.
            If a dict is supplied then the each value is treated as its own
            viaId and the returned value is a dict where each entry has been
            resolved.
        """
        self.target = target
        self.viaId = viaId

        self.keys = []
        keys = keys or 'inDataset'  # default to the top node in the graph
        if keys is not None:
            self.keys += helpers.listify(keys)
        self.keys.append(target)

    def __repr__(self):
        """Return a more complete string representation."""
        return u'Rule(%s, %s, %s, %s)' % (
            self.keys, self.values, self.target, self.viaId)

    @staticmethod
    def hasKeys(needles, haystack):
        """
        Checks if all the provided keys are present.

        :param needles: a list of strings
        :param haystack: a dict
        :return: bool
        """
        for n in needles:
            if n not in haystack.keys():
                return False
        return True

    @staticmethod
    def resolve_via_id(via_id, value, ids):
        """
        Resolve a viaId rule to return the resulting value.

        :param via_id: string|tuple of strings, viaId chains
        :param value: string the entry matching "target" of the rule.
        :param ids: a dict of all @id values
        :return: the matching value
        """
        # allow strings as input but handle them like tuples
        if isinstance(via_id, basestring):
            via_id = (via_id, )

        i = 0
        while len(via_id) > i:
            id_entry = via_id[i]
            if value in ids.keys() and id_entry in ids[value].keys():
                value = ids[value][id_entry]
                i += 1
            else:
                return None
        return value

    def resolve(self, entries, ids):
        """
        Resolve a rule to return the resulting value.

        :param entries: all the data under @graph
        :param ids: a dict of all @id values
        :return: the matching value
        """
        value = None
        if Rule.hasKeys(self.keys, entries):
            value = entries[self.target]

        # break here if no hit
        if not value:
            return None

        # convert values for viaId rules
        if self.viaId:
            # value can be either a single entry or a list
            values = helpers.listify(value)
            results = []
            for val in values:
                if isinstance(self.viaId, dict):
                    result = {}
                    for key, via_id in self.viaId.iteritems():
                        result[key] = Rule.resolve_via_id(via_id, val, ids)
                    if set(result.values()) == set([None]):
                        # if all entries are None
                        # @todo: should log these to spot schema changes
                        return None
                    results.append(result)
                else:  # self.viaId is either string or tuple
                    result = Rule.resolve_via_id(self.viaId, val, ids)
                    if not result:
                        # @todo: should log these to spot schema changes
                        return None
                    results.append(result)

            # reformat for output
            if len(results) == 1:
                value = results[0]  # undo listify
            else:
                value = results

        return value


class KulturnavBot(object):
    """
    A bot to enrich and create information on Wikidata based on KulturNav info
    """
    EDIT_SUMMARY = 'import using #Kulturnav data'
    KULTURNAV_ID_P = '1248'
    GEONAMES_ID_P = '1566'
    SWE_KOMMUNKOD_P = '525'
    SWE_COUNTYKOD_P = '507'
    PLACE_P = '276'
    TIME_P = '585'   # date
{'fraction_non_alphanumeric': 0.07272261151939839, 'fraction_numerical': 0.006665527260297385, 'mean_word_length': 1.8747236551215918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5416134', 'n_tokens_mistral': 3308, 'n_tokens_neox': 3154, 'n_words': 1243}