<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and source statements about Surface Water Bodies.

Author: Lokal_Profil
License: MIT

usage:
    python swb_import.py [OPTIONS]

&params;
"""
from __future__ import unicode_literals
from builtins import dict

import pywikibot

import wikidataStuff.helpers as helpers
from wikidataStuff.WikidataStuff import WikidataStuff as WdS

from WFD.WFDBase import WfdBot, UnmappedValueError

parameter_help = """\
SwbBot options (may be omitted):
-year              Year to which the WFD data applies.
-new               if present new items are created on Wikidata, otherwise
                   only updates are processed.
-in_file           path to the data file
-mappings          path to the mappings file (if not "mappings.json")
-cutoff            number items to process before stopping (if not then all)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-dir               directory in which user_config is located
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}
EDIT_SUMMARY = 'importing #SWB using data from #WFD'


class SwbBot(WfdBot):
    """Bot to enrich/create info on Wikidata for SWB objects."""

    def __init__(self, mappings, year, new=False, cutoff=None):
        """
        Initialise the SwbBot.

        :param mappings: dict holding data for any offline mappings
        :param year: year of the report (used to date certain statements).
        :param new: whether to also create new items
        :param cutoff: the number of items to process before stopping. None
            being interpreted as all.
        """
        super(SwbBot, self).__init__(mappings, year, new, cutoff,
                                     EDIT_SUMMARY)

        self.eu_swb_p = 'P2856'  # eu_cd
        self.eu_rbd_p = 'P2965'  # euRBDCode

        self.swb_cats = mappings.get('surfaceWaterBodyCategory')
        self.impact_types = mappings.get('swSignificantImpactType')

        self.load_known_items()

    def load_known_items(self):
        """
        Load existing eu_swb and eu_rbd items.

        The loaded values are Qids (without q-prefixes)
        """
        self.swb_items = helpers.fill_cache_wdqs(self.eu_swb_p)
        self.rbd_items = helpers.fill_cache_wdqs(self.eu_rbd_p)

    def set_common_values(self, data):
        """
        Set and validate values shared by every SWB in the dataset.

        :param data: dict of all the SWB:s in the RBD
        """
        super(SwbBot, self).set_common_values(data)
        try:
            rbd_q = self.rbd_items.get[data.get('euRBDCode')]
            self.rbd = self.wd.QtoItemPage(rbd_q)
        except KeyError:
            raise UnmappedValueError('online rbd objects',
                                     data.get('euRBDCode'))

        self.descriptions = self.mappings.get('descriptions').get('SWB')
        WfdBot.validate_mapping(self.descriptions, self.langs, 'descriptions')

    def process_all_swb(self, data):
        """
        Handle all the surface water bodies in a single RBD.

        Only increments counter when an swb is updated.

        :param data: dict of all the swb:s in the RBD
        """
        count = 0
        for entry_data in data:
            if self.cutoff and count >= self.cutoff:
                break
            eu_swb_code = entry_data.get('euSurfaceWaterBodyCode')
            item = None
            if eu_swb_code in self.swb_items:
                item = self.wd.QtoItemPage(self.swb_items[eu_swb_code])

            if item or self.new:
                self.process_single_swb(entry_data, item)
                count += 1

    def process_single_swb(self, data, item):
        """
        Process a swb (whether item exists or not).

        :param data: dict of data for a single swb
        :param item: Wikidata item associated with a swb, or None if one
            should be created.
        """
        item = item or self.create_new_swb_item(data)
        item.exists()  # load the item contents

        # Determine claims
        name = data.get('surfaceWaterBodyName')  # in English per page 33
        protoclaims = self.make_protoclaims(data)

        # Upload claims
        self.wd.addLabelOrAlias('en', name, item)
        self.commit_claims(protoclaims, item)

    def create_new_swb_item(self, data):
        """
        Create a new swb item with some basic info and return.

        The new item is created with a label, descriptions and
        an eu_swb_code claim. This allows the item to be found in a second
        pass should the script crash before adding the protoclaims.

        The eu_swb_code is added without reference (this is handled
        during the protoclaim stage)
        The added label is in english per p.33 of the WFD specifications.
        The protoclaim stage should recognise that the label has already been
        added and thus not add it again.

        :param data: dict of data for a single swb
        :return: pywikibot.ItemPage
        """<commit_msg>Do not use non-names for Label or Alias (#7)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and source statements about Surface Water Bodies.

Author: Lokal_Profil
License: MIT

usage:
    python swb_import.py [OPTIONS]

&params;
"""
from __future__ import unicode_literals
from builtins import dict

import pywikibot

import wikidataStuff.helpers as helpers
from wikidataStuff.WikidataStuff import WikidataStuff as WdS

from WFD.WFDBase import WfdBot, UnmappedValueError

parameter_help = """\
SwbBot options (may be omitted):
-year              Year to which the WFD data applies.
-new               if present new items are created on Wikidata, otherwise
                   only updates are processed.
-in_file           path to the data file
-mappings          path to the mappings file (if not "mappings.json")
-cutoff            number items to process before stopping (if not then all)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-dir               directory in which user_config is located
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}
EDIT_SUMMARY = 'importing #SWB using data from #WFD'


class SwbBot(WfdBot):
    """Bot to enrich/create info on Wikidata for SWB objects."""

    def __init__(self, mappings, year, new=False, cutoff=None):
        """
        Initialise the SwbBot.

        :param mappings: dict holding data for any offline mappings
        :param year: year of the report (used to date certain statements).
        :param new: whether to also create new items
        :param cutoff: the number of items to process before stopping. None
            being interpreted as all.
        """
        super(SwbBot, self).__init__(mappings, year, new, cutoff,
                                     EDIT_SUMMARY)

        self.eu_swb_p = 'P2856'  # eu_cd
        self.eu_rbd_p = 'P2965'  # euRBDCode

        self.swb_cats = mappings.get('surfaceWaterBodyCategory')
        self.impact_types = mappings.get('swSignificantImpactType')

        self.load_known_items()

    def load_known_items(self):
        """
        Load existing eu_swb and eu_rbd items.

        The loaded values are Qids (without q-prefixes)
        """
        self.swb_items = helpers.fill_cache_wdqs(self.eu_swb_p)
        self.rbd_items = helpers.fill_cache_wdqs(self.eu_rbd_p)

    def set_common_values(self, data):
        """
        Set and validate values shared by every SWB in the dataset.

        :param data: dict of all the SWB:s in the RBD
        """
        super(SwbBot, self).set_common_values(data)
        try:
            rbd_q = self.rbd_items.get[data.get('euRBDCode')]
            self.rbd = self.wd.QtoItemPage(rbd_q)
        except KeyError:
            raise UnmappedValueError('online rbd objects',
                                     data.get('euRBDCode'))

        self.descriptions = self.mappings.get('descriptions').get('SWB')
        WfdBot.validate_mapping(self.descriptions, self.langs, 'descriptions')

    def process_all_swb(self, data):
        """
        Handle all the surface water bodies in a single RBD.

        Only increments counter when an swb is updated.

        :param data: dict of all the swb:s in the RBD
        """
        count = 0
        for entry_data in data:
            if self.cutoff and count >= self.cutoff:
                break
            eu_swb_code = entry_data.get('euSurfaceWaterBodyCode')
            item = None
            if eu_swb_code in self.swb_items:
                item = self.wd.QtoItemPage(self.swb_items[eu_swb_code])

            if item or self.new:
                self.process_single_swb(entry_data, item)
                count += 1

    def process_single_swb(self, data, item):
        """
        Process a swb (whether item exists or not).

        :param data: dict of data for a single swb
        :param item: Wikidata item associated with a swb, or None if one
            should be created.
        """
        bad_names = ('not applicable', )  # known (lower case) non-names
        item = item or self.create_new_swb_item(data)
        item.exists()  # load the item contents

        # Determine claims
        name = data.get('surfaceWaterBodyName')  # in English per page 33
        if name and name.lower() in bad_names:
            name = None
        protoclaims = self.make_protoclaims(data)

        # Upload claims
        if name:
            self.wd.addLabelOrAlias('en', name, item)
        self.commit_claims(protoclaims, item)

    def create_new_swb_item(self, data):
        """
        Create a new swb item with some basic info and return.

        The new item is created with a label, descriptions and
        an eu_swb_code claim. This allows the item to be found in a second
        pass should the script crash before adding the protoclaims.

        The eu_swb_code is added without reference (this is handled
        during the protoclaim stage)
        The added label is in english per p.33 of the WFD specifications.
        The protoclaim stage should recognise that the label has already been
        added and thus not add it again.

        :param data: dict of data for a single swb
        :return: pywikibot.ItemPage
        """
{'fraction_non_alphanumeric': 0.07433730311179408, 'fraction_numerical': 0.002977333845562812, 'mean_word_length': 2.2520299812617113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2176495', 'n_tokens_mistral': 3114, 'n_tokens_neox': 2971, 'n_words': 1029}