<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import paintings from the Nationalmuseum (Sweden) to Wikidata.
    by Lokal_Profil

Based on http://git.wikimedia.org/summary/labs%2Ftools%2Fmultichill.git
    /bot/wikidata/rijksmuseum_import.py by Multichill

"""
import json
import pywikibot
from pywikibot import pagegenerators
import urllib
import pywikibot.data.wikidataquery as wdquery
import datetime
import config as config
import codecs

EDIT_SUMMARY = u'NationalmuseumBot'
COMMONS_Q = u'565'
INSTITUTION_Q = u'842858'
PAINTING_Q = u'3305213'
ICON_Q = u'132137'
MINIATURE_URL = u'http://partage.vocnet.org/part00814'
MAX_ROWS = 100  # max number of rows per request in Europeana API


class PaintingsBot:
    """
    A bot to enrich and create paintings on Wikidata
    """
    def __init__(self, dictGenerator, paintingIdProperty):
        """
        Arguments:
            * generator    - A generator that yields Dict objects.

        """
        self.generator = dictGenerator
        self.repo = pywikibot.Site().data_repository()
        self.commons = pywikibot.Site(u'commons', u'commons')

        self.paintingIdProperty = paintingIdProperty
        self.paintingIds = self.fillCache(self.paintingIdProperty)

    def fillCache(self, propertyId, queryoverride=u'', cacheMaxAge=0):
        """
        Query Wikidata to fill the cache of monuments we already have an object for
        """
        result = {}
        if queryoverride:
            query = queryoverride
        else:
            query = u'CLAIM[195:%s] AND CLAIM[%s]' % (INSTITUTION_Q, propertyId)  # collection
        wd_queryset = wdquery.QuerySet(query)

        wd_query = wdquery.WikidataQuery(cacheMaxAge=cacheMaxAge)
        data = wd_query.query(wd_queryset, props=[str(propertyId), ])

        if data.get('status').get('error') == 'OK':
            expectedItems = data.get('status').get('items')
            props = data.get('props').get(str(propertyId))
            for prop in props:
                # FIXME: This will overwrite id's that are used more than once.
                # Use with care and clean up your dataset first
                result[prop[2]] = prop[0]

            if expectedItems == len(result):
                pywikibot.output('I now have %s items in cache' % expectedItems)

        return result

    def run(self, addNew=True):
        """
        Starts the robot.
        """
        nationalmuseum = pywikibot.ItemPage(self.repo, u'Q%s' % INSTITUTION_Q)
        self.creators = {}

        # mapping prefixes to subcollections
        prefixMap = {u'NM': {u'subcol': None, u'place': u'Q%s' % INSTITUTION_Q},
                     u'NMB': {u'subcol': None, u'place': u'Q%s' % INSTITUTION_Q},
                     # u'NMI': {u'subcol': u'Q18573057', u'place': u'Q%s' % INSTITUTION_Q},
                     u'NMDrh': {u'subcol': u'Q18572999', u'place': u'Q208559'},
                     u'NMGrh': {u'subcol': u'Q2817221', u'place': u'Q714783'},
                     u'NMGu': {u'subcol': u'Q18573011', u'place': u'Q1556819'},
                     u'NMRbg': {u'subcol': u'Q18573027', u'place': u'Q1934091'},
                     u'NMStrh': {u'subcol': u'Q18573032', u'place': u'Q1416870'},
                     u'NMVst': {u'subcol': u'Q18573020', u'place': u'Q1757808'},
                     u'NMTiP': {u'subcol': u'Q18573041', u'place': u'Q927844'}
                     }

        for painting in self.generator:
            # Buh, for this one I know for sure it's in there

            # paintingId = painting['object']['proxies'][0]['about'].replace(u'/proxy/provider/90402/', u'').replace(u'_', u'-')
            ids = painting['object']['proxies'][0]['dcIdentifier']['def']
            paintingId = ids[0].replace('Inv. Nr.:', '').strip('( )')
            objId = ids[1]
            uri = u'http://emp-web-22.zetcom.ch/eMuseumPlus?service=ExternalInterface&module=collection&objectId=%s&viewType=detailView' % objId
            europeanaUrl = u'http://europeana.eu/portal/record/%s.html' % (painting['object']['about'],)
            images = self.fileFromExternalLink(uri)

            # the museum contains sevaral subcollections. Only deal with mapped ones
            if paintingId.split(' ')[0] not in prefixMap.keys():
                pywikibot.output(u'Skipped due to collection: %s' % paintingId)
                continue

            # for now skip all of the miniatures
            miniature = False
            for concept in painting['object']['concepts']:
                if concept[u'about'] == MINIATURE_URL:
                    # pywikibot.output(u'Skipping miniature')
                    miniature = True
                    break
            if miniature:
                continue

            try:
                dcCreatorName = painting['object']['proxies'][0]['dcCreator']['sv'][0].strip()
                # print dcCreatorName
            except KeyError:
                print 'skipped'
                continue

            paintingItem = None
            # newclaims = []
            if paintingId in self.paintingIds:
                paintingItemTitle = u'Q%s' % (self.paintingIds.get(paintingId),)
                # print paintingItemTitle
                paintingItem = pywikibot.ItemPage(self.repo, title=paintingItemTitle)

            elif addNew:
                # creating a new one
                data = {'labels': {},
                        'descriptions': {},
                        }

                for dcTitleLang, dcTitle in painting['object']['proxies'][0]['dcTitle'].iteritems():
                    data['labels'][dcTitleLang] = {'language': dcTitleLang,
                                                   'value': dcTitle[0]}

                if dcCreatorName:
                    if dcCreatorName == u'Okänd':
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting by unknown painter'}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij van onbekende schilder'}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning av okänd konstnär'}
                    elif dcCreatorName.startswith(u'Attributed to'):
                        attribName = dcCreatorName[len(u'Attributed to'):].strip()
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting attributed to %s' % (attribName,)}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij toegeschreven aan %s' % (attribName,)}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning tillskriven %s' % (attribName,)}
                    elif dcCreatorName.startswith(u'Manner of') or \
                         dcCreatorName.startswith(u'Copy after') or \
                         dcCreatorName.startswith(u'Workshop of') or \
                         dcCreatorName.startswith(u'Circle of'):
                        continue
                    else:
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting by %s' % (dcCreatorName,)}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij van %s' % (dcCreatorName,)}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning av %s' % (dcCreatorName,)}

                # print data
                # create new empty item and request Q-number
                identification = {}
                summary = u'%s: Creating new item with data from %s' % (EDIT_SUMMARY, europeanaUrl)
                pywikibot.output(summary)
                # monumentItem.editEntity(data, summary=summary)
                try:
                    result = self.repo.editEntity(identification, data, summary=summary)
                except pywikibot.data.api.APIError, e:
                    if e.code == u'modification-failed':
                        pywikibot.output(u'modification-failed error: skipping %s' % uri)
                        continue
                    else:
                        pywikibot.output(e)
                        exit(1)
                # print result
                paintingItemTitle = result.get(u'entity').get('id')
                paintingItem = pywikibot.ItemPage(self.repo, title=paintingItemTitle)

                # add identifier
                newclaim = pywikibot.Claim(self.repo, u'P%s' % (self.paintingIdProperty,))
                newclaim.setTarget(paintingId)
                pywikibot.output('Adding new id claim to %s' % paintingItem)
                paintingItem.addClaim(newclaim)
                self.addReference(paintingItem, newclaim, uri)

                newqualifier = pywikibot.Claim(self.repo, u'P195')  # Add collection, isQualifier=True
                newqualifier.setTarget(nationalmuseum)
                pywikibot.output('Adding new qualifier claim to %s' % paintingItem)
                newclaim.addQualifier(newqualifier)

                # add collection (and subcollection)
                newclaim = pywikibot.Claim(self.repo, u'P195')
                newclaim.setTarget(nationalmuseum)
                pywikibot.output('Adding collection claim to %s' % paintingItem)<commit_msg>Now implements disambiguation of descriptions, when needed<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import paintings from the Nationalmuseum (Sweden) to Wikidata.
    by Lokal_Profil

Based on http://git.wikimedia.org/summary/labs%2Ftools%2Fmultichill.git
    /bot/wikidata/rijksmuseum_import.py by Multichill

"""
import json
import pywikibot
from pywikibot import pagegenerators
import urllib
import pywikibot.data.wikidataquery as wdquery
import datetime
import config as config
import codecs

EDIT_SUMMARY = u'NationalmuseumBot'
COMMONS_Q = u'565'
INSTITUTION_Q = u'842858'
PAINTING_Q = u'3305213'
ICON_Q = u'132137'
MINIATURE_URL = u'http://partage.vocnet.org/part00814'
MAX_ROWS = 100  # max number of rows per request in Europeana API


class PaintingsBot:
    """
    A bot to enrich and create paintings on Wikidata
    """
    def __init__(self, dictGenerator, paintingIdProperty):
        """
        Arguments:
            * generator    - A generator that yields Dict objects.

        """
        self.generator = dictGenerator
        self.repo = pywikibot.Site().data_repository()
        self.commons = pywikibot.Site(u'commons', u'commons')

        self.paintingIdProperty = paintingIdProperty
        self.paintingIds = self.fillCache(self.paintingIdProperty)

    def fillCache(self, propertyId, queryoverride=u'', cacheMaxAge=0):
        """
        Query Wikidata to fill the cache of monuments we already have an object for
        """
        result = {}
        if queryoverride:
            query = queryoverride
        else:
            query = u'CLAIM[195:%s] AND CLAIM[%s]' % (INSTITUTION_Q, propertyId)  # collection
        wd_queryset = wdquery.QuerySet(query)

        wd_query = wdquery.WikidataQuery(cacheMaxAge=cacheMaxAge)
        data = wd_query.query(wd_queryset, props=[str(propertyId), ])

        if data.get('status').get('error') == 'OK':
            expectedItems = data.get('status').get('items')
            props = data.get('props').get(str(propertyId))
            for prop in props:
                # FIXME: This will overwrite id's that are used more than once.
                # Use with care and clean up your dataset first
                result[prop[2]] = prop[0]

            if expectedItems == len(result):
                pywikibot.output('I now have %s items in cache' % expectedItems)

        return result

    def run(self, addNew=True):
        """
        Starts the robot.
        """
        nationalmuseum = pywikibot.ItemPage(self.repo, u'Q%s' % INSTITUTION_Q)
        self.creators = {}

        # mapping prefixes to subcollections
        prefixMap = {u'NM': {u'subcol': None, u'place': u'Q%s' % INSTITUTION_Q},
                     u'NMB': {u'subcol': None, u'place': u'Q%s' % INSTITUTION_Q},
                     # u'NMI': {u'subcol': u'Q18573057', u'place': u'Q%s' % INSTITUTION_Q},
                     u'NMDrh': {u'subcol': u'Q18572999', u'place': u'Q208559'},
                     u'NMGrh': {u'subcol': u'Q2817221', u'place': u'Q714783'},
                     u'NMGu': {u'subcol': u'Q18573011', u'place': u'Q1556819'},
                     u'NMRbg': {u'subcol': u'Q18573027', u'place': u'Q1934091'},
                     u'NMStrh': {u'subcol': u'Q18573032', u'place': u'Q1416870'},
                     u'NMVst': {u'subcol': u'Q18573020', u'place': u'Q1757808'},
                     # u'NMHpd': Harpsund
                     u'NMTiP': {u'subcol': u'Q18573041', u'place': u'Q927844'}
                     }

        for painting in self.generator:
            # Buh, for this one I know for sure it's in there

            # paintingId = painting['object']['proxies'][0]['about'].replace(u'/proxy/provider/90402/', u'').replace(u'_', u'-')
            ids = painting['object']['proxies'][0]['dcIdentifier']['def']
            paintingId = ids[0].replace('Inv. Nr.:', '').strip('( )')
            objId = ids[1]
            uri = u'http://emp-web-22.zetcom.ch/eMuseumPlus?service=ExternalInterface&module=collection&objectId=%s&viewType=detailView' % objId
            europeanaUrl = u'http://europeana.eu/portal/record/%s.html' % (painting['object']['about'],)
            images = self.fileFromExternalLink(uri)

            # the museum contains sevaral subcollections. Only deal with mapped ones
            if paintingId.split(' ')[0] not in prefixMap.keys():
                pywikibot.output(u'Skipped due to collection: %s' % paintingId)
                continue

            # for now skip all of the miniatures
            miniature = False
            for concept in painting['object']['concepts']:
                if concept[u'about'] == MINIATURE_URL:
                    # pywikibot.output(u'Skipping miniature')
                    miniature = True
                    break
            if miniature:
                continue

            try:
                dcCreatorName = painting['object']['proxies'][0]['dcCreator']['sv'][0].strip()
                # print dcCreatorName
            except KeyError:
                print 'skipped'
                continue

            paintingItem = None
            # newclaims = []
            if paintingId in self.paintingIds:
                paintingItemTitle = u'Q%s' % (self.paintingIds.get(paintingId),)
                # print paintingItemTitle
                paintingItem = pywikibot.ItemPage(self.repo, title=paintingItemTitle)

            elif addNew:
                # creating a new one
                data = {'labels': {},
                        'descriptions': {},
                        }

                for dcTitleLang, dcTitle in painting['object']['proxies'][0]['dcTitle'].iteritems():
                    data['labels'][dcTitleLang] = {'language': dcTitleLang,
                                                   'value': dcTitle[0]}

                if dcCreatorName:
                    if dcCreatorName == u'Okänd':
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting by unknown painter'}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij van onbekende schilder'}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning av okänd konstnär'}
                    elif dcCreatorName.startswith(u'Attributed to'):
                        attribName = dcCreatorName[len(u'Attributed to'):].strip()
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting attributed to %s' % (attribName,)}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij toegeschreven aan %s' % (attribName,)}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning tillskriven %s' % (attribName,)}
                    elif dcCreatorName.startswith(u'Manner of') or \
                         dcCreatorName.startswith(u'Copy after') or \
                         dcCreatorName.startswith(u'Workshop of') or \
                         dcCreatorName.startswith(u'Circle of'):
                        continue
                    else:
                        data['descriptions']['en'] = {'language': u'en', 'value': u'painting by %s' % (dcCreatorName,)}
                        data['descriptions']['nl'] = {'language': u'nl', 'value': u'schilderij van %s' % (dcCreatorName,)}
                        data['descriptions']['sv'] = {'language': u'sv', 'value': u'målning av %s' % (dcCreatorName,)}

                # print data
                # create new empty item and request Q-number
                identification = {}
                summary = u'%s: Creating new item with data from %s' % (EDIT_SUMMARY, europeanaUrl)
                pywikibot.output(summary)
                # monumentItem.editEntity(data, summary=summary)
                try:
                    result = self.repo.editEntity(identification, data, summary=summary)
                except pywikibot.data.api.APIError, e:
                    if e.code == u'modification-failed':
                        # disambiguate and try again
                        pywikibot.output(u'DISAMBIGUATING')
                        for lang in data['descriptions'].keys():
                            data['descriptions'][lang]['value'] += u' (%s)' % paintingId
                        try:
                            result = self.repo.editEntity(identification, data, summary=summary)
                        except pywikibot.data.api.APIError, e:
                            if e.code == u'modification-failed':
                                pywikibot.output(u'modification-failed error: skipping %s' % uri)
                                continue
                            else:
                                pywikibot.output(e)
                                exit(1)
                    else:
                        pywikibot.output(e)
                        exit(1)
                # print result
                paintingItemTitle = result.get(u'entity').get('id')
                paintingItem = pywikibot.ItemPage(self.repo, title=paintingItemTitle)

                # add identifier
                newclaim = pywikibot.Claim(self.repo, u'P%s' % (self.paintingIdProperty,))
                newclaim.setTarget(paintingId)
                pywikibot.output('Adding new id claim to %s' % paintingItem)
                paintingItem.addClaim(newclaim)
                self.addReference(paintingItem, newclaim, uri)

                newqualifier = pywikibot.Claim(self.repo, u'P195')  # Add collection, isQualifier=True
                newqualifier.setTarget(nationalmuseum)
                pywikibot.output('Adding new qualifier claim to %s' % paintingItem)
                newclaim.addQualifier(newqualifier)

                # add collection (and subcollection)
                newclaim = pywikibot.Claim(self.repo, u'P195')
                newclaim.setTarget(nationalmuseum)
                pywikibot.output('Adding collection claim to %s' % paintingItem)
{'fraction_non_alphanumeric': 0.12401493156366653, 'fraction_numerical': 0.017783077561177935, 'mean_word_length': 1.9124263928733203, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359308', 'n_tokens_mistral': 5948, 'n_tokens_neox': 5449, 'n_words': 1188}