<commit_before>import difflib
import datetime
import json
from collections import defaultdict, Counter
from time import gmtime, strftime

import requests
from dateutil import parser as du

from wikidataintegrator.ref_handlers import update_retrieved_if_new
from . import wdi_core
from .wdi_core import WDItemEngine, WDApiError
from wikidataintegrator.wdi_config import config


class Release(object):
    """
    Create a release item
    Example: Ensembl Release 86 Q27613766

    A relase item has the following required properties:
        title: **user_defined**
        description: **user_defined**
        instance of (P31): edition (Q3331189)
        edition or translation of (P629): **user_defined** (wd_item)
        edition number (P393): **user_defined** (str)
    Optional properties:
        archive URL (P1065): (str)
        publication date (P577): (wd_date)

    Example usage:
    r = PBB_Helpers.Release("Ensembl Release 85", "Release 85 of Ensembl", "85", edition_of="Ensembl",
                            archive_url="http://jul2016.archive.ensembl.org/",
                            pub_date='+2016-07-01T00:00:00Z', date_precision=10)
    r.create(login)

    """

    _release_cache = defaultdict(dict)
    _database_cache = dict()

    def __init__(self, title, description, edition, edition_of=None, edition_of_wdid=None, archive_url=None,
                 pub_date=None, date_precision=11):
        """

        :param title: title of release item
        :type title: str
        :param description: description of release item
        :type description: str
        :param edition: edition number or unique identifier for the release
        :type edition: str
        :param edition_of: name of database. database wdid will automatically be looked up. Must pass either edition_of or edition_of_wdid
        :type edition_of: str
        :param edition_of_wdid: wikidata id of database
        :type edition_of_wdid: str
        :param archive_url: (optional)
        :type archive_url: str
        :param pub_date: (optional) Datetime will be converted to str
        :type pub_date: str or datetime
        :param date_precision: (optional) passed to PBB_Core.WDTime as is. default is 11 (day)
        :type date_precision: int
        """
        self.title = title
        self.description = description
        self.edition = str(edition)
        self.archive_url = archive_url
        if isinstance(pub_date, datetime.date):
            self.pub_date = pub_date.strftime('+%Y-%m-%dT%H:%M:%SZ')
        else:
            self.pub_date = pub_date
        self.date_precision = date_precision

        if (edition_of is None and edition_of_wdid is None) or (edition_of and edition_of_wdid):
            raise ValueError("must provide either edition_of or edition_of_wdid")
        if edition_of:
            self.edition_of_wdid = self.lookup_database(edition_of)
        else:
            self.edition_of_wdid = edition_of_wdid

        self.statements = None
        self.make_statements()

    @classmethod
    def lookup_database(cls, edition_of):
        if edition_of in cls._database_cache:
            return cls._database_cache[edition_of]

        query = """SELECT ?item ?itemLabel WHERE {
                    ?item wdt:P31 wd:Q4117139 .
                    SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
                }"""
        results = WDItemEngine.execute_sparql_query(query)
        db_item_map = {x['itemLabel']['value']: x['item']['value'].replace('http://www.wikidata.org/entity/', '') for x
                       in results['results']['bindings']}
        if edition_of not in db_item_map:
            raise ValueError("Database {} not found in wikidata. Please provide edition_of_wdid".format(edition_of))

        cls._database_cache[edition_of] = db_item_map[edition_of]
        return db_item_map[edition_of]

    def make_statements(self):
        s = []
        # instance of edition
        s.append(wdi_core.WDItemID('Q3331189', 'P31'))
        # edition or translation of
        s.append(wdi_core.WDItemID(self.edition_of_wdid, 'P629'))
        # edition number
        s.append(wdi_core.WDString(self.edition, 'P393'))

        if self.archive_url:
            s.append(wdi_core.WDUrl(self.archive_url, 'P1065'))

        if self.pub_date:
            s.append(wdi_core.WDTime(self.pub_date, 'P577', precision=self.date_precision))

        self.statements = s

    def get_or_create(self, login=None):

        # check in cache
        if self.edition_of_wdid in self._release_cache and self.edition in self._release_cache[self.edition_of_wdid]:
            return self._release_cache[self.edition_of_wdid][self.edition]

        # check in wikidata
        edition_dict = id_mapper("P393", (("P629", self.edition_of_wdid), ("P31", "Q3331189")))
        if edition_dict and self.edition in edition_dict:
            # add to cache
            self._release_cache[self.edition_of_wdid][self.edition] = edition_dict[self.edition]
            return edition_dict[self.edition]

        # create new
        if login is None:
            raise ValueError("login required to create item")
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="release")
        item.set_label(self.title)
        item.set_description(description=self.description, lang='en')
        write_success = try_write(item, self.edition + "|" + self.edition_of_wdid, 'P393|P629', login)
        if write_success:
            # add to cache
            self._release_cache[self.edition_of_wdid][self.edition] = item.wd_item_id
            return item.wd_item_id
        else:
            raise write_success


class PubmedItem(object):
    """
    Get or create a wikidata item stub given a pubmed ID

    Usage:
    PubmedItem(22674334).get_or_create(login)

    Data source: Europepmc
    API: https://europepmc.org/RestfulWebService
    Doc: https://europepmc.org/docs/EBI_Europe_PMC_Web_Service_Reference.pdf

    wikidata Item format
    general guidelines: https://www.wikidata.org/wiki/Help:Sources
    more specific to journal articles: https://github.com/mitar/bib2wikidata/issues/1#issuecomment-50998834
    modelling after: https://www.wikidata.org/wiki/Q21093381 because it has both authors and author name strings

    This will add authors, if they have orcid IDs AND the item already exists in wikidata

    https://www.wikidata.org/wiki/Wikidata:WikiProject_Source_MetaData/Bibliographic_metadata_for_scholarly_articles_in_Wikidata

    """

    _cache = dict()
    PROPS = {
        'instance of': 'P31',
        'title': 'P1476',
        'published in': 'P1433',
        'volume': 'P478',
        'publication date': 'P577',
        'page(s)': 'P304',
        'issue': 'P433',
        'author name string': 'P2093',
        'author': 'P50',
        'PubMed ID': 'P698',
        'MED': 'P698',
        'DOI': 'P356',
        'series ordinal': 'P1545',
        'PMCID': 'P932',
        'PMC': 'P932',
    }

    id_types = {"MED", "PMC", "EUROPEPMC", "PAT", "NBK", "HIR", "ETH", "CTX", "CBA", "AGR", "DOI"}

    pubtypes = {"research-article": "Q13442814",  # scientific article
                "Journal Article": "Q13442814",  # scientific article
                }
    descriptions = {'Q13442814': 'scientific article'}

    def __init__(self, ext_id, id_type="MED"):
        self.warnings = []
        self.errors = []
        assert id_type in self.id_types, "id_type must be in {}".format(self.id_types)
        self.ext_id = str(ext_id)
        self.id_type = id_type
        self.meta = {
            'title': None,
            'doi': None,
            'volume': None,
            'issue': None,
            'journal_wdid': None,
            'journal_issn': None,
            'type': None,
            'date': None,
            'pages': None,
            'authors': None,
            'pubtypes': None,
            'pubtype_qid': None
        }

        self.reference = None
        self.statements = None
        self.wdid = None
        self.article = None
        self.user_agent = config['USER_AGENT_DEFAULT']

    def get_article_info(self):
        if self.id_type == "PMC":
            url = 'http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMCID:{}&resulttype=core&format=json'
            url = url.format(self.ext_id)
        elif self.id_type == "DOI":
            url = "http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:%22{}%22&resulttype=core&format=json"
            url = url.format(self.ext_id)
        elif self.id_type != "DOI":
            url = 'http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=EXT_ID:{}%20AND%20SRC:{}&resulttype=core&format=json'
            url = url.format(self.ext_id, self.id_type)
        else:
            raise ValueError()
        headers = {
            'User-Agent': self.user_agent
        }
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        d = response.json()
        if d['hitCount'] != 1:
            raise ValueError("No results")
        self.article = d['resultList']['result'][0]

    def get_metadata(self):
        if not self.article:
            self.get_article_info()

        self.meta['pmid'] = self.article['pmid']
        self.meta['pmcid'] = self.article.get('pmcid')
        self.meta['title'] = self.article['title'][:249]
        if 'pubTypeList' in self.article:
            pubtypes = self.article['pubTypeList']['pubType']
            if not isinstance(pubtypes, list):
                pubtypes = [pubtypes]
            self.meta['pubtypes'] = pubtypes
        else:
            self.meta['pubtypes'] = ['research-article']
            self.warnings.append("unknown publication type, assuming scientific article")
            print("unknown publication type, assuming scientific article")
        self.meta['pubtype_qid'] = list(set(self.pubtypes[x] for x in self.meta['pubtypes'] if x in self.pubtypes))
        if len(self.meta['pubtype_qid']) < 1:
            self.errors.append("unknown publication type: {}".format(self.meta['pubtypes']))
            raise ValueError("unknown publication type: {}".format(self.meta['pubtypes']))

        if 'issn' in self.article['journalInfo']['journal']:
            self.meta['journal_issn'] = self.article['journalInfo']['journal']['issn']  # published in
        elif 'essn' in self.article['journalInfo']['journal']:
            self.meta['journal_issn'] = self.article['journalInfo']['journal']['essn']  # published in
        else:
            msg = "unknown journal: {}".format(self.article['journalInfo']['journal'])
            self.errors.append(msg)
            print(msg)

        self.meta['journal_wdid'] = prop2qid("P236", self.meta['journal_issn'])
        if not self.meta['journal_wdid']:
            raise ValueError("journal not found: {}".format(self.meta['journal_issn']))
        self.meta['date'] = du.parse(self.article['firstPublicationDate']).strftime('+%Y-%m-%dT%H:%M:%SZ')

        authors = []
        for author in self.article['authorList']['author']:
            if 'firstName' in author and 'lastName' in author:
                authors.append(author['firstName'] + ' ' + author['lastName'])
            elif 'fullName' in author:
                authors.append(author['fullName'])
            elif 'collectiveName' in author:
                authors.append(author['collectiveName'])
            else:
                msg = "unknown author: {}".format(author)
                print(msg)
                self.errors.append(msg)
                authors.append(None)

        self.meta['authors'] = authors
        self.meta['author_orcid'] = {author: x['authorId']['value'] for author, x in
                                     zip(authors, self.article['authorList']['author']) if
                                     'authorId' in x and x['authorId']['type'] == 'ORCID'}
        self.meta['author_qid'] = {author: PubmedItem.lookup_author(orcid) for author, orcid in
                                   self.meta['author_orcid'].items()}

        # optional
        self.meta['volume'] = self.article['journalInfo'].get('volume')
        self.meta['issue'] = self.article['journalInfo'].get('issue')
        self.meta['pages'] = self.article.get('pageInfo')
        self.meta['doi'] = self.article.get('doi')

    @staticmethod
    def lookup_author(orcid_id):
        return prop2qid("P496", orcid_id)

    def make_reference(self):
        if 'pmid' not in self.meta:
            self.get_metadata()
        stated_in = wdi_core.WDItemID(value='Q5412157', prop_nr='P248', is_reference=True)
        pmid = wdi_core.WDString(value=self.meta['pmid'], prop_nr='P698', is_reference=True)
        retrieved = wdi_core.WDTime(strftime("+%Y-%m-%dT00:00:00Z", gmtime()), prop_nr='P813', is_reference=True)
        self.reference = [stated_in, pmid, retrieved]

    def make_author_statements(self, ordinals=None):
        """
        ordinals is a list of ordinals to NOT include
        use this to update an existing item that may have authors, so you don't want to
        add the author string for them
        :param ordinals: if None, include all authors
        :return:
        """
        s = []
        for n, author in enumerate(self.meta['authors'], 1):
            if ordinals is not None and n in ordinals:
                continue
            series_ordinal = wdi_core.WDString(str(n), self.PROPS['series ordinal'], is_qualifier=True)
            if author in self.meta['author_orcid']:
                if self.meta['author_qid'][author]:
                    s.append(wdi_core.WDItemID(self.meta['author_qid'][author], self.PROPS['author'],
                                               references=[self.reference], qualifiers=[series_ordinal]))
                else:
                    msg = "No Author found for author: {}, orcid: {}".format(author, self.meta['author_orcid'][author])
                    self.warnings.append(msg)
                    print(msg)
                    s.append(wdi_core.WDString(author, self.PROPS['author name string'],
                                               references=[self.reference], qualifiers=[series_ordinal]))
            elif author:
                s.append(wdi_core.WDString(author, self.PROPS['author name string'],
                                           references=[self.reference], qualifiers=[series_ordinal]))<commit_msg>see #29<commit_after>import difflib
import datetime
import json
from collections import defaultdict, Counter
from time import gmtime, strftime

import requests
from dateutil import parser as du

from wikidataintegrator.ref_handlers import update_retrieved_if_new
from . import wdi_core
from .wdi_core import WDItemEngine, WDApiError
from wikidataintegrator.wdi_config import config


class Release(object):
    """
    Create a release item
    Example: Ensembl Release 86 Q27613766

    A relase item has the following required properties:
        title: **user_defined**
        description: **user_defined**
        instance of (P31): edition (Q3331189)
        edition or translation of (P629): **user_defined** (wd_item)
        edition number (P393): **user_defined** (str)
    Optional properties:
        archive URL (P1065): (str)
        publication date (P577): (wd_date)

    Example usage:
    r = PBB_Helpers.Release("Ensembl Release 85", "Release 85 of Ensembl", "85", edition_of="Ensembl",
                            archive_url="http://jul2016.archive.ensembl.org/",
                            pub_date='+2016-07-01T00:00:00Z', date_precision=10)
    r.create(login)

    """

    _release_cache = defaultdict(dict)
    _database_cache = dict()

    def __init__(self, title, description, edition, edition_of=None, edition_of_wdid=None, archive_url=None,
                 pub_date=None, date_precision=11):
        """

        :param title: title of release item
        :type title: str
        :param description: description of release item
        :type description: str
        :param edition: edition number or unique identifier for the release
        :type edition: str
        :param edition_of: name of database. database wdid will automatically be looked up. Must pass either edition_of or edition_of_wdid
        :type edition_of: str
        :param edition_of_wdid: wikidata id of database
        :type edition_of_wdid: str
        :param archive_url: (optional)
        :type archive_url: str
        :param pub_date: (optional) Datetime will be converted to str
        :type pub_date: str or datetime
        :param date_precision: (optional) passed to PBB_Core.WDTime as is. default is 11 (day)
        :type date_precision: int
        """
        self.title = title
        self.description = description
        self.edition = str(edition)
        self.archive_url = archive_url
        if isinstance(pub_date, datetime.date):
            self.pub_date = pub_date.strftime('+%Y-%m-%dT%H:%M:%SZ')
        else:
            self.pub_date = pub_date
        self.date_precision = date_precision

        if (edition_of is None and edition_of_wdid is None) or (edition_of and edition_of_wdid):
            raise ValueError("must provide either edition_of or edition_of_wdid")
        if edition_of:
            self.edition_of_wdid = self.lookup_database(edition_of)
        else:
            self.edition_of_wdid = edition_of_wdid

        self.statements = None
        self.make_statements()

    @classmethod
    def lookup_database(cls, edition_of):
        if edition_of in cls._database_cache:
            return cls._database_cache[edition_of]

        query = """SELECT ?item ?itemLabel WHERE {
                    ?item wdt:P31 wd:Q4117139 .
                    SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
                }"""
        results = WDItemEngine.execute_sparql_query(query)
        db_item_map = {x['itemLabel']['value']: x['item']['value'].replace('http://www.wikidata.org/entity/', '') for x
                       in results['results']['bindings']}
        if edition_of not in db_item_map:
            raise ValueError("Database {} not found in wikidata. Please provide edition_of_wdid".format(edition_of))

        cls._database_cache[edition_of] = db_item_map[edition_of]
        return db_item_map[edition_of]

    def make_statements(self):
        s = []
        # instance of edition
        s.append(wdi_core.WDItemID('Q3331189', 'P31'))
        # edition or translation of
        s.append(wdi_core.WDItemID(self.edition_of_wdid, 'P629'))
        # edition number
        s.append(wdi_core.WDString(self.edition, 'P393'))

        if self.archive_url:
            s.append(wdi_core.WDUrl(self.archive_url, 'P1065'))

        if self.pub_date:
            s.append(wdi_core.WDTime(self.pub_date, 'P577', precision=self.date_precision))

        self.statements = s

    def get_or_create(self, login=None):

        # check in cache
        if self.edition_of_wdid in self._release_cache and self.edition in self._release_cache[self.edition_of_wdid]:
            return self._release_cache[self.edition_of_wdid][self.edition]

        # check in wikidata
        edition_dict = id_mapper("P393", (("P629", self.edition_of_wdid), ("P31", "Q3331189")))
        if edition_dict and self.edition in edition_dict:
            # add to cache
            self._release_cache[self.edition_of_wdid][self.edition] = edition_dict[self.edition]
            return edition_dict[self.edition]

        # create new
        if login is None:
            raise ValueError("login required to create item")
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="release")
        item.set_label(self.title)
        item.set_description(description=self.description, lang='en')
        write_success = try_write(item, self.edition + "|" + self.edition_of_wdid, 'P393|P629', login)
        if write_success:
            # add to cache
            self._release_cache[self.edition_of_wdid][self.edition] = item.wd_item_id
            return item.wd_item_id
        else:
            raise write_success


class PubmedItem(object):
    """
    Get or create a wikidata item stub given a pubmed ID

    Usage:
    PubmedItem(22674334).get_or_create(login)

    Data source: Europepmc
    API: https://europepmc.org/RestfulWebService
    Doc: https://europepmc.org/docs/EBI_Europe_PMC_Web_Service_Reference.pdf

    wikidata Item format
    general guidelines: https://www.wikidata.org/wiki/Help:Sources
    more specific to journal articles: https://github.com/mitar/bib2wikidata/issues/1#issuecomment-50998834
    modelling after: https://www.wikidata.org/wiki/Q21093381 because it has both authors and author name strings

    This will add authors, if they have orcid IDs AND the item already exists in wikidata

    https://www.wikidata.org/wiki/Wikidata:WikiProject_Source_MetaData/Bibliographic_metadata_for_scholarly_articles_in_Wikidata

    """

    _cache = dict()
    PROPS = {
        'instance of': 'P31',
        'title': 'P1476',
        'published in': 'P1433',
        'volume': 'P478',
        'publication date': 'P577',
        'page(s)': 'P304',
        'issue': 'P433',
        'author name string': 'P2093',
        'author': 'P50',
        'PubMed ID': 'P698',
        'MED': 'P698',
        'DOI': 'P356',
        'series ordinal': 'P1545',
        'PMCID': 'P932',
        'PMC': 'P932',
    }

    id_types = {"MED", "PMC", "EUROPEPMC", "PAT", "NBK", "HIR", "ETH", "CTX", "CBA", "AGR", "DOI"}

    pubtypes = {"research-article": "Q13442814",  # scientific article
                "Journal Article": "Q13442814",  # scientific article
                }
    descriptions = {'Q13442814': 'scientific article'}

    def __init__(self, ext_id, id_type="MED"):
        self.warnings = []
        self.errors = []
        assert id_type in self.id_types, "id_type must be in {}".format(self.id_types)
        self.ext_id = str(ext_id)
        self.id_type = id_type
        self.meta = {
            'title': None,
            'doi': None,
            'volume': None,
            'issue': None,
            'journal_wdid': None,
            'journal_issn': None,
            'type': None,
            'date': None,
            'pages': None,
            'authors': None,
            'pubtypes': None,
            'pubtype_qid': None
        }
        if self.id_type == "PMC" and self.ext_id.startswith("PMC"):
            self.ext_id = self.ext_id.replace("PMC", "")

        self.reference = None
        self.statements = None
        self.wdid = None
        self.article = None
        self.user_agent = config['USER_AGENT_DEFAULT']

    def get_article_info(self):
        if self.id_type == "PMC":
            url = 'http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMCID:PMC{}&resulttype=core&format=json'
            url = url.format(self.ext_id)
        elif self.id_type == "DOI":
            url = "http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:%22{}%22&resulttype=core&format=json"
            url = url.format(self.ext_id)
        elif self.id_type != "DOI":
            url = 'http://www.ebi.ac.uk/europepmc/webservices/rest/search?query=EXT_ID:{}%20AND%20SRC:{}&resulttype=core&format=json'
            url = url.format(self.ext_id, self.id_type)
        else:
            raise ValueError()
        headers = {
            'User-Agent': self.user_agent
        }
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        d = response.json()
        if d['hitCount'] != 1:
            raise ValueError("No results")
        self.article = d['resultList']['result'][0]

    def get_metadata(self):
        if not self.article:
            self.get_article_info()

        self.meta['pmid'] = self.article['pmid']
        self.meta['pmcid'] = self.article.get('pmcid', '').replace("PMC", "")
        self.meta['title'] = self.article['title'][:249]
        if 'pubTypeList' in self.article:
            pubtypes = self.article['pubTypeList']['pubType']
            if not isinstance(pubtypes, list):
                pubtypes = [pubtypes]
            self.meta['pubtypes'] = pubtypes
        else:
            self.meta['pubtypes'] = ['research-article']
            self.warnings.append("unknown publication type, assuming scientific article")
            print("unknown publication type, assuming scientific article")
        self.meta['pubtype_qid'] = list(set(self.pubtypes[x] for x in self.meta['pubtypes'] if x in self.pubtypes))
        if len(self.meta['pubtype_qid']) < 1:
            self.errors.append("unknown publication type: {}".format(self.meta['pubtypes']))
            raise ValueError("unknown publication type: {}".format(self.meta['pubtypes']))

        if 'issn' in self.article['journalInfo']['journal']:
            self.meta['journal_issn'] = self.article['journalInfo']['journal']['issn']  # published in
        elif 'essn' in self.article['journalInfo']['journal']:
            self.meta['journal_issn'] = self.article['journalInfo']['journal']['essn']  # published in
        else:
            msg = "unknown journal: {}".format(self.article['journalInfo']['journal'])
            self.errors.append(msg)
            print(msg)

        self.meta['journal_wdid'] = prop2qid("P236", self.meta['journal_issn'])
        if not self.meta['journal_wdid']:
            raise ValueError("journal not found: {}".format(self.meta['journal_issn']))
        self.meta['date'] = du.parse(self.article['firstPublicationDate']).strftime('+%Y-%m-%dT%H:%M:%SZ')

        authors = []
        for author in self.article['authorList']['author']:
            if 'firstName' in author and 'lastName' in author:
                authors.append(author['firstName'] + ' ' + author['lastName'])
            elif 'fullName' in author:
                authors.append(author['fullName'])
            elif 'collectiveName' in author:
                authors.append(author['collectiveName'])
            else:
                msg = "unknown author: {}".format(author)
                print(msg)
                self.errors.append(msg)
                authors.append(None)

        self.meta['authors'] = authors
        self.meta['author_orcid'] = {author: x['authorId']['value'] for author, x in
                                     zip(authors, self.article['authorList']['author']) if
                                     'authorId' in x and x['authorId']['type'] == 'ORCID'}
        self.meta['author_qid'] = {author: PubmedItem.lookup_author(orcid) for author, orcid in
                                   self.meta['author_orcid'].items()}

        # optional
        self.meta['volume'] = self.article['journalInfo'].get('volume')
        self.meta['issue'] = self.article['journalInfo'].get('issue')
        self.meta['pages'] = self.article.get('pageInfo')
        self.meta['doi'] = self.article.get('doi')

    @staticmethod
    def lookup_author(orcid_id):
        return prop2qid("P496", orcid_id)

    def make_reference(self):
        if 'pmid' not in self.meta:
            self.get_metadata()
        if 'pmcid' in self.meta and self.meta['pmcid']:
            pmid = wdi_core.WDString(value=self.meta['pmcid'], prop_nr='P932', is_reference=True)
        else:
            pmid = wdi_core.WDString(value=self.meta['pmid'], prop_nr='P698', is_reference=True)
        stated_in = wdi_core.WDItemID(value='Q5412157', prop_nr='P248', is_reference=True)
        retrieved = wdi_core.WDTime(strftime("+%Y-%m-%dT00:00:00Z", gmtime()), prop_nr='P813', is_reference=True)
        self.reference = [stated_in, pmid, retrieved]

    def make_author_statements(self, ordinals=None):
        """
        ordinals is a list of ordinals to NOT include
        use this to update an existing item that may have authors, so you don't want to
        add the author string for them
        :param ordinals: if None, include all authors
        :return:
        """
        s = []
        for n, author in enumerate(self.meta['authors'], 1):
            if ordinals is not None and n in ordinals:
                continue
            series_ordinal = wdi_core.WDString(str(n), self.PROPS['series ordinal'], is_qualifier=True)
            if author in self.meta['author_orcid']:
                if self.meta['author_qid'][author]:
                    s.append(wdi_core.WDItemID(self.meta['author_qid'][author], self.PROPS['author'],
                                               references=[self.reference], qualifiers=[series_ordinal]))
                else:
                    msg = "No Author found for author: {}, orcid: {}".format(author, self.meta['author_orcid'][author])
                    self.warnings.append(msg)
                    print(msg)
                    s.append(wdi_core.WDString(author, self.PROPS['author name string'],
                                               references=[self.reference], qualifiers=[series_ordinal]))
            elif author:
                s.append(wdi_core.WDString(author, self.PROPS['author name string'],
                                           references=[self.reference], qualifiers=[series_ordinal]))
{'fraction_non_alphanumeric': 0.12263408289694357, 'fraction_numerical': 0.017695177465174385, 'mean_word_length': 2.4233157586408907, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5695983', 'n_tokens_mistral': 9200, 'n_tokens_neox': 8733, 'n_words': 1903}