<commit_before>import datetime
import functools
import json
import xml.etree.ElementTree as ET
from time import gmtime, strftime

import requests

from . import wdi_core
from .wdi_core import WDItemEngine, WDApiError


class Release(object):
    """
    Create a release item
    Example: Ensembl Release 86 Q27613766

    A relase item has the following required properties:
        title: **user_defined**
        description: **user_defined**
        instance of (P31): edition (Q3331189)
        edition or translation of (P629): **user_defined** (wd_item)
        edition number (P393): **user_defined** (str)
    Optional properties:
        archive URL (P1065): (str)
        publication date (P577): (wd_date)

    Example usage:
    r = PBB_Helpers.Release("Ensembl Release 85", "Release 85 of Ensembl", "85", edition_of="Ensembl",
                            archive_url="http://jul2016.archive.ensembl.org/",
                            pub_date='+2016-07-01T00:00:00Z', date_precision=10)
    r.create(login)

    """

    def __init__(self, title, description, edition, edition_of=None, edition_of_wdid=None, archive_url=None,
                 pub_date=None, date_precision=11):
        """

        :param title: title of release item
        :type title: str
        :param description: description of release item
        :type description: str
        :param edition: edition number or unique identifier for the release
        :type edition: str
        :param edition_of: name of database. database wdid will automatically be looked up. Must pass either edition_of or edition_of_wdid
        :type edition_of: str
        :param edition_of_wdid: wikidata id of database
        :type edition_of_wdid: str
        :param archive_url: (optional)
        :type archive_url: str
        :param pub_date: (optional) Datetime will be converted to str
        :type pub_date: str or datetime
        :param date_precision: (optional) passed to PBB_Core.WDTime as is. default is 11 (day)
        :type date_precision: int
        """
        self.title = title
        self.description = description
        self.edition = str(edition)
        self.archive_url = archive_url
        if isinstance(pub_date, datetime.date):
            self.pub_date = pub_date.strftime('+%Y-%m-%dT%H:%M:%SZ')
        else:
            self.pub_date = pub_date
        self.date_precision = date_precision

        if (edition_of is None and edition_of_wdid is None) or (edition_of and edition_of_wdid):
            raise ValueError("must provide either edition_of or edition_of_wdid")
        if edition_of:
            self.edition_of_wdid = self.lookup_database(edition_of)
        else:
            self.edition_of_wdid = edition_of_wdid

        self.statements = None
        self.make_statements()

    @staticmethod
    @functools.lru_cache()
    def lookup_database(edition_of):
        query = """SELECT ?item ?itemLabel WHERE {
                    ?item wdt:P31 wd:Q4117139 .
                    SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
                }"""
        results = WDItemEngine.execute_sparql_query(query)
        db_item_map = {x['itemLabel']['value']: x['item']['value'].replace('http://www.wikidata.org/entity/', '') for x
                       in results['results']['bindings']}
        if edition_of not in db_item_map:
            raise ValueError("Database {} not found in wikidata. Please provide edition_of_wdid".format(edition_of))
        return db_item_map[edition_of]

    @functools.lru_cache()
    def release_exists(self):
        edition_dict = id_mapper("P393", (("P629", self.edition_of_wdid), ("P31", "Q3331189")))
        return edition_dict.get(self.edition, False)

    def make_statements(self):
        s = []
        # instance of edition
        s.append(wdi_core.WDItemID('Q3331189', 'P31'))
        # edition or translation of
        s.append(wdi_core.WDItemID(self.edition_of_wdid, 'P629'))
        # edition number
        s.append(wdi_core.WDString(self.edition, 'P393'))

        if self.archive_url:
            s.append(wdi_core.WDUrl(self.archive_url, 'P1065'))

        if self.pub_date:
            s.append(wdi_core.WDTime(self.pub_date, 'P577', precision=self.date_precision))

        self.statements = s

    def get_or_create(self, login):
        re = self.release_exists()
        if re:
            return re
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="release")
        item.set_label(self.title)
        item.set_description(description=self.description, lang='en')
        try_write(item, self.edition + "|" + self.edition_of_wdid, 'P393|P629', login)
        return item.wd_item_id


class PubmedStub(object):
    """
    Get or create a wikidata item stub given a pubmed ID

    Usage:
    PubmedStub(22674334).create(login)

    """

    def __init__(self, pmid):
        self.pmid = str(pmid)
        self.title = None
        self.dois = None
        self.reference = None
        self.statements = None
        self._root = None
        self.wdid = prop2qid('P698', self.pmid)  # if doesn't exist, will be None

    def _validate_pmid(self):
        pubmed_url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=Pubmed&Retmode=xml&id={}'.format(
            self.pmid)
        r = requests.get(pubmed_url)
        self._root = ET.fromstring(r.text)
        return True if self._root.find(".//MedlineCitation/Article/ArticleTitle") is not None else False

    def _parse_ncbi(self):
        self.title = self._root.find(".//MedlineCitation/Article/ArticleTitle").text
        doiset = self._root.findall(".//ArticleIdList/ArticleId[@IdType='doi']")
        self.dois = [x.text for x in doiset]

    def _make_reference(self):
        refStatedIn = wdi_core.WDItemID(value='Q180686', prop_nr='P248', is_reference=True)
        refPubmedId = wdi_core.WDString(value=self.pmid, prop_nr='P698', is_reference=True)
        timeStringNow = strftime("+%Y-%m-%dT00:00:00Z", gmtime())
        refRetrieved = wdi_core.WDTime(timeStringNow, prop_nr='P813', is_reference=True)
        self.reference = [refStatedIn, refPubmedId, refRetrieved]

    def _make_statements(self):
        s = []
        # instance of scientific article
        s.append(wdi_core.WDItemID('Q13442814', 'P31', references=[self.reference]))
        # pmid
        s.append(wdi_core.WDExternalID(self.pmid, 'P698', references=[self.reference]))
        # title
        s.append(wdi_core.WDMonolingualText(self.title, 'P1476', references=[self.reference]))
        for doi in self.dois:
            s.append(wdi_core.WDExternalID(doi, 'P356', references=[self.reference]))
        self.statements = s

    def create(self, login):
        if self.wdid:
            return self.wdid
        if not self._validate_pmid():
            print("invalid pmid: {}".format(self.wdid))
            return None
        self._parse_ncbi()
        self._make_reference()
        self._make_statements()
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="scientific_article")
        item.set_label(self.title)<commit_msg>memoize pubmed stub creator. add published in statement<commit_after>import datetime
import functools
import json
import xml.etree.ElementTree as ET
from time import gmtime, strftime

import requests

from . import wdi_core
from .wdi_core import WDItemEngine, WDApiError


class Release(object):
    """
    Create a release item
    Example: Ensembl Release 86 Q27613766

    A relase item has the following required properties:
        title: **user_defined**
        description: **user_defined**
        instance of (P31): edition (Q3331189)
        edition or translation of (P629): **user_defined** (wd_item)
        edition number (P393): **user_defined** (str)
    Optional properties:
        archive URL (P1065): (str)
        publication date (P577): (wd_date)

    Example usage:
    r = PBB_Helpers.Release("Ensembl Release 85", "Release 85 of Ensembl", "85", edition_of="Ensembl",
                            archive_url="http://jul2016.archive.ensembl.org/",
                            pub_date='+2016-07-01T00:00:00Z', date_precision=10)
    r.create(login)

    """

    def __init__(self, title, description, edition, edition_of=None, edition_of_wdid=None, archive_url=None,
                 pub_date=None, date_precision=11):
        """

        :param title: title of release item
        :type title: str
        :param description: description of release item
        :type description: str
        :param edition: edition number or unique identifier for the release
        :type edition: str
        :param edition_of: name of database. database wdid will automatically be looked up. Must pass either edition_of or edition_of_wdid
        :type edition_of: str
        :param edition_of_wdid: wikidata id of database
        :type edition_of_wdid: str
        :param archive_url: (optional)
        :type archive_url: str
        :param pub_date: (optional) Datetime will be converted to str
        :type pub_date: str or datetime
        :param date_precision: (optional) passed to PBB_Core.WDTime as is. default is 11 (day)
        :type date_precision: int
        """
        self.title = title
        self.description = description
        self.edition = str(edition)
        self.archive_url = archive_url
        if isinstance(pub_date, datetime.date):
            self.pub_date = pub_date.strftime('+%Y-%m-%dT%H:%M:%SZ')
        else:
            self.pub_date = pub_date
        self.date_precision = date_precision

        if (edition_of is None and edition_of_wdid is None) or (edition_of and edition_of_wdid):
            raise ValueError("must provide either edition_of or edition_of_wdid")
        if edition_of:
            self.edition_of_wdid = self.lookup_database(edition_of)
        else:
            self.edition_of_wdid = edition_of_wdid

        self.statements = None
        self.make_statements()

    @staticmethod
    @functools.lru_cache()
    def lookup_database(edition_of):
        query = """SELECT ?item ?itemLabel WHERE {
                    ?item wdt:P31 wd:Q4117139 .
                    SERVICE wikibase:label { bd:serviceParam wikibase:language "en" }
                }"""
        results = WDItemEngine.execute_sparql_query(query)
        db_item_map = {x['itemLabel']['value']: x['item']['value'].replace('http://www.wikidata.org/entity/', '') for x
                       in results['results']['bindings']}
        if edition_of not in db_item_map:
            raise ValueError("Database {} not found in wikidata. Please provide edition_of_wdid".format(edition_of))
        return db_item_map[edition_of]

    @functools.lru_cache()
    def release_exists(self):
        edition_dict = id_mapper("P393", (("P629", self.edition_of_wdid), ("P31", "Q3331189")))
        return edition_dict.get(self.edition, False)

    def make_statements(self):
        s = []
        # instance of edition
        s.append(wdi_core.WDItemID('Q3331189', 'P31'))
        # edition or translation of
        s.append(wdi_core.WDItemID(self.edition_of_wdid, 'P629'))
        # edition number
        s.append(wdi_core.WDString(self.edition, 'P393'))

        if self.archive_url:
            s.append(wdi_core.WDUrl(self.archive_url, 'P1065'))

        if self.pub_date:
            s.append(wdi_core.WDTime(self.pub_date, 'P577', precision=self.date_precision))

        self.statements = s

    def get_or_create(self, login):
        re = self.release_exists()
        if re:
            return re
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="release")
        item.set_label(self.title)
        item.set_description(description=self.description, lang='en')
        try_write(item, self.edition + "|" + self.edition_of_wdid, 'P393|P629', login)
        return item.wd_item_id


class PubmedStub(object):
    """
    Get or create a wikidata item stub given a pubmed ID

    Usage:
    PubmedStub(22674334).create(login)

    """

    def __init__(self, pmid):
        self.pmid = str(pmid)
        self.title = None
        self.dois = None
        self.reference = None
        self.statements = None
        self._root = None
        self.wdid = None

    def _validate_pmid(self):
        pubmed_url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=Pubmed&Retmode=xml&id={}'.format(
            self.pmid)
        r = requests.get(pubmed_url)
        self._root = ET.fromstring(r.text)
        return True if self._root.find(".//MedlineCitation/Article/ArticleTitle") is not None else False

    def _parse_ncbi(self):
        self.title = self._root.find(".//MedlineCitation/Article/ArticleTitle").text
        doiset = self._root.findall(".//ArticleIdList/ArticleId[@IdType='doi']")
        self.dois = [x.text for x in doiset]
        self.journal_issns = self._root.findall(".//MedlineCitation/Article/Journal/ISSN")

    def _make_reference(self):
        refStatedIn = wdi_core.WDItemID(value='Q180686', prop_nr='P248', is_reference=True)
        refPubmedId = wdi_core.WDString(value=self.pmid, prop_nr='P698', is_reference=True)
        timeStringNow = strftime("+%Y-%m-%dT00:00:00Z", gmtime())
        refRetrieved = wdi_core.WDTime(timeStringNow, prop_nr='P813', is_reference=True)
        self.reference = [refStatedIn, refPubmedId, refRetrieved]

    def _make_statements(self):
        s = []
        # instance of scientific article
        s.append(wdi_core.WDItemID('Q13442814', 'P31', references=[self.reference]))
        # pmid
        s.append(wdi_core.WDExternalID(self.pmid, 'P698', references=[self.reference]))
        # title
        s.append(wdi_core.WDMonolingualText(self.title, 'P1476', references=[self.reference]))
        # dois
        for doi in self.dois:
            s.append(wdi_core.WDExternalID(doi, 'P356', references=[self.reference]))
        # published in *journal* #P1433
        journal_wdid = None
        for issn in self.journal_issns:
            journal_wdid = prop2qid("P236", issn.text)
            if journal_wdid:
                break
        if journal_wdid:
            s.append(wdi_core.WDItemID(journal_wdid, 'P1433', references=[self.reference]))

        self.statements = s

    def create(self, login):
        print("use PubmedStub.get_or_create instead")
        self.get_or_create(login)

    @functools.lru_cache()
    def get_or_create(self, login):
        self.wdid = prop2qid('P698', self.pmid)  # if doesn't exist, will be None
        if self.wdid:
            return self.wdid
        if not self._validate_pmid():
            print("invalid pmid: {}".format(self.wdid))
            return None
        self._parse_ncbi()
        self._make_reference()
        self._make_statements()
        item = wdi_core.WDItemEngine(item_name=self.title, data=self.statements, domain="scientific_article")
        item.set_label(self.title)
{'fraction_non_alphanumeric': 0.10988717537886374, 'fraction_numerical': 0.023099005274050337, 'mean_word_length': 2.656333902855748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2743742', 'n_tokens_mistral': 5009, 'n_tokens_neox': 4662, 'n_words': 1020}