<commit_before>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, urllib, os, itertools

import logging
logger = logging.getLogger('ti.providers.pubmed')

class Pubmed(Provider):  

    example_id = ("pmid", "22855908")

    url = "http://pubmed.gov"
    descr = "PubMed comprises more than 21 million citations for biomedical literature"
    provenance_url_pmc_citations_template = "http://www.ncbi.nlm.nih.gov/pubmed?linkname=pubmed_pubmed_citedin&from_uid=%s"
    provenance_url_pmc_citations_filtered_template = "http://www.ncbi.nlm.nih.gov/pubmed?term=%s&cmd=DetailsSearch"
    provenance_url_f1000_template = "http://f1000.com/pubmed/%s"

    metrics_url_template = None # have specific metrics urls instead
    metrics_pmc_citations_url_template = "http://www.pubmedcentral.nih.gov/utils/entrez2pmcciting.cgi?view=xml&id=%s&email=<EMAIL>&tool=total-impact"
    metrics_pmc_filter_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=(%s)&email=<EMAIL>&tool=total-impact"
    metrics_f1000_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=%s&cmd=llinks&email=<EMAIL>&tool=total-impact"

    aliases_from_doi_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?term=%s&email=<EMAIL>&tool=total-impact" 
    aliases_from_pmid_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=<EMAIL>&tool=total-<EMAIL>" 

    aliases_pubmed_url_template = "http://www.ncbi.nlm.nih.gov/pubmed/%s"

    biblio_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=<EMAIL>&tool=total-impact" 

    static_meta_dict = {
        "pmc_citations": {
            "display_name": "citations",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        }, 
        "pmc_citations_reviews": {
            "display_name": "citations: reviews",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by review papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        }, 
        "pmc_citations_editorials": {
            "display_name": "citations: editorials",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by editorials papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        },            
        "f1000": {
            "display_name": "reviewed",
            "provider": "F1000",
            "provider_url": "http://f1000.com",
            "description": "The article has been reviewed by F1000",
            "icon": "http://f1000.com/1344012469601/images/favicon.ico"
        }            
    }


    def __init__(self):
        super(Pubmed, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        relevant = (namespace=="pmid")
        return(relevant)

    # overriding default because overriding aliases method
    @property
    def provides_aliases(self):
        return True

    # overriding default because overriding aliases method
    @property
    def provides_metrics(self):
        return True

    def _extract_biblio(self, page, id=None):
        dict_of_keylists = {"year": ["PubmedArticleSet", "MedlineCitation", "Article", "Journal", "PubDate", "Year"], 
                            "title": ["PubmedArticleSet", "MedlineCitation", "Article", "ArticleTitle"],
                            "journal": ["PubmedArticleSet", "MedlineCitation", "Article", "Journal", "Title"],
                            }
        biblio_dict = provider._extract_from_xml(page, dict_of_keylists)
        dom_authors = provider._find_all_in_xml(page, "LastName")
        try:
            biblio_dict["authors"] = ", ".join([author.firstChild.data for author in dom_authors])
        except (AttributeError, TypeError):
            pass
        return biblio_dict  

    def _extract_aliases_from_doi(self, page, doi):
        dict_of_keylists = {"pmid": ["eSearchResult", "IdList", "Id"], 
                            "QueryTranslation": ["eSearchResult", "QueryTranslation"]}

        aliases_dict = provider._extract_from_xml(page, dict_of_keylists)
        aliases_list = []
        if aliases_dict:
            if aliases_dict["QueryTranslation"] == (doi + "[All Fields]"):
                aliases_list = [("pmid", str(aliases_dict["pmid"]))]
        return aliases_list


    def _extract_aliases_from_pmid(self, page, pmid):
        dict_of_keylists = {"doi": ["PubmedData", "ArticleIdList"]}

        (doc, lookup_function) = provider._get_doc_from_xml(page)
        doi = None
        try:
            articleidlist = doc.getElementsByTagName("ArticleIdList")[0]
            for articleid in articleidlist.getElementsByTagName("ArticleId"):
                if (articleid.getAttribute("IdType") == u"doi"):
                    doi = articleid.firstChild.data
        except (IndexError, TypeError):
            pass

        #sometimes no doi, or PMID has a doi-fragment in the doi field:
        aliases_list = []
        if doi:
            if "10." in doi:  
                aliases_list = [("doi", doi)]
        return aliases_list

    def _get_eutils_page(self, id, url, cache_enabled=True):
        logger.debug("%20s getting eutils page for %s" % (self.provider_name, id))

        response = self.http_get(url, cache_enabled=cache_enabled)
        if response.status_code != 200:
            logger.warning("%20s WARNING, status_code=%i getting %s" 
                % (self.provider_name, response.status_code, url))            
            if response.status_code == 404:
                return []
            else:
                self._get_error(response.status_code, response)
        return response.text

    # overriding so can look up in both directions
    def aliases(self, 
            aliases, 
            provider_url_template=None,
            cache_enabled=True):            

        new_aliases = []
        for alias in aliases:
            (namespace, nid) = alias
            if (namespace == "doi"):
                aliases_from_doi_url = self.aliases_from_doi_url_template %nid
                page = self._get_eutils_page(nid, aliases_from_doi_url, cache_enabled)
                new_aliases += self._extract_aliases_from_doi(page, nid)
            if (namespace == "pmid"):<commit_msg>fix uric that are too large<commit_after>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, urllib, os, itertools

import logging
logger = logging.getLogger('ti.providers.pubmed')

class Pubmed(Provider):  

    example_id = ("pmid", "22855908")

    url = "http://pubmed.gov"
    descr = "PubMed comprises more than 21 million citations for biomedical literature"
    provenance_url_pmc_citations_template = "http://www.ncbi.nlm.nih.gov/pubmed?linkname=pubmed_pubmed_citedin&from_uid=%s"
    provenance_url_pmc_citations_filtered_template = "http://www.ncbi.nlm.nih.gov/pubmed?term=%s&cmd=DetailsSearch"
    provenance_url_f1000_template = "http://f1000.com/pubmed/%s"

    metrics_url_template = None # have specific metrics urls instead
    metrics_pmc_citations_url_template = "http://www.pubmedcentral.nih.gov/utils/entrez2pmcciting.cgi?view=xml&id=%s&email=<EMAIL>&tool=total-impact"
    metrics_pmc_filter_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=(%s)&email=<EMAIL>&tool=total-impact"
    metrics_f1000_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=%s&cmd=llinks&email=<EMAIL>&tool=total-impact"

    aliases_from_doi_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?term=%s&email=<EMAIL>&tool=total-impact" 
    aliases_from_pmid_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=<EMAIL>&tool=total-impact" 

    aliases_pubmed_url_template = "http://www.ncbi.nlm.nih.gov/pubmed/%s"

    biblio_url_template = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=<EMAIL>&tool=total-impact" 

    static_meta_dict = {
        "pmc_citations": {
            "display_name": "citations",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        }, 
        "pmc_citations_reviews": {
            "display_name": "citations: reviews",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by review papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        }, 
        "pmc_citations_editorials": {
            "display_name": "citations: editorials",
            "provider": "PubMed Central",
            "provider_url": "http://pubmed.gov",
            "description": "The number of citations by editorials papers in PubMed Central",
            "icon": "http://www.ncbi.nlm.nih.gov/favicon.ico"
        },            
        "f1000": {
            "display_name": "reviewed",
            "provider": "F1000",
            "provider_url": "http://f1000.com",
            "description": "The article has been reviewed by F1000",
            "icon": "http://f1000.com/1344012469601/images/favicon.ico"
        }            
    }


    def __init__(self):
        super(Pubmed, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        relevant = (namespace=="pmid")
        return(relevant)

    # overriding default because overriding aliases method
    @property
    def provides_aliases(self):
        return True

    # overriding default because overriding aliases method
    @property
    def provides_metrics(self):
        return True

    def _extract_biblio(self, page, id=None):
        dict_of_keylists = {"year": ["PubmedArticleSet", "MedlineCitation", "Article", "Journal", "PubDate", "Year"], 
                            "title": ["PubmedArticleSet", "MedlineCitation", "Article", "ArticleTitle"],
                            "journal": ["PubmedArticleSet", "MedlineCitation", "Article", "Journal", "Title"],
                            }
        biblio_dict = provider._extract_from_xml(page, dict_of_keylists)
        dom_authors = provider._find_all_in_xml(page, "LastName")
        try:
            biblio_dict["authors"] = ", ".join([author.firstChild.data for author in dom_authors])
        except (AttributeError, TypeError):
            pass
        return biblio_dict  

    def _extract_aliases_from_doi(self, page, doi):
        dict_of_keylists = {"pmid": ["eSearchResult", "IdList", "Id"], 
                            "QueryTranslation": ["eSearchResult", "QueryTranslation"]}

        aliases_dict = provider._extract_from_xml(page, dict_of_keylists)
        aliases_list = []
        if aliases_dict:
            if aliases_dict["QueryTranslation"] == (doi + "[All Fields]"):
                aliases_list = [("pmid", str(aliases_dict["pmid"]))]
        return aliases_list


    def _extract_aliases_from_pmid(self, page, pmid):
        dict_of_keylists = {"doi": ["PubmedData", "ArticleIdList"]}

        (doc, lookup_function) = provider._get_doc_from_xml(page)
        doi = None
        try:
            articleidlist = doc.getElementsByTagName("ArticleIdList")[0]
            for articleid in articleidlist.getElementsByTagName("ArticleId"):
                if (articleid.getAttribute("IdType") == u"doi"):
                    doi = articleid.firstChild.data
        except (IndexError, TypeError):
            pass

        #sometimes no doi, or PMID has a doi-fragment in the doi field:
        aliases_list = []
        if doi:
            if "10." in doi:  
                aliases_list = [("doi", doi)]
        return aliases_list

    def _get_eutils_page(self, id, url, cache_enabled=True):
        logger.debug("%20s getting eutils page for %s" % (self.provider_name, id))

        response = self.http_get(url, cache_enabled=cache_enabled)
        if response.status_code != 200:
            logger.warning("%20s WARNING, status_code=%i getting %s" 
                % (self.provider_name, response.status_code, url))            
            if response.status_code == 404:
                return []
            if response.status_code == 414:  #Request-URI Too Large
                return []
            else:
                self._get_error(response.status_code, response)
        return response.text

    # overriding so can look up in both directions
    def aliases(self, 
            aliases, 
            provider_url_template=None,
            cache_enabled=True):            

        new_aliases = []
        for alias in aliases:
            (namespace, nid) = alias
            if (namespace == "doi"):
                aliases_from_doi_url = self.aliases_from_doi_url_template %nid
                page = self._get_eutils_page(nid, aliases_from_doi_url, cache_enabled)
                new_aliases += self._extract_aliases_from_doi(page, nid)
            if (namespace == "pmid"):
{'fraction_non_alphanumeric': 0.12360452389638818, 'fraction_numerical': 0.0102882159795695, 'mean_word_length': 2.6529850746268657, 'pattern_counts': {'":': 60, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 14, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5302020', 'n_tokens_mistral': 4538, 'n_tokens_neox': 4132, 'n_words': 763}