<commit_before>"""
Harvester of PubMed Central for the SHARE notification service

Example API call: http://www.pubmedcentral.nih.gov/oai/oai.cgi?verb=ListRecords&metadataPrefix=oai_dc&from=2015-04-13&until=2015-04-14
"""


from __future__ import unicode_literals

from scrapi.base import schemas
from scrapi.base import helpers
from scrapi.base import OAIHarvester


def oai_extract_url_pubmedcentral(header):
    PMC_ID = header.replace('oai:pubmedcentral.nih.gov:', '')
    return 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC' + PMC_ID


class PubMedCentralHarvester(OAIHarvester):
    short_name = 'pubmedcentral'
    long_name = 'PubMed Central'
    url = 'http://www.ncbi.nlm.nih.gov/pmc/'

    schema = helpers.updated_schema(
        schemas.OAISCHEMA,
        {
            "uris": {
                "canonicalUri": ('//ns0:header/ns0:identifier/node()', helpers.compose(oai_extract_url_pubmedcentral, helpers.single_result))
            }
        }
    )
<commit_msg>update pubmedcentral to match new schema override system<commit_after>"""
Harvester of PubMed Central for the SHARE notification service

Example API call: http://www.pubmedcentral.nih.gov/oai/oai.cgi?verb=ListRecords&metadataPrefix=oai_dc&from=2015-04-13&until=2015-04-14
"""


from __future__ import unicode_literals

from scrapi.base import helpers
from scrapi.base import OAIHarvester


def oai_extract_url_pubmedcentral(header):
    PMC_ID = header.replace('oai:pubmedcentral.nih.gov:', '')
    return 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC' + PMC_ID


class PubMedCentralHarvester(OAIHarvester):
    short_name = 'pubmedcentral'
    long_name = 'PubMed Central'
    url = 'http://www.ncbi.nlm.nih.gov/pmc/'

    @property
    def schema(self):
        return helpers.updated_schema(self._schema, {
            "uris": {
                "canonicalUri": ('//ns0:header/ns0:identifier/node()', helpers.compose(oai_extract_url_pubmedcentral, helpers.single_result))
            }
        })

{'fraction_non_alphanumeric': 0.12032355915065723, 'fraction_numerical': 0.01820020222446916, 'mean_word_length': 3.7458033573141485, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3610020', 'n_tokens_mistral': 733, 'n_tokens_neox': 654, 'n_words': 113}