<commit_before>import abc
import time
import logging

import pendulum
from furl import furl
from lxml import etree

from .harvester import Harvester

logger = logging.getLogger(__name__)


class OAIHarvester(Harvester, metaclass=abc.ABCMeta):

    metadata_prefix = 'oai_dc'
    namespaces = {
        'dc': 'http://purl.org/dc/elements/1.1/',
        'ns0': 'http://www.openarchives.org/OAI/2.0/',
        'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
    }
    url = None
    time_granularity = True
    from_param = 'from'
    until_param = 'until'

    def __init__(self, app_config):
        super().__init__(app_config)

        self.url = getattr(self.config, 'url', self.url)
        if not self.url:
            raise NotImplementedError('url')

        self.time_granularity = getattr(self.config, 'time_granularity', self.time_granularity)
        self.from_param = getattr(self.config, 'from_param', self.from_param)
        self.until_param = getattr(self.config, 'until_param', self.until_param)

    def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> list:
        url = furl(self.url)
        url.args['verb'] = 'ListRecords'
        url.args['metadataPrefix'] = self.metadata_prefix

        if self.time_granularity:
            url.args[self.from_param] = start_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
            url.args[self.until_param] = end_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
        else:
            url.args[self.from_param] = start_date.date().isoformat()
            url.args[self.until_param] = end_date.date().isoformat()

        return self.fetch_records(url)

    def fetch_records(self, url: furl) -> list:
        records, token = self.fetch_page(url, token=None)

        while True:
            for record in records:
                yield (
                    record.xpath('ns0:header/ns0:identifier', namespaces=self.namespaces)[0].text,
                    etree.tostring(record),
                )

            records, token = self.fetch_page(url, token=token)

            if not token or not records:
                break

    def fetch_page(self, url: furl, token: str=None) -> (list, str):
        if token:
            url.remove(self.from_param)
            url.remove(self.until_param)
            url.remove('metadataPrefix')
            url.args['resumptionToken'] = token

        while True:
            logger.info('Making request to {}'.format(url.url))
            resp = self.requests.get(url.url)
            if resp.ok:
                break
            if resp.status_code == 503:
                sleep = int(resp.headers.get('retry-after', 5)) + 2  # additional 2 seconds for good measure
                logger.warning('Server responded with %s. Waiting %s seconds.', resp, sleep)
                time.sleep(sleep)
                continue
            resp.raise_for_status()

        parsed = etree.fromstring(resp.content)

        records = parsed.xpath('//ns0:record', namespaces=self.namespaces)<commit_msg>[SHARE-393][Feature]  Ensure only verb and resumption token when token is present (#563)<commit_after>import abc
import time
import logging

import pendulum
from furl import furl
from lxml import etree

from .harvester import Harvester

logger = logging.getLogger(__name__)


class OAIHarvester(Harvester, metaclass=abc.ABCMeta):

    metadata_prefix = 'oai_dc'
    namespaces = {
        'dc': 'http://purl.org/dc/elements/1.1/',
        'ns0': 'http://www.openarchives.org/OAI/2.0/',
        'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
    }
    url = None
    time_granularity = True
    from_param = 'from'
    until_param = 'until'

    def __init__(self, app_config):
        super().__init__(app_config)

        self.url = getattr(self.config, 'url', self.url)
        if not self.url:
            raise NotImplementedError('url')

        self.time_granularity = getattr(self.config, 'time_granularity', self.time_granularity)
        self.from_param = getattr(self.config, 'from_param', self.from_param)
        self.until_param = getattr(self.config, 'until_param', self.until_param)

    def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> list:
        url = furl(self.url)
        url.args['verb'] = 'ListRecords'
        url.args['metadataPrefix'] = self.metadata_prefix

        if self.time_granularity:
            url.args[self.from_param] = start_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
            url.args[self.until_param] = end_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
        else:
            url.args[self.from_param] = start_date.date().isoformat()
            url.args[self.until_param] = end_date.date().isoformat()

        return self.fetch_records(url)

    def fetch_records(self, url: furl) -> list:
        records, token = self.fetch_page(url, token=None)

        while True:
            for record in records:
                yield (
                    record.xpath('ns0:header/ns0:identifier', namespaces=self.namespaces)[0].text,
                    etree.tostring(record),
                )

            records, token = self.fetch_page(url, token=token)

            if not token or not records:
                break

    def fetch_page(self, url: furl, token: str=None) -> (list, str):
        if token:
            url.args = {'resumptionToken': token, 'verb': 'ListRecords'}

        while True:
            logger.info('Making request to {}'.format(url.url))
            resp = self.requests.get(url.url)
            if resp.ok:
                break
            if resp.status_code == 503:
                sleep = int(resp.headers.get('retry-after', 5)) + 2  # additional 2 seconds for good measure
                logger.warning('Server responded with %s. Waiting %s seconds.', resp, sleep)
                time.sleep(sleep)
                continue
            resp.raise_for_status()

        parsed = etree.fromstring(resp.content)

        records = parsed.xpath('//ns0:record', namespaces=self.namespaces)
{'fraction_non_alphanumeric': 0.1234890558640967, 'fraction_numerical': 0.010454099967330937, 'mean_word_length': 2.34407427635172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2717397', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1884, 'n_words': 376}