<commit_before># coding: utf-8
"""
    Sickle
    ~~~~~~

    An OAI-PMH client.

    :copyright: Copright 2013 <NAME>
"""

import requests
from lxml import etree

from .models import Set, Record, Header, MetadataFormat, Identify
import oaiexceptions


import logging

# Logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

OAI_NAMESPACE = '{http://www.openarchives.org/OAI/%s/}'
XMLParser = etree.XMLParser(remove_blank_text=True, recover=True)


class Sickle(object):
    """Client for harvesting OAI interfaces.

    Use it like this::

        >>> sickle = Sickle('http://elis.da.ulcc.ac.uk/cgi/oai2')
        >>> records = sickle.ListRecords(metadataPrefix='oai_dc')
        >>> records.next()
        <Record oai:eprints.rclis.org:3780>

    :param endpoint: The endpoint of the OAI interface.
    :type endpoint: str
    :param http_method: Method used for requests (GET or POST, default: GET).
    :type http_method: str
    :param protocol_version: The OAI protocol version.
    :type protocol_version: str
    """
    def __init__(self, endpoint, http_method='GET', protocol_version='2.0'):
        super(Sickle, self).__init__()
        self.endpoint = endpoint
        self.http_method = http_method
        self.protocol_version = protocol_version
        self.oai_namespace = OAI_NAMESPACE % self.protocol_version
        self.last_response = None

    def harvest(self, **kwargs):
        """Make an HTTP request to the OAI server.

        :rtype: :class:`sickle.app.OAIResponse`
        """
        if self.http_method == 'GET':
            return OAIResponse(requests.get(self.endpoint, params=kwargs), 
                params=kwargs)
        elif self.http_method == 'POST':
            return OAIResponse(requests.post(self.endpoint, data=kwargs), 
                params=kwargs)

    def ListRecords(self, ignore_deleted=False, **kwargs):
        """Issue a ListRecords request.

        :param ignore_deleted: If set to :obj:`True`, the resulting 
                              :class:`sickle.app.OAIIterator` will skip records
                              flagged as deleted.
        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListRecords'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self, ignore_deleted=ignore_deleted)

    def ListIdentifiers(self, ignore_deleted=False, **kwargs):
        """Issue a ListIdentifiers request.

        :param ignore_deleted: If set to :obj:`True`, the resulting 
                              :class:`sickle.app.OAIIterator` will skip records
                              flagged as deleted.
        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListIdentifiers'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self, ignore_deleted=ignore_deleted)

    def ListSets(self, **kwargs):
        """Issue a ListSets request.

        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListSets'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self)

    def Identify(self):
        """Issue an Identify request.

        :rtype: :class:`sickle.models.Identify`
        """
        params = {'verb': 'Identify'}
        self.last_response = self.harvest(**params)
        return Identify(self.last_response)

    def GetRecord(self, **kwargs):
        """Issue a ListSets request.

        :rtype: :class:`sickle.app.OAIResponse`
        """
        params = kwargs
        params.update({'verb': 'GetRecord'})
        self.last_response = self.harvest(**params)
        # GetRecord is treated as a special case of ListRecords: 
        # by creating an OAIIterator and returning the first and 
        # only record.
        return OAIIterator(self.last_response, self).next()

    def ListMetadataFormats(self, **kwargs):
        """Issue a ListMetadataFormats request.

        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListMetadataFormats'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self)


class OAIResponse(object):
    """A response from an OAI server.

    Provides access to the returned data on different abstraction
    levels.

    :param response: The original HTTP response.
    :param params: The OAI parameters for the request.
    :type params: dict
    """
    def __init__(self, response, params):
        self.params = params
        self.response = response

    @property
    def raw(self):
        """The server's response as unicode."""
        return self.response.text

    @property
    def xml(self):
        """The server's response as parsed XML."""
        return etree.XML(self.response.text.encode("utf8"), parser=XMLParser)

    def __repr__(self):
        return '<OAIResponse %s>' % self.params.get('verb')


class OAIIterator(object):
    """Iterator over OAI records/identifiers/sets transparently aggregated via
    OAI-PMH.

    Can be used to conveniently iterate through the records of a repository.
    
    :param oai_response: The first OAI response.
    :type oai_response: :class:`sickle.app.OAIResponse`
    :param sickle: The Sickle object that issued the first request.
    :type sickle: :class:`sickle.app.Sickle`
    :param ignore_deleted: Flag for whether to ignore deleted records.
    :type ignore_deleted: bool
    """
    def __init__(self, oai_response, sickle, ignore_deleted=False):
        self.oai_response = oai_response
        self.sickle = sickle
        self.verb = self.oai_response.params.get("verb")
        # Determine on what element to iterate (records, headers, or sets)
        if self.verb == 'ListRecords' or self.verb == 'GetRecord':
            self.element = 'record'
            self.mapper = Record
        elif self.verb == 'ListIdentifiers':
            self.element = 'header'
            self.mapper = Header
        elif self.verb == 'ListSets':
            self.element = 'set'
            self.mapper = Set
        elif self.verb == 'ListMetadataFormats':
            self.element = 'metadataFormat'
            self.mapper = MetadataFormat
        error = self.oai_response.xml.find('.//' + self.sickle.oai_namespace + 'error')
        if error is not None:
            code = error.attrib.get('code', 'UNKNOWN')
            description = error.text or ''
            try:
                raise getattr(oaiexceptions, code[0].upper() + code[1:]), description
            except AttributeError:
                raise oaiexceptions.OAIError, description

        self._items = self.oai_response.xml.iterfind(
            './/' + self.sickle.oai_namespace + self.element)
        self.resumption_token = self._get_resumption_token()
        self.ignore_deleted = ignore_deleted

    def __iter__(self):
        return self

    def __repr__(self):
        return '<OAIIterator %s>' % self.verb

    def _get_resumption_token(self):
        """Extract and store the resumptionToken from the last response."""
        resumption_token = self.oai_response.xml.find(
            './/' + self.sickle.oai_namespace + 'resumptionToken')
        if resumption_token is None:
            return None
        else:
            return resumption_token.text
<commit_msg>Refactored mapping from verbs to classes; made mapping configurable<commit_after># coding: utf-8
"""
    Sickle
    ~~~~~~

    An OAI-PMH client.

    :copyright: Copright 2013 <NAME>
"""

import requests
from lxml import etree

from .models import Set, Record, Header, MetadataFormat, Identify
import oaiexceptions

import logging

# Logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

OAI_NAMESPACE = '{http://www.openarchives.org/OAI/%s/}'
XMLParser = etree.XMLParser(remove_blank_text=True, recover=True)


# Map OAI verbs to class representations
DEFAULT_CLASS_MAPPING = {
    'GetRecord': Record,
    'ListRecords': Record,
    'ListIdentifiers': Header,
    'ListSets': Set,
    'ListMetadataFormats': MetadataFormat,
    'Identify': Identify,
}

# Map OAI verbs to the XML elements 
VERBS_ELEMENTS = {
    'GetRecord': 'record',
    'ListRecords': 'record',
    'ListIdentifiers': 'header',
    'ListSets': 'set',
    'ListMetadataFormats': 'metadataFormat',
    'Identify': 'Identify',
}


class Sickle(object):
    """Client for harvesting OAI interfaces.

    Use it like this::

        >>> sickle = Sickle('http://elis.da.ulcc.ac.uk/cgi/oai2')
        >>> records = sickle.ListRecords(metadataPrefix='oai_dc')
        >>> records.next()
        <Record oai:eprints.rclis.org:3780>

    :param endpoint: The endpoint of the OAI interface.
    :type endpoint: str
    :param http_method: Method used for requests (GET or POST, default: GET).
    :type http_method: str
    :param protocol_version: The OAI protocol version.
    :type protocol_version: str
    :param class_mapping: A dictionary that maps OAI verbs to classes representing OAI items.
                          If not provided, :data:`sickle.app.DEFAULT_CLASS_MAPPING` is used.
    :type class_mapping: dict
    """
    def __init__(self, endpoint, http_method='GET', protocol_version='2.0',
        class_mapping=None):
        self.endpoint = endpoint
        self.http_method = http_method
        self.protocol_version = protocol_version
        self.oai_namespace = OAI_NAMESPACE % self.protocol_version
        if class_mapping is None:
            self.class_mapping = DEFAULT_CLASS_MAPPING
        else:
            self.class_mapping = class_mapping
        self.last_response = None

    def harvest(self, **kwargs):
        """Make an HTTP request to the OAI server.

        :rtype: :class:`sickle.app.OAIResponse`
        """
        if self.http_method == 'GET':
            return OAIResponse(requests.get(self.endpoint, params=kwargs), 
                params=kwargs)
        elif self.http_method == 'POST':
            return OAIResponse(requests.post(self.endpoint, data=kwargs), 
                params=kwargs)

    def ListRecords(self, ignore_deleted=False, **kwargs):
        """Issue a ListRecords request.

        :param ignore_deleted: If set to :obj:`True`, the resulting 
                              :class:`sickle.app.OAIIterator` will skip records
                              flagged as deleted.
        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListRecords'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self, ignore_deleted=ignore_deleted)

    def ListIdentifiers(self, ignore_deleted=False, **kwargs):
        """Issue a ListIdentifiers request.

        :param ignore_deleted: If set to :obj:`True`, the resulting 
                              :class:`sickle.app.OAIIterator` will skip records
                              flagged as deleted.
        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListIdentifiers'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self, ignore_deleted=ignore_deleted)

    def ListSets(self, **kwargs):
        """Issue a ListSets request.

        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListSets'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self)

    def Identify(self):
        """Issue an Identify request.

        :rtype: :class:`sickle.models.Identify`
        """
        params = {'verb': 'Identify'}
        self.last_response = self.harvest(**params)
        return Identify(self.last_response)

    def GetRecord(self, **kwargs):
        """Issue a ListSets request.

        :rtype: :class:`sickle.models.Record`
        """
        params = kwargs
        params.update({'verb': 'GetRecord'})
        self.last_response = self.harvest(**params)
        # GetRecord is treated as a special case of ListRecords: 
        # by creating an OAIIterator and returning the first and 
        # only record.
        return OAIIterator(self.last_response, self).next()

    def ListMetadataFormats(self, **kwargs):
        """Issue a ListMetadataFormats request.

        :rtype: :class:`sickle.app.OAIIterator`
        """
        params = kwargs
        params.update({'verb': 'ListMetadataFormats'})
        self.last_response = self.harvest(**params)
        return OAIIterator(self.last_response, self)


class OAIResponse(object):
    """A response from an OAI server.

    Provides access to the returned data on different abstraction
    levels.

    :param response: The original HTTP response.
    :param params: The OAI parameters for the request.
    :type params: dict
    """
    def __init__(self, response, params):
        self.params = params
        self.response = response

    @property
    def raw(self):
        """The server's response as unicode."""
        return self.response.text

    @property
    def xml(self):
        """The server's response as parsed XML."""
        return etree.XML(self.response.text.encode("utf8"), parser=XMLParser)

    def __repr__(self):
        return '<OAIResponse %s>' % self.params.get('verb')


class OAIIterator(object):
    """Iterator over OAI records/identifiers/sets transparently aggregated via
    OAI-PMH.

    Can be used to conveniently iterate through the records of a repository.
    
    :param oai_response: The first OAI response.
    :type oai_response: :class:`sickle.app.OAIResponse`
    :param sickle: The Sickle object that issued the first request.
    :type sickle: :class:`sickle.app.Sickle`
    :param ignore_deleted: Flag for whether to ignore deleted records.
    :type ignore_deleted: bool
    """
    def __init__(self, oai_response, sickle, ignore_deleted=False):
        self.oai_response = oai_response
        self.sickle = sickle
        self.verb = self.oai_response.params.get("verb")
        # Determine on what element to iterate (records, headers, or sets)
        self.element = VERBS_ELEMENTS[self.verb]
        # Get the reflection class for the elements returned by this verb
        self.mapper = self.sickle.class_mapping[self.verb]
        error = self.oai_response.xml.find('.//' + self.sickle.oai_namespace + 'error')
        if error is not None:
            code = error.attrib.get('code', 'UNKNOWN')
            description = error.text or ''
            try:
                raise getattr(oaiexceptions, code[0].upper() + code[1:]), description
            except AttributeError:
                raise oaiexceptions.OAIError, description

        self._items = self.oai_response.xml.iterfind(
            './/' + self.sickle.oai_namespace + self.element)
        self.resumption_token = self._get_resumption_token()
        self.ignore_deleted = ignore_deleted

    def __iter__(self):
        return self

    def __repr__(self):
        return '<OAIIterator %s>' % self.verb

    def _get_resumption_token(self):
        """Extract and store the resumptionToken from the last response."""
        resumption_token = self.oai_response.xml.find(
            './/' + self.sickle.oai_namespace + 'resumptionToken')
        if resumption_token is None:
            return None
        else:
            return resumption_token.text

{'fraction_non_alphanumeric': 0.10858683095879862, 'fraction_numerical': 0.0019252984212552945, 'mean_word_length': 2.565088080530771, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53207', 'n_tokens_mistral': 4659, 'n_tokens_neox': 4497, 'n_words': 1143}