<commit_before>import logging

from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect

from lr.controllers.harvest import HarvestController

from lr.lib import helpers as h
from lr.lib.base import BaseController, render
from lr.lib.oaipmh import oaipmh
from datetime import datetime
import json, iso8601
from couchdb.http import ResourceNotFound
from lr.lib.oaipmherrors import *


log = logging.getLogger(__name__)

class OaiPmhController(HarvestController):
    """REST Controller styled on the Atom Publishing Protocol"""
    # To properly map this controller, ensure your config/routing.py
    # file has a resource setup:
    #     map.resource('OAI-PMH', 'OAI-PMH')
    def _isTrue(self, value):
        return str(value).lower() in ['true', 't', '1', 'y', 'yes']
    
    def _getParams(self):
        req_params = {}
        
        if request.method == "POST":
            req_params = dict(request.params.copy(), **self._parameterizePOSTBody())
        else:
            req_params = request.params.copy()
        return req_params
    
    def _parameterizePOSTBody(self):
        from urlparse import parse_qs
        req_params = {}
        try:
            params = parse_qs(qs=request.body)
            for key in params.keys():
                req_params[key] = params[key][0]
        except:
            log.exception("Unable to parse POST.")
        return req_params
    
    def _parseParams(self):
        params = {}
        req_params = self._getParams()
        if (req_params.has_key('verb') == False) :
            raise BadVerbError()
        
        verb = params["verb"] = req_params['verb']
        
        if verb not in ["GetRecord", "ListRecords", "ListIdentifiers", "Identify", "ListMetadataFormats", "ListSets"]:
            raise BadVerbError()
        
        if verb == 'GetRecord' or verb == 'ListRecords' or verb == 'ListIdentifiers':        
            if req_params.has_key('metadataPrefix') == False:
                raise BadArgumentError('metadataPrefix is a required parameter.', verb)
            params["metadataPrefix"] = metadataPrefix = req_params['metadataPrefix']
        
        if verb == 'GetRecord' or verb == 'ListMetadataFormats':
            if req_params.has_key('by_doc_ID') and req_params.has_key('by_resource_ID'):
                if self._isTrue(req_params['by_doc_ID']) == self._isTrue(req_params['by_resource_ID']):
                    raise BadArgumentError('by_doc_ID and by_resource_ID have conflicting values.', verb)
                
            if req_params.has_key('by_doc_ID'):
                params['by_doc_ID'] = self._isTrue(req_params['by_doc_ID'])
                params['by_resource_ID'] = not params['by_doc_ID']
            else:
                params['by_doc_ID'] = False
                params['by_resource_ID'] = not params['by_doc_ID']
            
            if req_params.has_key('by_resource_ID'):
                params['by_resource_ID'] = self._isTrue(req_params['by_resource_ID'])
                params['by_doc_ID'] = not params['by_resource_ID']
        
        if verb == 'ListRecords' or verb == 'ListIdentifiers':
            from_date_gran = None
            until_date_gran = None
            if req_params.has_key('from'):
                try:
                    from_date = iso8601.parse_date(req_params['from'])
                    params['from'] = h.convertToISO8601UTC(from_date)
                    from_date_gran = h.getISO8601Granularity(req_params['from'])
                except:
                    raise BadArgumentError('from does not parse to ISO 8601.', verb)
            else:
                params['from'] = None
            
            if req_params.has_key('until'):
                try:
                    until_date = iso8601.parse_date(req_params['until'])
                    params['until'] = h.convertToISO8601UTC(until_date)
                    until_date_gran = h.getISO8601Granularity(req_params['until'])
                except:
                    raise BadArgumentError('until does not parse to ISO 8601.', verb)
            else:
                params['until'] = None
                
            if params['from'] != None and params['until'] != None:
                if params['from'] > params['until']:
                    raise BadArgumentError('until cannot preceed from.', verb)
                if from_date_gran != until_date_gran:
                    raise BadArgumentError('from and until parameters do not use the same granularity.', verb)
            
            harvestServiceGranularity = h.getHarvestServiceGranularity()
            
            if params['from'] != None and from_date_gran > harvestServiceGranularity:
                raise BadArgumentError('from is more granular than Harvest service permits', verb)
        
            if params['until'] != None and until_date_gran > harvestServiceGranularity:
                raise BadArgumentError('until is more granular than Harvest service permits', verb)
            
            if (from_date_gran != None and from_date_gran.granule != "day" and from_date_gran.granule != "second") or \
                (until_date_gran != None and until_date_gran.granule != "day" and until_date_gran.granule != "second"):
                formatSupport = "YYYY-MM-DD"
                if harvestServiceGranularity.granule == "second":
                    formatSupport = "YYYY-MM-DD and YYYY-MM-DDThh:mm:ssZ"
                raise BadArgumentError('from and until support {0} formats' % (formatSupport, ), verb)
            
        
        if verb in ['ListMetadataFormats', 'GetRecord']  and req_params.has_key('identifier'):
            params['identifier'] = req_params['identifier']
        elif verb == 'ListMetadataFormats':
            params['identifier'] = None
            params['by_doc_ID'] = None
            params['by_resource_ID'] = None
        
        if verb == 'GetRecord' and req_params.has_key('identifier') == False:
            raise BadArgumentError('identifier is a required parameter.', verb)
        
        return params

    def _returnResponse(self, body):
        response.headers["Content-Type"] = "text/xml; charset=utf-8"
        return body
                                 
    
    def _initRender(self,params):
        c.datetime_now = datetime.utcnow().isoformat()
        c.path_url = request.path_url
        if params.has_key("by_doc_ID"):
            c.by_doc_ID = params["by_doc_ID"]
        if params.has_key("by_resource_ID"):
            c.by_resource_ID = params["by_resource_ID"]
        if params.has_key("metadataPrefix"):
            c.metadataPrefix = params["metadataPrefix"]
        if params.has_key("from"):
            c.from_date = params["from"]
        if params.has_key("until"):
            c.until_date = params["until"]
        if params.has_key("identifier"):
            c.identifier = params["identifier"]
        
    def _handleOAIRequest(self, format='html'):
        o = oaipmh()
        
        def GetRecord(params):
            try:
                c.identifier = params["identifier"]
                if params["by_doc_ID"]  == True:
                    c.docList = [o.get_record(params["identifier"])]
                else:
                    c.docList = o.get_records_by_resource(params["identifier"])
                    if len(c.docList) == 0:
                        raise IdDoesNotExistError(params['verb'])
            except ResourceNotFound:
                raise IdDoesNotExistError(params['verb'])
            
            self._initRender(params)
            
            if c.docList != None:
                for doc in c.docList:
                    if c.metadataPrefix not in doc["payload_schema"]:
                        raise CannotDisseminateFormatError(params['verb'])

            body = "<error/>"
            try:
                body = render("/oaipmh-GetRecord.mako")
            except:
                log.exception("Unable to render template")
            return self._returnResponse(body)
        <commit_msg>fixed a bug caused by base harvest get_records_by_resource no longer returning a list but a generator.<commit_after>import logging

from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect

from lr.controllers.harvest import HarvestController

from lr.lib import helpers as h
from lr.lib.base import BaseController, render
from lr.lib.oaipmh import oaipmh
from datetime import datetime
import json, iso8601
from couchdb.http import ResourceNotFound
from lr.lib.oaipmherrors import *


log = logging.getLogger(__name__)

class OaiPmhController(HarvestController):
    """REST Controller styled on the Atom Publishing Protocol"""
    # To properly map this controller, ensure your config/routing.py
    # file has a resource setup:
    #     map.resource('OAI-PMH', 'OAI-PMH')
    def _isTrue(self, value):
        return str(value).lower() in ['true', 't', '1', 'y', 'yes']
    
    def _getParams(self):
        req_params = {}
        
        if request.method == "POST":
            req_params = dict(request.params.copy(), **self._parameterizePOSTBody())
        else:
            req_params = request.params.copy()
        return req_params
    
    def _parameterizePOSTBody(self):
        from urlparse import parse_qs
        req_params = {}
        try:
            params = parse_qs(qs=request.body)
            for key in params.keys():
                req_params[key] = params[key][0]
        except:
            log.exception("Unable to parse POST.")
        return req_params
    
    def _parseParams(self):
        params = {}
        req_params = self._getParams()
        if (req_params.has_key('verb') == False) :
            raise BadVerbError()
        
        verb = params["verb"] = req_params['verb']
        
        if verb not in ["GetRecord", "ListRecords", "ListIdentifiers", "Identify", "ListMetadataFormats", "ListSets"]:
            raise BadVerbError()
        
        if verb == 'GetRecord' or verb == 'ListRecords' or verb == 'ListIdentifiers':        
            if req_params.has_key('metadataPrefix') == False:
                raise BadArgumentError('metadataPrefix is a required parameter.', verb)
            params["metadataPrefix"] = metadataPrefix = req_params['metadataPrefix']
        
        if verb == 'GetRecord' or verb == 'ListMetadataFormats':
            if req_params.has_key('by_doc_ID') and req_params.has_key('by_resource_ID'):
                if self._isTrue(req_params['by_doc_ID']) == self._isTrue(req_params['by_resource_ID']):
                    raise BadArgumentError('by_doc_ID and by_resource_ID have conflicting values.', verb)
                
            if req_params.has_key('by_doc_ID'):
                params['by_doc_ID'] = self._isTrue(req_params['by_doc_ID'])
                params['by_resource_ID'] = not params['by_doc_ID']
            else:
                params['by_doc_ID'] = False
                params['by_resource_ID'] = not params['by_doc_ID']
            
            if req_params.has_key('by_resource_ID'):
                params['by_resource_ID'] = self._isTrue(req_params['by_resource_ID'])
                params['by_doc_ID'] = not params['by_resource_ID']
        
        if verb == 'ListRecords' or verb == 'ListIdentifiers':
            from_date_gran = None
            until_date_gran = None
            if req_params.has_key('from'):
                try:
                    from_date = iso8601.parse_date(req_params['from'])
                    params['from'] = h.convertToISO8601UTC(from_date)
                    from_date_gran = h.getISO8601Granularity(req_params['from'])
                except:
                    raise BadArgumentError('from does not parse to ISO 8601.', verb)
            else:
                params['from'] = None
            
            if req_params.has_key('until'):
                try:
                    until_date = iso8601.parse_date(req_params['until'])
                    params['until'] = h.convertToISO8601UTC(until_date)
                    until_date_gran = h.getISO8601Granularity(req_params['until'])
                except:
                    raise BadArgumentError('until does not parse to ISO 8601.', verb)
            else:
                params['until'] = None
                
            if params['from'] != None and params['until'] != None:
                if params['from'] > params['until']:
                    raise BadArgumentError('until cannot preceed from.', verb)
                if from_date_gran != until_date_gran:
                    raise BadArgumentError('from and until parameters do not use the same granularity.', verb)
            
            harvestServiceGranularity = h.getHarvestServiceGranularity()
            
            if params['from'] != None and from_date_gran > harvestServiceGranularity:
                raise BadArgumentError('from is more granular than Harvest service permits', verb)
        
            if params['until'] != None and until_date_gran > harvestServiceGranularity:
                raise BadArgumentError('until is more granular than Harvest service permits', verb)
            
            if (from_date_gran != None and from_date_gran.granule != "day" and from_date_gran.granule != "second") or \
                (until_date_gran != None and until_date_gran.granule != "day" and until_date_gran.granule != "second"):
                formatSupport = "YYYY-MM-DD"
                if harvestServiceGranularity.granule == "second":
                    formatSupport = "YYYY-MM-DD and YYYY-MM-DDThh:mm:ssZ"
                raise BadArgumentError('from and until support {0} formats' % (formatSupport, ), verb)
            
        
        if verb in ['ListMetadataFormats', 'GetRecord']  and req_params.has_key('identifier'):
            params['identifier'] = req_params['identifier']
        elif verb == 'ListMetadataFormats':
            params['identifier'] = None
            params['by_doc_ID'] = None
            params['by_resource_ID'] = None
        
        if verb == 'GetRecord' and req_params.has_key('identifier') == False:
            raise BadArgumentError('identifier is a required parameter.', verb)
        
        return params

    def _returnResponse(self, body):
        response.headers["Content-Type"] = "text/xml; charset=utf-8"
        return body
                                 
    
    def _initRender(self,params):
        c.datetime_now = datetime.utcnow().isoformat()
        c.path_url = request.path_url
        if params.has_key("by_doc_ID"):
            c.by_doc_ID = params["by_doc_ID"]
        if params.has_key("by_resource_ID"):
            c.by_resource_ID = params["by_resource_ID"]
        if params.has_key("metadataPrefix"):
            c.metadataPrefix = params["metadataPrefix"]
        if params.has_key("from"):
            c.from_date = params["from"]
        if params.has_key("until"):
            c.until_date = params["until"]
        if params.has_key("identifier"):
            c.identifier = params["identifier"]
        
    def _handleOAIRequest(self, format='html'):
        o = oaipmh()
        
        def GetRecord(params):
            try:
                c.identifier = params["identifier"]
                if params["by_doc_ID"]  == True:
                    c.docList = [o.get_record(params["identifier"])]
                else:
                    c.docList = list(o.get_records_by_resource(params["identifier"]))
                    if len(c.docList) == 0:
                        raise IdDoesNotExistError(params['verb'])
            except ResourceNotFound:
                raise IdDoesNotExistError(params['verb'])
            
            self._initRender(params)
            
            if c.docList != None:
                for doc in c.docList:
                    if c.metadataPrefix not in doc["payload_schema"]:
                        raise CannotDisseminateFormatError(params['verb'])

            body = "<error/>"
            try:
                body = render("/oaipmh-GetRecord.mako")
            except:
                log.exception("Unable to render template")
            return self._returnResponse(body)
        
{'fraction_non_alphanumeric': 0.10409629675121292, 'fraction_numerical': 0.005035927040471657, 'mean_word_length': 2.002765996680804, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5601644', 'n_tokens_mistral': 4563, 'n_tokens_neox': 4354, 'n_words': 1001}