<commit_before>from pylons import config
from ckan.lib.helpers import url_for
from datetime import datetime

from xmltodict import unparse
import collections
import sys

from ckanext.package_converter.model.metadata_format import MetadataFormats, XMLMetadataFormat
from ckanext.package_converter.model.record import XMLRecord, Record

import ckan.plugins as plugins

import oaipmh_error
from record_access import RecordAccessService

from logging import getLogger
log = getLogger(__name__)

class OAIPMHRepository(plugins.SingletonPlugin):

    def __init__(self):
        self.dateformat = config.get('oaipmh_repository.dateformat', '%Y-%m-%dT%H:%M:%SZ')
        self.verb_handlers = {
            'Identify': self.identify,
            'GetRecord': self.get_record,
            'ListIdentifiers': self.list_identifiers,
            'ListMetadataFormats': self.list_metadata_formats,
            'ListRecords': self.list_records,
            'ListSets': self.list_sets
        }
        self.id_prefix = config.get('oaipmh_repository.id_prefix', 'oai:ckan:id:')
        self.id_field = config.get('oaipmh_repository.id_field', 'name')
        self.regex = config.get('oaipmh_repository.regex', '*')
        self.record_access = RecordAccessService(self.dateformat, self.id_prefix, self.id_field, self.regex)
        log.debug(self)

    def handle_request(self, verb, params, url):
        oaipmh_verb = 'error'
        try:
            handler = self.verb_handlers[verb]
            content = handler(params)
            oaipmh_verb = verb
        except oaipmh_error.OAIPMHError as e:
            content = e.as_xml_dict()
        except KeyError as e:
             content = oaipmh_error.BadVerbError().as_xml_dict()
        except:
           e = sys.exc_info()[1]
           code = type(e).__name__
           message = str(e)
           content = oaipmh_error.OAIPMHError(code,  message).as_xml_dict()

        xmldict = self._envelop(oaipmh_verb, params, url, content)

        if not self._is_valid_oai_pmh_record(xmldict, metadata_prefix=params.get('metadataPrefix')):
            log.error('OAI-PMH Response Validation FAILED')
        else:
            log.debug('OAI-PMH Response Validation SUCCESS')
            log.debug('OAI-PMH Response is ERROR = {0}'.format(self._is_error_oai_pmh_record(xmldict)))

        return unparse(xmldict, pretty=True)

    def identify(self, params={}):
        if params:
            raise oaipmh_error.BadArgumentError()
        
        identify_dict = collections.OrderedDict()
        identify_dict['repositoryName'] = config.get('site.title') if config.get('site.title') else 'repository'
        identify_dict['baseURL'] = url_for(controller='ckanext.oaipmh_repository.controller:OAIPMHController', action='index')
        identify_dict['protocolVersion'] = '2.0'
        identify_dict['adminEmail'] = config.get('email_to', '<EMAIL>')
        identify_dict['earliestDatestamp'] = datetime(2004, 1, 1).strftime(self.dateformat)
        identify_dict['deletedRecord'] = 'no'
        identify_dict['granularity'] ='YYYY-MM-DD'
        return identify_dict

    def get_record(self, params):
        if set(params.keys()).difference(['identifier','metadataPrefix']):
            raise oaipmh_error.BadArgumentError()
        return(self.record_access.get_record(params.get('identifier'), params.get('metadataPrefix')))

    def list_identifiers(self, params):
        self._validate_params_list(params)
        return(self.record_access.list_identifiers(params.get('metadataPrefix'), 
                                                   params.get('from'), 
                                               	   params.get('until')))

    def list_metadata_formats(self, params):<commit_msg>fixed base URL in identify<commit_after>from pylons import config
from ckan.lib.helpers import url_for
from datetime import datetime

from xmltodict import unparse
import collections
import sys

from ckanext.package_converter.model.metadata_format import MetadataFormats, XMLMetadataFormat
from ckanext.package_converter.model.record import XMLRecord, Record

import ckan.plugins as plugins

import oaipmh_error
from record_access import RecordAccessService

from logging import getLogger
log = getLogger(__name__)

class OAIPMHRepository(plugins.SingletonPlugin):

    def __init__(self):
        self.dateformat = config.get('oaipmh_repository.dateformat', '%Y-%m-%dT%H:%M:%SZ')
        self.verb_handlers = {
            'Identify': self.identify,
            'GetRecord': self.get_record,
            'ListIdentifiers': self.list_identifiers,
            'ListMetadataFormats': self.list_metadata_formats,
            'ListRecords': self.list_records,
            'ListSets': self.list_sets
        }
        self.id_prefix = config.get('oaipmh_repository.id_prefix', 'oai:ckan:id:')
        self.id_field = config.get('oaipmh_repository.id_field', 'name')
        self.regex = config.get('oaipmh_repository.regex', '*')
        self.record_access = RecordAccessService(self.dateformat, self.id_prefix, self.id_field, self.regex)
        log.debug(self)

    def handle_request(self, verb, params, url):
        oaipmh_verb = 'error'
        try:
            handler = self.verb_handlers[verb]
            content = handler(params)
            oaipmh_verb = verb
        except oaipmh_error.OAIPMHError as e:
            content = e.as_xml_dict()
        except KeyError as e:
             content = oaipmh_error.BadVerbError().as_xml_dict()
        except:
           e = sys.exc_info()[1]
           code = type(e).__name__
           message = str(e)
           content = oaipmh_error.OAIPMHError(code,  message).as_xml_dict()

        xmldict = self._envelop(oaipmh_verb, params, url, content)

        if not self._is_valid_oai_pmh_record(xmldict, metadata_prefix=params.get('metadataPrefix')):
            log.error('OAI-PMH Response Validation FAILED')
        else:
            log.debug('OAI-PMH Response Validation SUCCESS')
            log.debug('OAI-PMH Response is ERROR = {0}'.format(self._is_error_oai_pmh_record(xmldict)))

        return unparse(xmldict, pretty=True)

    def identify(self, params={}):
        if params:
            raise oaipmh_error.BadArgumentError()
        
        identify_dict = collections.OrderedDict()
        identify_dict['repositoryName'] = config.get('site.title') if config.get('site.title') else 'repository'
        identify_dict['baseURL'] = config.get('ckan.site_url') + url_for(controller='ckanext.oaipmh_repository.controller:OAIPMHController', action='index')
        identify_dict['protocolVersion'] = '2.0'
        identify_dict['adminEmail'] = config.get('email_to', '<EMAIL>')
        identify_dict['earliestDatestamp'] = datetime(2004, 1, 1).strftime(self.dateformat)
        identify_dict['deletedRecord'] = 'no'
        identify_dict['granularity'] ='YYYY-MM-DD'
        return identify_dict

    def get_record(self, params):
        if set(params.keys()).difference(['identifier','metadataPrefix']):
            raise oaipmh_error.BadArgumentError()
        return(self.record_access.get_record(params.get('identifier'), params.get('metadataPrefix')))

    def list_identifiers(self, params):
        self._validate_params_list(params)
        return(self.record_access.list_identifiers(params.get('metadataPrefix'), 
                                                   params.get('from'), 
                                               	   params.get('until')))

    def list_metadata_formats(self, params):
{'fraction_non_alphanumeric': 0.11001729413329786, 'fraction_numerical': 0.002660635891978183, 'mean_word_length': 2.96309963099631, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '666474', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2226, 'n_words': 392}