<commit_before># -*- coding: ISO-8859-15 -*-
# =================================================================
#
# $Id$
#
# Authors: <NAME> <<EMAIL>>
#                <NAME> <<EMAIL>>
#
# Copyright (c) 2010 <NAME>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================

import os
import sys
import cgi
import urllib2
from lxml import etree
import config, fes, log, profile, repository, util

class Csw(object):
    ''' Base CSW server '''
    def __init__(self, configfile=None):
        ''' Initialize CSW '''
        # load user configuration
        self.config = config.get_config(configfile)

        # configure transaction support, if specified in config
        self._gen_manager()

        # init kvp
        self.kvp = {}

        self.gzip = False
        self.soap = False
        self.request = None
        self.exception = False
        self.profiles = None

        # configure logging
        try:
            self.log = log.initlog(self.config)
        except Exception, err:
            self.response = self.exceptionreport(
            'NoApplicableCode', 'service', str(err))

        # generate domain model
        config.MODEL['operations']['GetDomain'] = config.gen_domains()

        # set OGC schemas location
        if self.config['server'].has_key('ogc_schemas_base') is False:
            self.config['server']['ogc_schemas_base'] = config.OGC_SCHEMAS_BASE

        # set XML pretty print
        if (self.config['server'].has_key('xml_pretty_print') and
        self.config['server']['xml_pretty_print'] == 'true'):
            self.xml_pretty_print = 1
        else:
            self.xml_pretty_print = 0

        # set compression level
        if self.config['server'].has_key('gzip_compresslevel'):
            self.gzip_compresslevel = \
            int(self.config['server']['gzip_compresslevel'])
        else:
            self.gzip_compresslevel = 9

        # generate distributed search model, if specified in config
        if self.config['server'].has_key('federatedcatalogues'):
            self.log.debug('Configuring distributed search.')
            config.MODEL['constraints']['FederatedCatalogues'] = {}
            config.MODEL['constraints']['FederatedCatalogues']['values'] = []
            for fedcat in \
            self.config['server']['federatedcatalogues'].split(','):
                config.MODEL\
                ['constraints']['FederatedCatalogues']['values'].append(fedcat)

        self.log.debug('Configuration: %s.' % self.config)
        self.log.debug('Model: %s.' % config.MODEL)

        # load profiles

        # turn on APISO by default
        #self.config['server']['profiles'] = 'apiso'

        self.log.debug('Loading profiles.')

        if self.config['server'].has_key('profiles'):
            self.profiles = profile.load_profiles(
            os.path.join('server', 'profiles'), profile.Profile,
            self.config['server']['profiles'])
    
            for prof in self.profiles['plugins'].keys():
                tmp = self.profiles['plugins'][prof]()
                key = tmp.outputschema  # to ref by outputschema
                self.profiles['loaded'][key] = tmp
                self.profiles['loaded'][key].extend_core(
                config.MODEL, config.NAMESPACES, self.config)
    
            self.log.debug('Profiles loaded: %s.' %
            self.profiles['loaded'].keys())

        # init repository
        self.repository = \
        repository.Repository(self.config['repository'], 'records',
        config.MODEL['typenames'])

        self.log.debug('Repository loaded: %s.' % self.repository)

    def dispatch(self):
        ''' Handle incoming HTTP request '''
        error = 0

        if hasattr(self,'response'):
            self._write_response()
            return

        cgifs = cgi.FieldStorage(keep_blank_values=1)
    
        if cgifs.file:  # it's a POST request
            postdata = cgifs.file.read()
            self.requesttype = 'POST'
            self.request = postdata
            self.log.debug('Request type: POST.  Request:\n%s\n', self.request)
            self.kvp = self.parse_postdata(postdata)
            if isinstance(self.kvp, str):  # it's an exception
                error = 1
                locator = 'service'
                code = 'InvalidRequest'
                text = self.kvp
    
        else:  # it's a GET request
            self.requesttype = 'GET'
            self.request = 'http://%s%s' % \
            (os.environ['HTTP_HOST'], os.environ['REQUEST_URI'])
            self.log.debug('Request type: GET.  Request:\n%s\n', self.request)
            for key in cgifs.keys():
                self.kvp[key.lower()] = cgifs[key].value

        # check if Accept-Encoding was passed as an HTTP header
        self.log.debug('HTTP Headers:\n%s.' % os.environ)

        if (os.environ.has_key('HTTP_ACCEPT_ENCODING') and
           os.environ['HTTP_ACCEPT_ENCODING'].find('gzip') != -1):
           # set for gzip compressed response 
            self.gzip = True

        self.log.debug('Parsed request parameters: %s' % self.kvp)

        if error == 0:
            # test for the basic keyword values (service, version, request)
            for k in ['service', 'version', 'request']:
                if self.kvp.has_key(k) == False:
                    if (k == 'version' and self.kvp.has_key('request') and
                    self.kvp['request'] == 'GetCapabilities'):
                        pass
                    else:
                        error   = 1
                        locator = k
                        code = 'MissingParameterValue'
                        text = 'Missing keyword: %s' % k
                        break
        
            # test each of the basic keyword values
            if error == 0:
                # test service
                if self.kvp['service'] != 'CSW':
                    error = 1
                    locator = 'service'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for service: %s.\
                    Value MUST be CSW' % self.kvp['service']
        
                # test version
                if (self.kvp.has_key('version') and
                    util.get_version_integer(self.kvp['version']) !=
                    util.get_version_integer('2.0.2') and
                    self.kvp['request'] != 'GetCapabilities'):
                    error = 1
                    locator = 'version'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for version: %s.\
                    Value MUST be 2.0.2' % self.kvp['version']
        
                # check for GetCapabilities acceptversions
                if self.kvp.has_key('acceptversions'):
                    for vers in self.kvp['acceptversions'].split(','):
                        if (util.get_version_integer(vers) ==
                            util.get_version_integer('2.0.2')):
                            break
                        else:
                            error = 1
                            locator = 'acceptversions'
                            code = 'VersionNegotiationFailed'
                            text = 'Invalid parameter value in acceptversions:\
                            %s. Value MUST be 2.0.2' % \
                            self.kvp['acceptversions']
        
                # test request
                if self.kvp['request'] not in config.MODEL['operations'].keys():
                    error = 1
                    locator = 'request'
                    if self.kvp['request'] in ['Transaction','Harvest']:
                        code = 'OperationNotSupported'
                        text = '%s operations are not supported' % \
                        self.kvp['request']
                    else:
                        code = 'InvalidParameterValue'
                        text = 'Invalid value for request: %s' % \
                        self.kvp['request']

        if error == 1:  # return an ExceptionReport
            self.response = self.exceptionreport(code, locator, text)
    
        else:  # process per the request value
            if self.kvp['request'] == 'GetCapabilities':
                self.response = self.getcapabilities()
            elif self.kvp['request'] == 'DescribeRecord':
                self.response = self.describerecord()
            elif self.kvp['request'] == 'GetDomain':
                self.response = self.getdomain()
            elif self.kvp['request'] == 'GetRecords':
                self.response = self.getrecords()
            elif self.kvp['request'] == 'GetRecordById':
                self.response = self.getrecordbyid()
            elif self.kvp['request'] == 'GetRepositoryItem':
                self.response = self.getrepositoryitem()
            elif self.kvp['request'] == 'Transaction':
                self.response = self.transaction()
            elif self.kvp['request'] == 'Harvest':
                self.response = self.harvest()
            else:
                self.response = self.exceptionreport('InvalidParameterValue',
                'request', 'Invalid request parameter: %s' %
                self.kvp['request'])

        self._write_response()

    def exceptionreport(self, code, locator, text):
        ''' Generate ExceptionReport '''
        self.exception = True

        node = etree.Element(util.nspath_eval('ows:ExceptionReport'),
        nsmap=config.NAMESPACES, version='1.0.2',
        language=self.config['server']['language'])

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/ows/1.0.0/owsExceptionReport.xsd' % \
        (config.NAMESPACES['ows'], self.config['server']['ogc_schemas_base'])

        exception = etree.SubElement(node, util.nspath_eval('ows:Exception'),
        exceptionCode=code, locator=locator)

        etree.SubElement(exception,
        util.nspath_eval('ows:ExceptionText')).text=text

        return node
    
    def getcapabilities(self):
        ''' Handle GetCapabilities request '''
        serviceidentification = True
        serviceprovider = True
        operationsmetadata = True
        if self.kvp.has_key('sections'):
            serviceidentification = False
            serviceprovider = False
            operationsmetadata = False
            for section in self.kvp['sections'].split(','):
                if section == 'ServiceIdentification':
                    serviceidentification = True
                if section == 'ServiceProvider':
                    serviceprovider = True
                if section == 'OperationsMetadata':
                    operationsmetadata = True

        # check extra parameters that may be def'd by profiles
        if self.profiles is not None:
            for prof in self.profiles['loaded'].keys():
                result = \
                self.profiles['loaded'][prof].check_parameters(self.kvp)
                if result is not None:
                    return self.exceptionreport(result['code'],
                    result['locator'], result['text'])

        # @updateSequence: get latest update to repository

        updatesequence = \
        util.get_time_iso2unix(self.repository.query_latest_insert())

        node = etree.Element(util.nspath_eval('csw:Capabilities'),
        nsmap=config.NAMESPACES, version='2.0.2',
        updateSequence=str(updatesequence))

        if self.kvp.has_key('updatesequence'):
            if int(self.kvp['updatesequence']) == updatesequence:
                return node
            elif int(self.kvp['updatesequence']) > updatesequence:
                return self.exceptionreport('InvalidUpdateSequence',
                'updatesequence', 
                'outputsequence specified (%s) is higher than server\'s \
                updatesequence (%s)' % (self.kvp['updatesequence'], 
                updatesequence))

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        if serviceidentification:
            self.log.debug('Writing section ServiceIdentification.')

            serviceidentification = etree.SubElement(node, \
            util.nspath_eval('ows:ServiceIdentification'))

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Title')).text = \
            self.config['metadata:main']['identification_title']

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Abstract')).text = \
            self.config['metadata:main']['identification_abstract']

            keywords = etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Keywords'))

            for k in \
            self.config['metadata:main']['identification_keywords'].split(','):
                etree.SubElement(
                keywords, util.nspath_eval('ows:Keyword')).text = k

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:ServiceType'), codeSpace='OGC').text = 'CSW'

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:ServiceTypeVersion')).text = '2.0.2'

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Fees')).text = \
            self.config['metadata:main']['identification_fees']

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:AccessConstraints')).text = \
            self.config['metadata:main']['identification_accessconstraints']

        if serviceprovider:
            self.log.debug('Writing section ServiceProvider.')
            serviceprovider = etree.SubElement(node,
            util.nspath_eval('ows:ServiceProvider'))

            etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ProviderName')).text = \
            self.config['metadata:main']['provider_name']

            providersite = etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ProviderSite'))

            providersite.attrib[util.nspath_eval('xlink:type')] = 'simple'
            providersite.attrib[util.nspath_eval('xlink:href')] = \
            self.config['metadata:main']['provider_url']

            servicecontact = etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ServiceContact'))

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:IndividualName')).text = \
            self.config['metadata:main']['contact_name']

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:PositionName')).text = \
            self.config['metadata:main']['contact_position']

            contactinfo = etree.SubElement(servicecontact,
            util.nspath_eval('ows:ContactInfo'))

            phone = etree.SubElement(contactinfo, util.nspath_eval('ows:Phone'))
            etree.SubElement(phone, util.nspath_eval('ows:Voice')).text = \
            self.config['metadata:main']['contact_phone']

            etree.SubElement(phone, util.nspath_eval('ows:Facsimile')).text = \
            self.config['metadata:main']['contact_fax']

            address = etree.SubElement(contactinfo,
            util.nspath_eval('ows:Address'))

            etree.SubElement(address,
            util.nspath_eval('ows:DeliveryPoint')).text = \
            self.config['metadata:main']['contact_address']

            etree.SubElement(address, util.nspath_eval('ows:City')).text = \
            self.config['metadata:main']['contact_city']

            etree.SubElement(address,
            util.nspath_eval('ows:AdministrativeArea')).text = \
            self.config['metadata:main']['contact_stateorprovince']

            etree.SubElement(address,
            util.nspath_eval('ows:PostalCode')).text = \
            self.config['metadata:main']['contact_postalcode']

            etree.SubElement(address, util.nspath_eval('ows:Country')).text = \
            self.config['metadata:main']['contact_country']

            etree.SubElement(address,
            util.nspath_eval('ows:ElectronicMailAddress')).text = \
            self.config['metadata:main']['contact_email']

            url = etree.SubElement(contactinfo,
            util.nspath_eval('ows:OnlineResource'))

            url.attrib[util.nspath_eval('xlink:type')] = 'simple'
            url.attrib[util.nspath_eval('xlink:href')] = \
            self.config['metadata:main']['contact_url']

            etree.SubElement(contactinfo,
                             util.nspath_eval('ows:HoursOfService')).text = \
                             self.config['metadata:main']['contact_hours']

            etree.SubElement(contactinfo,
            util.nspath_eval('ows:ContactInstructions')).text = \
            self.config['metadata:main']['contact_instructions']

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:Role')).text = \
            self.config['metadata:main']['contact_role']

        if operationsmetadata:
            self.log.debug('Writing section OperationsMetadata.')
            operationsmetadata = etree.SubElement(node,
            util.nspath_eval('ows:OperationsMetadata'))

            for operation in config.MODEL['operations'].keys():
                oper = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Operation'), name = operation)

                dcp = etree.SubElement(oper, util.nspath_eval('ows:DCP'))
                http = etree.SubElement(dcp, util.nspath_eval('ows:HTTP'))

                if config.MODEL['operations'][operation]['methods']['get']:
                    get = etree.SubElement(http, util.nspath_eval('ows:Get'))
                    get.attrib[util.nspath_eval('xlink:type')] = 'simple'
                    get.attrib[util.nspath_eval('xlink:href')] = \
                    self.config['server']['url']

                if config.MODEL['operations'][operation]['methods']['post']:
                    post = etree.SubElement(http, util.nspath_eval('ows:Post'))
                    post.attrib[util.nspath_eval('xlink:type')] = 'simple'
                    post.attrib[util.nspath_eval('xlink:href')] = \
                    self.config['server']['url']

                for parameter in \
                config.MODEL['operations'][operation]['parameters']:
                    param = etree.SubElement(oper,
                    util.nspath_eval('ows:Parameter'), name = parameter)

                    for val in \
                    config.MODEL['operations'][operation]\
                    ['parameters'][parameter]['values']:
                        etree.SubElement(param,
                        util.nspath_eval('ows:Value')).text = val

                if operation == 'GetRecords':  # advertise queryables
                    for qbl in self.repository.queryables.keys():
                        if qbl != '_all':
                            param = etree.SubElement(oper,
                            util.nspath_eval('ows:Constraint'), name = qbl)
    
                            for qbl2 in self.repository.queryables[qbl]:
                                etree.SubElement(param,
                                util.nspath_eval('ows:Value')).text = qbl2

                    if self.profiles is not None:
                        for con in config.MODEL[\
                        'operations']['GetRecords']['constraints'].keys():
                            param = etree.SubElement(oper,
                            util.nspath_eval('ows:Constraint'), name = con)
                            for val in config.MODEL['operations']\
                            ['GetRecords']['constraints'][con]['values']:
                                etree.SubElement(param,
                                util.nspath_eval('ows:Value')).text = val

            for parameter in config.MODEL['parameters'].keys():
                param = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Parameter'), name = parameter)
                for val in config.MODEL['parameters'][parameter]['values']:
                    etree.SubElement(param,
                    util.nspath_eval('ows:Value')).text = val

            for constraint in config.MODEL['constraints'].keys():
                param = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Constraint'), name = constraint)
                for val in config.MODEL['constraints'][constraint]['values']:
                    etree.SubElement(param,
                    util.nspath_eval('ows:Value')).text = val
            
            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    ecnode = \
                    self.profiles['loaded'][prof].get_extendedcapabilities()
                    if ecnode is not None:
                        operationsmetadata.append(ecnode)

        # always write out Filter_Capabilities
        self.log.debug('Writing section Filter_Capabilities.')
        fltcaps = etree.SubElement(node,
        util.nspath_eval('ogc:Filter_Capabilities'))

        spatialcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Spatial_Capabilities'))

        geomops = etree.SubElement(spatialcaps,
        util.nspath_eval('ogc:GeometryOperands'))

        for geomtype in \
        fes.MODEL['GeometryOperands']['values']:
            etree.SubElement(geomops,
            util.nspath_eval('ogc:GeometryOperand')).text = geomtype
    
        spatialops = etree.SubElement(spatialcaps,
        util.nspath_eval('ogc:SpatialOperators'))

        for spatial_comparison in \
        fes.MODEL['SpatialOperators']['values']:
            etree.SubElement(spatialops,
            util.nspath_eval('ogc:SpatialOperator'), name = spatial_comparison)
    
        scalarcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Scalar_Capabilities'))

        etree.SubElement(scalarcaps, util.nspath_eval('ogc:LogicalOperators'))
        
        cmpops = etree.SubElement(scalarcaps,
        util.nspath_eval('ogc:ComparisonOperators'))
    
        for cmpop in fes.MODEL['ComparisonOperators'].keys():
            etree.SubElement(cmpops,
            util.nspath_eval('ogc:ComparisonOperator')).text = \
            fes.MODEL['ComparisonOperators'][cmpop]['opname']
    
        idcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Id_Capabilities'))

        for idcap in fes.MODEL['Ids']['values']:
            etree.SubElement(idcaps, util.nspath_eval('ogc:%s' % idcap))

        return node
    
    def describerecord(self):
        ''' Handle DescribeRecord request '''
    
        if self.kvp.has_key('typename') is False or \
        len(self.kvp['typename']) == 0:  # missing typename
        # set to return all typenames
            self.kvp['typename'] = []
            self.kvp['typename'].append('csw:Record')

            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    self.kvp['typename'].append(
                    self.profiles['loaded'][prof].typename)

        if self.requesttype == 'GET':  # pass via GET
            self.kvp['typename'] = self.kvp['typename'].split(',')

        if (self.kvp.has_key('outputformat') and
            self.kvp['outputformat'] not in
            config.MODEL['operations']['DescribeRecord']
            ['parameters']['outputFormat']['values']):  # bad outputformat
            return self.exceptionreport('InvalidParameterValue',
            'outputformat', 'Invalid value for outputformat: %s' %
            self.kvp['outputformat'])
    
        if (self.kvp.has_key('schemalanguage') and
            self.kvp['schemalanguage'] not in
            config.MODEL['operations']['DescribeRecord']['parameters']
            ['schemaLanguage']['values']):  # bad schemalanguage
            return self.exceptionreport('InvalidParameterValue',
            'schemalanguage', 'Invalid value for schemalanguage: %s' %
            self.kvp['schemalanguage'])

        node = etree.Element(util.nspath_eval('csw:DescribeRecordResponse'),
        nsmap = config.NAMESPACES)

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        for typename in self.kvp['typename']:
            if typename == 'csw:Record':   # load core schema    
                self.log.debug('Writing csw:Record schema.')
                schemacomponent = etree.SubElement(node,
                util.nspath_eval('csw:SchemaComponent'),
                schemaLanguage='XMLSCHEMA',
                targetNamespace = config.NAMESPACES['csw'])
                dublincore = etree.parse(os.path.join(
                self.config['server']['home'],
                'etc', 'schemas', 'ogc', 'csw',
                '2.0.2', 'record.xsd')).getroot()

                schemacomponent.append(dublincore)

            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    if self.profiles['loaded'][prof].typename == typename:
                        scnodes = \
                        self.profiles['loaded'][prof].get_schemacomponents()
                        if scnodes is not None:
                            for scn in scnodes:
                                node.append(scn)
        return node
    
    def getdomain(self):
        ''' Handle GetDomain request '''
        if (self.kvp.has_key('parametername') is False and
            self.kvp.has_key('propertyname') is False):
            return self.exceptionreport('MissingParameterValue',
            'parametername', 'Missing value. \
            One of propertyname or parametername must be specified')
        
        node = etree.Element(util.nspath_eval('csw:GetDomainResponse'),
        nsmap = config.NAMESPACES)

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        if self.kvp.has_key('parametername'):
            for pname in self.kvp['parametername'].split(','):
                self.log.debug('Parsing parametername %s.' % pname)
                domainvalue = etree.SubElement(node,
                util.nspath_eval('csw:DomainValues'), type = 'csw:Record')
                etree.SubElement(domainvalue,
                util.nspath_eval('csw:ParameterName')).text = pname
                operation, parameter = pname.split('.')
                if (operation in config.MODEL['operations'].keys() and
                    parameter in
                    config.MODEL['operations'][operation]['parameters'].keys()):
                    listofvalues = etree.SubElement(domainvalue,
                    util.nspath_eval('csw:ListOfValues'))
                    for val in \
                    config.MODEL['operations'][operation]\
                    ['parameters'][parameter]['values']:
                        etree.SubElement(listofvalues,
                        util.nspath_eval('csw:Value')).text = val

        if self.kvp.has_key('propertyname'):
            for pname in self.kvp['propertyname'].split(','):
                self.log.debug('Parsing propertyname %s.' % pname)

                if pname.find('/') == 0:  # it's an XPath
                    pname2 = pname
                else:  # it's a core queryable, map to internal typename model
                    pname2 = self.repository.queryables['_all'][pname]

                # TODO: smarter way of handling domain value type
                if pname2.find('apiso:') != -1 or pname2.find('gmd:') != -1:
                    dvtype = 'gmd:MD_Metadata'
                else:
                    dvtype = 'csw:Record'
                domainvalue = etree.SubElement(node,
                util.nspath_eval('csw:DomainValues'), type = dvtype)
                etree.SubElement(domainvalue,
                util.nspath_eval('csw:PropertyName')).text = pname

                try:
                    self.log.debug('Querying repository property %s.' % pname2)
        
                    results = self.repository.query_domain(pname2,[dvtype])
                    self.log.debug('Results: %s' % str(len(results)))
                    listofvalues = etree.SubElement(domainvalue,
                    util.nspath_eval('csw:ListOfValues'))
                    for result in results:
                        if result[0] is not None:  # drop null values
                            etree.SubElement(listofvalues,
                            util.nspath_eval('csw:Value')).text = result[0]
                except Exception, err:
                    self.log.debug('No results for propertyname %s: %s.' %
                    (pname2, str(err)))
        return node

    def getrecords(self):
        ''' Handle GetRecords request '''

        timestamp = util.get_today_and_now()

        if (self.kvp.has_key('elementsetname') is False and
            self.kvp.has_key('elementname') is False):
            # mutually exclusive required
            return self.exceptionreport('MissingParameterValue',
            'elementsetname',
            'Missing one of ElementSetName or ElementName parameter(s)')

        if self.kvp.has_key('outputschema') is False:
            self.kvp['outputschema'] = config.NAMESPACES['csw']

        if (self.kvp['outputschema'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['outputSchema']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'outputschema', 'Invalid outputSchema parameter value: %s' %
            self.kvp['outputschema'])

        if self.kvp.has_key('outputformat') is False:
            self.kvp['outputformat'] = 'application/xml'
     
        if (self.kvp['outputformat'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['outputFormat']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'outputformat', 'Invalid outputFormat parameter value: %s' %
            self.kvp['outputformat'])

        if self.kvp.has_key('resulttype') is False:
            self.kvp['resulttype'] = 'hits'

        if self.kvp['resulttype'] is not None:
            if (self.kvp['resulttype'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['resultType']['values']):
                return self.exceptionreport('InvalidParameterValue',
                'resulttype', 'Invalid resultType parameter value: %s' %
                self.kvp['resulttype'])

        if ((self.kvp.has_key('elementname') is False or 
             len(self.kvp['elementname']) == 0) and
             self.kvp['elementsetname'] not in
             config.MODEL['operations']['GetRecords']['parameters']
             ['ElementSetName']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'elementsetname', 'Invalid ElementSetName parameter value: %s' %
            self.kvp['elementsetname'])

        if (self.kvp.has_key('elementname') and
            self.requesttype == 'GET'):  # passed via GET
            self.kvp['elementname'] = self.kvp['elementname'].split(',')
            self.kvp['elementsetname'] = 'summary'
        
        if (self.kvp.has_key('typenames') and
            self.requesttype == 'GET'):  # passed via GET
            self.kvp['typenames'] = self.kvp['typenames'].split(',')

        if self.kvp.has_key('typenames'):
            for tname in self.kvp['typenames']:
                if (tname not in config.MODEL['operations']['GetRecords']
                    ['parameters']['typeNames']['values']):
                    return self.exceptionreport('InvalidParameterValue',
                    'typenames', 'Invalid typeNames parameter value: %s' %
                    tname)

        # check elementname's
        if self.kvp.has_key('elementname'):
            for ename in self.kvp['elementname']:
                enamelist = self.repository.queryables['_all'].keys()
                if ename not in enamelist:
                    return self.exceptionreport('InvalidParameterValue',
                    'elementname', 'Invalid ElementName parameter value: %s' %
                    ename)

        if self.kvp['resulttype'] == 'validate':
            return self._write_acknowledgement()

        if self.kvp.has_key('maxrecords') is False:
            self.kvp['maxrecords'] = self.config['server']['maxrecords']

        if self.kvp.has_key('constraint') and self.requesttype == 'GET':
            # GET request
            self.log.debug('csw:Constraint passed over HTTP GET.')
            if self.kvp.has_key('constraintlanguage') is False:
                return self.exceptionreport('MissingParameterValue',
                'constraintlanguage',
                'constraintlanguage required when constraint specified')
            if (self.kvp['constraintlanguage'] not in
            config.MODEL['operations']['GetRecords']['parameters']
            ['CONSTRAINTLANGUAGE']['values']):
                return self.exceptionreport('InvalidParameterValue',
                'constraintlanguage', 'Invalid constraintlanguage: %s'
                % self.kvp['constraintlanguage'])
            if self.kvp['constraintlanguage'] == 'CQL_TEXT':
                tmp = self.kvp['constraint']
                self.kvp['constraint'] = {}
                self.kvp['constraint']['type'] = 'cql'
                self.kvp['constraint']['where'] = \
                self._cql_update_queryables_mappings(tmp,
                self.repository.queryables['_all'])
            elif self.kvp['constraintlanguage'] == 'FILTER':
                # validate filter XML
                try:
                    schema = os.path.join(self.config['server']['home'],
                    'etc', 'schemas', 'ogc', 'filter', '1.1.0', 'filter.xsd')
                    self.log.debug('Validating Filter %s.' %
                    self.kvp['constraint'])
                    schema = etree.XMLSchema(etree.parse(schema))
                    parser = etree.XMLParser(schema=schema)
                    doc = etree.fromstring(self.kvp['constraint'], parser)
                    self.log.debug('Filter is valid XML.')
                    self.kvp['constraint'] = {}
                    self.kvp['constraint']['type'] = 'filter'
                    self.kvp['constraint']['where'] = \
                    fes.parse(doc,
                    self.repository.queryables['_all'].keys(),
                    self.repository.dbtype)
                except Exception, err:
                    errortext = \
                    'Exception: document not valid.\nError: %s.' % str(err)

                    self.log.debug(errortext)
                    return self.exceptionreport('InvalidParameterValue',
                    'constraint', 'Invalid Filter query: %s' % errortext)

        if self.kvp.has_key('sortby') is False:
            self.kvp['sortby'] = None

        if self.kvp.has_key('startposition') is False:
            self.kvp['startposition'] = 1

        # query repository
        self.log.debug('Querying repository with constraint : %s,\
        sortby: %s, typenames: %s.' %
        (self.kvp['constraint'], self.kvp['sortby'], self.kvp['typenames']))

        try:
            results = self.repository.query(constraint=self.kvp['constraint'],
            sortby=self.kvp['sortby'], typenames=self.kvp['typenames'])
        except Exception, err:
            return self.exceptionreport('InvalidParameterValue', 'constraint',
            'Invalid query: %s' % err)

        if len(results) == 0:
            matched = '0'
            returned = '0'
            nextrecord = '0'

        else:
            matched = str(len(results))
            if int(matched) < int(self.kvp['maxrecords']):
                returned = str(int(matched))
                nextrecord = '0'
            else:
                returned = str(self.kvp['maxrecords'])
                nextrecord = str(int(self.kvp['startposition']) + \
                int(self.kvp['maxrecords']))
            if int(self.kvp['maxrecords']) == 0:
                nextrecord = '1'

        dsresults = []

        if (self.config['server'].has_key('federatedcatalogues') and
            self.kvp['distributedsearch'] == 'TRUE' and
            self.kvp['hopcount'] > 0):  # do distributed search

            self.log.debug('DistributedSearch specified (hopCount: %s).' %
            self.kvp['hopcount'])

            from owslib.csw import CatalogueServiceWeb
            for fedcat in \
            self.config['server']['federatedcatalogues'].split(','):
                self.log.debug('Performing distributed search on federated \
                catalogue: %s.' % fedcat)
                remotecsw = CatalogueServiceWeb(fedcat)
                remotecsw.getrecords(xml=self.request)
                self.log.debug('Distributed search results from catalogue \
                %s: %s.' % (fedcat, remotecsw.results))
                matched = str(int(matched) + int(remotecsw.results['matches']))
                dsresults.append(remotecsw.records)

        self.log.debug('Results: matched: %s, returned: %s, next: %s.' % \
        (matched, returned, nextrecord))

        node = etree.Element(util.nspath_eval('csw:GetRecordsResponse'),
        nsmap = config.NAMESPACES, version='2.0.2')

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])
<commit_msg>fix GET constraint and distributedsearch handling<commit_after># -*- coding: ISO-8859-15 -*-
# =================================================================
#
# $Id$
#
# Authors: <NAME> <<EMAIL>>
#                <NAME> <<EMAIL>>
#
# Copyright (c) 2010 <NAME>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================

import os
import sys
import cgi
import urllib2
from lxml import etree
import config, fes, log, profile, repository, util

class Csw(object):
    ''' Base CSW server '''
    def __init__(self, configfile=None):
        ''' Initialize CSW '''
        # load user configuration
        self.config = config.get_config(configfile)

        # configure transaction support, if specified in config
        self._gen_manager()

        # init kvp
        self.kvp = {}

        self.gzip = False
        self.soap = False
        self.request = None
        self.exception = False
        self.profiles = None

        # configure logging
        try:
            self.log = log.initlog(self.config)
        except Exception, err:
            self.response = self.exceptionreport(
            'NoApplicableCode', 'service', str(err))

        # generate domain model
        config.MODEL['operations']['GetDomain'] = config.gen_domains()

        # set OGC schemas location
        if self.config['server'].has_key('ogc_schemas_base') is False:
            self.config['server']['ogc_schemas_base'] = config.OGC_SCHEMAS_BASE

        # set XML pretty print
        if (self.config['server'].has_key('xml_pretty_print') and
        self.config['server']['xml_pretty_print'] == 'true'):
            self.xml_pretty_print = 1
        else:
            self.xml_pretty_print = 0

        # set compression level
        if self.config['server'].has_key('gzip_compresslevel'):
            self.gzip_compresslevel = \
            int(self.config['server']['gzip_compresslevel'])
        else:
            self.gzip_compresslevel = 9

        # generate distributed search model, if specified in config
        if self.config['server'].has_key('federatedcatalogues'):
            self.log.debug('Configuring distributed search.')
            config.MODEL['constraints']['FederatedCatalogues'] = {}
            config.MODEL['constraints']['FederatedCatalogues']['values'] = []
            for fedcat in \
            self.config['server']['federatedcatalogues'].split(','):
                config.MODEL\
                ['constraints']['FederatedCatalogues']['values'].append(fedcat)

        self.log.debug('Configuration: %s.' % self.config)
        self.log.debug('Model: %s.' % config.MODEL)

        # load profiles

        # turn on APISO by default
        #self.config['server']['profiles'] = 'apiso'

        self.log.debug('Loading profiles.')

        if self.config['server'].has_key('profiles'):
            self.profiles = profile.load_profiles(
            os.path.join('server', 'profiles'), profile.Profile,
            self.config['server']['profiles'])
    
            for prof in self.profiles['plugins'].keys():
                tmp = self.profiles['plugins'][prof]()
                key = tmp.outputschema  # to ref by outputschema
                self.profiles['loaded'][key] = tmp
                self.profiles['loaded'][key].extend_core(
                config.MODEL, config.NAMESPACES, self.config)
    
            self.log.debug('Profiles loaded: %s.' %
            self.profiles['loaded'].keys())

        # init repository
        self.repository = \
        repository.Repository(self.config['repository'], 'records',
        config.MODEL['typenames'])

        self.log.debug('Repository loaded: %s.' % self.repository)

    def dispatch(self):
        ''' Handle incoming HTTP request '''
        error = 0

        if hasattr(self,'response'):
            self._write_response()
            return

        cgifs = cgi.FieldStorage(keep_blank_values=1)
    
        if cgifs.file:  # it's a POST request
            postdata = cgifs.file.read()
            self.requesttype = 'POST'
            self.request = postdata
            self.log.debug('Request type: POST.  Request:\n%s\n', self.request)
            self.kvp = self.parse_postdata(postdata)
            if isinstance(self.kvp, str):  # it's an exception
                error = 1
                locator = 'service'
                code = 'InvalidRequest'
                text = self.kvp
    
        else:  # it's a GET request
            self.requesttype = 'GET'
            self.request = 'http://%s%s' % \
            (os.environ['HTTP_HOST'], os.environ['REQUEST_URI'])
            self.log.debug('Request type: GET.  Request:\n%s\n', self.request)
            for key in cgifs.keys():
                self.kvp[key.lower()] = cgifs[key].value

        # check if Accept-Encoding was passed as an HTTP header
        self.log.debug('HTTP Headers:\n%s.' % os.environ)

        if (os.environ.has_key('HTTP_ACCEPT_ENCODING') and
           os.environ['HTTP_ACCEPT_ENCODING'].find('gzip') != -1):
           # set for gzip compressed response 
            self.gzip = True

        self.log.debug('Parsed request parameters: %s' % self.kvp)

        if error == 0:
            # test for the basic keyword values (service, version, request)
            for k in ['service', 'version', 'request']:
                if self.kvp.has_key(k) == False:
                    if (k == 'version' and self.kvp.has_key('request') and
                    self.kvp['request'] == 'GetCapabilities'):
                        pass
                    else:
                        error   = 1
                        locator = k
                        code = 'MissingParameterValue'
                        text = 'Missing keyword: %s' % k
                        break
        
            # test each of the basic keyword values
            if error == 0:
                # test service
                if self.kvp['service'] != 'CSW':
                    error = 1
                    locator = 'service'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for service: %s.\
                    Value MUST be CSW' % self.kvp['service']
        
                # test version
                if (self.kvp.has_key('version') and
                    util.get_version_integer(self.kvp['version']) !=
                    util.get_version_integer('2.0.2') and
                    self.kvp['request'] != 'GetCapabilities'):
                    error = 1
                    locator = 'version'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for version: %s.\
                    Value MUST be 2.0.2' % self.kvp['version']
        
                # check for GetCapabilities acceptversions
                if self.kvp.has_key('acceptversions'):
                    for vers in self.kvp['acceptversions'].split(','):
                        if (util.get_version_integer(vers) ==
                            util.get_version_integer('2.0.2')):
                            break
                        else:
                            error = 1
                            locator = 'acceptversions'
                            code = 'VersionNegotiationFailed'
                            text = 'Invalid parameter value in acceptversions:\
                            %s. Value MUST be 2.0.2' % \
                            self.kvp['acceptversions']
        
                # test request
                if self.kvp['request'] not in config.MODEL['operations'].keys():
                    error = 1
                    locator = 'request'
                    if self.kvp['request'] in ['Transaction','Harvest']:
                        code = 'OperationNotSupported'
                        text = '%s operations are not supported' % \
                        self.kvp['request']
                    else:
                        code = 'InvalidParameterValue'
                        text = 'Invalid value for request: %s' % \
                        self.kvp['request']

        if error == 1:  # return an ExceptionReport
            self.response = self.exceptionreport(code, locator, text)
    
        else:  # process per the request value
            if self.kvp['request'] == 'GetCapabilities':
                self.response = self.getcapabilities()
            elif self.kvp['request'] == 'DescribeRecord':
                self.response = self.describerecord()
            elif self.kvp['request'] == 'GetDomain':
                self.response = self.getdomain()
            elif self.kvp['request'] == 'GetRecords':
                self.response = self.getrecords()
            elif self.kvp['request'] == 'GetRecordById':
                self.response = self.getrecordbyid()
            elif self.kvp['request'] == 'GetRepositoryItem':
                self.response = self.getrepositoryitem()
            elif self.kvp['request'] == 'Transaction':
                self.response = self.transaction()
            elif self.kvp['request'] == 'Harvest':
                self.response = self.harvest()
            else:
                self.response = self.exceptionreport('InvalidParameterValue',
                'request', 'Invalid request parameter: %s' %
                self.kvp['request'])

        self._write_response()

    def exceptionreport(self, code, locator, text):
        ''' Generate ExceptionReport '''
        self.exception = True

        node = etree.Element(util.nspath_eval('ows:ExceptionReport'),
        nsmap=config.NAMESPACES, version='1.0.2',
        language=self.config['server']['language'])

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/ows/1.0.0/owsExceptionReport.xsd' % \
        (config.NAMESPACES['ows'], self.config['server']['ogc_schemas_base'])

        exception = etree.SubElement(node, util.nspath_eval('ows:Exception'),
        exceptionCode=code, locator=locator)

        etree.SubElement(exception,
        util.nspath_eval('ows:ExceptionText')).text=text

        return node
    
    def getcapabilities(self):
        ''' Handle GetCapabilities request '''
        serviceidentification = True
        serviceprovider = True
        operationsmetadata = True
        if self.kvp.has_key('sections'):
            serviceidentification = False
            serviceprovider = False
            operationsmetadata = False
            for section in self.kvp['sections'].split(','):
                if section == 'ServiceIdentification':
                    serviceidentification = True
                if section == 'ServiceProvider':
                    serviceprovider = True
                if section == 'OperationsMetadata':
                    operationsmetadata = True

        # check extra parameters that may be def'd by profiles
        if self.profiles is not None:
            for prof in self.profiles['loaded'].keys():
                result = \
                self.profiles['loaded'][prof].check_parameters(self.kvp)
                if result is not None:
                    return self.exceptionreport(result['code'],
                    result['locator'], result['text'])

        # @updateSequence: get latest update to repository

        updatesequence = \
        util.get_time_iso2unix(self.repository.query_latest_insert())

        node = etree.Element(util.nspath_eval('csw:Capabilities'),
        nsmap=config.NAMESPACES, version='2.0.2',
        updateSequence=str(updatesequence))

        if self.kvp.has_key('updatesequence'):
            if int(self.kvp['updatesequence']) == updatesequence:
                return node
            elif int(self.kvp['updatesequence']) > updatesequence:
                return self.exceptionreport('InvalidUpdateSequence',
                'updatesequence', 
                'outputsequence specified (%s) is higher than server\'s \
                updatesequence (%s)' % (self.kvp['updatesequence'], 
                updatesequence))

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        if serviceidentification:
            self.log.debug('Writing section ServiceIdentification.')

            serviceidentification = etree.SubElement(node, \
            util.nspath_eval('ows:ServiceIdentification'))

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Title')).text = \
            self.config['metadata:main']['identification_title']

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Abstract')).text = \
            self.config['metadata:main']['identification_abstract']

            keywords = etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Keywords'))

            for k in \
            self.config['metadata:main']['identification_keywords'].split(','):
                etree.SubElement(
                keywords, util.nspath_eval('ows:Keyword')).text = k

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:ServiceType'), codeSpace='OGC').text = 'CSW'

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:ServiceTypeVersion')).text = '2.0.2'

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:Fees')).text = \
            self.config['metadata:main']['identification_fees']

            etree.SubElement(serviceidentification,
            util.nspath_eval('ows:AccessConstraints')).text = \
            self.config['metadata:main']['identification_accessconstraints']

        if serviceprovider:
            self.log.debug('Writing section ServiceProvider.')
            serviceprovider = etree.SubElement(node,
            util.nspath_eval('ows:ServiceProvider'))

            etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ProviderName')).text = \
            self.config['metadata:main']['provider_name']

            providersite = etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ProviderSite'))

            providersite.attrib[util.nspath_eval('xlink:type')] = 'simple'
            providersite.attrib[util.nspath_eval('xlink:href')] = \
            self.config['metadata:main']['provider_url']

            servicecontact = etree.SubElement(serviceprovider,
            util.nspath_eval('ows:ServiceContact'))

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:IndividualName')).text = \
            self.config['metadata:main']['contact_name']

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:PositionName')).text = \
            self.config['metadata:main']['contact_position']

            contactinfo = etree.SubElement(servicecontact,
            util.nspath_eval('ows:ContactInfo'))

            phone = etree.SubElement(contactinfo, util.nspath_eval('ows:Phone'))
            etree.SubElement(phone, util.nspath_eval('ows:Voice')).text = \
            self.config['metadata:main']['contact_phone']

            etree.SubElement(phone, util.nspath_eval('ows:Facsimile')).text = \
            self.config['metadata:main']['contact_fax']

            address = etree.SubElement(contactinfo,
            util.nspath_eval('ows:Address'))

            etree.SubElement(address,
            util.nspath_eval('ows:DeliveryPoint')).text = \
            self.config['metadata:main']['contact_address']

            etree.SubElement(address, util.nspath_eval('ows:City')).text = \
            self.config['metadata:main']['contact_city']

            etree.SubElement(address,
            util.nspath_eval('ows:AdministrativeArea')).text = \
            self.config['metadata:main']['contact_stateorprovince']

            etree.SubElement(address,
            util.nspath_eval('ows:PostalCode')).text = \
            self.config['metadata:main']['contact_postalcode']

            etree.SubElement(address, util.nspath_eval('ows:Country')).text = \
            self.config['metadata:main']['contact_country']

            etree.SubElement(address,
            util.nspath_eval('ows:ElectronicMailAddress')).text = \
            self.config['metadata:main']['contact_email']

            url = etree.SubElement(contactinfo,
            util.nspath_eval('ows:OnlineResource'))

            url.attrib[util.nspath_eval('xlink:type')] = 'simple'
            url.attrib[util.nspath_eval('xlink:href')] = \
            self.config['metadata:main']['contact_url']

            etree.SubElement(contactinfo,
                             util.nspath_eval('ows:HoursOfService')).text = \
                             self.config['metadata:main']['contact_hours']

            etree.SubElement(contactinfo,
            util.nspath_eval('ows:ContactInstructions')).text = \
            self.config['metadata:main']['contact_instructions']

            etree.SubElement(servicecontact,
            util.nspath_eval('ows:Role')).text = \
            self.config['metadata:main']['contact_role']

        if operationsmetadata:
            self.log.debug('Writing section OperationsMetadata.')
            operationsmetadata = etree.SubElement(node,
            util.nspath_eval('ows:OperationsMetadata'))

            for operation in config.MODEL['operations'].keys():
                oper = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Operation'), name = operation)

                dcp = etree.SubElement(oper, util.nspath_eval('ows:DCP'))
                http = etree.SubElement(dcp, util.nspath_eval('ows:HTTP'))

                if config.MODEL['operations'][operation]['methods']['get']:
                    get = etree.SubElement(http, util.nspath_eval('ows:Get'))
                    get.attrib[util.nspath_eval('xlink:type')] = 'simple'
                    get.attrib[util.nspath_eval('xlink:href')] = \
                    self.config['server']['url']

                if config.MODEL['operations'][operation]['methods']['post']:
                    post = etree.SubElement(http, util.nspath_eval('ows:Post'))
                    post.attrib[util.nspath_eval('xlink:type')] = 'simple'
                    post.attrib[util.nspath_eval('xlink:href')] = \
                    self.config['server']['url']

                for parameter in \
                config.MODEL['operations'][operation]['parameters']:
                    param = etree.SubElement(oper,
                    util.nspath_eval('ows:Parameter'), name = parameter)

                    for val in \
                    config.MODEL['operations'][operation]\
                    ['parameters'][parameter]['values']:
                        etree.SubElement(param,
                        util.nspath_eval('ows:Value')).text = val

                if operation == 'GetRecords':  # advertise queryables
                    for qbl in self.repository.queryables.keys():
                        if qbl != '_all':
                            param = etree.SubElement(oper,
                            util.nspath_eval('ows:Constraint'), name = qbl)
    
                            for qbl2 in self.repository.queryables[qbl]:
                                etree.SubElement(param,
                                util.nspath_eval('ows:Value')).text = qbl2

                    if self.profiles is not None:
                        for con in config.MODEL[\
                        'operations']['GetRecords']['constraints'].keys():
                            param = etree.SubElement(oper,
                            util.nspath_eval('ows:Constraint'), name = con)
                            for val in config.MODEL['operations']\
                            ['GetRecords']['constraints'][con]['values']:
                                etree.SubElement(param,
                                util.nspath_eval('ows:Value')).text = val

            for parameter in config.MODEL['parameters'].keys():
                param = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Parameter'), name = parameter)
                for val in config.MODEL['parameters'][parameter]['values']:
                    etree.SubElement(param,
                    util.nspath_eval('ows:Value')).text = val

            for constraint in config.MODEL['constraints'].keys():
                param = etree.SubElement(operationsmetadata,
                util.nspath_eval('ows:Constraint'), name = constraint)
                for val in config.MODEL['constraints'][constraint]['values']:
                    etree.SubElement(param,
                    util.nspath_eval('ows:Value')).text = val
            
            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    ecnode = \
                    self.profiles['loaded'][prof].get_extendedcapabilities()
                    if ecnode is not None:
                        operationsmetadata.append(ecnode)

        # always write out Filter_Capabilities
        self.log.debug('Writing section Filter_Capabilities.')
        fltcaps = etree.SubElement(node,
        util.nspath_eval('ogc:Filter_Capabilities'))

        spatialcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Spatial_Capabilities'))

        geomops = etree.SubElement(spatialcaps,
        util.nspath_eval('ogc:GeometryOperands'))

        for geomtype in \
        fes.MODEL['GeometryOperands']['values']:
            etree.SubElement(geomops,
            util.nspath_eval('ogc:GeometryOperand')).text = geomtype
    
        spatialops = etree.SubElement(spatialcaps,
        util.nspath_eval('ogc:SpatialOperators'))

        for spatial_comparison in \
        fes.MODEL['SpatialOperators']['values']:
            etree.SubElement(spatialops,
            util.nspath_eval('ogc:SpatialOperator'), name = spatial_comparison)
    
        scalarcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Scalar_Capabilities'))

        etree.SubElement(scalarcaps, util.nspath_eval('ogc:LogicalOperators'))
        
        cmpops = etree.SubElement(scalarcaps,
        util.nspath_eval('ogc:ComparisonOperators'))
    
        for cmpop in fes.MODEL['ComparisonOperators'].keys():
            etree.SubElement(cmpops,
            util.nspath_eval('ogc:ComparisonOperator')).text = \
            fes.MODEL['ComparisonOperators'][cmpop]['opname']
    
        idcaps = etree.SubElement(fltcaps,
        util.nspath_eval('ogc:Id_Capabilities'))

        for idcap in fes.MODEL['Ids']['values']:
            etree.SubElement(idcaps, util.nspath_eval('ogc:%s' % idcap))

        return node
    
    def describerecord(self):
        ''' Handle DescribeRecord request '''
    
        if self.kvp.has_key('typename') is False or \
        len(self.kvp['typename']) == 0:  # missing typename
        # set to return all typenames
            self.kvp['typename'] = []
            self.kvp['typename'].append('csw:Record')

            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    self.kvp['typename'].append(
                    self.profiles['loaded'][prof].typename)

        if self.requesttype == 'GET':  # pass via GET
            self.kvp['typename'] = self.kvp['typename'].split(',')

        if (self.kvp.has_key('outputformat') and
            self.kvp['outputformat'] not in
            config.MODEL['operations']['DescribeRecord']
            ['parameters']['outputFormat']['values']):  # bad outputformat
            return self.exceptionreport('InvalidParameterValue',
            'outputformat', 'Invalid value for outputformat: %s' %
            self.kvp['outputformat'])
    
        if (self.kvp.has_key('schemalanguage') and
            self.kvp['schemalanguage'] not in
            config.MODEL['operations']['DescribeRecord']['parameters']
            ['schemaLanguage']['values']):  # bad schemalanguage
            return self.exceptionreport('InvalidParameterValue',
            'schemalanguage', 'Invalid value for schemalanguage: %s' %
            self.kvp['schemalanguage'])

        node = etree.Element(util.nspath_eval('csw:DescribeRecordResponse'),
        nsmap = config.NAMESPACES)

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        for typename in self.kvp['typename']:
            if typename == 'csw:Record':   # load core schema    
                self.log.debug('Writing csw:Record schema.')
                schemacomponent = etree.SubElement(node,
                util.nspath_eval('csw:SchemaComponent'),
                schemaLanguage='XMLSCHEMA',
                targetNamespace = config.NAMESPACES['csw'])
                dublincore = etree.parse(os.path.join(
                self.config['server']['home'],
                'etc', 'schemas', 'ogc', 'csw',
                '2.0.2', 'record.xsd')).getroot()

                schemacomponent.append(dublincore)

            if self.profiles is not None:
                for prof in self.profiles['loaded'].keys():
                    if self.profiles['loaded'][prof].typename == typename:
                        scnodes = \
                        self.profiles['loaded'][prof].get_schemacomponents()
                        if scnodes is not None:
                            for scn in scnodes:
                                node.append(scn)
        return node
    
    def getdomain(self):
        ''' Handle GetDomain request '''
        if (self.kvp.has_key('parametername') is False and
            self.kvp.has_key('propertyname') is False):
            return self.exceptionreport('MissingParameterValue',
            'parametername', 'Missing value. \
            One of propertyname or parametername must be specified')
        
        node = etree.Element(util.nspath_eval('csw:GetDomainResponse'),
        nsmap = config.NAMESPACES)

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

        if self.kvp.has_key('parametername'):
            for pname in self.kvp['parametername'].split(','):
                self.log.debug('Parsing parametername %s.' % pname)
                domainvalue = etree.SubElement(node,
                util.nspath_eval('csw:DomainValues'), type = 'csw:Record')
                etree.SubElement(domainvalue,
                util.nspath_eval('csw:ParameterName')).text = pname
                operation, parameter = pname.split('.')
                if (operation in config.MODEL['operations'].keys() and
                    parameter in
                    config.MODEL['operations'][operation]['parameters'].keys()):
                    listofvalues = etree.SubElement(domainvalue,
                    util.nspath_eval('csw:ListOfValues'))
                    for val in \
                    config.MODEL['operations'][operation]\
                    ['parameters'][parameter]['values']:
                        etree.SubElement(listofvalues,
                        util.nspath_eval('csw:Value')).text = val

        if self.kvp.has_key('propertyname'):
            for pname in self.kvp['propertyname'].split(','):
                self.log.debug('Parsing propertyname %s.' % pname)

                if pname.find('/') == 0:  # it's an XPath
                    pname2 = pname
                else:  # it's a core queryable, map to internal typename model
                    pname2 = self.repository.queryables['_all'][pname]

                # TODO: smarter way of handling domain value type
                if pname2.find('apiso:') != -1 or pname2.find('gmd:') != -1:
                    dvtype = 'gmd:MD_Metadata'
                else:
                    dvtype = 'csw:Record'
                domainvalue = etree.SubElement(node,
                util.nspath_eval('csw:DomainValues'), type = dvtype)
                etree.SubElement(domainvalue,
                util.nspath_eval('csw:PropertyName')).text = pname

                try:
                    self.log.debug('Querying repository property %s.' % pname2)
        
                    results = self.repository.query_domain(pname2,[dvtype])
                    self.log.debug('Results: %s' % str(len(results)))
                    listofvalues = etree.SubElement(domainvalue,
                    util.nspath_eval('csw:ListOfValues'))
                    for result in results:
                        if result[0] is not None:  # drop null values
                            etree.SubElement(listofvalues,
                            util.nspath_eval('csw:Value')).text = result[0]
                except Exception, err:
                    self.log.debug('No results for propertyname %s: %s.' %
                    (pname2, str(err)))
        return node

    def getrecords(self):
        ''' Handle GetRecords request '''

        timestamp = util.get_today_and_now()

        if (self.kvp.has_key('elementsetname') is False and
            self.kvp.has_key('elementname') is False):
            # mutually exclusive required
            return self.exceptionreport('MissingParameterValue',
            'elementsetname',
            'Missing one of ElementSetName or ElementName parameter(s)')

        if self.kvp.has_key('outputschema') is False:
            self.kvp['outputschema'] = config.NAMESPACES['csw']

        if (self.kvp['outputschema'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['outputSchema']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'outputschema', 'Invalid outputSchema parameter value: %s' %
            self.kvp['outputschema'])

        if self.kvp.has_key('outputformat') is False:
            self.kvp['outputformat'] = 'application/xml'
     
        if (self.kvp['outputformat'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['outputFormat']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'outputformat', 'Invalid outputFormat parameter value: %s' %
            self.kvp['outputformat'])

        if self.kvp.has_key('resulttype') is False:
            self.kvp['resulttype'] = 'hits'

        if self.kvp['resulttype'] is not None:
            if (self.kvp['resulttype'] not in config.MODEL['operations']
            ['GetRecords']['parameters']['resultType']['values']):
                return self.exceptionreport('InvalidParameterValue',
                'resulttype', 'Invalid resultType parameter value: %s' %
                self.kvp['resulttype'])

        if ((self.kvp.has_key('elementname') is False or 
             len(self.kvp['elementname']) == 0) and
             self.kvp['elementsetname'] not in
             config.MODEL['operations']['GetRecords']['parameters']
             ['ElementSetName']['values']):
            return self.exceptionreport('InvalidParameterValue',
            'elementsetname', 'Invalid ElementSetName parameter value: %s' %
            self.kvp['elementsetname'])

        if (self.kvp.has_key('elementname') and
            self.requesttype == 'GET'):  # passed via GET
            self.kvp['elementname'] = self.kvp['elementname'].split(',')
            self.kvp['elementsetname'] = 'summary'
        
        if (self.kvp.has_key('typenames') and
            self.requesttype == 'GET'):  # passed via GET
            self.kvp['typenames'] = self.kvp['typenames'].split(',')

        if self.kvp.has_key('typenames'):
            for tname in self.kvp['typenames']:
                if (tname not in config.MODEL['operations']['GetRecords']
                    ['parameters']['typeNames']['values']):
                    return self.exceptionreport('InvalidParameterValue',
                    'typenames', 'Invalid typeNames parameter value: %s' %
                    tname)

        # check elementname's
        if self.kvp.has_key('elementname'):
            for ename in self.kvp['elementname']:
                enamelist = self.repository.queryables['_all'].keys()
                if ename not in enamelist:
                    return self.exceptionreport('InvalidParameterValue',
                    'elementname', 'Invalid ElementName parameter value: %s' %
                    ename)

        if self.kvp['resulttype'] == 'validate':
            return self._write_acknowledgement()

        if self.kvp.has_key('maxrecords') is False:
            self.kvp['maxrecords'] = self.config['server']['maxrecords']

        if self.requesttype == 'GET':
            if self.kvp.has_key('constraint'):
                # GET request
                self.log.debug('csw:Constraint passed over HTTP GET.')
                if self.kvp.has_key('constraintlanguage') is False:
                    return self.exceptionreport('MissingParameterValue',
                    'constraintlanguage',
                    'constraintlanguage required when constraint specified')
                if (self.kvp['constraintlanguage'] not in
                config.MODEL['operations']['GetRecords']['parameters']
                ['CONSTRAINTLANGUAGE']['values']):
                    return self.exceptionreport('InvalidParameterValue',
                    'constraintlanguage', 'Invalid constraintlanguage: %s'
                    % self.kvp['constraintlanguage'])
                if self.kvp['constraintlanguage'] == 'CQL_TEXT':
                    tmp = self.kvp['constraint']
                    self.kvp['constraint'] = {}
                    self.kvp['constraint']['type'] = 'cql'
                    self.kvp['constraint']['where'] = \
                    self._cql_update_queryables_mappings(tmp,
                    self.repository.queryables['_all'])
                elif self.kvp['constraintlanguage'] == 'FILTER':
                    # validate filter XML
                    try:
                        schema = os.path.join(self.config['server']['home'],
                        'etc', 'schemas', 'ogc', 'filter', '1.1.0', 'filter.xsd')
                        self.log.debug('Validating Filter %s.' %
                        self.kvp['constraint'])
                        schema = etree.XMLSchema(etree.parse(schema))
                        parser = etree.XMLParser(schema=schema)
                        doc = etree.fromstring(self.kvp['constraint'], parser)
                        self.log.debug('Filter is valid XML.')
                        self.kvp['constraint'] = {}
                        self.kvp['constraint']['type'] = 'filter'
                        self.kvp['constraint']['where'] = \
                        fes.parse(doc,
                        self.repository.queryables['_all'].keys(),
                        self.repository.dbtype)
                    except Exception, err:
                        errortext = \
                        'Exception: document not valid.\nError: %s.' % str(err)
    
                        self.log.debug(errortext)
                        return self.exceptionreport('InvalidParameterValue',
                        'constraint', 'Invalid Filter query: %s' % errortext)
            else:
                self.kvp['constraint'] = {}

        if self.kvp.has_key('sortby') is False:
            self.kvp['sortby'] = None

        if self.kvp.has_key('startposition') is False:
            self.kvp['startposition'] = 1

        # query repository
        self.log.debug('Querying repository with constraint : %s,\
        sortby: %s, typenames: %s.' %
        (self.kvp['constraint'], self.kvp['sortby'], self.kvp['typenames']))

        try:
            results = self.repository.query(constraint=self.kvp['constraint'],
            sortby=self.kvp['sortby'], typenames=self.kvp['typenames'])
        except Exception, err:
            return self.exceptionreport('InvalidParameterValue', 'constraint',
            'Invalid query: %s' % err)

        if len(results) == 0:
            matched = '0'
            returned = '0'
            nextrecord = '0'

        else:
            matched = str(len(results))
            if int(matched) < int(self.kvp['maxrecords']):
                returned = str(int(matched))
                nextrecord = '0'
            else:
                returned = str(self.kvp['maxrecords'])
                nextrecord = str(int(self.kvp['startposition']) + \
                int(self.kvp['maxrecords']))
            if int(self.kvp['maxrecords']) == 0:
                nextrecord = '1'

        dsresults = []

        if (self.config['server'].has_key('federatedcatalogues') and
            self.kvp.has_key('distributedsearch') and
            self.kvp['distributedsearch'] == 'TRUE' and
            self.kvp['hopcount'] > 0):  # do distributed search

            self.log.debug('DistributedSearch specified (hopCount: %s).' %
            self.kvp['hopcount'])

            from owslib.csw import CatalogueServiceWeb
            for fedcat in \
            self.config['server']['federatedcatalogues'].split(','):
                self.log.debug('Performing distributed search on federated \
                catalogue: %s.' % fedcat)
                remotecsw = CatalogueServiceWeb(fedcat)
                remotecsw.getrecords(xml=self.request)
                self.log.debug('Distributed search results from catalogue \
                %s: %s.' % (fedcat, remotecsw.results))
                matched = str(int(matched) + int(remotecsw.results['matches']))
                dsresults.append(remotecsw.records)

        self.log.debug('Results: matched: %s, returned: %s, next: %s.' % \
        (matched, returned, nextrecord))

        node = etree.Element(util.nspath_eval('csw:GetRecordsResponse'),
        nsmap = config.NAMESPACES, version='2.0.2')

        node.attrib[util.nspath_eval('xsi:schemaLocation')] = \
        '%s %s/csw/2.0.2/CSW-discovery.xsd' % \
        (config.NAMESPACES['csw'], self.config['server']['ogc_schemas_base'])

{'fraction_non_alphanumeric': 0.12679487179487178, 'fraction_numerical': 0.0024871794871794872, 'mean_word_length': 1.924123711340206, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3900015', 'n_tokens_mistral': 21312, 'n_tokens_neox': 20404, 'n_words': 3839}