<commit_before>import traceback
from owslib.csw import CatalogueServiceWeb
from lxml import etree

namespaces = {
    'atom': 'http://www.w3.org/2005/Atom',
    'che': 'http://www.geocat.ch/2008/che',
    'csw': 'http://www.opengis.net/cat/csw/2.0.2',
    'dc' : 'http://purl.org/dc/elements/1.1/',
    'dct': 'http://purl.org/dc/terms/',
    'dif': 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/',
    'fgdc': 'http://www.opengis.net/cat/csw/csdgm',
    'gco': 'http://www.isotc211.org/2005/gco',
    'gmd': 'http://www.isotc211.org/2005/gmd',
    'gml': 'http://www.opengis.net/gml',
    'ogc': 'http://www.opengis.net/ogc',
    'ows': 'http://www.opengis.net/ows',
    'rim': 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0',
    'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
    'xs' : 'http://www.w3.org/2001/XMLSchema',
    'xs2': 'http://www.w3.org/XML/Schema',
    'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
}

class Attribute(object):
    def __init__(self, config, **kwargs):
        self._config = config
        self.env = kwargs

    def get_value(self, **kwargs):
        """ Abstract method to return the value of the attribute """
        raise NotImplementedError

class StringAttribute(Attribute):
    def get_value(self, **kwargs):
        return self._config

class XmlAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        xml = self.env['xml']
        return etree.tostring(xml)

class XPathAttribute(Attribute):
    def get_element(self, xml):
        return xml.find(self._config, namespaces)

    def get_value(self, **kwargs):
        self.env.update(kwargs)
        xml = self.env['xml']
        try:
            # this should probably return a XPathTextAttribute
            value = self.get_element(xml)
        except:
            value = ''
        return value

class XPathMultiAttribute(XPathAttribute):
    def get_element(self, xml):
        return xml.findall(self._config, namespaces)

class XPathTextAttribute(XPathAttribute):
    def get_value(self, **kwargs):
        value = super(XPathTextAttribute, self).get_value(**kwargs)
        return value.text if hasattr(value, 'text') else value

class XPathMultiTextAttribute(XPathMultiAttribute):
    def get_value(self, **kwargs):
        value = super(XPathMultiTextAttribute, self).get_value(**kwargs)
        return value.text if hasattr(value, 'text') else value

class CombinedAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        value = ''
        separator = self.env['separator'] if self.env.has_key('separator') else ' '
        for attribute in self._config:
            new_value = attribute.get_value(**kwargs)
            value = value + attribute.get_value(**kwargs) + separator if new_value != None else value
        return value.strip(separator)

class MultiAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        value = ''
        separator = self.env['separator'] if self.env.has_key('separator') else ' '
        for attribute in self._config:
            new_value = attribute.get_value(**kwargs)
            try:
                iterator = iter(new_value)
                for inner_attribute in iterator:
                    # it should be possible to call inner_attribute.get_value and the right thing(tm) happens'
                    value = value + (inner_attribute.text if hasattr(inner_attribute, 'text') else inner_attribute) + separator
            except TypeError:
                value = value + new_value + separator
        return value.strip(separator)
    

class FirstInOrderAttribute(CombinedAttribute):
    def get_value(self, **kwargs):
        for attribute in self._config:
            value = attribute.get_value(**kwargs)
            if value != '':
                return value
        return ''

class CkanMetadata(object):
    """ Provides general access to CSW for CKAN """
    def __init__(self, url, schema, version='2.0.2', lang='en-US'):
        self.schema = schema
        self.catalog = CatalogueServiceWeb(url, lang, version, timeout=10, skip_caps=True)
        self.metadata = dict.fromkeys([
            'id', 
            'name', 
            'title', 
            'url', 
            'author', 
            'author_email', 
            'maintainer', 
            'maintainer_email',
            'license',
            'version',
            'notes',
            'tags',
            'metadata_url',
            'metadata_raw',
        ])

    def get_by_search(self, searchterm, propertyname='csw:AnyText'):
        """ Returns the found csw dataset with the given searchterm """
        self.catalog.getrecords(keywords=[searchterm], propertyname=propertyname)
        if self.catalog.response == None or self.catalog.results['matches'] == 0:
            raise DatasetNotFoundError("No dataset for the given searchterm '%s' (%s) found" % (searchterm, propertyname))
        return self.catalog.records

    def get_by_id(self, id):
        """ Returns the csw dataset with the given id """
        self.catalog.getrecordbyid(id=[id], outputschema=self.schema)
        return self.catalog.response

    def get_id_by_dataset_name(self, dataset_name):
        """ 
            Returns the id of a dataset identified by it's name.
            If there are multiple datasets with the given name,
            only the id of the first one is returned.
        """
        return self.get_by_search(dataset_name, 'title').itervalues().next().identifier

    def get_attribute(self, dataset_name, ckan_attribute):
        """ Abstract method to define the mapping of a ckan attribute to a csw attribute """
        raise NotImplementedError

    def get_xml(self, id):
        dataset_xml_string = self.get_by_id(id)
        if dataset_xml_string == None:
            raise DatasetNotFoundError("Dataset with id %s not found" % id)
        return dataset_xml_string

    def get_ckan_metadata(self, dataset_name):
        """ Returns the requested dataset mapped to CKAN attributes """
        id = self.get_id_by_dataset_name(dataset_name)
        dataset_xml = etree.fromstring(self.get_xml(id))
        for key in self.metadata:
            attribute = self.get_attribute(dataset_name, key)
            self.metadata[key] = attribute.get_value(xml=dataset_xml)
        return self.metadata


class SwisstopoCkanMetadata(CkanMetadata):
    """ Provides access to the csw service of swisstopo """

    default_mapping = {
        'id': XPathTextAttribute('.//gmd:fileIdentifier/gco:CharacterString'), 
        'name': XPathTextAttribute(".//gmd:identificationInfo//gmd:citation//gmd:title//gmd:textGroup/gmd:LocalisedCharacterString[@locale='#DE']"), 
        'title': XPathTextAttribute(".//gmd:identificationInfo//gmd:citation//gmd:alternateTitle//gmd:textGroup/gmd:LocalisedCharacterString[@locale='#DE']"), 
        'url': XPathTextAttribute(".//gmd:distributionInfo//gmd:transferOptions//gmd:onLine//gmd:linkage//che:URLGroup/che:LocalisedURL[@locale='#DE']"), 
        'author': CombinedAttribute([<commit_msg>Use xpath method instead of find/findall to enable 'namespaces' param<commit_after>import traceback
from owslib.csw import CatalogueServiceWeb
from lxml import etree
import logging
log = logging.getLogger(__name__)

namespaces = {
    'atom': 'http://www.w3.org/2005/Atom',
    'che': 'http://www.geocat.ch/2008/che',
    'csw': 'http://www.opengis.net/cat/csw/2.0.2',
    'dc' : 'http://purl.org/dc/elements/1.1/',
    'dct': 'http://purl.org/dc/terms/',
    'dif': 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/',
    'fgdc': 'http://www.opengis.net/cat/csw/csdgm',
    'gco': 'http://www.isotc211.org/2005/gco',
    'gmd': 'http://www.isotc211.org/2005/gmd',
    'gml': 'http://www.opengis.net/gml',
    'ogc': 'http://www.opengis.net/ogc',
    'ows': 'http://www.opengis.net/ows',
    'rim': 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0',
    'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
    'xs' : 'http://www.w3.org/2001/XMLSchema',
    'xs2': 'http://www.w3.org/XML/Schema',
    'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
}

class Attribute(object):
    def __init__(self, config, **kwargs):
        self._config = config
        self.env = kwargs

    def get_value(self, **kwargs):
        """ Abstract method to return the value of the attribute """
        raise NotImplementedError

class StringAttribute(Attribute):
    def get_value(self, **kwargs):
        return self._config

class XmlAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        xml = self.env['xml']
        return etree.tostring(xml)

class XPathAttribute(Attribute):
    def get_element(self, xml):
        return xml.xpath(self._config, namespaces=namespaces)[0]

    def get_value(self, **kwargs):
        self.env.update(kwargs)
        xml = self.env['xml']
        try:
            # this should probably return a XPathTextAttribute
            value = self.get_element(xml)
        except Exception as e:
            log.debug(e)
            value = ''
        return value

class XPathMultiAttribute(XPathAttribute):
    def get_element(self, xml):
        return xml.xpath(self._config, namespaces=namespaces)

class XPathTextAttribute(XPathAttribute):
    def get_value(self, **kwargs):
        value = super(XPathTextAttribute, self).get_value(**kwargs)
        return value.text if hasattr(value, 'text') else value

class XPathMultiTextAttribute(XPathMultiAttribute):
    def get_value(self, **kwargs):
        value = super(XPathMultiTextAttribute, self).get_value(**kwargs)
        return value.text if hasattr(value, 'text') else value

class CombinedAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        value = ''
        separator = self.env['separator'] if self.env.has_key('separator') else ' '
        for attribute in self._config:
            new_value = attribute.get_value(**kwargs)
            value = value + attribute.get_value(**kwargs) + separator if new_value != None else value
        return value.strip(separator)

class MultiAttribute(Attribute):
    def get_value(self, **kwargs):
        self.env.update(kwargs)
        value = ''
        separator = self.env['separator'] if self.env.has_key('separator') else ' '
        for attribute in self._config:
            new_value = attribute.get_value(**kwargs)
            try:
                iterator = iter(new_value)
                for inner_attribute in iterator:
                    # it should be possible to call inner_attribute.get_value and the right thing(tm) happens'
                    value = value + (inner_attribute.text if hasattr(inner_attribute, 'text') else inner_attribute) + separator
            except TypeError:
                value = value + new_value + separator
        return value.strip(separator)
    

class FirstInOrderAttribute(CombinedAttribute):
    def get_value(self, **kwargs):
        for attribute in self._config:
            value = attribute.get_value(**kwargs)
            if value != '':
                return value
        return ''

class CkanMetadata(object):
    """ Provides general access to CSW for CKAN """
    def __init__(self, url, schema, version='2.0.2', lang='en-US'):
        self.schema = schema
        self.catalog = CatalogueServiceWeb(url, lang, version, timeout=10, skip_caps=True)
        self.metadata = dict.fromkeys([
            'id', 
            'name', 
            'title', 
            'url', 
            'author', 
            'author_email', 
            'maintainer', 
            'maintainer_email',
            'license',
            'version',
            'notes',
            'tags',
            'metadata_url',
            'metadata_raw',
        ])

    def get_by_search(self, searchterm, propertyname='csw:AnyText'):
        """ Returns the found csw dataset with the given searchterm """
        self.catalog.getrecords(keywords=[searchterm], propertyname=propertyname)
        if self.catalog.response == None or self.catalog.results['matches'] == 0:
            raise DatasetNotFoundError("No dataset for the given searchterm '%s' (%s) found" % (searchterm, propertyname))
        return self.catalog.records

    def get_by_id(self, id):
        """ Returns the csw dataset with the given id """
        self.catalog.getrecordbyid(id=[id], outputschema=self.schema)
        return self.catalog.response

    def get_id_by_dataset_name(self, dataset_name):
        """ 
            Returns the id of a dataset identified by it's name.
            If there are multiple datasets with the given name,
            only the id of the first one is returned.
        """
        return self.get_by_search(dataset_name, 'title').itervalues().next().identifier

    def get_attribute(self, dataset_name, ckan_attribute):
        """ Abstract method to define the mapping of a ckan attribute to a csw attribute """
        raise NotImplementedError

    def get_xml(self, id):
        dataset_xml_string = self.get_by_id(id)
        if dataset_xml_string == None:
            raise DatasetNotFoundError("Dataset with id %s not found" % id)
        return dataset_xml_string

    def get_ckan_metadata(self, dataset_name):
        """ Returns the requested dataset mapped to CKAN attributes """
        id = self.get_id_by_dataset_name(dataset_name)
        log.debug("Dataset ID: %s" % id)

        dataset_xml = etree.fromstring(self.get_xml(id))
        for key in self.metadata:
            log.debug("Metadata key: %s" % key)
            attribute = self.get_attribute(dataset_name, key)
            self.metadata[key] = attribute.get_value(xml=dataset_xml)
        return self.metadata


class SwisstopoCkanMetadata(CkanMetadata):
    """ Provides access to the csw service of swisstopo """

    default_mapping = {
        'id': XPathTextAttribute('.//gmd:fileIdentifier/gco:CharacterString'), 
        'name': XPathTextAttribute(".//gmd:identificationInfo//gmd:citation//gmd:title//gmd:textGroup/gmd:LocalisedCharacterString[@locale='#DE']"), 
        'title': XPathTextAttribute(".//gmd:identificationInfo//gmd:citation//gmd:alternateTitle//gmd:textGroup/gmd:LocalisedCharacterString[@locale='#DE']"), 
        'url': XPathTextAttribute(".//gmd:distributionInfo//gmd:transferOptions//gmd:onLine//gmd:linkage//che:URLGroup/che:LocalisedURL[@locale='#DE']"), 
        'author': CombinedAttribute([
{'fraction_non_alphanumeric': 0.12859021385563016, 'fraction_numerical': 0.007959028306457195, 'mean_word_length': 2.7271085891152955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 26, 'xml': 42}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1770538', 'n_tokens_mistral': 4459, 'n_tokens_neox': 4282, 'n_words': 940}