<commit_before>import requests
import logging
from pylons import config

from rdflib.namespace import Namespace

from ckan.plugins import toolkit
from ckanext.dcat.profiles import RDFProfile

log = logging.getLogger(__name__)


DCT = Namespace("http://purl.org/dc/terms/")


class NIArcGISProfile(RDFProfile):
    '''
    An RDF profile for the Northern Ireland ArcGIS harvester

    '''

    def parse_dataset(self, dataset_dict, dataset_ref):

        # TODO: if there is more than one source with different defaults,
        # modify accordingly
        dataset_dict['frequency'] = 'notPlanned'
        dataset_dict['topic_category'] = 'location'
        dataset_dict['lineage'] = '-'
        dataset_dict['contact_name'] = 'OSNI Mapping Helpdesk'
        dataset_dict['contact_email'] = '<EMAIL>'
        dataset_dict['license_id'] = 'uk-ogl'

        _remove_extra('contact_name', dataset_dict)
        _remove_extra('contact_email', dataset_dict)

        # Ping the ArcGIS server so the processing of the files
        # starts
        identifier = None
        avoid = []

        if toolkit.asbool(
                config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):

            for extra in dataset_dict.get('extras', []):
                if extra['key'] == 'identifier' and extra['value']:
                    identifier = extra['value']
            if identifier:
                query = toolkit.get_action('package_search')(
                    {}, {'q': 'guid:"{0}"'.format(identifier)})
                if query['count']:
                    current_dataset = query['results'][0]
                    for current_resource in current_dataset.get('resources',
                                                                []):
                        if ('requested' in current_resource and
                                toolkit.asbool(current_resource['requested'])):
                            avoid.append(current_resource['url'])

            for resource in dataset_dict.get('resources', []):
                if resource['format'] == 'OGC WMS':
                    resource['format'] = 'WMS'

                resource['requested'] = False
                file_formats = ('geojson', 'kml', 'zip', 'csv')

                if resource['url'] in avoid:
                    resource['requested'] = True
                elif resource['format'].lower() in file_formats:
                    try:
                        requests.head(resource['url'])

                        resource['requested'] = True
                        log.debug(
                            'Requested resource to start the processing: {0}'
                            .format(resource['url']))
                    except Exception, e:
                        log.debug(
                            'Error requesting resource: {0}\n{1}'
                            .format(resource['url'], e))
                        pass

        return dataset_dict


class CausewayProfile(RDFProfile):

    def parse_dataset(self, dataset_dict, dataset_ref):

        dataset_dict['frequency'] = 'notPlanned'
        dataset_dict['topic_category'] = 'location'
        dataset_dict['lineage'] = '-'
        dataset_dict['contact_name'] = '<NAME>'
        dataset_dict['contact_email'] = '<EMAIL>'
        dataset_dict['license_id'] = 'uk-ogl'

        _remove_extra('contact_name', dataset_dict)
        _remove_extra('contact_email', dataset_dict)

        # Ping the ArcGIS server so the processing of the files
        # starts
        identifier = None
        avoid = []

        if toolkit.asbool(
                config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):

            for extra in dataset_dict.get('extras', []):
                if extra['key'] == 'identifier' and extra['value']:
                    identifier = extra['value']
            if identifier:
                query = toolkit.get_action('package_search')(
                    {}, {'q': 'guid:"{0}"'.format(identifier)})
                if query['count']:
                    current_dataset = query['results'][0]
                    for current_resource in current_dataset.get('resources',
                                                                []):
                        if ('requested' in current_resource and
                                toolkit.asbool(current_resource['requested'])):
                            avoid.append(current_resource['url'])

            for resource in dataset_dict.get('resources', []):
                if resource['format'] == 'OGC WMS':
                    resource['format'] = 'WMS'
                if resource['format'] == 'zip':
                    resource['format'] = 'SHP'

                resource['requested'] = False
                file_formats = ('geojson', 'kml', 'zip', 'csv')

                if resource['url'] in avoid:
                    resource['requested'] = True
                elif resource['format'].lower() in file_formats:
                    try:
                        requests.head(resource['url'])

                        resource['requested'] = True
                        log.debug(
                            'Requested resource to start the processing: {0}'
                            .format(resource['url']))
                    except Exception, e:
                        log.debug(
                            'Error requesting resource: {0}\n{1}'
                            .format(resource['url'], e))
                        pass

        return dataset_dict

<commit_msg>Use uppercase for format<commit_after>import requests
import logging
from pylons import config

from rdflib.namespace import Namespace

from ckan.plugins import toolkit
from ckanext.dcat.profiles import RDFProfile

log = logging.getLogger(__name__)


DCT = Namespace("http://purl.org/dc/terms/")


class NIArcGISProfile(RDFProfile):
    '''
    An RDF profile for the Northern Ireland ArcGIS harvester

    '''

    def parse_dataset(self, dataset_dict, dataset_ref):

        # TODO: if there is more than one source with different defaults,
        # modify accordingly
        dataset_dict['frequency'] = 'notPlanned'
        dataset_dict['topic_category'] = 'location'
        dataset_dict['lineage'] = '-'
        dataset_dict['contact_name'] = 'OSNI Mapping Helpdesk'
        dataset_dict['contact_email'] = '<EMAIL>'
        dataset_dict['license_id'] = 'uk-ogl'

        _remove_extra('contact_name', dataset_dict)
        _remove_extra('contact_email', dataset_dict)

        # Ping the ArcGIS server so the processing of the files
        # starts
        identifier = None
        avoid = []

        if toolkit.asbool(
                config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):

            for extra in dataset_dict.get('extras', []):
                if extra['key'] == 'identifier' and extra['value']:
                    identifier = extra['value']
            if identifier:
                query = toolkit.get_action('package_search')(
                    {}, {'q': 'guid:"{0}"'.format(identifier)})
                if query['count']:
                    current_dataset = query['results'][0]
                    for current_resource in current_dataset.get('resources',
                                                                []):
                        if ('requested' in current_resource and
                                toolkit.asbool(current_resource['requested'])):
                            avoid.append(current_resource['url'])

            for resource in dataset_dict.get('resources', []):
                if resource['format'] == 'OGC WMS':
                    resource['format'] = 'WMS'

                resource['requested'] = False
                file_formats = ('geojson', 'kml', 'zip', 'csv')

                if resource['url'] in avoid:
                    resource['requested'] = True
                elif resource['format'].lower() in file_formats:
                    try:
                        requests.head(resource['url'])

                        resource['requested'] = True
                        log.debug(
                            'Requested resource to start the processing: {0}'
                            .format(resource['url']))
                    except Exception, e:
                        log.debug(
                            'Error requesting resource: {0}\n{1}'
                            .format(resource['url'], e))
                        pass

        return dataset_dict


class CausewayProfile(RDFProfile):

    def parse_dataset(self, dataset_dict, dataset_ref):

        dataset_dict['frequency'] = 'notPlanned'
        dataset_dict['topic_category'] = 'location'
        dataset_dict['lineage'] = '-'
        dataset_dict['contact_name'] = '<NAME>'
        dataset_dict['contact_email'] = '<EMAIL>'
        dataset_dict['license_id'] = 'uk-ogl'

        _remove_extra('contact_name', dataset_dict)
        _remove_extra('contact_email', dataset_dict)

        # Ping the ArcGIS server so the processing of the files
        # starts
        identifier = None
        avoid = []

        if toolkit.asbool(
                config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):

            for extra in dataset_dict.get('extras', []):
                if extra['key'] == 'identifier' and extra['value']:
                    identifier = extra['value']
            if identifier:
                query = toolkit.get_action('package_search')(
                    {}, {'q': 'guid:"{0}"'.format(identifier)})
                if query['count']:
                    current_dataset = query['results'][0]
                    for current_resource in current_dataset.get('resources',
                                                                []):
                        if ('requested' in current_resource and
                                toolkit.asbool(current_resource['requested'])):
                            avoid.append(current_resource['url'])

            for resource in dataset_dict.get('resources', []):
                if resource['format'] == 'OGC WMS':
                    resource['format'] = 'WMS'
                if resource['format'] == 'ZIP':
                    resource['format'] = 'SHP'

                resource['requested'] = False
                file_formats = ('geojson', 'kml', 'zip', 'csv')

                if resource['url'] in avoid:
                    resource['requested'] = True
                elif resource['format'].lower() in file_formats:
                    try:
                        requests.head(resource['url'])

                        resource['requested'] = True
                        log.debug(
                            'Requested resource to start the processing: {0}'
                            .format(resource['url']))
                    except Exception, e:
                        log.debug(
                            'Error requesting resource: {0}\n{1}'
                            .format(resource['url'], e))
                        pass

        return dataset_dict


{'fraction_non_alphanumeric': 0.11264305668198613, 'fraction_numerical': 0.0018022889069117779, 'mean_word_length': 1.5419147961520843, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2069655', 'n_tokens_mistral': 2802, 'n_tokens_neox': 2626, 'n_words': 601}