<commit_before>import json
import requests

from pylons import config

import ckan.plugins as p

from ckanext.dcat.interfaces import IDCATRDFHarvester
from ckanext.sweden.dcat import template_helpers


VALIDATION_SERVICE = 'https://validator.dcat-editor.com/service'


class SwedenDCATRDFHarvester(p.SingletonPlugin):

    p.implements(IDCATRDFHarvester, inherit=True)
    p.implements(p.IConfigurer)
    p.implements(p.ITemplateHelpers)

    def after_download(self, content, harvest_job):

        if not p.toolkit.asbool(config.get('ckanext.sweden.harvest.use_validation', True)):
            return content, []

        validation_service = config.get('ckanext.sweden.harvest.validation_service', VALIDATION_SERVICE)

        stop_on_errors = p.toolkit.asbool(config.get('ckanext.sweden.harvest.stop_on_validation_errors', False))

        errors = []
        try:
            r = requests.post(validation_service, data=content)
        except requests.exceptions.RequestException, e:
            errors.append(
                'Error contacting the validation service: {0}'.format(str(e)))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

        if r.status_code != 200:

            errors.append(
                'The validation service returned an error: {0}'.format(
                    r.status_code))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

        else:
            response = r.json()

            if not all([response.get('rdfError'),
                        response.get('errors'),
                        response.get('warnings')]):
                # All clear
                return content, []

            if response.get('rdfError'):
                errors.append(response.get('rdfError'))
            else:
                if response.get('mandatoryError'):
                    for _class in response['mandatoryError']:
                        errors.append(
                            'Mandatory class {0} missing'.format(_class))

                for resource in response.get('resources', []):
                    errors.append(json.dumps(resource))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

    # IConfigurer
    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

    # ITemplateHelpers
    def get_helpers(self):
        return {
            'json_loads': template_helpers.json_loads,
        }<commit_msg>Fix response check (again)<commit_after>import json
import requests

from pylons import config

import ckan.plugins as p

from ckanext.dcat.interfaces import IDCATRDFHarvester
from ckanext.sweden.dcat import template_helpers


VALIDATION_SERVICE = 'https://validator.dcat-editor.com/service'


class SwedenDCATRDFHarvester(p.SingletonPlugin):

    p.implements(IDCATRDFHarvester, inherit=True)
    p.implements(p.IConfigurer)
    p.implements(p.ITemplateHelpers)

    def after_download(self, content, harvest_job):

        if not p.toolkit.asbool(config.get('ckanext.sweden.harvest.use_validation', True)):
            return content, []

        validation_service = config.get('ckanext.sweden.harvest.validation_service', VALIDATION_SERVICE)

        stop_on_errors = p.toolkit.asbool(config.get('ckanext.sweden.harvest.stop_on_validation_errors', False))

        errors = []
        try:
            r = requests.post(validation_service, data=content)
        except requests.exceptions.RequestException, e:
            errors.append(
                'Error contacting the validation service: {0}'.format(str(e)))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

        if r.status_code != 200:

            errors.append(
                'The validation service returned an error: {0}'.format(
                    r.status_code))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

        else:
            response = r.json()

            if not any([response.get('rdfError'),
                        response.get('errors'),
                        response.get('warnings')]):
                # All clear
                return content, []

            if response.get('rdfError'):
                errors.append(response.get('rdfError'))
            else:
                if response.get('mandatoryError'):
                    for _class in response['mandatoryError']:
                        errors.append(
                            'Mandatory class {0} missing'.format(_class))

                for resource in response.get('resources', []):
                    errors.append(json.dumps(resource))

            if stop_on_errors:
                return None, errors
            else:
                return content, errors

    # IConfigurer
    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

    # ITemplateHelpers
    def get_helpers(self):
        return {
            'json_loads': template_helpers.json_loads,
        }
{'fraction_non_alphanumeric': 0.09196275888276648, 'fraction_numerical': 0.0022800684020520615, 'mean_word_length': 1.8954895489548955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2619310', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1391, 'n_words': 287}