<commit_before># coding: utf-8

import os, re
import datetime
import difflib
import shutil
import traceback
import uuid
from lxml import etree
from cgi import FieldStorage
from pylons import config
from ckan import model
from ckan.model import Session
from ckan.logic import action, get_action
import ckan.plugins.toolkit as tk
from ckan import plugins as p
from ckan.lib.helpers import json
from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.harvest.model import HarvestObject
from ckanext.stadtzhtheme.plugin import StadtzhThemePlugin

import logging
log = logging.getLogger(__name__)


class InvalidCommentError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)


class StadtzhHarvester(HarvesterBase):
    '''
    BaseHarvester for the City of Zurich
    '''

    ORGANIZATION = {
        'de': u'Stadt Zürich',
        'fr': u'fr_Stadt Zürich',
        'it': u'it_Stadt Zürich',
        'en': u'en_Stadt Zürich',
    }
    LANG_CODES = ['de', 'fr', 'it', 'en']

    config = {
        'user': u'harvest'
    }

    META_DIR = ''
    BUCKET = 'default'

    def __init__(self, **kwargs):
        HarvesterBase.__init__(self, **kwargs)
        try:
            self.INTERNAL_SITE_URL = config['ckan.site_url_internal']
            self.CKAN_SITE_URL = config['ckan.site_url']
            self.DIFF_PATH = config['metadata.diffpath']
        except KeyError as e:
            raise Exception("'%s' not found in config" % e.message)

    def gather_stage(self, harvest_job):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def fetch_stage(self, harvest_object):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def import_stage(self, harvest_object):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def _gather_datasets(self, harvest_job):
        ids = []
        try:
            # list directories in dropzone folder
            datasets = self._remove_hidden_files(os.listdir(self.DATA_PATH))
            log.debug("Directories in %s: %s" % (self.DATA_PATH, datasets))

            # foreach -> meta.xml -> create entry
            for dataset in datasets:
                log.debug(self._validate_package_id(dataset))
                dataset_id = self._validate_package_id(dataset)
                if dataset_id:
                    meta_xml_file_path = os.path.join(self.DATA_PATH, dataset, self.META_DIR, 'meta.xml')
                    if os.path.exists(meta_xml_file_path):
                        try:
                            with open(meta_xml_file_path, 'r') as meta_xml:
                                parser = etree.XMLParser(encoding='utf-8')
                                dataset_node = etree.fromstring(meta_xml.read(), parser=parser).find('datensatz')
                            metadata = self._dropzone_get_metadata(dataset, dataset_node)
                        except Exception, e:
                            log.exception(e)
                            self._save_gather_error(
                                'Could not parse metadata in %s: %s / %s'
                                % (meta_xml_file_path, str(e), traceback.format_exc()),
                                harvest_job
                            )
                            continue
                    else:
                        metadata = {
                            'datasetID': dataset_id,
                            'title': dataset,
                            'url': None,
                            'resources': self._generate_resources_dict_array(dataset),
                        }
                    if not os.path.isdir(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset)):
                        os.makedirs(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset))

                    with open(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset, 'metadata-' + str(datetime.date.today())), 'w') as meta_json:
                        meta_json.write(json.dumps(metadata, sort_keys=True, indent=4, separators=(',', ': ')))
                        log.debug('Metadata JSON created')

                    id = self._save_harvest_object(metadata, harvest_job)
                    ids.append(id)

            self._create_notifications_for_deleted_datasets()
            return ids
        except Exception, e:
            log.exception(e)
            self._save_gather_error(
                'Unable to get content from folder: %s: %s / %s'
                % (self.DATA_PATH, str(e), traceback.format_exc()),
                harvest_job
            )
	    return []

    def _fetch_datasets(self, harvest_object):
        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False
        # Get the URL
        datasetID = json.loads(harvest_object.content)['datasetID']
        log.debug(harvest_object.content)

        # Get contents
        try:
            harvest_object.save()
            log.debug('successfully processed ' + datasetID)
            return True
        except Exception, e:
            log.exception(e)
            self._save_object_error(
                (
                    'Unable to get content for package: %s: %r / %s'
                    % (datasetID, e, traceback.format_exc())
                ),
                harvest_object
            )
            return False

    def _import_datasets(self, harvest_object):
        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False

        try:
            self._import_package(harvest_object)
            Session.commit()
            return True
        except Exception, e:
            log.exception(e)
            self._save_object_error(
                (
                    'Unable to get content for package: %s: %r / %s'
                    % (harvest_object.guid, e, traceback.format_exc())
                ),
                harvest_object
            )
            return False

    def _import_package(self, harvest_object):
        package_dict = json.loads(harvest_object.content)
        package_dict['id'] = harvest_object.guid
        package_dict['name'] = munge_title_to_name(package_dict[u'datasetID'])
        user = model.User.get(self.config['user'])
        context = {
            'model': model,
            'session': Session,
            'user': self.config['user']
        }

        # get existing package if it exists
        try:
            existing_package = self._find_existing_package(package_dict)
        except tk.ObjectNotFound:
            existing_package = None  # package does not exist

        # remove showcases from pkg_dict
        showcases = package_dict['related']
        del package_dict['related']

        # update existing resources, delete old ones, create new ones
        action_dict = {} 
        if existing_package:
            old_resources = existing_package['resources']
            new_resources = self._generate_resources_dict_array(package_dict['id'], include_files=True)

            for r in new_resources:
                action = {'action': 'create', 'new_resource': r, 'old_resource': None}
                for old in old_resources:
                    if old['name'] == r['name']:
                        action['action'] = 'update'
                        action['old_resource'] = old
                        break
                action_dict[r['name']] = action<commit_msg>Change comment to be less confusing<commit_after># coding: utf-8

import os, re
import datetime
import difflib
import shutil
import traceback
import uuid
from lxml import etree
from cgi import FieldStorage
from pylons import config
from ckan import model
from ckan.model import Session
from ckan.logic import action, get_action
import ckan.plugins.toolkit as tk
from ckan import plugins as p
from ckan.lib.helpers import json
from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.harvest.model import HarvestObject
from ckanext.stadtzhtheme.plugin import StadtzhThemePlugin

import logging
log = logging.getLogger(__name__)


class InvalidCommentError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)


class StadtzhHarvester(HarvesterBase):
    '''
    BaseHarvester for the City of Zurich
    '''

    ORGANIZATION = {
        'de': u'Stadt Zürich',
        'fr': u'fr_Stadt Zürich',
        'it': u'it_Stadt Zürich',
        'en': u'en_Stadt Zürich',
    }
    LANG_CODES = ['de', 'fr', 'it', 'en']

    config = {
        'user': u'harvest'
    }

    META_DIR = ''
    BUCKET = 'default'

    def __init__(self, **kwargs):
        HarvesterBase.__init__(self, **kwargs)
        try:
            self.INTERNAL_SITE_URL = config['ckan.site_url_internal']
            self.CKAN_SITE_URL = config['ckan.site_url']
            self.DIFF_PATH = config['metadata.diffpath']
        except KeyError as e:
            raise Exception("'%s' not found in config" % e.message)

    def gather_stage(self, harvest_job):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def fetch_stage(self, harvest_object):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def import_stage(self, harvest_object):
        raise NotImplementedError("This is only a base harvester, use one of its childs") 

    def _gather_datasets(self, harvest_job):
        ids = []
        try:
            # list directories in dropzone folder
            datasets = self._remove_hidden_files(os.listdir(self.DATA_PATH))
            log.debug("Directories in %s: %s" % (self.DATA_PATH, datasets))

            # foreach -> meta.xml -> create entry
            for dataset in datasets:
                log.debug(self._validate_package_id(dataset))
                dataset_id = self._validate_package_id(dataset)
                if dataset_id:
                    meta_xml_file_path = os.path.join(self.DATA_PATH, dataset, self.META_DIR, 'meta.xml')
                    if os.path.exists(meta_xml_file_path):
                        try:
                            with open(meta_xml_file_path, 'r') as meta_xml:
                                parser = etree.XMLParser(encoding='utf-8')
                                dataset_node = etree.fromstring(meta_xml.read(), parser=parser).find('datensatz')
                            metadata = self._dropzone_get_metadata(dataset, dataset_node)
                        except Exception, e:
                            log.exception(e)
                            self._save_gather_error(
                                'Could not parse metadata in %s: %s / %s'
                                % (meta_xml_file_path, str(e), traceback.format_exc()),
                                harvest_job
                            )
                            continue
                    else:
                        metadata = {
                            'datasetID': dataset_id,
                            'title': dataset,
                            'url': None,
                            'resources': self._generate_resources_dict_array(dataset),
                        }
                    if not os.path.isdir(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset)):
                        os.makedirs(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset))

                    with open(os.path.join(self.DIFF_PATH, self.METADATA_DIR, dataset, 'metadata-' + str(datetime.date.today())), 'w') as meta_json:
                        meta_json.write(json.dumps(metadata, sort_keys=True, indent=4, separators=(',', ': ')))
                        log.debug('Metadata JSON created')

                    id = self._save_harvest_object(metadata, harvest_job)
                    ids.append(id)

            self._create_notifications_for_deleted_datasets()
            return ids
        except Exception, e:
            log.exception(e)
            self._save_gather_error(
                'Unable to get content from folder: %s: %s / %s'
                % (self.DATA_PATH, str(e), traceback.format_exc()),
                harvest_job
            )
	    return []

    def _fetch_datasets(self, harvest_object):
        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False
        # Get the URL
        datasetID = json.loads(harvest_object.content)['datasetID']
        log.debug(harvest_object.content)

        # Get contents
        try:
            harvest_object.save()
            log.debug('successfully processed ' + datasetID)
            return True
        except Exception, e:
            log.exception(e)
            self._save_object_error(
                (
                    'Unable to get content for package: %s: %r / %s'
                    % (datasetID, e, traceback.format_exc())
                ),
                harvest_object
            )
            return False

    def _import_datasets(self, harvest_object):
        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False

        try:
            self._import_package(harvest_object)
            Session.commit()
            return True
        except Exception, e:
            log.exception(e)
            self._save_object_error(
                (
                    'Unable to get content for package: %s: %r / %s'
                    % (harvest_object.guid, e, traceback.format_exc())
                ),
                harvest_object
            )
            return False

    def _import_package(self, harvest_object):
        package_dict = json.loads(harvest_object.content)
        package_dict['id'] = harvest_object.guid
        package_dict['name'] = munge_title_to_name(package_dict[u'datasetID'])
        user = model.User.get(self.config['user'])
        context = {
            'model': model,
            'session': Session,
            'user': self.config['user']
        }

        # get existing package if it exists
        try:
            existing_package = self._find_existing_package(package_dict)
        except tk.ObjectNotFound:
            existing_package = None  # package does not exist

        # remove related from pkg_dict
        showcases = package_dict['related']
        del package_dict['related']

        # update existing resources, delete old ones, create new ones
        action_dict = {} 
        if existing_package:
            old_resources = existing_package['resources']
            new_resources = self._generate_resources_dict_array(package_dict['id'], include_files=True)

            for r in new_resources:
                action = {'action': 'create', 'new_resource': r, 'old_resource': None}
                for old in old_resources:
                    if old['name'] == r['name']:
                        action['action'] = 'update'
                        action['old_resource'] = old
                        break
                action_dict[r['name']] = action
{'fraction_non_alphanumeric': 0.08909641595153962, 'fraction_numerical': 0.0003785966683493185, 'mean_word_length': 1.7354159475319295, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '802390', 'n_tokens_mistral': 4351, 'n_tokens_neox': 4218, 'n_words': 1000}