<commit_before>import logging
import os

from ckan import model
from ckan.model.types import make_uuid
from ckan import plugins as p
from ckan.lib.celery_app import celery
from ckanext.report.interfaces import IReport
from ckanext.archiver.interfaces import IPipe

log = logging.getLogger(__name__)

class ArchiverPlugin(p.SingletonPlugin):
    """
    Registers to be notified whenever CKAN resources are created or their URLs
    change, and will create a new ckanext.archiver celery task to archive the
    resource.
    """
    p.implements(p.IDomainObjectModification, inherit=True)
    p.implements(p.IResourceUrlChange)
    p.implements(IReport)
    p.implements(p.IConfigurer, inherit=True)

    # IDomainObjectModification / IResourceUrlChange

    def notify(self, entity, operation=None):
        if not isinstance(entity, model.Package):
            return

        log.debug('Notified of package event: %s %s', entity.id, operation)

        create_package_archiver_task(entity, 'priority')

    # IReport

    def register_reports(self):
        """Register details of an extension's reports"""
        from ckanext.archiver import reports
        return [reports.broken_links_report_info,
                ]

    # IConfigurer

    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

def create_archiver_task(resource, queue):
    from pylons import config
    package = resource.resource_group.package
    task_id = '%s/%s/%s' % (package.name, resource.id[:4], make_uuid()[:4])
    ckan_ini_filepath = os.path.abspath(config.__file__)
    celery.send_task('archiver.update', args=[ckan_ini_filepath, resource.id, queue],
                     task_id=task_id, queue=queue)
    log.debug('Archival of resource put into celery queue %s: %s/%s url=%r', queue, package.name, resource.id, resource.url)<commit_msg>Do not need to listen to resource url changes any more - domain object ones are enough.<commit_after>import logging
import os

from ckan import model
from ckan.model.types import make_uuid
from ckan import plugins as p
from ckan.lib.celery_app import celery
from ckanext.report.interfaces import IReport
from ckanext.archiver.interfaces import IPipe

log = logging.getLogger(__name__)


class ArchiverPlugin(p.SingletonPlugin):
    """
    Registers to be notified whenever CKAN resources are created or their URLs
    change, and will create a new ckanext.archiver celery task to archive the
    resource.
    """
    p.implements(p.IDomainObjectModification, inherit=True)
    p.implements(IReport)
    p.implements(p.IConfigurer, inherit=True)

    # IDomainObjectModification

    def notify(self, entity, operation=None):
        if not isinstance(entity, model.Package):
            return

        log.debug('Notified of package event: %s %s', entity.id, operation)

        create_package_archiver_task(entity, 'priority')

    # IReport

    def register_reports(self):
        """Register details of an extension's reports"""
        from ckanext.archiver import reports
        return [reports.broken_links_report_info,
                ]

    # IConfigurer

    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

def create_archiver_task(resource, queue):
    from pylons import config
    package = resource.resource_group.package
    task_id = '%s/%s/%s' % (package.name, resource.id[:4], make_uuid()[:4])
    ckan_ini_filepath = os.path.abspath(config.__file__)
    celery.send_task('archiver.update', args=[ckan_ini_filepath, resource.id, queue],
                     task_id=task_id, queue=queue)
    log.debug('Archival of resource put into celery queue %s: %s/%s url=%r', queue, package.name, resource.id, resource.url)
{'fraction_non_alphanumeric': 0.09309149106428381, 'fraction_numerical': 0.0010669511869831954, 'mean_word_length': 3.641089108910891, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2102271', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1104, 'n_words': 298}