<commit_before>import logging
import json
import hashlib
import datetime
import uuid

from pylons import config
import requests

from ckan import plugins as p
from ckan import model

from ckanext.harvest.model import HarvestObject

from ckanext.glasgow.logic.action import _get_api_endpoint, _expire_task_status

import ckanext.glasgow.logic.schema as custom_schema
from ckanext.glasgow.model import HarvestLastAudit
from ckanext.glasgow.harvesters import (
    EcHarvester,
    get_dataset_name_from_task,
    get_initial_dataset_name,
    get_dataset_name_from_id,
    get_task_for_request_id,
    get_org_name,
)


log = logging.getLogger(__name__)


def save_last_audit_id(audit_id, harvest_job_id=None):

    new_last_audit = HarvestLastAudit(
        audit_id=audit_id,
        harvest_job_id=harvest_job_id,
    )
    new_last_audit.save()


class EcChangelogHarvester(EcHarvester):

    force_import = False

    def info(self):
        return {
            'name': 'ec_changelog_harvester',
            'title': 'EC Changelog Harvester',
            'description': 'Harvester for regular Changelog imports from ' +
                           'the CTPEC API',

        }

    def gather_stage(self, harvest_job):
        log.debug('In ChangelogHarvester gather_stage')

        # Get the last harvested AuditId
        last_audit = model.Session.query(HarvestLastAudit) \
            .order_by(HarvestLastAudit.created.desc()) \
            .first()

        if last_audit:
            audit_id = last_audit.audit_id
        else:
            audit_id = '0'

        # Get all Audits
        audits = p.toolkit.get_action('changelog_show')(
            {'ignore_auth': True},
            {'audit_id': audit_id, 'top': 1000})

        # Check if there are any new audits to process
        if not len(audits) or (
           len(audits) == 1 and
           audits[0]['AuditId'] == audit_id):
            log.debug(
                'No new audits to process since last run ' +
                '(Last audit id {0})'.format(audit_id))
            return []

        # Ignore the first audit if an audit id was defined as start,
        # as this one will be included in the results
        audits = audits[1:] if audit_id != '0' and len(audits) > 1 else audits

        ids = []
        update_audits = {}
        for audit in audits:
            # We only want to use the most recent update per object per run
            # Store the most recent audit against a hash of the id fields
            if 'update' in audit['Command'].lower():
                m = hashlib.md5()
                m.update(json.dumps(audit['CustomProperties']))
                ids_hash = m.hexdigest()
                update_audits[ids_hash] = audit
            else:
                obj = HarvestObject(guid=audit['AuditId'], job=harvest_job,
                                    content=json.dumps(audit))
                obj.save()
                ids.append(obj.id)

        # Save the last AuditId to know where to start in the next run
        save_last_audit_id(audit['AuditId'], harvest_job.id)

        for key, audit in update_audits.iteritems():
            obj = HarvestObject(guid=audit['AuditId'], job=harvest_job,
                                content=json.dumps(audit))
            obj.save()
            ids.append(obj.id)

        return ids

    def fetch_stage(self, harvest_object):
        return True

    def import_stage(self, harvest_object):

        audit = json.loads(harvest_object.content)

        log.debug('Import stage for audit "{0}"'.format(audit.get('AuditId')))

        command = audit.get('Command')

        handler = get_audit_command_handler(command)
        if not handler:
            log.warning(
                'No handler for command {0}, skipping ...'.format(command))
            return False

        context = {
            'model': model,
            'ignore_auth': True,
            'user': self._get_user_name(),
            'local_action': True,
        }

        log.debug('Calling handler for command "{0}"'.format(command))<commit_msg>[#183] change harvest user to Owner from audit<commit_after>import logging
import json
import hashlib
import datetime
import uuid

from pylons import config
import requests

from ckan import plugins as p
from ckan import model

from ckanext.harvest.model import HarvestObject

from ckanext.glasgow.logic.action import _get_api_endpoint, _expire_task_status

import ckanext.glasgow.logic.schema as custom_schema
from ckanext.glasgow.model import HarvestLastAudit
from ckanext.glasgow.harvesters import (
    EcHarvester,
    get_dataset_name_from_task,
    get_initial_dataset_name,
    get_dataset_name_from_id,
    get_task_for_request_id,
    get_org_name,
)


log = logging.getLogger(__name__)


def save_last_audit_id(audit_id, harvest_job_id=None):

    new_last_audit = HarvestLastAudit(
        audit_id=audit_id,
        harvest_job_id=harvest_job_id,
    )
    new_last_audit.save()


class EcChangelogHarvester(EcHarvester):

    force_import = False

    def info(self):
        return {
            'name': 'ec_changelog_harvester',
            'title': 'EC Changelog Harvester',
            'description': 'Harvester for regular Changelog imports from ' +
                           'the CTPEC API',

        }

    def get_username_from_audit(self, audit):
        username = audit.get('Owner', None)
        if username:
            try:
                p.toolkit.get_action('user_show', {'id': username})
            except p.toolkit.ObjectNotFound:
                username = self._get_user_name()
        else:
            username = self._get_user_name()
        return username

    def gather_stage(self, harvest_job):
        log.debug('In ChangelogHarvester gather_stage')

        # Get the last harvested AuditId
        last_audit = model.Session.query(HarvestLastAudit) \
            .order_by(HarvestLastAudit.created.desc()) \
            .first()

        if last_audit:
            audit_id = last_audit.audit_id
        else:
            audit_id = '0'

        # Get all Audits
        audits = p.toolkit.get_action('changelog_show')(
            {'ignore_auth': True},
            {'audit_id': audit_id, 'top': 1000})

        # Check if there are any new audits to process
        if not len(audits) or (
           len(audits) == 1 and
           audits[0]['AuditId'] == audit_id):
            log.debug(
                'No new audits to process since last run ' +
                '(Last audit id {0})'.format(audit_id))
            return []

        # Ignore the first audit if an audit id was defined as start,
        # as this one will be included in the results
        audits = audits[1:] if audit_id != '0' and len(audits) > 1 else audits

        ids = []
        update_audits = {}
        for audit in audits:
            # We only want to use the most recent update per object per run
            # Store the most recent audit against a hash of the id fields
            if 'update' in audit['Command'].lower():
                m = hashlib.md5()
                m.update(json.dumps(audit['CustomProperties']))
                ids_hash = m.hexdigest()
                update_audits[ids_hash] = audit
            else:
                obj = HarvestObject(guid=audit['AuditId'], job=harvest_job,
                                    content=json.dumps(audit))
                obj.save()
                ids.append(obj.id)

        # Save the last AuditId to know where to start in the next run
        save_last_audit_id(audit['AuditId'], harvest_job.id)

        for key, audit in update_audits.iteritems():
            obj = HarvestObject(guid=audit['AuditId'], job=harvest_job,
                                content=json.dumps(audit))
            obj.save()
            ids.append(obj.id)

        return ids

    def fetch_stage(self, harvest_object):
        return True

    def import_stage(self, harvest_object):

        audit = json.loads(harvest_object.content)

        log.debug('Import stage for audit "{0}"'.format(audit.get('AuditId')))

        command = audit.get('Command')

        handler = get_audit_command_handler(command)
        if not handler:
            log.warning(
                'No handler for command {0}, skipping ...'.format(command))
            return False

        context = {
            'model': model,
            'ignore_auth': True,
            'user': self.get_username_from_audit(audit),
            'local_action': True,
        }

        log.debug('Calling handler for command "{0}"'.format(command))
{'fraction_non_alphanumeric': 0.09049562682215744, 'fraction_numerical': 0.003848396501457726, 'mean_word_length': 2.0848920863309353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945946', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2505, 'n_words': 635}