<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Schedules jobs."""

import logging
from uuid import UUID
from hiispider.metacomponents.scheduler import Scheduler
from hiispider.components import Logger, MySQL, JobQueue
from twisted.internet.defer import inlineCallbacks
from hiispider.components.base import shared


LOGGER = logging.getLogger(__name__)


def get_uuid_bytes(uuid):
    try:
        # Make sure the uuid is in bytes
        uuid_bytes = UUID(uuid).bytes
        return uuid_bytes
    except ValueError:
        LOGGER.error("Could not turn UUID into bytes: %s" % uuid)
        return None


class JobScheduler(Scheduler):

    removed_jobs = set([])
    requires = [Logger, MySQL, JobQueue]

    def __init__(self, server, config, server_mode, **kwargs):
        super(JobScheduler, self).__init__(server, config, server_mode,
            server.jobqueue, **kwargs)
        self.service_mapping = config["service_mapping"]
        self.server.expose(self.enqueue_uuid)
        self.server.expose(self.add_uuid)
        self.server.expose(self.remove_uuid)

    def is_valid(self, item):
        if item in self.removed_jobs:
            self.removed_jobs.remove(item)
            return False
        return True

    @shared
    def enqueue_uuid(self, uuid):
        self.queue.publish(get_uuid_bytes(uuid))

    @shared
    def add_uuid(self, uuid, job_type):
        return self._add_uuid(self, uuid, job_type)

    def _add_uuid(self, uuid, job_type):
        if job_type in self.service_mapping:
            job_type = self.service_mapping[job_type]
        if job_type in self.server.functions:
            interval = self.server.functions[job_type]['interval']
            bytes = get_uuid_bytes(uuid)
            if bytes:
                self.add(bytes, int(interval))

    @shared
    def remove_uuid(self, uuid):<commit_msg>don't pass self because that is silly<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Schedules jobs."""

import logging
from uuid import UUID
from hiispider.metacomponents.scheduler import Scheduler
from hiispider.components import Logger, MySQL, JobQueue
from twisted.internet.defer import inlineCallbacks
from hiispider.components.base import shared


LOGGER = logging.getLogger(__name__)


def get_uuid_bytes(uuid):
    try:
        # Make sure the uuid is in bytes
        uuid_bytes = UUID(uuid).bytes
        return uuid_bytes
    except ValueError:
        LOGGER.error("Could not turn UUID into bytes: %s" % uuid)
        return None


class JobScheduler(Scheduler):

    removed_jobs = set([])
    requires = [Logger, MySQL, JobQueue]

    def __init__(self, server, config, server_mode, **kwargs):
        super(JobScheduler, self).__init__(server, config, server_mode,
            server.jobqueue, **kwargs)
        self.service_mapping = config["service_mapping"]
        self.server.expose(self.enqueue_uuid)
        self.server.expose(self.add_uuid)
        self.server.expose(self.remove_uuid)

    def is_valid(self, item):
        if item in self.removed_jobs:
            self.removed_jobs.remove(item)
            return False
        return True

    @shared
    def enqueue_uuid(self, uuid):
        self.queue.publish(get_uuid_bytes(uuid))

    @shared
    def add_uuid(self, uuid, job_type):
        return self._add_uuid(uuid, job_type)

    def _add_uuid(self, uuid, job_type):
        if job_type in self.service_mapping:
            job_type = self.service_mapping[job_type]
        if job_type in self.server.functions:
            interval = self.server.functions[job_type]['interval']
            bytes = get_uuid_bytes(uuid)
            if bytes:
                self.add(bytes, int(interval))

    @shared
    def remove_uuid(self, uuid):
{'fraction_non_alphanumeric': 0.09643047416089505, 'fraction_numerical': 0.0005327650506126798, 'mean_word_length': 2.839468302658487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4925844', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1132, 'n_words': 274}