<commit_before>'''A custom rq worker class to add start & stop SNS messages to all jobs'''

import logging
from rq.worker import Worker
from harvester.sns_message import publish_to_harvesting

logger = logging.getLogger(__name__)

class SNSWorker(Worker):
    def execute_job(self, job, queue):
        """Spawns a work horse to perform the actual work and passes it a job.
        The worker will wait for the work horse and make sure it executes
        within the given timeout bounds, or will end the work horse with
        SIGALRM.
        """
        worker_name = (self.key.rsplit(':', 1)[1]).rsplit('.', 1)[0]
        subject = 'Starting job {} on worker {}'.format(job.description,
                worker_name)
        publish_to_harvesting(subject, subject)
        self.set_state('busy')
        self.fork_work_horse(job, queue)
        self.monitor_work_horse(job)
        subject = 'Finished job {} on worker {}'.format(job.description,
                worker_name)
        publish_to_harvesting(subject, subject)
        self.set_state('idle')
<commit_msg>put worker ip first in message<commit_after>'''A custom rq worker class to add start & stop SNS messages to all jobs'''

import logging
from rq.worker import Worker
from harvester.sns_message import publish_to_harvesting

logger = logging.getLogger(__name__)


class SNSWorker(Worker):
    def execute_job(self, job, queue):
        """Spawns a work horse to perform the actual work and passes it a job.
        The worker will wait for the work horse and make sure it executes
        within the given timeout bounds, or will end the work horse with
        SIGALRM.
        """
        worker_name = (self.key.rsplit(':', 1)[1]).rsplit('.', 1)[0]
        subject = 'Worker {} starting job {}'.format(
            worker_name,
            job.description)
        publish_to_harvesting(subject, subject)
        self.set_state('busy')
        self.fork_work_horse(job, queue)
        self.monitor_work_horse(job)
        subject = 'Worker {} finished job {}'.format(
            worker_name,
            job.description)
        publish_to_harvesting(subject, subject)
        self.set_state('idle')
{'fraction_non_alphanumeric': 0.09396591432519576, 'fraction_numerical': 0.0036849378166743437, 'mean_word_length': 2.797202797202797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2990561', 'n_tokens_mistral': 628, 'n_tokens_neox': 613, 'n_words': 203}