<commit_before>import os
import sys
import signal
import logging

from optparse import make_option

from django.utils.daemonize import become_daemon
from django.core.management.base import NoArgsCommand

from ...utils import get_backend, get_middleware, set_process_title

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list + (
        make_option('--pidfile', action='store', dest='pidfile', default=None,
            help="Fork and write pidfile to this file."),
        make_option('--logfile', action='store', dest='logfile', default=None,
            help="Log to the specified file."),
    )

    def handle_noargs(self, **options):
        level = {
            '0': logging.WARNING,
            '1': logging.INFO,
            '2': logging.DEBUG,
        }[options['verbosity']]

        set_process_title("Starting")

        logging.basicConfig(
            level=level,
            format='%(asctime)-15s %(process)d %(levelname).1s: %(message)s',
            filename=options['logfile'],
        )

        logging.info("Starting queue runner")

        backend = get_backend()
        logging.info("Started backend %s", backend)

        get_middleware()
        logging.info("Loaded middleware")

        # fork() only after we have started enough to catch failure, including
        # being able to write to our pidfile.
        if options['pidfile']:
            with open(options['pidfile'], 'w') as f:
                become_daemon(our_home_dir='/')
                print >>f, os.getpid()

        self.should_run = True
        def handle_term(signum, stack):
            logging.info("Caught TERM signal; exiting")
            self.should_run = False
        signal.signal(signal.SIGTERM, handle_term)

        while self.should_run:
            logging.debug("Checking backend for items")
            set_process_title("Waiting for items")

            try:
                job = backend.dequeue(1)

                if job is not None:
                    set_process_title("Running a job: %s" % job)
                    job.run()
            except KeyboardInterrupt:
                sys.exit(1)<commit_msg>Log when running a job.<commit_after>import os
import sys
import signal
import logging

from optparse import make_option

from django.utils.daemonize import become_daemon
from django.core.management.base import NoArgsCommand

from ...utils import get_backend, get_middleware, set_process_title

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list + (
        make_option('--pidfile', action='store', dest='pidfile', default=None,
            help="Fork and write pidfile to this file."),
        make_option('--logfile', action='store', dest='logfile', default=None,
            help="Log to the specified file."),
    )

    def handle_noargs(self, **options):
        level = {
            '0': logging.WARNING,
            '1': logging.INFO,
            '2': logging.DEBUG,
        }[options['verbosity']]

        set_process_title("Starting")

        logging.basicConfig(
            level=level,
            format='%(asctime)-15s %(process)d %(levelname).1s: %(message)s',
            filename=options['logfile'],
        )

        logging.info("Starting queue runner")

        backend = get_backend()
        logging.info("Started backend %s", backend)

        get_middleware()
        logging.info("Loaded middleware")

        # fork() only after we have started enough to catch failure, including
        # being able to write to our pidfile.
        if options['pidfile']:
            with open(options['pidfile'], 'w') as f:
                become_daemon(our_home_dir='/')
                print >>f, os.getpid()

        self.should_run = True
        def handle_term(signum, stack):
            logging.info("Caught TERM signal; exiting")
            self.should_run = False
        signal.signal(signal.SIGTERM, handle_term)

        while self.should_run:
            logging.debug("Checking backend for items")
            set_process_title("Waiting for items")

            try:
                job = backend.dequeue(1)

                if job is not None:
                    set_process_title("Running a job: %s" % job)
                    logging.info("Running a job: %s" % job)
                    job.run()
                    logging.info("Finished running a job: %s" % job)
            except KeyboardInterrupt:
                sys.exit(1)
{'fraction_non_alphanumeric': 0.1092096374690385, 'fraction_numerical': 0.0036027921639270434, 'mean_word_length': 2.1347918136908963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2140832', 'n_tokens_mistral': 1249, 'n_tokens_neox': 1197, 'n_words': 312}