<commit_before>import atexit
import logging
import multiprocessing
import platform as _platform
import os
import socket
import sys
import warnings

from celery import __version__
from celery import platforms
from celery import signals
from celery.exceptions import ImproperlyConfigured
from celery.routes import Router
from celery.task import discard_all
from celery.utils import info, get_full_cls_name, LOG_LEVELS
from celery.worker import WorkController


SYSTEM = _platform.system()
IS_OSX = SYSTEM == "Darwin"

STARTUP_INFO_FMT = """
Configuration ->
    . broker -> %(conninfo)s
    . queues ->
%(queues)s
    . concurrency -> %(concurrency)s
    . loader -> %(loader)s
    . logfile -> %(logfile)s@%(loglevel)s
    . events -> %(events)s
    . beat -> %(celerybeat)s
%(tasks)s
""".strip()

TASK_LIST_FMT = """    . tasks ->\n%s"""


class Worker(object):
    WorkController = WorkController

    def __init__(self, concurrency=None, loglevel=None, logfile=None,
            hostname=None, discard=False, run_clockservice=False,
            schedule=None, task_time_limit=None, task_soft_time_limit=None,
            max_tasks_per_child=None, queues=None, events=False, db=None,
            include=None, defaults=None, pidfile=None, **kwargs):
        if defaults is None:
            from celery import conf
            defaults = conf
        self.defaults = defaults
        self.concurrency = (concurrency or
                            defaults.CELERYD_CONCURRENCY or
                            multiprocessing.cpu_count())
        self.loglevel = loglevel or defaults.CELERYD_LOG_LEVEL
        self.logfile = logfile or defaults.CELERYD_LOG_FILE
        self.hostname = hostname or socket.gethostname()
        self.discard = discard
        self.run_clockservice = run_clockservice
        self.schedule = schedule or defaults.CELERYBEAT_SCHEDULE_FILENAME
        self.events = events
        self.task_time_limit = (task_time_limit or
                                defaults.CELERYD_TASK_TIME_LIMIT)
        self.task_soft_time_limit = (task_soft_time_limit or
                                     defaults.CELERYD_TASK_SOFT_TIME_LIMIT)
        self.max_tasks_per_child = (max_tasks_per_child or
                                    defaults.CELERYD_MAX_TASKS_PER_CHILD)
        self.db = db
        self.queues = queues or []
        self.include = include or []
        self.pidfile = pidfile
        self._isatty = sys.stdout.isatty()

        if isinstance(self.queues, basestring):
            self.queues = self.queues.split(",")
        if isinstance(self.include, basestring):
            self.include = self.include.split(",")

        if not isinstance(self.loglevel, int):
            self.loglevel = LOG_LEVELS[self.loglevel.upper()]

    def run(self):
        self.init_loader()
        self.init_queues()
        self.worker_init()
        self.redirect_stdouts_to_logger()
        print("celery@%s v%s is starting." % (self.hostname, __version__))

        if getattr(self.settings, "DEBUG", False):
            warnings.warn("Using settings.DEBUG leads to a memory leak, "
                    "never use this setting in a production environment!")

        if self.discard:
            self.purge_messages()

        # Dump configuration to screen so we have some basic information
        # for when users sends bug reports.
        print(self.startup_info())
        self.set_process_status("Running...")

        self.run_worker()

    def on_listener_ready(self, listener):
        signals.worker_ready.send(sender=listener)
        print("celery@%s has started." % self.hostname)

    def init_queues(self):
        conf = self.defaults
        if self.queues:
            conf.QUEUES = dict((queue, options)
                                for queue, options in conf.QUEUES.items()
                                    if queue in self.queues)
            for queue in self.queues:<commit_msg>celeryd now emits a warning if running as root<commit_after>import atexit
import logging
import multiprocessing
import platform as _platform
import os
import socket
import sys
import warnings

from celery import __version__
from celery import platforms
from celery import signals
from celery.exceptions import ImproperlyConfigured
from celery.routes import Router
from celery.task import discard_all
from celery.utils import info, get_full_cls_name, LOG_LEVELS
from celery.worker import WorkController


SYSTEM = _platform.system()
IS_OSX = SYSTEM == "Darwin"

STARTUP_INFO_FMT = """
Configuration ->
    . broker -> %(conninfo)s
    . queues ->
%(queues)s
    . concurrency -> %(concurrency)s
    . loader -> %(loader)s
    . logfile -> %(logfile)s@%(loglevel)s
    . events -> %(events)s
    . beat -> %(celerybeat)s
%(tasks)s
""".strip()

TASK_LIST_FMT = """    . tasks ->\n%s"""


class Worker(object):
    WorkController = WorkController

    def __init__(self, concurrency=None, loglevel=None, logfile=None,
            hostname=None, discard=False, run_clockservice=False,
            schedule=None, task_time_limit=None, task_soft_time_limit=None,
            max_tasks_per_child=None, queues=None, events=False, db=None,
            include=None, defaults=None, pidfile=None, **kwargs):
        if defaults is None:
            from celery import conf
            defaults = conf
        self.defaults = defaults
        self.concurrency = (concurrency or
                            defaults.CELERYD_CONCURRENCY or
                            multiprocessing.cpu_count())
        self.loglevel = loglevel or defaults.CELERYD_LOG_LEVEL
        self.logfile = logfile or defaults.CELERYD_LOG_FILE
        self.hostname = hostname or socket.gethostname()
        self.discard = discard
        self.run_clockservice = run_clockservice
        self.schedule = schedule or defaults.CELERYBEAT_SCHEDULE_FILENAME
        self.events = events
        self.task_time_limit = (task_time_limit or
                                defaults.CELERYD_TASK_TIME_LIMIT)
        self.task_soft_time_limit = (task_soft_time_limit or
                                     defaults.CELERYD_TASK_SOFT_TIME_LIMIT)
        self.max_tasks_per_child = (max_tasks_per_child or
                                    defaults.CELERYD_MAX_TASKS_PER_CHILD)
        self.db = db
        self.queues = queues or []
        self.include = include or []
        self.pidfile = pidfile
        self._isatty = sys.stdout.isatty()

        if isinstance(self.queues, basestring):
            self.queues = self.queues.split(",")
        if isinstance(self.include, basestring):
            self.include = self.include.split(",")

        if not isinstance(self.loglevel, int):
            self.loglevel = LOG_LEVELS[self.loglevel.upper()]

    def run(self):
        self.init_loader()
        self.init_queues()
        self.worker_init()
        self.redirect_stdouts_to_logger()
        print("celery@%s v%s is starting." % (self.hostname, __version__))

        if getattr(os, "geteuid", None) and os.geteuid() == 0:
            warnings.warn(
                "Running celeryd with superuser privileges is not encouraged!")

        if getattr(self.settings, "DEBUG", False):
            warnings.warn("Using settings.DEBUG leads to a memory leak, "
                    "never use this setting in a production environment!")

        if self.discard:
            self.purge_messages()

        # Dump configuration to screen so we have some basic information
        # for when users sends bug reports.
        print(self.startup_info())
        self.set_process_status("Running...")

        self.run_worker()

    def on_listener_ready(self, listener):
        signals.worker_ready.send(sender=listener)
        print("celery@%s has started." % self.hostname)

    def init_queues(self):
        conf = self.defaults
        if self.queues:
            conf.QUEUES = dict((queue, options)
                                for queue, options in conf.QUEUES.items()
                                    if queue in self.queues)
            for queue in self.queues:
{'fraction_non_alphanumeric': 0.08901562112571287, 'fraction_numerical': 0.0001239771881973717, 'mean_word_length': 2.32659793814433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '770860', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2324, 'n_words': 551}