<commit_before>from datetime import datetime, timezone
from logging import getLogger
import threading

from .task import Tasks, Batch, Schedulable
from .utils import run_forever, handle_sigterm
from .job import Job, JobStatus, advance_job_status
from .brokers.base import Broker
from .const import DEFAULT_QUEUE, DEFAULT_NAMESPACE, DEFAULT_WORKER_NUMBER
from .worker import Workers
from . import exc


logger = getLogger(__name__)


class Engine:
    """Spinach Engine coordinating a broker with workers.

    :arg broker: instance of a :class:`Broker`
    :arg namespace: name of the namespace used by the Engine. When different
         Engines use the same Redis server, they must use different<commit_msg>Document how internal threads interact with each other<commit_after>from datetime import datetime, timezone
from logging import getLogger
import threading

from .task import Tasks, Batch, Schedulable
from .utils import run_forever, handle_sigterm
from .job import Job, JobStatus, advance_job_status
from .brokers.base import Broker
from .const import DEFAULT_QUEUE, DEFAULT_NAMESPACE, DEFAULT_WORKER_NUMBER
from .worker import Workers
from . import exc


logger = getLogger(__name__)


class Engine:
    """Spinach Engine coordinating a broker with workers.

    This class does the orchestration of all components, it is the one that
    starts and terminates the whole machinery.

    The Engine can be run in two modes:
    - client: synchronously submits jobs.
    - worker: asynchronously executes jobs.

    Submitting jobs is quite easy, so running the Engine in client mode doesn't
    require spawning any thread.

    Executing jobs however is a bit more involved, so running the Engine in
    worker mode ends up spawning a few threads:
    - a few worker threads: they are only responsible for executing the task
      function and advancing the job status once it is finished.
    - a result notifier thread: sends back the result of job executions to the
      Broker backend, acts basically as a client.
    - an arbiter thread: fetches jobs from the Broker and gives them to the
      workers as well as doing some periodic bookkeeping.
    - a Broker subscriber thread: receives notifications from the backend when
      something happens, typically a job is enqueued.
    - the process main thread: starts all the above threads, then does nothing
      waiting for the signal to terminate the threads it started.

    This means that a Spinach worker process has at least 5 threads.

    :arg broker: instance of a :class:`Broker`
    :arg namespace: name of the namespace used by the Engine. When different
         Engines use the same Redis server, they must use different
{'fraction_non_alphanumeric': 0.03960029607698001, 'fraction_numerical': 0.0003700962250185048, 'mean_word_length': 3.8440860215053765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1256992', 'n_tokens_mistral': 686, 'n_tokens_neox': 659, 'n_words': 361}