<commit_before>require 'resque'

require 'resqued/backoff'
require 'resqued/logging'

module Resqued
  # Models a worker process.
  class Worker
    include Resqued::Logging

    def initialize(options)
      @queues = options.fetch(:queues)
      @backoff = Backoff.new
    end

    # Public: The pid of the worker process.
    attr_reader :pid

    # Private.
    attr_reader :queues

    # Public: True if there is no worker process mapped to this object.
    def idle?
      pid.nil?
    end

    # Public: Checks if this worker works on jobs from the queue.
    def queue_key
      queues.sort.join(';')
    end

    # Public: Claim this worker for another listener's worker.
    def wait_for(pid)
      raise "Already running #{@pid} (can't wait for #{pid})" if @pid
      @self_started = nil
      @pid = pid
    end

    # Public: The old worker process finished!
    def finished!(process_status)
      @pid = nil
      @backoff.finished
    end

    # Public: The amount of time we need to wait before starting a new worker.
    def backing_off_for
      @pid ? nil : @backoff.how_long?
    end

    # Public: Start a job, if there's one waiting in one of my queues.
    def try_start
      return if @backoff.wait?
      @backoff.started
      @self_started = true
      if @pid = fork
        # still in the listener
      else
        # In case we get a signal before the process is all the way up.
        [:QUIT, :TERM, :INT].each { |signal| trap(signal) { exit 1 } }
        $0 = "STARTING RESQUE FOR #{queues.join(',')}"
        if Resque.respond_to?("logger=") && ! log_to_stdout?
          Resque.logger = Resque.logger.class.new(logging_io)
        end
        resque_worker = Resque::Worker.new(*queues)
        resque_worker.log "Starting worker #{resque_worker}"
        resque_worker.term_child = true # Hopefully do away with those warnings!
        resque_worker.work(5)
        exit 0
      end
    end

    # Public: Shut this worker down.
    #
    # We are using these signal semantics:
    # HUP: restart (QUIT workers)
    # INT/TERM: immediately exit
    # QUIT: graceful shutdown
    #
    # Resque uses these (compatible) signal semantics:
    # TERM: Shutdown immediately, stop processing jobs.
    #  INT: Shutdown immediately, stop processing jobs.
    # QUIT: Shutdown after the current job has finished processing.
    # USR1: Kill the forked child immediately, continue processing jobs.
    # USR2: Don't process any new jobs
    # CONT: Start processing jobs again after a USR2
    def kill(signal)
      Process.kill(signal.to_s, pid) if pid && @self_started
    end
  end
end<commit_msg>Try to get all resques to write their logs to our log file.<commit_after>require 'resque'

require 'resqued/backoff'
require 'resqued/logging'

module Resqued
  # Models a worker process.
  class Worker
    include Resqued::Logging

    def initialize(options)
      @queues = options.fetch(:queues)
      @backoff = Backoff.new
    end

    # Public: The pid of the worker process.
    attr_reader :pid

    # Private.
    attr_reader :queues

    # Public: True if there is no worker process mapped to this object.
    def idle?
      pid.nil?
    end

    # Public: Checks if this worker works on jobs from the queue.
    def queue_key
      queues.sort.join(';')
    end

    # Public: Claim this worker for another listener's worker.
    def wait_for(pid)
      raise "Already running #{@pid} (can't wait for #{pid})" if @pid
      @self_started = nil
      @pid = pid
    end

    # Public: The old worker process finished!
    def finished!(process_status)
      @pid = nil
      @backoff.finished
    end

    # Public: The amount of time we need to wait before starting a new worker.
    def backing_off_for
      @pid ? nil : @backoff.how_long?
    end

    # Public: Start a job, if there's one waiting in one of my queues.
    def try_start
      return if @backoff.wait?
      @backoff.started
      @self_started = true
      if @pid = fork
        # still in the listener
      else
        # In case we get a signal before the process is all the way up.
        [:QUIT, :TERM, :INT].each { |signal| trap(signal) { exit 1 } }
        $0 = "STARTING RESQUE FOR #{queues.join(',')}"
        if ! log_to_stdout?
          lf = logging_io
          if Resque.respond_to?("logger=")
            Resque.logger = Resque.logger.class.new(lf)
          else
            $stdout.reopen(lf)
            lf.close
          end
        end
        resque_worker = Resque::Worker.new(*queues)
        resque_worker.log "Starting worker #{resque_worker}"
        resque_worker.term_child = true # Hopefully do away with those warnings!
        resque_worker.work(5)
        exit 0
      end
    end

    # Public: Shut this worker down.
    #
    # We are using these signal semantics:
    # HUP: restart (QUIT workers)
    # INT/TERM: immediately exit
    # QUIT: graceful shutdown
    #
    # Resque uses these (compatible) signal semantics:
    # TERM: Shutdown immediately, stop processing jobs.
    #  INT: Shutdown immediately, stop processing jobs.
    # QUIT: Shutdown after the current job has finished processing.
    # USR1: Kill the forked child immediately, continue processing jobs.
    # USR2: Don't process any new jobs
    # CONT: Start processing jobs again after a USR2
    def kill(signal)
      Process.kill(signal.to_s, pid) if pid && @self_started
    end
  end
end
{'fraction_non_alphanumeric': 0.08396383096512272, 'fraction_numerical': 0.002583502491234545, 'mean_word_length': 2.4066624764299185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '196908', 'n_tokens_mistral': 1690, 'n_tokens_neox': 1611, 'n_words': 586}