<commit_before>class Resque
  class Worker
    attr_reader :resque

    def initialize(server, *queues)
      @resque = Resque.new(server)
      @queues = queues
      validate_queues
    end

    class NoQueueError < RuntimeError; end

    def validate_queues
      if @queues.nil? || @queues.empty?
        raise NoQueueError.new("Please give each worker at least one queue.")
      end
    end

    def work(interval = 5)
      register_signal_handlers

      loop do
        break if @shutdown

        if job = reserve
          log "Got #{job.inspect}"
          process job
        else
          log "Sleeping"
          sleep interval.to_i
        end
      end
    end

    def process(job = nil)
      return unless job ||= reserve

      begin
        job.perform
      rescue Object => e
        log "#{job.inspect} failed: #{e.inspect}"
        job.fail(e, self)
      else
        log "#{job.inspect} done processing"
        job.done
      end
    end

    def register_signal_handlers
      trap('TERM') { shutdown }
      trap('INT')  { shutdown }
    end

    def shutdown
      log 'Exiting...'
      @shutdown = true
    end

    def reserve
      @queues.each do |queue|
        if job = @resque.reserve(queue)
          return job
        end
      end

      nil
    end

    def inspect
      "#<Worker #{to_s}>"
    end

    def to_s
      @to_s ||= "#{`hostname`.chomp}:#{Process.pid}"
    end

    def log(message)
      puts "*** #{message}"
    end
  end
end<commit_msg>logger is an option, off by default<commit_after>class Resque
  class Worker
    attr_reader   :resque
    attr_accessor :logger

    def initialize(server, *queues)
      @resque = Resque.new(server)
      @queues = queues
      validate_queues
    end

    class NoQueueError < RuntimeError; end

    def validate_queues
      if @queues.nil? || @queues.empty?
        raise NoQueueError.new("Please give each worker at least one queue.")
      end
    end

    def work(interval = 5)
      register_signal_handlers

      loop do
        break if @shutdown

        if job = reserve
          log "Got #{job.inspect}"
          process job
        else
          log "Sleeping"
          sleep interval.to_i
        end
      end
    end

    def process(job = nil)
      return unless job ||= reserve

      begin
        job.perform
      rescue Object => e
        log "#{job.inspect} failed: #{e.inspect}"
        job.fail(e, self)
      else
        log "#{job.inspect} done processing"
        job.done
      end
    end

    def register_signal_handlers
      trap('TERM') { shutdown }
      trap('INT')  { shutdown }
    end

    def shutdown
      log 'Exiting...'
      @shutdown = true
    end

    def reserve
      @queues.each do |queue|
        if job = @resque.reserve(queue)
          return job
        end
      end

      nil
    end

    def inspect
      "#<Worker #{to_s}>"
    end

    def to_s
      @to_s ||= "#{`hostname`.chomp}:#{Process.pid}"
    end

    def log(message)
      puts "*** #{message}" if logger
    end
  end
end
{'fraction_non_alphanumeric': 0.0915032679738562, 'fraction_numerical': 0.00065359477124183, 'mean_word_length': 1.6969162995594713, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2930668', 'n_tokens_mistral': 969, 'n_tokens_neox': 940, 'n_words': 271}