<commit_before>require 'resque'

module Resque
  def self.prune_dead_workers
    Worker.all.each do |worker|
      worker.unregister_worker if worker.last_heartbeat_before?(5)
    end
  end

  class Worker
    def startup_with_heartbeat
      startup_without_heartbeat
      Thread.new do
        loop do
          sleep(2)
          heartbeat!
        end
      end
    end
    alias startup_without_heartbeat startup
    alias startup startup_with_heartbeat

    def heartbeat!
      redis.sadd(:workers, self)
      redis.set("worker:#{self}:heartbeat", Time.now.to_s)
    end

    def last_heartbeat_before?(seconds)
      Time.parse(last_heartbeat).utc < (Time.now.utc - seconds)
    end

    def last_heartbeat
      redis.get("worker:#{self}:heartbeat") || started
    end
  end
end<commit_msg>workaround multi-threading issues with redis connection<commit_after>require 'resque'

module Resque
  def self.prune_dead_workers
    Worker.all.each do |worker|
      worker.unregister_worker if worker.last_heartbeat_before?(5)
    end
  end

  class Worker
    def startup_with_heartbeat
      startup_without_heartbeat
      Thread.new do
        loop do
          sleep(2)
          heartbeat!
        end
      end
    end
    alias startup_without_heartbeat startup
    alias startup startup_with_heartbeat

    # apparently the Redis connection is not thread-safe, so we connect another instance
    def heartbeat_redis
      @heartbeat_redis ||= Redis.new("#{redis.port}:#{redis.port}")
    end

    def heartbeat!
      heartbeat_redis.sadd(:workers, self)
      heartbeat_redis.set("worker:#{self}:heartbeat", Time.now.to_s)
    end

    def last_heartbeat_before?(seconds)
      Time.parse(last_heartbeat).utc < (Time.now.utc - seconds)
    end

    def last_heartbeat
      redis.get("worker:#{self}:heartbeat") || started
    end
  end
end
{'fraction_non_alphanumeric': 0.07982740021574973, 'fraction_numerical': 0.002157497303128371, 'mean_word_length': 2.8405797101449277, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675320', 'n_tokens_mistral': 589, 'n_tokens_neox': 584, 'n_words': 129}