<commit_before># Stats on Pace
module Pace
  class Info
    WORKER_EXPIRE = 600 # 10 minutes

    class << self
      def log(queue, job)
        reset unless @initialized

        update_queue(queue)
        update_class(job)
        update_processed
      end

      def save
        save_info
        save_classes
        save_queues
        save_worker
        reset
        Pace.log "saved info to redis (worker #{uuid})"
      end

      def uuid
        @uuid ||= $uuid.generate
      end

      def processed
        @processed || 0
      end

      def queues
        @queues || {}
      end

      def classes
        @classes || {}
      end

      def reset
        @initialized = true
        @processed = 0
        @queues = {}
        @classes = {}
      end

      private

      def redis
        @redis ||= Pace.redis_connect
      end

      def k(key)
        "resque:pace:#{key}"
      end

      def update_processed
        @processed += 1
        @total ||= 0
        @total += 1
      end

      def update_class(job)
        @classes[job["class"]] ||= 0
        @classes[job["class"]] += 1
      end

      def update_queue(queue)
        @queues[queue] ||= {}
        @queues[queue][:processed] ||= 0
        @queues[queue] = {
          :last_job_at  => Time.now.to_i,
          :processed    => @queues[queue][:processed] + 1
        }
      end

      def save_info
        redis.hset(k("info"), "updated_at", Time.now.to_i)
        redis.hincrby(k("info"), "processed", processed)
      end

      def save_classes
        classes.each do |klass, count|
          redis.hincrby(k("info:classes"), klass, count)
        end
      end

      def save_queues
        queues.each do |queue, info|
          redis.hset(k("info:queues:#{queue}"), "updated_at", Time.now.to_i)
          redis.hset(k("info:queues:#{queue}"), "last_job_at", info[:last_job_at])
          redis.hincrby(k("info:queues:#{queue}"), "processed", info[:processed])
        end
      end

      def save_worker
        key = k("info:workers:#{uuid}")
        redis.hset(key, "created_at", Time.now.to_i) unless @initialized
        redis.hset(key, "updated_at", Time.now.to_i)
        redis.hset(key, "queues", @queues && @queues.keys.join(', '))
        redis.hset(key, "processed", @total)
        redis.expire(key, WORKER_EXPIRE)
      end
    end
  end
end<commit_msg>Fix created_at<commit_after># Stats on Pace
module Pace
  class Info
    WORKER_EXPIRE = 600 # 10 minutes

    class << self
      def log(queue, job)
        reset unless @initialized

        update_queue(queue)
        update_class(job)
        update_processed
      end

      def save
        save_info
        save_classes
        save_queues
        save_worker
        reset
        Pace.log "saved info to redis (worker #{uuid})"
      end

      def uuid
        @uuid ||= $uuid.generate
      end

      def processed
        @processed || 0
      end

      def queues
        @queues || {}
      end

      def classes
        @classes || {}
      end

      def reset
        @initialized = true
        @processed = 0
        @queues = {}
        @classes = {}
      end

      private

      def redis
        @redis ||= Pace.redis_connect
      end

      def k(key)
        "resque:pace:#{key}"
      end

      def update_processed
        @processed += 1
        @total ||= 0
        @total += 1
      end

      def update_class(job)
        @classes[job["class"]] ||= 0
        @classes[job["class"]] += 1
      end

      def update_queue(queue)
        @queues[queue] ||= {}
        @queues[queue][:processed] ||= 0
        @queues[queue] = {
          :last_job_at  => Time.now.to_i,
          :processed    => @queues[queue][:processed] + 1
        }
      end

      def save_info
        redis.hset(k("info"), "updated_at", Time.now.to_i)
        redis.hincrby(k("info"), "processed", processed)
      end

      def save_classes
        classes.each do |klass, count|
          redis.hincrby(k("info:classes"), klass, count)
        end
      end

      def save_queues
        queues.each do |queue, info|
          redis.hset(k("info:queues:#{queue}"), "updated_at", Time.now.to_i)
          redis.hset(k("info:queues:#{queue}"), "last_job_at", info[:last_job_at])
          redis.hincrby(k("info:queues:#{queue}"), "processed", info[:processed])
        end
      end

      def save_worker
        key = k("info:workers:#{uuid}")
        unless @created
          redis.hset(key, "created_at", Time.now.to_i)
          @created = true
        end
        redis.hset(key, "updated_at", Time.now.to_i)
        redis.hset(key, "queues", queues && queues.keys.join(', '))
        redis.hset(key, "processed", @total)
        redis.expire(key, WORKER_EXPIRE)
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.12450386463338207, 'fraction_numerical': 0.005849174848548151, 'mean_word_length': 1.7612456747404843, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3168861', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1563, 'n_words': 335}