<commit_before>require 'time'
require 'resque'
require 'resque/server'

module Resque
  module Plugins
    # ResqueCleaner class provides useful functionalities to retry or clean
    # failed jobs. Let's clean up your failed list!
    class ResqueCleaner
      include Resque::Helpers
      # ResqueCleaner fetches all elements from Redis and checks them
      # by linear when filtering them. Since there is a performance concern,
      # ResqueCleaner handles only the latest x(default 1000) jobs.
      #
      # You can change the value through limiter attribute.
      # e.g. cleaner.limiter.maximum = 5000
      attr_reader :limiter

      # Set false if you don't show any message.
      attr_accessor :print_message

      # Initializes instance
      def initialize
        @failure = Resque::Failure.backend
        @print_message = true
        @limiter = Limiter.new self
      end

      # Returns redis instance.
      def redis
        Resque.redis
      end

      # Returns failure backend. Only supports redis backend.
      def failure
        @failure
      end

      # Stats by date.
      def stats_by_date(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          date = job["failed_at"][0,10]
          stats[date] ||= 0
          stats[date] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Stats by class.
      def stats_by_class(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          klass = job["payload"] && job["payload"]["class"] ? job["payload"]["class"] : "UNKNOWN"
          stats[klass] ||= 0
          stats[klass] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Stats by exception.
      def stats_by_exception(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          exception = job["exception"]
          stats[exception] ||= 0
          stats[exception] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Print stats
      def print_stats(stats)
        log too_many_message if @limiter.on?
        stats.keys.sort.each do |k|
          log "%15s: %4d" % [k,stats[k]]
        end
        log "%15s: %4d" % ["total", @limiter.count]
      end

      # Returns every jobs for which block evaluates to true.
      def select(&block)
        jobs = @limiter.jobs
        block_given? ? @limiter.jobs.select(&block) : jobs
      end
      alias :failure_jobs :select

      # Clears every jobs for which block evaluates to true.
      def clear(&block)
        cleared = 0
        @limiter.lock do
          @limiter.jobs.each_with_index do |job,i|
            if !block_given? || block.call(job)
              index = @limiter.start_index + i - cleared
              # fetches again since you can't ensure that it is always true:
              # a == endode(decode(a))
              value = redis.lindex(:failed, index)
              redis.lrem(:failed, 1, value)
              cleared += 1
            end
          end
        end
        cleared
      end

      # Retries every jobs for which block evaluates to true.
      def requeue(clear_after_requeue=false, options={}, &block)
        requeued = 0
        queue = options["queue"] || options[:queue]
        @limiter.lock do
          @limiter.jobs.each_with_index do |job,i|
            if !block_given? || block.call(job)
              index = @limiter.start_index + i - requeued

              value = redis.lindex(:failed, index)
              redis.multi do
                Job.create(queue||job['queue'], job['payload']['class'], *job['payload']['args'])

                if clear_after_requeue
                  # remove job
                  redis.lrem(:failed, 1, value)
                else
                  # mark retried
                  job['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
                  redis.lset(:failed, @limiter.start_index+i, Resque.encode(job))
                end
              end

              requeued += 1
            end
          end
        end
        requeued
      end

      # Clears all jobs except the last X jobs
      def clear_stale<commit_msg>Add a note<commit_after>require 'time'
require 'resque'
require 'resque/server'

module Resque
  module Plugins
    # ResqueCleaner class provides useful functionalities to retry or clean
    # failed jobs. Let's clean up your failed list!
    class ResqueCleaner
      include Resque::Helpers
      # ResqueCleaner fetches all elements from Redis and checks them
      # by linear when filtering them. Since there is a performance concern,
      # ResqueCleaner handles only the latest x(default 1000) jobs.
      #
      # You can change the value through limiter attribute.
      # e.g. cleaner.limiter.maximum = 5000
      attr_reader :limiter

      # Set false if you don't show any message.
      attr_accessor :print_message

      # Initializes instance
      def initialize
        @failure = Resque::Failure.backend
        @print_message = true
        @limiter = Limiter.new self
      end

      # Returns redis instance.
      def redis
        Resque.redis
      end

      # Returns failure backend. Only supports redis backend.
      def failure
        @failure
      end

      # Stats by date.
      def stats_by_date(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          date = job["failed_at"][0,10]
          stats[date] ||= 0
          stats[date] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Stats by class.
      def stats_by_class(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          klass = job["payload"] && job["payload"]["class"] ? job["payload"]["class"] : "UNKNOWN"
          stats[klass] ||= 0
          stats[klass] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Stats by exception.
      def stats_by_exception(&block)
        jobs, stats = select(&block), {}
        jobs.each do |job|
          exception = job["exception"]
          stats[exception] ||= 0
          stats[exception] += 1
        end

        print_stats(stats) if print?
        stats
      end

      # Print stats
      def print_stats(stats)
        log too_many_message if @limiter.on?
        stats.keys.sort.each do |k|
          log "%15s: %4d" % [k,stats[k]]
        end
        log "%15s: %4d" % ["total", @limiter.count]
      end

      # Returns every jobs for which block evaluates to true.
      def select(&block)
        jobs = @limiter.jobs
        block_given? ? @limiter.jobs.select(&block) : jobs
      end
      alias :failure_jobs :select

      # Clears every jobs for which block evaluates to true.
      def clear(&block)
        cleared = 0
        @limiter.lock do
          @limiter.jobs.each_with_index do |job,i|
            if !block_given? || block.call(job)
              index = @limiter.start_index + i - cleared
              # fetches again since you can't ensure that it is always true:
              # a == endode(decode(a))
              value = redis.lindex(:failed, index)
              redis.lrem(:failed, 1, value)
              cleared += 1
            end
          end
        end
        cleared
      end

      # Retries every jobs for which block evaluates to true.
      def requeue(clear_after_requeue=false, options={}, &block)
        requeued = 0
        queue = options["queue"] || options[:queue]
        @limiter.lock do
          @limiter.jobs.each_with_index do |job,i|
            if !block_given? || block.call(job)
              index = @limiter.start_index + i - requeued

              value = redis.lindex(:failed, index)
              redis.multi do
                Job.create(queue||job['queue'], job['payload']['class'], *job['payload']['args'])

                if clear_after_requeue
                  # remove job
                  # TODO: should use ltrim. not sure why i used lrem here...
                  redis.lrem(:failed, 1, value)
                else
                  # mark retried
                  job['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
                  redis.lset(:failed, @limiter.start_index+i, Resque.encode(job))
                end
              end

              requeued += 1
            end
          end
        end
        requeued
      end

      # Clears all jobs except the last X jobs
      def clear_stale
{'fraction_non_alphanumeric': 0.09625794304542246, 'fraction_numerical': 0.006825135325959049, 'mean_word_length': 1.7214217098943323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7181896', 'n_tokens_mistral': 2499, 'n_tokens_neox': 2381, 'n_words': 712}