<commit_before># Similar to the standard library's Queue class in terms of synchronizing
# access, but keeps jobs in the order we want to work them. Assumes there's
# many threads competing to retrieve jobs.

module Que
  class JobQueue
    def initialize
      @stop  = false
      @array = []
      @mutex = Mutex.new
      @cv    = ConditionVariable.new
    end

    def push(*jobs)
      jobs.flatten!

      @mutex.synchronize do
        # At some point, for large queue sizes and small numbers of jobs to
        # insert, it may be worth investigating an insertion by binary search.
        @array.push(*jobs).sort_by! { |job| job.values_at(:priority, :run_at, :job_id) }
        @cv.broadcast
      end
    end

    # Looping/ConditionVariable technique borrowed from the rubysl-thread gem.
    def shift(priority = Float::INFINITY)
      loop do
        @mutex.synchronize do
          if @stop
            @cv.signal
            return :stop
          elsif (pk = @array.first) && pk[:priority] <= priority
            item = @array.shift
            @cv.signal
            return item
          else
            @cv.wait(@mutex)
          end
        end
      end
    end

    def count
      @array.count
    end

    def to_a
      @array.dup
    end

    def stop
      @mutex.synchronize do
        @stop = true
        @cv.signal
      end
    end

    def clear
      @mutex.synchronize do
        ids = @array.map { |pk| pk[:job_id] }
        @array.clear
        @cv.signal
        ids
      end<commit_msg>Use ConditionVariable#broadcast where appropriate.<commit_after># Similar to the standard library's Queue class in terms of synchronizing
# access, but keeps jobs in the order we want to work them. Assumes there's
# many threads competing to retrieve jobs.

module Que
  class JobQueue
    def initialize
      @stop  = false
      @array = []
      @mutex = Mutex.new
      @cv    = ConditionVariable.new
    end

    def push(*jobs)
      jobs.flatten!

      @mutex.synchronize do
        # At some point, for large queue sizes and small numbers of jobs to
        # insert, it may be worth investigating an insertion by binary search.
        @array.push(*jobs).sort_by! { |job| job.values_at(:priority, :run_at, :job_id) }
        @cv.broadcast
      end
    end

    # Looping/ConditionVariable technique borrowed from the rubysl-thread gem.
    def shift(priority = Float::INFINITY)
      loop do
        @mutex.synchronize do
          if @stop
            return :stop
          elsif (pk = @array.first) && pk[:priority] <= priority
            item = @array.shift
            @cv.signal
            return item
          else
            @cv.wait(@mutex)
          end
        end
      end
    end

    def count
      @array.count
    end

    def to_a
      @array.dup
    end

    def stop
      @mutex.synchronize do
        @stop = true
        @cv.broadcast
      end
    end

    def clear
      @mutex.synchronize do
        ids = @array.map { |pk| pk[:job_id] }
        @array.clear
        @cv.signal
        ids
      end
{'fraction_non_alphanumeric': 0.07749918593292088, 'fraction_numerical': 0.0, 'mean_word_length': 1.7502238137869293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2716966', 'n_tokens_mistral': 907, 'n_tokens_neox': 878, 'n_words': 295}