<commit_before># The priority queue is a queue of queues--not really a queue in and of itself
class PriorityQueue
  PRIORITY_RANGE = (1..10).to_a

  def initialize(repos = nil, enqueue: false, rescore: false)
    @queue_base_name = ENV['REDIS_PRIORITY_QUEUE_BASE_NAME']
    @repos = repos
    enqueue_redis(rescore) if enqueue

    @unique_queues = generate_queues
    @queues = skewed_dist_of_queues(@unique_queues)
  end

  def next
    # get leftmost queue and push to the end
    queue = @queues.shift
    @queues.push(queue)
    # return the next repository
    queue.next
  end

  def enqueue_redis(rescore)
    puts "Scoring and ordering..."
    calculate_scores(@repos) if rescore
    scored_repos = @repos.order(:score)

    puts "Creating prioritized queues..."
    enqueue_prioritized_sub_queues(scored_repos)
  end

  private

  def enqueue_prioritized_sub_queues(tracked_repos)
    bucket_size = (tracked_repos.count.to_f / PRIORITY_RANGE.length).ceil

    # Enqueue a sub queue for each priority level
    index = 0
    tracked_repos.in_batches(of: bucket_size) do |batch|
      priority = PRIORITY_RANGE[index]
      queue_name = sub_queue_name(priority.to_s)
      index += 1

      sub_queue = CircularRedisQueue.new(batch, queue_name: queue_name)
      sub_queue.enqueue_redis
    end
  end

  # For enqueing
  def calculate_scores
    count = 0
    tracked_repos.in_batches do |batch|
      batch.each do |repo|
        repo.update_score
        count += 1
        puts "#{count} repos scored"
      end
    end
  end

  # For operation
  def generate_queues
    priority_tags = PRIORITY_RANGE

    unique_queues = []
    priority_tags.each do |tag|<commit_msg>converted in_batches to in_groups_of because in_batches doesn't work as expected for a collection, it is supposed to be used directly on a queury. It will grab all the data from a query even if the relation you returned holds a subset using a limit on the query. Is this a bug solved in rails 5?<commit_after># The priority queue is a queue of queues--not really a queue in and of itself
class PriorityQueue
  PRIORITY_RANGE = (1..10).to_a

  def initialize(repos = nil, enqueue: false, rescore: false)
    @queue_base_name = ENV['REDIS_PRIORITY_QUEUE_BASE_NAME']
    @repos = repos
    enqueue_redis(rescore) if enqueue

    @unique_queues = generate_queues
    @queues = skewed_dist_of_queues(@unique_queues)
  end

  def next
    # get leftmost queue and push to the end
    queue = @queues.shift
    @queues.push(queue)
    # return the next repository
    queue.next
  end

  def enqueue_redis(rescore)
    puts "Scoring and ordering..."
    calculate_scores if rescore
    scored_repos = @repos.order(:score)

    puts "Creating prioritized queues..."
    enqueue_prioritized_sub_queues(scored_repos)
  end

  private

  def enqueue_prioritized_sub_queues(tracked_repos)
    bucket_size = (tracked_repos.count.to_f / PRIORITY_RANGE.length).ceil

    # Enqueue a sub queue for each priority level
    index = 0
    tracked_repos.in_groups_of(bucket_size, false) do |batch|
      priority = PRIORITY_RANGE[index]
      queue_name = sub_queue_name(priority.to_s)
      index += 1

      sub_queue = CircularRedisQueue.new(batch, queue_name: queue_name)
      sub_queue.enqueue_redis
    end
  end

  # For enqueing
  def calculate_scores
    count = 0
    @repos.each do |repo|
      repo.update_score
      count += 1
    end
    puts "#{count} repos scored"
  end

  # For operation
  def generate_queues
    priority_tags = PRIORITY_RANGE

    unique_queues = []
    priority_tags.each do |tag|
{'fraction_non_alphanumeric': 0.07094972067039106, 'fraction_numerical': 0.004189944134078212, 'mean_word_length': 3.263095238095238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4909494', 'n_tokens_mistral': 1245, 'n_tokens_neox': 1177, 'n_words': 329}