<commit_before>require_relative '../queue/redis_wrapper'
require_relative 'github_repo_scraper'

class ScraperDispatcher
  # Dispatches scrapers to use various redis queues and scrape for different
  # types of data

  @current_repo = nil
  @redis_wrapper = RedisWrapper.new

  # TODO: refactor rake tasks
  # TODO: Add a pathway to scrape repos based on the prioritized queue

  def self.scrape(queue_name: RedisWrapper::REDIS_ACTIVE_QUEUE_NAME, commits_on: true, issues_on: false)
    if queue_name == RedisWrapper::REDIS_ACTIVE_QUEUE_NAME ||
       queue_name == RedisWrapper::REDIS_PRIORITY_QUEUE_NAME

      scraper_handler(queue_name) do
        GithubRepoScraper.commits(repositories: [@current_repo]) if commits_on
        GithubRepoScraper.issues(repositories: [@current_repo]) if issues_on
      end
    elsif queue_name == RedisWrapper::REDIS_BLANK_QUEUE_NAME
      # TODO: Debug and test
      scraper_handler(queue_name) { GithubRepoScraper.update_repo_data([@current_repo]) }
    else
      raise "That queue name does not exist. Exiting Dispatcher."
    end
  end

  private

  def self.scraper_handler(queue_name)
    start_time = Time.now
    scrape_count = 0

    loop do
      repo_data = @redis_wrapper.next_repo(queue_name)

      @current_repo = Repository.find(repo_data['id'])

      puts "Scraping: #{repo_data['url']}"

      yield if block_given?

      scrape_count += 1
      puts "Scraped #{scrape_count} in #{((Time.now - start_time) / 60).round(2)} mins"
    end
  end
end
<commit_msg>added a priority queue path that uses multiple queues, one per priority level, for scraping<commit_after>require_relative '../queue/redis_wrapper'
require_relative 'github_repo_scraper'

class ScraperDispatcher
  # Dispatches scrapers to use various redis queues and scrape for different
  # types of data

  @current_repo = nil
  @redis_wrapper = RedisWrapper.new

  def self.scrape(queue_name: RedisWrapper::REDIS_ACTIVE_QUEUE_NAME, commits_on: true, issues_on: false)
    if queue_name == RedisWrapper::REDIS_ACTIVE_QUEUE_NAME
      scraper_handler(queue_name) do
        GithubRepoScraper.commits(repositories: [@current_repo]) if commits_on
        GithubRepoScraper.issues(repositories: [@current_repo]) if issues_on
      end
    elsif queue_name == RedisWrapper::REDIS_PRIORITY_QUEUE_NAME
      priority_scraper_handler(queue_name)
    elsif queue_name == RedisWrapper::REDIS_BLANK_QUEUE_NAME
      # TODO: Debug and test
      scraper_handler(queue_name) { GithubRepoScraper.update_repo_data([@current_repo]) }
    else
      raise "That queue name does not exist. Exiting Dispatcher."
    end
  end

  private

  def self.scraper_handler(queue_name)
    start_time = Time.now
    scrape_count = 0

    loop do
      # TODO: dispatcher should work directly with a queue that uses redis underneath,
      # not the other way around
      repo_data = @redis_wrapper.next_repo(queue_name)

      @current_repo = Repository.find(repo_data['id'])

      puts "Scraping: #{repo_data['url']}"

      yield if block_given?

      scrape_count += 1
      puts "Scraped #{scrape_count} in #{((Time.now - start_time) / 60).round(2)} mins"
    end
  end

  def self.priority_handler
    priority_tags = RedisWrapper::PRIORITY_RANGE
    base_queue_name = RedisWrapper::REDIS_PRIORITY_QUEUE_NAME

    queues = []
    priority_tags.each do |tag|
      queue_name = base_queue_name + '_' + tag.to_s
      queues << CircularRedisQueue.new(@redis_wrapper.redis, queue_name)
    end

    # add them to a skewed distribution list

    loop do
      # get leftmost queue and push to the end
      queue = queues.shift
      queues.push(queue)

      @current_repo = Repository.find(queue.next['id'])

      GithubRepoScraper.commits(repositories: [@current_repo])
    end
  end
end

{'fraction_non_alphanumeric': 0.08212305254819117, 'fraction_numerical': 0.0026406126221283337, 'mean_word_length': 3.758793969849246, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5980988', 'n_tokens_mistral': 1281, 'n_tokens_neox': 1232, 'n_words': 292}