<commit_before>class ScraperDispatcher
  def self.scrape_commits
    # TODO: set url to env variable = heroku host
    conn = Faraday.new(url: 'http://4f4f870d.ngrok.io')
    loop do
      response = conn.get '/next_job'
      puts JSON.parse(response.body)
      gem_id = JSON.parse(response.body)['gem_id']
      GithubScraper.lib_commits(libraries: [ RubyGem.find(gem_id) ])
    end
  end
end<commit_msg>added connection to redis queue<commit_after>class ScraperDispatcher
  def self.scrape_commits
    # refactor this to open a remote redis connection
    loop do
      gem_id = next_gem_id
      GithubScraper.lib_commits(libraries: [ RubyGem.find(gem_id) ])
    end
  end

  private

  def redis
    # TODO: convert to environment variable
    url = '192.168.3.11'
    @redis ||= Redis.new(url: url)
  end

  def next_gem_id
    next_id = redis.lpop('rubygems')
    redis.rpush('rubygems', next_id)
    next_id
  end
end
{'fraction_non_alphanumeric': 0.09611231101511879, 'fraction_numerical': 0.01511879049676026, 'mean_word_length': 3.175675675675676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5404600', 'n_tokens_mistral': 352, 'n_tokens_neox': 338, 'n_words': 75}