<commit_before>STDOUT.sync = true

require "open-uri"
require "json"
require "net/http"
require "logger"
require "parallel"
require "retriable"
require "uri"
require_relative "./bestgems_api"
require_relative "./rubygems_api"

class PutGemDetail
  def initialize(bestgems_api_base:, api_key:)
    @bestgems_api_base, @api_key = bestgems_api_base, api_key
  end

  def api_key
    @api_key
  end

  def logger
    @logger ||= Logger.new(STDOUT)
  end

  def execute
    import_gem_detail
  end

  def process_gem(gem_name)
    logger.info(gem_name: gem_name)

    info = retry_with { rubygems_api.info(gem_name) }

    unless info
      logger.error(type: :fetch_failed, gem_name: gem_name)

      return
    end

    retry_with { bestgems_api.put_detail(info) }
    retry_with { bestgems_api.put_dependencies(info) }
  end

  def import_gem_detail
    logger.info("import_gem_detail")

    page = 1

    loop do
      logger.info(page)

      gems = retry_with { bestgems_api.gems(page) }

      break unless gems.count > 0

      Parallel.each(gems, in_processes: 4) do |gem|
        process_gem(gem["name"])
      end

      page += 1
    end
  end

  def retry_with
    Retriable.retriable(tries: 20) do
      begin
        yield
      rescue => e
        @logger.warn(error_class: e.class.name, error_message: e.message)<commit_msg>Import versions<commit_after>STDOUT.sync = true

require "open-uri"
require "json"
require "net/http"
require "logger"
require "parallel"
require "retriable"
require "uri"
require_relative "./bestgems_api"
require_relative "./rubygems_api"

class PutGemDetail
  def initialize(bestgems_api_base:, api_key:)
    @bestgems_api_base, @api_key = bestgems_api_base, api_key
  end

  def api_key
    @api_key
  end

  def logger
    @logger ||= Logger.new(STDOUT)
  end

  def execute
    import_gem_detail
  end

  def process_gem(gem_name)
    logger.info(gem_name: gem_name)

    info = retry_with { rubygems_api.info(gem_name) }

    unless info
      logger.error(type: :fetch_info_failed, gem_name: gem_name)

      return
    end

    retry_with { bestgems_api.put_detail(info) }
    retry_with { bestgems_api.put_dependencies(info) }

    versions = retry_with { rubygems_api.versions(gem_name) }

    unless versions
      logger.error(type: :fetch_versions_failed, gem_name: gem_name)

      return
    end

    retry_with { bestgems_api.put_versions(gem_name, versions) }
  end

  def import_gem_detail
    logger.info("import_gem_detail")

    page = 1

    loop do
      logger.info(page)

      gems = retry_with { bestgems_api.gems(page) }

      break unless gems.count > 0

      Parallel.each(gems, in_processes: 2) do |gem|
        process_gem(gem["name"])
      end

      page += 1
    end
  end

  def retry_with
    Retriable.retriable(tries: 20) do
      begin
        yield
      rescue => e
        @logger.warn(error_class: e.class.name, error_message: e.message)
{'fraction_non_alphanumeric': 0.09143835616438356, 'fraction_numerical': 0.00410958904109589, 'mean_word_length': 3.0910364145658265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2198484', 'n_tokens_mistral': 1079, 'n_tokens_neox': 1005, 'n_words': 218}