<commit_before>class CratesCrawler < Versioneye::Crawl

  API_HOST  = "https://crates.io"
  API_URL   = "https://crates.io/api/v1"

  def self.logger
    if !defined?(@@log) || @@log.nil?
      @@log = Versioneye::DynLog.new("log/crates.log", 10).log
    end
    @@log
  end

  def self.crawl(api_key)
    api_key = api_key.to_s.strip

    if api_key.empty?
      logger.error "crates api_key is not set - will stop crawler"
      return
    end

    logger.info "Going to crawl rust packages on the crates.io"
    crawl_product_list(api_key)
    logger.info "Done!"
  end


  def self.crawl_product_list(api_key, page_nr = 1, per_page = 100)
    logger.info "going to crawl product list from #{page_nr}, per_page: #{per_page}"

    n = 0
    loop do
      products = fetch_product_list(api_key, page_nr, per_page)
      if products.to_a.empty?
        logger.info "Stopping crawl_product_list after #{page_nr} due empty response"
        break
      end

      products.to_a.each do |product|
        crawl_product_details api_key, product[:id], product[:max_version]
        n += 1<commit_msg>Read API key from db if not given<commit_after>class CratesCrawler < Versioneye::Crawl

  API_HOST  = "https://crates.io"
  API_URL   = "https://crates.io/api/v1"

  def self.logger
    if !defined?(@@log) || @@log.nil?
      @@log = Versioneye::DynLog.new("log/crates.log", 10).log
    end
    @@log
  end

  def self.crawl( api_key = nil )
    if api_key.to_s.empty?
      env     = Settings.instance.environment
      api_key = GlobalSetting.get env, 'cratesio_api_key'
    end

    if api_key.to_s.empty?
      logger.error "crates api_key is not set - will stop crawler"
      return
    end

    logger.info "Going to crawl rust packages on the crates.io"
    crawl_product_list(api_key)
    logger.info "Done!"
  end


  def self.crawl_product_list(api_key, page_nr = 1, per_page = 100)
    logger.info "going to crawl product list from #{page_nr}, per_page: #{per_page}"

    n = 0
    loop do
      products = fetch_product_list(api_key, page_nr, per_page)
      if products.to_a.empty?
        logger.info "Stopping crawl_product_list after #{page_nr} due empty response"
        break
      end

      products.to_a.each do |product|
        crawl_product_details api_key, product[:id], product[:max_version]
        n += 1
{'fraction_non_alphanumeric': 0.10734220695577501, 'fraction_numerical': 0.007728638900815801, 'mean_word_length': 2.8322368421052633, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '239527', 'n_tokens_mistral': 864, 'n_tokens_neox': 805, 'n_words': 201}