<commit_before>require 'net/http'
require 'set'
require 'open-uri'
require 'json'

require 'ghtorrent/logging'
require 'ghtorrent/settings'
require 'ghtorrent/time'

module GHTorrent
  module APIClient
    include GHTorrent::Logging
    include GHTorrent::Settings

    # This is to fix an annoying bug in JRuby's SSL not being able to
    # verify a valid certificate.
    if defined? JRUBY_VERSION
      OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
    end

    # A paged request. Used when the result can expand to more than one
    # result pages.
    def paged_api_request(url, pages = -1)

      data = api_request_raw(url)

      return [] if data.nil?

      unless data.meta['link'].nil?
        links = parse_links(data.meta['link'])

        if pages > 0
          pages = pages - 1
          if pages == 0
            return parse_request_result(data)
          end
        end

        if links['next'].nil?
          parse_request_result(data)
        else
          parse_request_result(data) | paged_api_request(links['next'], pages)
        end
      else
        parse_request_result(data)
      end
    end

    # A normal request. Returns a hash or an array of hashes representing the
    # parsed JSON result.
    def api_request(url)
      parse_request_result api_request_raw(url)
    end

    private

    # Parse a Github link header
    def parse_links(links)
      links.split(/,/).reduce({}) do |acc, x|
        matches = x.strip.match(/<(.*)>; rel=\"(.*)\"/)
        acc[matches[2]] = matches[1]
        acc
      end
    end

    # Parse the JSON result array
    def parse_request_result(result)
      if result.nil?
        []
      else
        json = result.read

        if json.nil?
          []
        else
          JSON.parse(json)
        end
      end
    end

    # Do the actual request and return the result object
    def api_request_raw(url)
      @num_api_calls ||= 0
      @ts ||= Time.now().tv_sec()
      @cache ||= config(:cache)
      @cache_dir ||= config(:cache_dir)

      #Rate limiting to avoid error requests
      if Time.now().tv_sec() - @ts < 60 then
        if @num_api_calls >= @settings['mirror']['reqrate'].to_i
          sleep = 60 - (Time.now().tv_sec() - @ts)
          debug "APIClient: Sleeping for #{sleep}"
          sleep (sleep)
          @num_api_calls = 0
          @ts = Time.now().tv_sec()
        end
      else
        debug "APIClient: Tick, num_calls = #{@num_api_calls}, zeroing"
        @num_api_calls = 0
        @ts = Time.now().tv_sec()
      end

      begin
        start_time = Time.now
        from_cache = false

        contents = if @cache
          sig = Digest::SHA1.hexdigest url
          file = File.join(@cache_dir, sig)

          if File.exist?(file)
            f = File.open(file, 'r')
            from_cache = true
            YAML::load(f)
          else
            result = do_request(url)
            @num_api_calls += 1
            tocache = Cachable.new(result)
            f = File.open(file, 'w')
            YAML::dump tocache, f
            f.close
            tocache
          end
        else
          do_request(url)
          @num_api_calls += 1
        end

        total = Time.now.to_ms - start_time.to_ms
        debug "APIClient: Request: #{url} (#{@num_api_calls} calls,#{if from_cache then " from cache, " end} Total: #{total} ms)"
        contents
      rescue OpenURI::HTTPError => e
        case e.io.status[0].to_i
          # The following indicate valid Github return codes
          when 400, # Bad request<commit_msg>Create the cache directory if not exists<commit_after>require 'open-uri'
require 'net/http'
require 'fileutils'
require 'json'

require 'ghtorrent/logging'
require 'ghtorrent/settings'
require 'ghtorrent/time'

module GHTorrent
  module APIClient
    include GHTorrent::Logging
    include GHTorrent::Settings

    # This is to fix an annoying bug in JRuby's SSL not being able to
    # verify a valid certificate.
    if defined? JRUBY_VERSION
      OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
    end

    # A paged request. Used when the result can expand to more than one
    # result pages.
    def paged_api_request(url, pages = -1)

      data = api_request_raw(url)

      return [] if data.nil?

      unless data.meta['link'].nil?
        links = parse_links(data.meta['link'])

        if pages > 0
          pages = pages - 1
          if pages == 0
            return parse_request_result(data)
          end
        end

        if links['next'].nil?
          parse_request_result(data)
        else
          parse_request_result(data) | paged_api_request(links['next'], pages)
        end
      else
        parse_request_result(data)
      end
    end

    # A normal request. Returns a hash or an array of hashes representing the
    # parsed JSON result.
    def api_request(url)
      parse_request_result api_request_raw(url)
    end

    private

    # Parse a Github link header
    def parse_links(links)
      links.split(/,/).reduce({}) do |acc, x|
        matches = x.strip.match(/<(.*)>; rel=\"(.*)\"/)
        acc[matches[2]] = matches[1]
        acc
      end
    end

    # Parse the JSON result array
    def parse_request_result(result)
      if result.nil?
        []
      else
        json = result.read

        if json.nil?
          []
        else
          JSON.parse(json)
        end
      end
    end

    # Do the actual request and return the result object
    def api_request_raw(url)
      @num_api_calls ||= 0
      @ts ||= Time.now().tv_sec()
      @cache ||= config(:cache)
      @cache_dir ||= config(:cache_dir)

      #Rate limiting to avoid error requests
      if Time.now().tv_sec() - @ts < 60 then
        if @num_api_calls >= @settings['mirror']['reqrate'].to_i
          sleep = 60 - (Time.now().tv_sec() - @ts)
          debug "APIClient: Sleeping for #{sleep}"
          sleep (sleep)
          @num_api_calls = 0
          @ts = Time.now().tv_sec()
        end
      else
        debug "APIClient: Tick, num_calls = #{@num_api_calls}, zeroing"
        @num_api_calls = 0
        @ts = Time.now().tv_sec()
      end

      begin
        start_time = Time.now
        from_cache = false

        contents = if @cache
          FileUtils.mkdir_p(@cache_dir)
          sig = Digest::SHA1.hexdigest url
          file = File.join(@cache_dir, sig)

          if File.exist?(file)
            f = File.open(file, 'r')
            from_cache = true
            YAML::load(f)
          else
            result = do_request(url)
            @num_api_calls += 1
            tocache = Cachable.new(result)
            f = File.open(file, 'w')
            YAML::dump tocache, f
            f.close
            tocache
          end
        else
          do_request(url)
          @num_api_calls += 1
        end

        total = Time.now.to_ms - start_time.to_ms
        debug "APIClient: Request: #{url} (#{@num_api_calls} calls,#{if from_cache then " from cache, " end} Total: #{total} ms)"
        contents
      rescue OpenURI::HTTPError => e
        case e.io.status[0].to_i
          # The following indicate valid Github return codes
          when 400, # Bad request
{'fraction_non_alphanumeric': 0.10362550354215863, 'fraction_numerical': 0.005556327267676066, 'mean_word_length': 1.890405459654757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2104780', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2224, 'n_words': 600}