<commit_before>require 'net/http'

class APICache
  # Wraps up querying the API.
  # 
  # Ensures that the API is not called more frequently than every +period+ seconds, and times out API requests after +timeout+ seconds.
  #
  class API
    @query_times = {}
    class << self
      attr_reader :query_times
    end

    # Takes the following options
    #
    # period:: Maximum frequency to call the API
    # timeout:: Timeout when calling api (either to the proviced url or
    #           excecuting the passed block)
    # block:: If passed then the block is excecuted instead of HTTP GET 
    #         against the provided key
    #
    def initialize(key, options, &block)
      @key, @block = key, block
      @timeout = options[:timeout]
      @period = options[:period]
    end

    # Checks whether the API can be queried (i.e. whether :period has passed
    # since the last query to the API).
    #
    # If :period is 0 then there is no limit on how frequently queries can
    # be made to the API.
    #
    def queryable?
      if query_times[@key]
        if Time.now - query_times[@key] > @period
          APICache.logger.debug "Queryable: true - retry_time has passed"
          true
        else
          APICache.logger.debug "Queryable: false - queried too recently"
          false
        end
      else
        APICache.logger.debug "Queryable: true - never used API before"
        true
      end
    end

    # Fetch data from the API.
    #
    # If no block is given then the key is assumed to be a URL and which will
    # be queried expecting a 200 response. Otherwise the return value of the
    # block will be used.
    #
    # If the block is unable to fetch the value from the API it should raise
    # APICache::Invalid.
    #
    def get
      APICache.logger.debug "Fetching data from the API"
      query_times[@key] = Time.now
      Timeout::timeout(@timeout) do
        if @block
          # This should raise APICache::Invalid if it is not correct
          @block.call
        else
          get_key_via_http
        end
      end
    rescue Timeout::Error, APICache::Invalid => e
      raise APICache::CannotFetch, e.message
    end

    private

    def get_key_via_http
      response = redirecting_get(@key)
      case response
      when Net::HTTPSuccess
        # 2xx response code
        response.body
      else
        raise APICache::Invalid, "Invalid http response: #{response.code}"
      end
    end

    def redirecting_get(url)
      r = Net::HTTP.get_response(URI.parse(url))
      r.header['location'] ? redirecting_get(r.header['location']) : r
    end

    def query_times
      self.class.query_times<commit_msg>Store API query times in the cache store rather than in memory<commit_after>require 'net/http'

class APICache
  # Wraps up querying the API.
  # 
  # Ensures that the API is not called more frequently than every +period+ seconds, and times out API requests after +timeout+ seconds.
  #
  class API
    # Takes the following options
    #
    # period:: Maximum frequency to call the API
    # timeout:: Timeout when calling api (either to the proviced url or
    #           excecuting the passed block)
    # block:: If passed then the block is excecuted instead of HTTP GET 
    #         against the provided key
    #
    def initialize(key, options, &block)
      @key, @block = key, block
      @timeout = options[:timeout]
      @period = options[:period]
    end

    # Checks whether the API can be queried (i.e. whether :period has passed
    # since the last query to the API).
    #
    # If :period is 0 then there is no limit on how frequently queries can
    # be made to the API.
    #
    def queryable?
      if previously_queried?
        if Time.now - queried_at > @period
          APICache.logger.debug "Queryable: true - retry_time has passed"
          true
        else
          APICache.logger.debug "Queryable: false - queried too recently"
          false
        end
      else
        APICache.logger.debug "Queryable: true - never used API before"
        true
      end
    end

    # Fetch data from the API.
    #
    # If no block is given then the key is assumed to be a URL and which will
    # be queried expecting a 200 response. Otherwise the return value of the
    # block will be used.
    #
    # If the block is unable to fetch the value from the API it should raise
    # APICache::Invalid.
    #
    def get
      APICache.logger.debug "Fetching data from the API"
      set_queried_at
      Timeout::timeout(@timeout) do
        if @block
          # This should raise APICache::Invalid if it is not correct
          @block.call
        else
          get_key_via_http
        end
      end
    rescue Timeout::Error, APICache::Invalid => e
      raise APICache::CannotFetch, e.message
    end

    private

    def get_key_via_http
      response = redirecting_get(@key)
      case response
      when Net::HTTPSuccess
        # 2xx response code
        response.body
      else
        raise APICache::Invalid, "Invalid http response: #{response.code}"
      end
    end

    def redirecting_get(url)
      r = Net::HTTP.get_response(URI.parse(url))
      r.header['location'] ? redirecting_get(r.header['location']) : r
    end

    def previously_queried?
      APICache.store.exists?("#{@key}_queried_at")
    end
    
    def queried_at
      APICache.store.get("#{@key}_queried_at")
    end
    
    def set_queried_at
      APICache.store.set("#{@key}_queried_at", Time.now)
{'fraction_non_alphanumeric': 0.07306018854242205, 'fraction_numerical': 0.0018129079042784628, 'mean_word_length': 2.2169096209912538, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471259', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1584, 'n_words': 590}