<commit_before>require 'net/http'

# The Http module defines a 
# 
#   Http.get(url)
#
# method.
module Http
  extend self

  # the default expiration time for get requests.
  module MaxAge
    def self.config
      @config ||= (App.config["cache-max-age"] || {}).to_a
    end
    
    def self.for(url)
      config.each do |pattern, max_age|
        return max_age if url.index(pattern)
      end
      nil
    end
  end
  
  def get(url, max_age = MaxAge.for(url))
    App.logger.benchmark("[GET] #{url}", :minimum => 20) do 
      App.logger.debug "[GET] #{url}"
      App.cached(url, max_age) do get_(url) end
    end
  end

  private
  
  def get_(uri_str, limit = 10)
    raise 'too many redirections' if limit == 0

    response = Net::HTTP.get_response(URI(uri_str))

    case response
    when Net::HTTPSuccess then
      response.body
    when Net::HTTPRedirection then
      location = response['location']
      App.logger.debug "redirected to #{location}"
      get_(location, limit - 1)
    else  
      response.value
    end
  end
end<commit_msg>Adds HTTPS support to Http.get<commit_after>require 'net/http'

# The Http module defines a 
# 
#   Http.get(url)
#
# method.
module Http
  extend self

  # the default expiration time for get requests.
  module MaxAge
    def self.config
      @config ||= (App.config["cache-max-age"] || {}).to_a
    end
    
    def self.for(url)
      config.each do |pattern, max_age|
        return max_age if url.index(pattern)
      end
      nil
    end
  end
  
  def get(url, max_age = MaxAge.for(url))
    App.logger.benchmark("[GET] #{url}", :minimum => 20) do 
      App.cached(url, max_age) do 
        App.logger.debug "[GET] #{url}"
        get_(url) 
      end
    end
  end

  private
  
  def get_(uri_str, limit = 10)
    raise 'too many redirections' if limit == 0

    uri = URI.parse(uri_str)
    
    http = Net::HTTP.new(uri.host, uri.port)
    if uri.scheme == "https"
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end
    request = Net::HTTP::Get.new(uri.request_uri)
    response = http.request(request)

    case response
    when Net::HTTPSuccess then
      response.body
    when Net::HTTPRedirection then
      location = response['location']
      App.logger.debug "redirected to #{location}"
      get_(location, limit - 1)
    else  
      response.value
    end
  end
end
{'fraction_non_alphanumeric': 0.11469233989116785, 'fraction_numerical': 0.0050230221850146506, 'mean_word_length': 2.428981348637016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '476333', 'n_tokens_mistral': 824, 'n_tokens_neox': 795, 'n_words': 214}