<commit_before>require 'net/http'
require 'umakadata/logging/http_log'

module Umakadata

  module HTTPHelper

    def http_get(uri, args = {})
      args = {
        :time_out => 10
      }.merge(args)

      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)

      http = Net::HTTP.new(uri.host, uri.port)
      http.open_timeout = args[:time_out]

      request = Net::HTTP::Get.new(uri.path.empty? ? '/' : uri.path, args[:headers])

      http_log = Umakadata::Logging::HttpLog.new(uri, request)

      execute_request(http, request, http_log, args)
    end

    def http_post(uri, form_data, args = {})
      args = {
        :time_out => 10
      }.merge(args)

      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)

      http = Net::HTTP.new(uri.host, uri.port)
      http.open_timeout = args[:time_out]

      request = Net::HTTP::Post.new(uri.path.empty? ? '/' : uri.path, args[:headers])
      request.set_form_data(form_data, ';')

      http_log = Umakadata::Logging::HttpLog.new(uri, request)

      execute_request(http, request, http_log, args)
    end

    def execute_request(http, request, http_log, args = {})
      begin
        response = http_log.response = http.start { |h|
          h.request(request)
        }
      rescue => e
        http_log.error = e
      end

      # append a log object to log container
      log = args[:logger]
      log.push http_log unless log.nil?

      response
    end

    def http_get_recursive(uri, headers = {}, time_out = 10, limit = 10)
      raise RuntimeError, 'HTTP redirect too deep' if limit == 0

      http = Net::HTTP.new(uri.host, uri.port)
      http.open_timeout = time_out

      begin
        resource = uri.path
        resource += "?" + uri.query unless uri.query.nil?
        response = http.get(resource, headers)
      rescue => e
        puts e
        return e.message
      end

      case response
        when Net::HTTPSuccess
          return response
        when Net::HTTPRedirection
          return http_get_recursive(URI(response['location']), headers, time_out, limit - 1)
        else
          return response
      end
    end

  end

end<commit_msg>Change the definition of http_get_recursive to http_helper<commit_after>require 'net/http'
require 'umakadata/logging/http_log'

module Umakadata

  module HTTPHelper

    def http_get(uri, args = {})
      args = {
        :time_out => 10
      }.merge(args)

      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)

      http = Net::HTTP.new(uri.host, uri.port)
      http.open_timeout = args[:time_out]

      request = Net::HTTP::Get.new(uri.path.empty? ? '/' : uri.path, args[:headers])

      http_log = Umakadata::Logging::HttpLog.new(uri, request)

      execute_request(http, request, http_log, args)
    end

    def http_post(uri, form_data, args = {})
      args = {
        :time_out => 10
      }.merge(args)

      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)

      http = Net::HTTP.new(uri.host, uri.port)
      http.open_timeout = args[:time_out]

      request = Net::HTTP::Post.new(uri.path.empty? ? '/' : uri.path, args[:headers])
      request.set_form_data(form_data, ';')

      http_log = Umakadata::Logging::HttpLog.new(uri, request)

      execute_request(http, request, http_log, args)
    end

    def execute_request(http, request, http_log, args = {})
      begin
        response = http_log.response = http.start { |h|
          h.request(request)
        }
      rescue => e
        http_log.error = e
      end

      # append a log object to log container
      log = args[:logger]
      log.push http_log unless log.nil?

      response
    end

    def http_get_recursive(uri, args = {}, limit = 10)
      raise RuntimeError, 'HTTP redirect too deep' if limit == 0

      resource = uri.path
      resource += "?" + uri.query unless uri.query.nil?
      uri = URI.join(uri.to_s, resource)
      response = http_get(uri, args)

      return http_get_recursive(URI(response['location']), args, limit - 1) if response.is_a? Net::HTTPRedirection
      return response
    end

  end

end
{'fraction_non_alphanumeric': 0.12340216322517207, 'fraction_numerical': 0.004424778761061947, 'mean_word_length': 2.448305084745763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7655642', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1326, 'n_words': 297}