<commit_before>module Rmega
  module Net
    include Loggable
    include Options

    def survive(retries = options.max_retries, &block)
      yield
    rescue ServerError
      raise
    rescue Exception => error
      retries -= 1
      raise(error) if retries < 0
      logger.debug("[#{error.class}] #{error.message}. #{retries} attempt(s) left.")
      sleep(options.retry_interval)
      retry
    end

    def http_get_content(url)
      uri = URI(url)
      req = ::Net::HTTP::Get.new(uri)
      return send_http_request(uri, req).body
    end

    def http_post(url, data)
      uri = URI(url)
      req = ::Net::HTTP::Post.new(uri)
      req.body = data
      logger.debug("REQ POST #{url} #{cut_string(data)}")
      response = send_http_request(uri, req)
      logger.debug("REP #{response.code} #{cut_string(response.body)}")
      return response
    end

    private

    def send_http_request(uri, req)
      http = ::Net::HTTP.new(uri.host, uri.port)
      apply_http_options(http)
      http.use_ssl = true if uri.scheme == 'https'
      return http.request(req)
    end

    def apply_http_options(http)
      options.http.each do |name, value|
        http.__send__("#{name}=", value)
      end<commit_msg>fix Get and Post args<commit_after>module Rmega
  module Net
    include Loggable
    include Options

    def survive(retries = options.max_retries, &block)
      yield
    rescue ServerError
      raise
    rescue Exception => error
      retries -= 1
      raise(error) if retries < 0
      logger.debug("[#{error.class}] #{error.message}. #{retries} attempt(s) left.")
      sleep(options.retry_interval)
      retry
    end

    def http_get_content(url)
      uri = URI(url)
      req = ::Net::HTTP::Get.new(uri.request_uri)
      return send_http_request(uri, req).body
    end

    def http_post(url, data)
      uri = URI(url)
      req = ::Net::HTTP::Post.new(uri.request_uri)
      req.body = data
      logger.debug("REQ POST #{url} #{cut_string(data)}")
      response = send_http_request(uri, req)
      logger.debug("REP #{response.code} #{cut_string(response.body)}")
      return response
    end

    private

    def send_http_request(uri, req)
      http = ::Net::HTTP.new(uri.host, uri.port)
      apply_http_options(http)
      http.use_ssl = true if uri.scheme == 'https'
      return http.request(req)
    end

    def apply_http_options(http)
      options.http.each do |name, value|
        http.__send__("#{name}=", value)
      end
{'fraction_non_alphanumeric': 0.12389380530973451, 'fraction_numerical': 0.0016090104585679806, 'mean_word_length': 2.6519823788546257, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5239247', 'n_tokens_mistral': 809, 'n_tokens_neox': 802, 'n_words': 183}