<commit_before>module Tom
  module Http

    # Makes a http request of the given method to the given url.
    # Passes the options on to EM::HttpRequest.put (or whatever
    # method has to be called) and does some error handling and
    # works around some EM:HttpRequest oddities (see handle_errors).
    #
    # @param method [Symbol] Request method, lowercase, you know
    #   the deal
    # @param url [String] Full url (e.g. host + path)
    # @param options [Hash] Options to pass to EM::HttpRequest's 
    #   get/put/post/delete method
    def self.make_request(method, url, options = {})
      Tom::LOG.info "     curl -X#{method.upcase} -d '#{options[:body]}' #{url}"

      conn = EM::HttpRequest.new(url, connection_options)
      result  = conn.send(method, options)
      handle_errors(method, url, result)

      result
    end

    private

    def self.connection_options
      { connect_timeout: Tom.config[:timeouts][:connect_timeout],
        inactivity_timeout: Tom.config[:timeouts][:inactivity_timeout]}
    end

    def self.handle_errors(method, url, result)
      result.errback do
        raise "Tom::Adapter.forward_request error #{method} #{url}"
      end
      return unless result.response_header.status == 0
      raise "EM::HttpRequest returned response code 0 for #{url} - timeout?"
    end
  end
end<commit_msg>According to the em-http-request examples this yields more info<commit_after>module Tom
  module Http

    # Makes a http request of the given method to the given url.
    # Passes the options on to EM::HttpRequest.put (or whatever
    # method has to be called) and does some error handling and
    # works around some EM:HttpRequest oddities (see handle_errors).
    #
    # @param method [Symbol] Request method, lowercase, you know
    #   the deal
    # @param url [String] Full url (e.g. host + path)
    # @param options [Hash] Options to pass to EM::HttpRequest's 
    #   get/put/post/delete method
    def self.make_request(method, url, options = {})
      Tom::LOG.info "     curl -X#{method.upcase} -d '#{options[:body]}' #{url}"

      conn = EM::HttpRequest.new(url, connection_options)
      result  = conn.send(method, options)
      handle_errors(method, url, result)

      result
    end

    private

    def self.connection_options
      { connect_timeout: Tom.config[:timeouts][:connect_timeout],
        inactivity_timeout: Tom.config[:timeouts][:inactivity_timeout]}
    end

    def self.handle_errors(method, url, result)
      result.errback do
        raise "Tom::Adapter.forward_request error '#{result.error}' for #{method} #{url}"
      end
      return unless result.response_header.status == 0
      raise "EM::HttpRequest returned response code 0 for #{url} - timeout?"
    end
  end
end
{'fraction_non_alphanumeric': 0.10469314079422383, 'fraction_numerical': 0.001444043321299639, 'mean_word_length': 3.075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285485', 'n_tokens_mistral': 817, 'n_tokens_neox': 797, 'n_words': 263}