<commit_before>module Basil
  # Handles simple and no-so-simple HTTP requests. If options is a Hash,
  # you must provide :host. Optionally, :path, :port, :user, and
  # :password can be specified. If options is not a Hash it is expected
  # to be a simple url (ex "http://google.com").
  #
  # Currently, https is used if :port is specified as 443 or a url is
  # passed that begins with "https". Basic authentication is used if
  # :username or :password is given.
  module HTTP
    class << self
      def get(options)
        logger.debug "GET"
        logger.debug options

        resp = if options.is_a? Hash
          host     = options['host']
          port     = options['port'] || 80
          path     = options['path'] || '/'
          username = options['user']     # may be nil
          password = options['password'] # may be nil
          secure   = port == 443

          require(secure ? 'net/https' : 'net/http')
          net = Net::HTTP.new(host, port)

          if secure
            net.use_ssl = true

            # An explicit cert file is needed if run on OSX, provided by the
            # curl-ca-bundle cert package. value might be nil
            net.ca_file = Config.https_cert_file #if Config.https_cert_file
          end

          net.start do |http|
            req = Net::HTTP::Get.new(path)
            req.basic_auth(username, password) if username || password

            logger.debug req
            http.request(req)
          end
        else
          url = options
          require(url =~ /^https/ ? 'net/https' : 'net/http')

          logger.debug url
          Net::HTTP.get_response(URI.parse(url))
        end

        unless resp.is_a?(Net::HTTPOK)
          logger.warn 'Non-200 HTTP Response'
          logger.warn resp
        end

        resp
      end

      private

      def logger
        @logger ||= Loggers['http']
      end
    end
  end
end
<commit_msg>cleanup HTTP module a bit<commit_after>module Basil
  # Handles simple and no-so-simple HTTP requests. Not used directly by
  # Plugins, but indirectly by Utils#get_http and friends.
  module HTTP
    class << self
      #
      #   get(url) OR
      #
      #   get(options)
      #     options['host'] - required
      #     options['port'] - optional, defaults to 80
      #     options['path'] - optional, defaults to /
      #     options['username'] - optional
      #     options['password'] - optional
      #
      def get(options)
        logger.debug "GET"
        logger.debug options

        resp = case options
               when Hash   then complex_get(options)
               when String then simple_get(options)
               else raise ArgumentError, "get method accepts only Hash or String"
               end

        unless resp.is_a?(Net::HTTPOK)
          logger.warn 'Non-200 HTTP Response'
          logger.warn resp
        end

        resp
      end

      private

      def simple_get(url)
        require_net_http(url =~ /^https/)

        Net::HTTP.get_response(URI.parse(url))
      end

      def complex_get(options)
        host     = options.fetch('host') { raise ArgumentError, "options['host'] is required" }
        port     = options.fetch('port', 80)
        path     = options.fetch('path', '/')
        username = options['user']
        password = options['password']
        secure   = port == 443

        require_net_http(secure)

        net = Net::HTTP.new(host, port)

        if secure
          net.use_ssl = true
          net.ca_file = Config.https_cert_file # OSX fix
        end

        net.start do |http|
          req = Net::HTTP::Get.new(path)
          req.basic_auth(username, password) if username || password

          http.request(req)
        end
      end

      def require_net_http(secure)
        require(secure ? 'net/https' : 'net/http')
      end

      def logger
        @logger ||= Loggers['http']
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.0979145473041709, 'fraction_numerical': 0.005340793489318413, 'mean_word_length': 1.7580645161290323, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1500238', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1090, 'n_words': 337}