<commit_before>require 'net/http'
require 's3_streamer/buffered_writer'

module S3Streamer
  class Downstream
    DEFAULT_PART_SIZE = 10 * 1024 * 1024

    def initialize(uri, options={})
      @uri = uri
      @options = options
    end

    def each
      Net::HTTP.start(@uri.host) do |http|
        http.request(Net::HTTP::Get.new @uri) do |response|
          buffer = BufferedWriter.new(min_part_size: part_size) do |part|
            yield part
          end

          response.read_body { |chunk| buffer.push chunk }

          buffer.finish
        end
      end
    end

    private

    def part_size
      @options[:part_size] || DEFAULT_PART_SIZE
    end
  end
end<commit_msg>feat: use ssl for https requests <commit_after>require 'net/http'
require 's3_streamer/buffered_writer'

module S3Streamer
  class Downstream
    DEFAULT_PART_SIZE = 10 * 1024 * 1024

    attr_reader :uri

    def initialize(uri, options={})
      @uri = uri
      @options = options
    end

    def each
      Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
        http.request get_uri do |response|
          buffer = BufferedWriter.new(min_part_size: part_size) do |part|
            yield part
          end

          response.read_body { |chunk| buffer.push chunk }

          buffer.finish
        end
      end
    end

    private

    def use_ssl
      uri.scheme == 'https'
    end

    def get_uri
      Net::HTTP::Get.new uri
    end

    def part_size
      @options[:part_size] || DEFAULT_PART_SIZE
    end
  end
end
{'fraction_non_alphanumeric': 0.08512020792722547, 'fraction_numerical': 0.015594541910331383, 'mean_word_length': 2.1818181818181817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2735825', 'n_tokens_mistral': 513, 'n_tokens_neox': 475, 'n_words': 126}