<commit_before># frozen-string-literal: true

require "open-uri"
require "net/https"

require "down/backend"

require "tempfile"
require "fileutils"

module Down
  class NetHttp < Backend
    def initialize(options = {})
      @options = {
        "User-Agent" => "Down/#{Down::VERSION}",
        max_redirects: 2,
        open_timeout:  30,
        read_timeout:  30,
      }.merge(options)
    end

    def download(url, options = {})
      options = @options.merge(options)

      max_size            = options.delete(:max_size)
      max_redirects       = options.delete(:max_redirects)
      progress_proc       = options.delete(:progress_proc)
      content_length_proc = options.delete(:content_length_proc)
      destination         = options.delete(:destination)

      open_uri_options = {
        content_length_proc: proc { |size|
          if size && max_size && size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          content_length_proc.call(size) if content_length_proc
        },
        progress_proc: proc { |current_size|
          if max_size && current_size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          progress_proc.call(current_size) if progress_proc
        },
        redirect: false,
      }

      if options[:proxy]
        proxy    = URI(options.delete(:proxy))
        user     = proxy.user
        password = proxy.password

        if user || password
          proxy.user     = nil
          proxy.password = nil

          open_uri_options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
        else
          open_uri_options[:proxy] = proxy.to_s
        end
      end

      open_uri_options.merge!(options)

      uri = ensure_uri(url)

      if uri.user || uri.password
        open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
        uri.user = nil
        uri.password = <PASSWORD>
      end

      open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)

      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
      OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
      tempfile.extend Down::NetHttp::DownloadedFile

      download_result(tempfile, destination)
    end

    def open(url, options = {})
      options = @options.merge(options)

      uri = ensure_uri(url)

      request = Fiber.new do
        net_http_request(uri, options) do |response|
          Fiber.yield response
        end
      end

      response = request.resume

      response_error!(response) unless response.is_a?(Net::HTTPSuccess)

      Down::ChunkedIO.new(
        chunks:     enum_for(:stream_body, response),
        size:       response["Content-Length"] && response["Content-Length"].to_i,
        encoding:   response.type_params["charset"],
        rewindable: options.fetch(:rewindable, true),
        on_close:   -> { request.resume }, # close HTTP connnection
        data: {
          status:   response.code.to_i,
          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
                      name = downcased_name.split("-").map(&:capitalize).join("-")
                      headers.merge!(name => value)
                    },
          response: response,
        },
      )
    end

    private

    def open_uri(uri, options, follows_remaining: 0)
      downloaded_file = uri.open(options)
    rescue OpenURI::HTTPRedirect => exception
      raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0

      uri = exception.uri

      if !exception.io.meta["set-cookie"].to_s.empty?
        options["Cookie"] = exception.io.meta["set-cookie"]
      end

      follows_remaining -= 1
      retry
    rescue OpenURI::HTTPError => exception
      code, message = exception.io.status
      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
      response = response_class.new(nil, code, message)
      exception.io.metas.each do |name, values|
        values.each { |value| response.add_field(name, value) }
      end<commit_msg>Remove unnecessary local variable assignment<commit_after># frozen-string-literal: true

require "open-uri"
require "net/https"

require "down/backend"

require "tempfile"
require "fileutils"

module Down
  class NetHttp < Backend
    def initialize(options = {})
      @options = {
        "User-Agent" => "Down/#{Down::VERSION}",
        max_redirects: 2,
        open_timeout:  30,
        read_timeout:  30,
      }.merge(options)
    end

    def download(url, options = {})
      options = @options.merge(options)

      max_size            = options.delete(:max_size)
      max_redirects       = options.delete(:max_redirects)
      progress_proc       = options.delete(:progress_proc)
      content_length_proc = options.delete(:content_length_proc)
      destination         = options.delete(:destination)

      open_uri_options = {
        content_length_proc: proc { |size|
          if size && max_size && size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          content_length_proc.call(size) if content_length_proc
        },
        progress_proc: proc { |current_size|
          if max_size && current_size > max_size
            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
          end
          progress_proc.call(current_size) if progress_proc
        },
        redirect: false,
      }

      if options[:proxy]
        proxy    = URI(options.delete(:proxy))
        user     = proxy.user
        password = proxy.password

        if user || password
          proxy.user     = nil
          proxy.password = nil

          open_uri_options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
        else
          open_uri_options[:proxy] = proxy.to_s
        end
      end

      open_uri_options.merge!(options)

      uri = ensure_uri(url)

      if uri.user || uri.password
        open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
        uri.user = nil
        uri.password = nil
      end

      open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)

      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
      OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
      tempfile.extend Down::NetHttp::DownloadedFile

      download_result(tempfile, destination)
    end

    def open(url, options = {})
      options = @options.merge(options)

      uri = ensure_uri(url)

      request = Fiber.new do
        net_http_request(uri, options) do |response|
          Fiber.yield response
        end
      end

      response = request.resume

      response_error!(response) unless response.is_a?(Net::HTTPSuccess)

      Down::ChunkedIO.new(
        chunks:     enum_for(:stream_body, response),
        size:       response["Content-Length"] && response["Content-Length"].to_i,
        encoding:   response.type_params["charset"],
        rewindable: options.fetch(:rewindable, true),
        on_close:   -> { request.resume }, # close HTTP connnection
        data: {
          status:   response.code.to_i,
          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
                      name = downcased_name.split("-").map(&:capitalize).join("-")
                      headers.merge!(name => value)
                    },
          response: response,
        },
      )
    end

    private

    def open_uri(uri, options, follows_remaining: 0)
      uri.open(options)
    rescue OpenURI::HTTPRedirect => exception
      raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0

      uri = exception.uri

      if !exception.io.meta["set-cookie"].to_s.empty?
        options["Cookie"] = exception.io.meta["set-cookie"]
      end

      follows_remaining -= 1
      retry
    rescue OpenURI::HTTPError => exception
      code, message = exception.io.status
      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
      response = response_class.new(nil, code, message)
      exception.io.metas.each do |name, values|
        values.each { |value| response.add_field(name, value) }
      end
{'fraction_non_alphanumeric': 0.1066080342286665, 'fraction_numerical': 0.0057047777513667694, 'mean_word_length': 2.3808758537565287, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471286', 'n_tokens_mistral': 2521, 'n_tokens_neox': 2454, 'n_words': 508}