<commit_before>require 'fileutils'
require 'net/http'
require 'net/https'
require 'uri'

require 'pathname'
require Pathname.new(__FILE__).dirname.dirname.expand_path + 'remote_file'

Puppet::Type.type(:remote_file).provide(:ruby, :parent => Puppet::Provider::Remote_file) do
  desc "remote_file using Net::HTTP from Ruby's standard library."

  mk_resource_methods

  def create
    get @resource[:source]
    validate_checksum if checksum_specified?
  end

  private

  def validate_checksum
    raise Puppet::Error.new "Inconsistent checksums. Checksum of fetched file is #{calculated_checksum}. You specified #{specified_checksum}" if calculated_checksum != specified_checksum
  end

  def specified_checksum
    @resource[:checksum]
  end

  def calculated_checksum
    Digest::MD5.file(@resource[:name]) 
  end

  def checksum_specified?
    ! specified_checksum.nil?
  end

  def get(url, i=0)
    p = URI.parse url
    case p.scheme
    when /https?/
      http_get p, i
    when "file"
      FileUtils.copy p.path, @resource[:path]
    end
  end

  def http_get(p, i=0)
    if i > 5
      raise Puppet::Error.new "Redirected more than 5 times when trying to download #{@resource[:path]}, aborting."
    end
    c = Net::HTTP.new(p.host, p.port)
    c.use_ssl = p.scheme == "https" ? true : false
    if c.use_ssl? and @resource[:verify_peer] == false
      c.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end
    c.request_get(p.request_uri) do |req|
      case req.code
      when /30[12]/
        get req['location'], i+1
      when "200"
        File.open(@resource[:path], 'w') do |fh|
          req.read_body { |buf| fh.write buf }
          fh.flush
        end
      else
        raise Puppet::Error.new "Unexpected response code #{req.code}: #{req.read_body}"
      end
    end
  end
end
<commit_msg>Generalize http method logic, use tempfile<commit_after>require 'fileutils'
require 'net/http'
require 'net/https'
require 'uri'

require 'pathname'
require Pathname.new(__FILE__).dirname.dirname.expand_path + 'remote_file'

Puppet::Type.type(:remote_file).provide(:ruby, :parent => Puppet::Provider::Remote_file) do
  desc "remote_file using Net::HTTP from Ruby's standard library."

  mk_resource_methods

  REQUEST_TYPES = {
    'get'  => Net::HTTP::Get,
    'head' => Net::HTTP::Head,
  }

  def create
    get @resource[:source]
    validate_checksum if checksum_specified?
  end

  private

  def validate_checksum
    raise Puppet::Error.new "Inconsistent checksums. Checksum of fetched file is #{calculated_checksum}. You specified #{specified_checksum}" if calculated_checksum != specified_checksum
  end

  def specified_checksum
    @resource[:checksum]
  end

  def calculated_checksum
    Digest::MD5.file(@resource[:name]) 
  end

  def checksum_specified?
    ! specified_checksum.nil?
  end

  # Determine and begin the appropriate method of getting the target file.
  #
  def get(url)
    p = URI.parse url
    case p.scheme
    when /https?/
      http_get p, @resource[:path]
    when "file"
      FileUtils.copy p.path, @resource[:path]
    end
  end

  # Perform an HTTP GET request, saving the body in the specified download
  # path, and return the result.
  #
  def http_get(uri, download_path)

    # We'll save to a tempfile in case something goes wrong in the download
    # process. This avoids accidentally overwriting an old version, or
    # leaving a partially downloaded file in place
    begin
      tempfile = Tempfile.new('remote_file')
      response = http(uri, tempfile)

      # If download was successful, copy the tempfile over to the resource path.
      if response.kind_of?(Net::HTTPSuccess)
        tempfile.flush
        FileUtils.mv(tempfile.path, download_path)
      end
    ensure
      tempfile.close
      tempfile.unlink
      response
    end
  end

  # Use Net::HTTP to perform a request against a webserver, following
  # redirects, and return the final response. This method accepts a uri, an io
  # object to which the response body will be saved in the event of an
  # HTTPSuccess response from the webserver, and an optional hash of options.
  # The method will follow HTTPRedirect codes up to 10 times, and will return
  # the final HTTPResponse.
  #
  # @param uri [URI::HTTP] the uri to perform the request against
  # @param io_ready_to_write [IO] an open file or other IO object ready to
  #        write. The response body will be written to this object.
  # @param options [Hash] a hash of options to adjust behavior
  #
  def http(uri, io_ready_to_write, options = {})
    verb    = options[:http_method] || 'get'
    limit   = options[:limit]       || 10

    raise ArgumentError, 'HTTP redirect too deep' if limit == 0

    # Create the Net::HTTP connection and  request objects
    connection = Net::HTTP.new(uri.host, uri.port)
    request    = REQUEST_TYPES[verb.to_s.downcase].new(uri.request_uri)

    # Configure the Net::HTTP connection object
    if uri.scheme == 'https'
      connection.use_ssl = true
    end

    if connection.use_ssl? and @resource[:verify_peer] == false
      connection.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end

    # Configure the Net::HTTPRequest object
    if options[:headers]
      options[:headers].each {|key,value| request[key] = value }
    end

    # Connect and perform the request
    http_method = connection.method("request_#{verb.downcase}".to_sym)
    recursive_response = nil
    response = connection.start do |http|
      http.request(request) do |resp|
        # Determine and react to the request result
        case resp
        when Net::HTTPRedirection
          next_opts = options.merge(:limit => limit - 1)
          next_loc  = URI.parse(resp['location'])
          recursive_response = http(next_loc, io_ready_to_write, next_opts)
        when Net::HTTPSuccess
          resp.read_body do |chunk|
            io_ready_to_write.write(chunk)
          end
        else
          raise Puppet::Error.new "Unexpected response code #{resp.code}: #{resp.read_body}"
        end
      end
    end

    recursive_response || response
  end
end

{'fraction_non_alphanumeric': 0.09458128078817735, 'fraction_numerical': 0.003284072249589491, 'mean_word_length': 3.1155405405405405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6330986', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1846, 'n_words': 567}