<commit_before>require 'net/http'
require 'link_header'
require 'time'
require 'openssl'

module RDF; module Util
  ##
  # Wrapper for retrieving RDF resources from HTTP(S) and file: scheme locations.
  # 
  # By default, HTTP(S) resources are retrieved using Net::HTTP. However, 
  # If the [Rest Client](https://rubygems.org/gems/rest-client) gem is included,
  # it will be used for retrieving resources, allowing for
  # sophisticated HTTP caching using [REST Client Components](https://rubygems.org/gems/rest-client-components)
  # allowing the use of `Rack::Cache` to avoid network access.
  #
  # To use other HTTP clients, consumers can subclass 
  # {RDF::Util::File::HttpAdapter} and set the {RDF::Util::File.http_adapter}.
  #
  # Also supports the file: scheme for access to local files.
  #
  #
  # @since 0.2.4
  module File

    ##
    # @abstract Subclass and override {.open_url} to implement a custom adapter
    # @since 1.2
    class HttpAdapter
      ##
      # @param [Array, String] headers
      #   HTTP Request headers
      # @return [Hash] A hash of HTTP request headers
      def self.headers headers: {}
        headers['Accept'] ||= default_accept_header
        headers['User-Agent'] ||= default_user_agent
        headers
      end

      ##
      # @return [String] the value for an Accept header
      def self.default_accept_header
        (RDF::Format.accept_types + %w(*/*;q=0.1)).join(", ")
      end

      ##
      # @return [String] the default User-Agent used when fetching resources.
      def self.default_user_agent
        "Ruby RDF.rb/#{RDF::VERSION}"
      end

      ##
      # @abstract
      # @param [String] base_uri to open
      # @param [String] proxy
      #   HTTP Proxy to use for requests.
      # @param [Array, String] headers ({})
      #   HTTP Request headers
      #
      #   Defaults `Accept` header based on available reader content types to allow for content negotiation based on available readers.
      #
      #   Defaults  `User-Agent` header, unless one is specified.
      # @param [Boolean] verify_none (false)
      #   Don't verify SSL certificates
      # @param  [Hash{Symbol => Object}] options
      #   options are ignored in this implementation. Applications are encouraged
      #   to override this implementation to provide more control over HTTP
      #   headers and redirect following.
      # @return [RemoteDocument, Object] A {RemoteDocument}. If a block is given, the result of evaluating the block is returned.
      # @raise [IOError] if not found
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        raise NoMethodError.new("#{self.inspect} does not implement required method `open_url` for ", "open_url")
      end
    end

    ##
    # If the [Rest Client](https://rubygems.org/gems/rest-client) gem is included,
    # it will be used for retrieving resources allowing for
    # sophisticated HTTP caching using [REST Client Components](https://rubygems.org/gems/rest-client-components)
    # allowing the use of `Rack::Cache` to avoid network access.
    # @since 1.2
    class RestClientAdapter < HttpAdapter
      # (see HttpAdapter.open_url)
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        ssl_verify = verify_none ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER

        # If RestClient is loaded, prefer it
        RestClient.proxy = proxy.to_s if proxy
        client = RestClient::Resource.new(base_uri, verify_ssl: ssl_verify)
        client.get(headers(headers: headers)) do |response, request, res, &blk|
          case response.code
          when 200..299
            # found object

            # If a Location is returned, it defines the base resource for this file, not it's actual ending location
            document_options = {
              base_uri:     RDF::URI(response.headers.fetch(:location, base_uri)),
              code:         response.code.to_i,
              headers:      response.headers
            }

            remote_document = RemoteDocument.new(response.body, document_options)
          when 300..399
            # Document base is redirected location
            # Location may be relative
            base_uri = ::URI.join(base_uri, response.headers[:location].to_s).to_s
            response.follow_redirection(&blk)
          else
            raise IOError, "<#{base_uri}>: #{response.code}"
          end
        end
      end
    end

    ##
    # Net::HTTP adapter to retrieve resources without additional dependencies
    # @since 1.2
    class NetHttpAdapter < HttpAdapter
      # (see HttpAdapter.open_url)
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        ssl_verify = verify_none ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER

        redirect_count = 0
        max_redirects = 5
        parsed_url = ::URI.parse(base_uri)
        parsed_proxy = ::URI.parse(proxy.to_s)
        base_uri = parsed_url.to_s
        remote_document = nil

        until remote_document do
          Net::HTTP::start(parsed_url.host, parsed_url.port,
                          parsed_proxy.host, parsed_proxy.port,
                          open_timeout: 60 * 1000,
                          use_ssl: parsed_url.scheme == 'https',
                          verify_mode: ssl_verify
          ) do |http|
            request = Net::HTTP::Get.new(parsed_url.request_uri, headers(headers: headers))
            http.request(request) do |response|<commit_msg>Use RDF::URI.parse instead of ::URI.parse when using NetHttpAdapter.<commit_after>require 'net/http'
require 'link_header'
require 'time'
require 'openssl'

module RDF; module Util
  ##
  # Wrapper for retrieving RDF resources from HTTP(S) and file: scheme locations.
  # 
  # By default, HTTP(S) resources are retrieved using Net::HTTP. However, 
  # If the [Rest Client](https://rubygems.org/gems/rest-client) gem is included,
  # it will be used for retrieving resources, allowing for
  # sophisticated HTTP caching using [REST Client Components](https://rubygems.org/gems/rest-client-components)
  # allowing the use of `Rack::Cache` to avoid network access.
  #
  # To use other HTTP clients, consumers can subclass 
  # {RDF::Util::File::HttpAdapter} and set the {RDF::Util::File.}.
  #
  # Also supports the file: scheme for access to local files.
  #
  #
  # @since 0.2.4
  module File

    ##
    # @abstract Subclass and override {.open_url} to implement a custom adapter
    # @since 1.2
    class HttpAdapter
      ##
      # @param [Array, String] headers
      #   HTTP Request headers
      # @return [Hash] A hash of HTTP request headers
      def self.headers headers: {}
        headers['Accept'] ||= default_accept_header
        headers['User-Agent'] ||= default_user_agent
        headers
      end

      ##
      # @return [String] the value for an Accept header
      def self.default_accept_header
        (RDF::Format.accept_types + %w(*/*;q=0.1)).join(", ")
      end

      ##
      # @return [String] the default User-Agent used when fetching resources.
      def self.default_user_agent
        "Ruby RDF.rb/#{RDF::VERSION}"
      end

      ##
      # @abstract
      # @param [String] base_uri to open
      # @param [String] proxy
      #   HTTP Proxy to use for requests.
      # @param [Array, String] headers ({})
      #   HTTP Request headers
      #
      #   Defaults `Accept` header based on available reader content types to allow for content negotiation based on available readers.
      #
      #   Defaults  `User-Agent` header, unless one is specified.
      # @param [Boolean] verify_none (false)
      #   Don't verify SSL certificates
      # @param  [Hash{Symbol => Object}] options
      #   options are ignored in this implementation. Applications are encouraged
      #   to override this implementation to provide more control over HTTP
      #   headers and redirect following.
      # @return [RemoteDocument, Object] A {RemoteDocument}. If a block is given, the result of evaluating the block is returned.
      # @raise [IOError] if not found
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        raise NoMethodError.new("#{self.inspect} does not implement required method `open_url` for ", "open_url")
      end
    end

    ##
    # If the [Rest Client](https://rubygems.org/gems/rest-client) gem is included,
    # it will be used for retrieving resources allowing for
    # sophisticated HTTP caching using [REST Client Components](https://rubygems.org/gems/rest-client-components)
    # allowing the use of `Rack::Cache` to avoid network access.
    # @since 1.2
    class RestClientAdapter < HttpAdapter
      # (see HttpAdapter.open_url)
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        ssl_verify = verify_none ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER

        # If RestClient is loaded, prefer it
        RestClient.proxy = proxy.to_s if proxy
        client = RestClient::Resource.new(base_uri, verify_ssl: ssl_verify)
        client.get(headers(headers: headers)) do |response, request, res, &blk|
          case response.code
          when 200..299
            # found object

            # If a Location is returned, it defines the base resource for this file, not it's actual ending location
            document_options = {
              base_uri:     RDF::URI(response.headers.fetch(:location, base_uri)),
              code:         response.code.to_i,
              headers:      response.headers
            }

            remote_document = RemoteDocument.new(response.body, document_options)
          when 300..399
            # Document base is redirected location
            # Location may be relative
            base_uri = ::URI.join(base_uri, response.headers[:location].to_s).to_s
            response.follow_redirection(&blk)
          else
            raise IOError, "<#{base_uri}>: #{response.code}"
          end
        end
      end
    end

    ##
    # Net::HTTP adapter to retrieve resources without additional dependencies
    # @since 1.2
    class NetHttpAdapter < HttpAdapter
      # (see HttpAdapter.open_url)
      def self.open_url(base_uri, proxy: nil, headers: {}, verify_none: false, **options)
        ssl_verify = verify_none ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER

        redirect_count = 0
        max_redirects = 5
        parsed_url = RDF::URI.parse(base_uri)
        parsed_proxy = RDF::URI.parse(proxy.to_s)
        base_uri = parsed_url.to_s
        remote_document = nil

        until remote_document do
          Net::HTTP::start(parsed_url.host, parsed_url.port,
                          parsed_proxy.host, parsed_proxy.port,
                          open_timeout: 60 * 1000,
                          use_ssl: parsed_url.scheme == 'https',
                          verify_mode: ssl_verify
          ) do |http|
            request = Net::HTTP::Get.new(parsed_url.request_uri, headers(headers: headers))
            http.request(request) do |response|
{'fraction_non_alphanumeric': 0.0978563099829581, 'fraction_numerical': 0.005561036864292762, 'mean_word_length': 2.500784929356358, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2056476', 'n_tokens_mistral': 3193, 'n_tokens_neox': 3079, 'n_words': 956}