<commit_before>module Addressable
  class URI
    module CharacterClasses
      USERINFO = UNRESERVED + SUB_DELIMS + "\\:"
    end
  end
end

module WebMock

  module Util

    class URI

      def self.normalize_uri(uri)
        return uri if uri.is_a?(Regexp)
        uri = 'http://' + uri unless uri.match('^https?://') if uri.is_a?(String)
        normalized_uri = Addressable::URI.heuristic_parse(uri)
        normalized_uri.query_values = Hash[*normalized_uri.query_values.sort.flatten] if normalized_uri.query_values
        normalized_uri = normalized_uri.normalize #normalize! is slower 
        normalized_uri.port = normalized_uri.inferred_port unless normalized_uri.port
        normalized_uri
      end

      def self.variations_of_uri_as_strings(uri_object)
        normalized_uri = normalize_uri(uri_object.dup).freeze
        uris = [ normalized_uri ]

        if normalized_uri.port == Addressable::URI.port_mapping[normalized_uri.scheme]
          uris = uris_with_inferred_port_and_without(uris)
        end

        if normalized_uri.scheme == "http"
          uris = uris_with_scheme_and_without(uris)
        end

        if normalized_uri.path == '/'
          uris = uris_with_trailing_slash_and_without(uris)
        end

        uris = uris_encoded_and_unencoded(uris)

        uris.map {|uri| uri.to_s.gsub(/^\/\//,'') }.uniq
      end

      def self.strip_default_port_from_uri_string(uri_string)
        case uri_string
        when %r{^http://}  then uri_string.sub(%r{:80(/|$)}, '\1')
        when %r{^https://} then uri_string.sub(%r{:443(/|$)}, '\1')
        else uri_string
        end
      end

      def self.encode_unsafe_chars_in_userinfo(userinfo)
        Addressable::URI.encode_component(userinfo, Addressable::URI::CharacterClasses::USERINFO)
      end

      private

      def self.uris_with_inferred_port_and_without(uris)
        uris.map { |uri| [ uri, uri.omit(:port).freeze ] }.flatten
      end

      def self.uris_encoded_and_unencoded(uris)
        uris.map do |uri|
          [ uri, Addressable::URI.heuristic_parse(Addressable::URI.unencode(uri)).freeze ]
        end.flatten
      end

      def self.uris_with_scheme_and_without(uris)
        uris.map { |uri| [ uri, uri.omit(:scheme).freeze ] }.flatten
      end

      def self.uris_with_trailing_slash_and_without(uris)
        uris = uris.map { |uri| [ uri, uri.omit(:path).freeze ] }.flatten
      end

    end
  end

end<commit_msg>URI matching against stubs with regular expressions is now 2 times faster.<commit_after>module Addressable
  class URI
    module CharacterClasses
      USERINFO = UNRESERVED + SUB_DELIMS + "\\:"
    end
  end
end

module WebMock

  module Util

    class URI

      def self.normalize_uri(uri)
        return uri if uri.is_a?(Regexp)
        uri = 'http://' + uri unless uri.match('^https?://') if uri.is_a?(String)
        normalized_uri = Addressable::URI.heuristic_parse(uri)
        normalized_uri.query_values = Hash[*normalized_uri.query_values.sort.flatten] if normalized_uri.query_values
        normalized_uri = normalized_uri.normalize #normalize! is slower 
        normalized_uri.port = normalized_uri.inferred_port unless normalized_uri.port
        normalized_uri
      end

      def self.variations_of_uri_as_strings(uri_object)
        normalized_uri = normalize_uri(uri_object.dup).freeze
        uris = [ normalized_uri ]

        if normalized_uri.path == '/'
          uris = uris_with_trailing_slash_and_without(uris)
        end

        uris = uris_encoded_and_unencoded(uris)

        if normalized_uri.port == Addressable::URI.port_mapping[normalized_uri.scheme]
          uris = uris_with_inferred_port_and_without(uris)
        end
        
        if normalized_uri.scheme == "http"
          uris = uris_with_scheme_and_without(uris)
        end

        uris.map {|uri| uri.to_s.gsub(/^\/\//,'') }.uniq
      end

      def self.strip_default_port_from_uri_string(uri_string)
        case uri_string
        when %r{^http://}  then uri_string.sub(%r{:80(/|$)}, '\1')
        when %r{^https://} then uri_string.sub(%r{:443(/|$)}, '\1')
        else uri_string
        end
      end

      def self.encode_unsafe_chars_in_userinfo(userinfo)
        Addressable::URI.encode_component(userinfo, Addressable::URI::CharacterClasses::USERINFO)
      end

      private

      def self.uris_with_inferred_port_and_without(uris)
        uris.map { |uri| [ uri, uri.gsub(%r{(:80)|(:443)}, "").freeze ] }.flatten
      end

      def self.uris_encoded_and_unencoded(uris)
        uris.map do |uri|
          [ uri.to_s, Addressable::URI.unencode(uri, String).freeze ]
        end.flatten
      end

      def self.uris_with_scheme_and_without(uris)
        uris.map { |uri| [ uri, uri.gsub(%r{^https?://},"").freeze ] }.flatten
      end

      def self.uris_with_trailing_slash_and_without(uris)
        uris = uris.map { |uri| [ uri, uri.omit(:path).freeze ] }.flatten
      end

    end
  end

end
{'fraction_non_alphanumeric': 0.10947876836385591, 'fraction_numerical': 0.004024954719259408, 'mean_word_length': 2.6842105263157894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '886089', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1651, 'n_words': 290}