<commit_before># encoding:utf-8
#--
# Copyright (C) <NAME>
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#++


require "addressable/version"
require "addressable/idna"
require "public_suffix"

##
# Addressable is a library for processing links and URIs.
module Addressable
  ##
  # This is an implementation of a URI parser based on
  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>,
  # <a href="http://www.ietf.org/rfc/rfc3987.txt">RFC 3987</a>.
  class URI
    ##
    # Raised if something other than a uri is supplied.
    class InvalidURIError < StandardError
    end

    ##
    # Container for the character classes specified in
    # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
    module CharacterClasses
      ALPHA = "a-zA-Z"
      DIGIT = "0-9"
      GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
      SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
      RESERVED = GEN_DELIMS + SUB_DELIMS
      UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
      PCHAR = UNRESERVED + SUB_DELIMS + "\\:\\@"
      SCHEME = ALPHA + DIGIT + "\\-\\+\\."
      HOST = UNRESERVED + SUB_DELIMS + "\\[\\:\\]"
      AUTHORITY = PCHAR
      PATH = PCHAR + "\\/"
      QUERY = PCHAR + "\\/\\?"
      FRAGMENT = PCHAR + "\\/\\?"
    end

    SLASH = '/'
    EMPTY_STR = ''

    URIREGEX = /^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?$/

    PORT_MAPPING = {
      "http" => 80,
      "https" => 443,
      "ftp" => 21,
      "tftp" => 69,
      "sftp" => 22,
      "ssh" => 22,
      "svn+ssh" => 22,
      "telnet" => 23,
      "nntp" => 119,
      "gopher" => 70,
      "wais" => 210,
      "ldap" => 389,
      "prospero" => 1525
    }

    ##
    # Returns a URI object based on the parsed string.
    #
    # @param [String, Addressable::URI, #to_str] uri
    #   The URI string to parse.
    #   No parsing is performed if the object is already an
    #   <code>Addressable::URI</code>.
    #
    # @return [Addressable::URI] The parsed URI.
    def self.parse(uri)
      # If we were given nil, return nil.
      return nil unless uri
      # If a URI object is passed, just return itself.
      return uri.dup if uri.kind_of?(self)

      # If a URI object of the Ruby standard library variety is passed,
      # convert it to a string, then parse the string.
      # We do the check this way because we don't want to accidentally
      # cause a missing constant exception to be thrown.
      if uri.class.name =~ /^URI\b/
        uri = uri.to_s
      end

      # Otherwise, convert to a String
      begin
        uri = uri.to_str
      rescue TypeError, NoMethodError
        raise TypeError, "Can't convert #{uri.class} into String."
      end if not uri.is_a? String

      # This Regexp supplied as an example in RFC 3986, and it works great.
      scan = uri.scan(URIREGEX)
      fragments = scan[0]
      scheme = fragments[1]
      authority = fragments[3]
      path = fragments[4]
      query = fragments[6]
      fragment = fragments[8]
      user = nil
      password = <PASSWORD>
      host = nil
      port = nil
      if authority != nil
        # The Regexp above doesn't split apart the authority.
        userinfo = authority[/^([^\[\]]*)@/, 1]
        if userinfo != nil
          user = userinfo.strip[/^([^:]*):?/, 1]
          password = userinfo.strip[/:(.*)$/, 1]
        end
        host = authority.sub(
          /^([^\[\]]*)@/, EMPTY_STR
        ).sub(
          /:([^:@\[\]]*?)$/, EMPTY_STR
        )
        port = authority[/:([^:@\[\]]*?)$/, 1]
      end
      if port == EMPTY_STR
        port = nil
      end

      return new(
        :scheme => scheme,
        :user => user,
        :password => password,
        :host => host,
        :port => port,
        :path => path,
        :query => query,
        :fragment => fragment
      )
    end

    ##
    # Converts an input to a URI. The input does not have to be a valid
    # URI — the method will use heuristics to guess what URI was intended.
    # This is not standards-compliant, merely user-friendly.
    #
    # @param [String, Addressable::URI, #to_str] uri
    #   The URI string to parse.
    #   No parsing is performed if the object is already an
    #   <code>Addressable::URI</code>.
    # @param [Hash] hints
    #   A <code>Hash</code> of hints to the heuristic parser.
    #   Defaults to <code>{:scheme => "http"}</code>.
    #
    # @return [Addressable::URI] The parsed URI.
    def self.heuristic_parse(uri, hints={})
      # If we were given nil, return nil.
      return nil unless uri
      # If a URI object is passed, just return itself.
      return uri.dup if uri.kind_of?(self)

      # If a URI object of the Ruby standard library variety is passed,
      # convert it to a string, then parse the string.
      # We do the check this way because we don't want to accidentally
      # cause a missing constant exception to be thrown.
      if uri.class.name =~ /^URI\b/
        uri = uri.to_s
      end

      if !uri.respond_to?(:to_str)
        raise TypeError, "Can't convert #{uri.class} into String."
      end
      # Otherwise, convert to a String
      uri = uri.to_str.dup.strip
      hints = {
        :scheme => "http"
      }.merge(hints)
      case uri
      when /^http:\/+/
        uri.sub!(/^http:\/+/, "http://")
      when /^https:\/+/
        uri.sub!(/^https:\/+/, "https://")
      when /^feed:\/+http:\/+/
        uri.sub!(/^feed:\/+http:\/+/, "feed:http://")
      when /^feed:\/+/
        uri.sub!(/^feed:\/+/, "feed://")
      when /^file:\/+/
        uri.sub!(/^file:\/+/, "file:///")
      when /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
        uri.sub!(/^/, hints[:scheme] + "://")
      end
      match = uri.match(URIREGEX)
      fragments = match.captures
      authority = fragments[3]
      if authority && authority.length > 0
        new_authority = authority.gsub(/\\/, '/').gsub(/ /, '%20')
        # NOTE: We want offset 4, not 3!
        offset = match.offset(4)
        uri[offset[0]...offset[1]] = new_authority
      end
      parsed = self.parse(uri)
      if parsed.scheme =~ /^[^\/?#\.]+\.[^\/?#]+$/
        parsed = self.parse(hints[:scheme] + "://" + uri)
      end
      if parsed.path.include?(".")
        new_host = parsed.path[/^([^\/]+\.[^\/]*)/, 1]
        if new_host
          parsed.defer_validation do<commit_msg>remove extraneous repetition in regexes<commit_after># encoding:utf-8
#--
# Copyright (C) <NAME>
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#++


require "addressable/version"
require "addressable/idna"
require "public_suffix"

##
# Addressable is a library for processing links and URIs.
module Addressable
  ##
  # This is an implementation of a URI parser based on
  # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>,
  # <a href="http://www.ietf.org/rfc/rfc3987.txt">RFC 3987</a>.
  class URI
    ##
    # Raised if something other than a uri is supplied.
    class InvalidURIError < StandardError
    end

    ##
    # Container for the character classes specified in
    # <a href="http://www.ietf.org/rfc/rfc3986.txt">RFC 3986</a>.
    module CharacterClasses
      ALPHA = "a-zA-Z"
      DIGIT = "0-9"
      GEN_DELIMS = "\\:\\/\\?\\#\\[\\]\\@"
      SUB_DELIMS = "\\!\\$\\&\\'\\(\\)\\*\\+\\,\\;\\="
      RESERVED = GEN_DELIMS + SUB_DELIMS
      UNRESERVED = ALPHA + DIGIT + "\\-\\.\\_\\~"
      PCHAR = UNRESERVED + SUB_DELIMS + "\\:\\@"
      SCHEME = ALPHA + DIGIT + "\\-\\+\\."
      HOST = UNRESERVED + SUB_DELIMS + "\\[\\:\\]"
      AUTHORITY = PCHAR
      PATH = PCHAR + "\\/"
      QUERY = PCHAR + "\\/\\?"
      FRAGMENT = PCHAR + "\\/\\?"
    end

    SLASH = '/'
    EMPTY_STR = ''

    URIREGEX = /^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?$/

    PORT_MAPPING = {
      "http" => 80,
      "https" => 443,
      "ftp" => 21,
      "tftp" => 69,
      "sftp" => 22,
      "ssh" => 22,
      "svn+ssh" => 22,
      "telnet" => 23,
      "nntp" => 119,
      "gopher" => 70,
      "wais" => 210,
      "ldap" => 389,
      "prospero" => 1525
    }

    ##
    # Returns a URI object based on the parsed string.
    #
    # @param [String, Addressable::URI, #to_str] uri
    #   The URI string to parse.
    #   No parsing is performed if the object is already an
    #   <code>Addressable::URI</code>.
    #
    # @return [Addressable::URI] The parsed URI.
    def self.parse(uri)
      # If we were given nil, return nil.
      return nil unless uri
      # If a URI object is passed, just return itself.
      return uri.dup if uri.kind_of?(self)

      # If a URI object of the Ruby standard library variety is passed,
      # convert it to a string, then parse the string.
      # We do the check this way because we don't want to accidentally
      # cause a missing constant exception to be thrown.
      if uri.class.name =~ /^URI\b/
        uri = uri.to_s
      end

      # Otherwise, convert to a String
      begin
        uri = uri.to_str
      rescue TypeError, NoMethodError
        raise TypeError, "Can't convert #{uri.class} into String."
      end if not uri.is_a? String

      # This Regexp supplied as an example in RFC 3986, and it works great.
      scan = uri.scan(URIREGEX)
      fragments = scan[0]
      scheme = fragments[1]
      authority = fragments[3]
      path = fragments[4]
      query = fragments[6]
      fragment = fragments[8]
      user = nil
      password = <PASSWORD>
      host = nil
      port = nil
      if authority != nil
        # The Regexp above doesn't split apart the authority.
        userinfo = authority[/^([^\[\]]*)@/, 1]
        if userinfo != nil
          user = userinfo.strip[/^([^:]*):?/, 1]
          password = userinfo.strip[/:(.*)$/, 1]
        end
        host = authority.sub(
          /^([^\[\]]*)@/, EMPTY_STR
        ).sub(
          /:([^:@\[\]]*?)$/, EMPTY_STR
        )
        port = authority[/:([^:@\[\]]*?)$/, 1]
      end
      if port == EMPTY_STR
        port = nil
      end

      return new(
        :scheme => scheme,
        :user => user,
        :password => password,
        :host => host,
        :port => port,
        :path => path,
        :query => query,
        :fragment => fragment
      )
    end

    ##
    # Converts an input to a URI. The input does not have to be a valid
    # URI — the method will use heuristics to guess what URI was intended.
    # This is not standards-compliant, merely user-friendly.
    #
    # @param [String, Addressable::URI, #to_str] uri
    #   The URI string to parse.
    #   No parsing is performed if the object is already an
    #   <code>Addressable::URI</code>.
    # @param [Hash] hints
    #   A <code>Hash</code> of hints to the heuristic parser.
    #   Defaults to <code>{:scheme => "http"}</code>.
    #
    # @return [Addressable::URI] The parsed URI.
    def self.heuristic_parse(uri, hints={})
      # If we were given nil, return nil.
      return nil unless uri
      # If a URI object is passed, just return itself.
      return uri.dup if uri.kind_of?(self)

      # If a URI object of the Ruby standard library variety is passed,
      # convert it to a string, then parse the string.
      # We do the check this way because we don't want to accidentally
      # cause a missing constant exception to be thrown.
      if uri.class.name =~ /^URI\b/
        uri = uri.to_s
      end

      if !uri.respond_to?(:to_str)
        raise TypeError, "Can't convert #{uri.class} into String."
      end
      # Otherwise, convert to a String
      uri = uri.to_str.dup.strip
      hints = {
        :scheme => "http"
      }.merge(hints)
      case uri
      when /^http:\//
        uri.sub!(/^http:\/+/, "http://")
      when /^https:\//
        uri.sub!(/^https:\/+/, "https://")
      when /^feed:\/+http:\//
        uri.sub!(/^feed:\/+http:\/+/, "feed:http://")
      when /^feed:\//
        uri.sub!(/^feed:\/+/, "feed://")
      when /^file:\//
        uri.sub!(/^file:\/+/, "file:///")
      when /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
        uri.sub!(/^/, hints[:scheme] + "://")
      end
      match = uri.match(URIREGEX)
      fragments = match.captures
      authority = fragments[3]
      if authority && authority.length > 0
        new_authority = authority.gsub(/\\/, '/').gsub(/ /, '%20')
        # NOTE: We want offset 4, not 3!
        offset = match.offset(4)
        uri[offset[0]...offset[1]] = new_authority
      end
      parsed = self.parse(uri)
      if parsed.scheme =~ /^[^\/?#\.]+\.[^\/?#]+$/
        parsed = self.parse(hints[:scheme] + "://" + uri)
      end
      if parsed.path.include?(".")
        new_host = parsed.path[/^([^\/]+\.[^\/]*)/, 1]
        if new_host
          parsed.defer_validation do
{'fraction_non_alphanumeric': 0.16410293796019537, 'fraction_numerical': 0.013851425238754829, 'mean_word_length': 1.9673372269089335, 'pattern_counts': {'":': 4, '<': 37, '<?xml version=': 0, '>': 83, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3790311', 'n_tokens_mistral': 4768, 'n_tokens_neox': 4479, 'n_words': 1337}