<commit_before>package rl

import util.parsing.combinator._
import java.net.IDN

sealed trait URINode

//object UserInfo {
//  def apply(userInfo: String): Option[UserInfo] = {
//    userInfo.toOption map  { uif =>
//      val Array(user, secret) = if (uif.indexOf(":") > -1) (uif.toString split ':') else Array(uif, "")
//      UserInfo(user, secret)
//    }
//  }
//}
//case class UserInfo(user: String, secret: String) extends URINode {
//  override def toString = (user /: secret.toOption) { _ + ":" + _ }
//}
//
//object Authority {
//  def apply(authority: String): Authority = {
//    val `has @` = (authority indexOf '@') > -1
//    val Array(uif, auu) = if (`has @`) authority split '@' else Array("", authority)
//    val au = if (auu.startsWith("@")) auu substring 1 else auu
//
//    val uinf = UserInfo(uif)
//    val `has :` = au.indexOf(':') > -1
//    if (`has :`) {
//      val Array(h, port) = au split ':'
//      new Authority(uinf, h, Some(port.toInt))
//    } else new Authority(uinf, au, None)
//  }
//}
//
//case class Authority(userInfo: Option[UserInfo], host: String, port: Option[Int]) extends URINode {
//
//  override def toString = {
////    ((userInfo map { _.toString }) :\ ((host /: port) { _ + ":" + _ })) { _ + "@" + _ }
//    (userInfo map { _.toString + "@" } getOrElse "") + host + (port map { ":" + _ } getOrElse "") //expresses intent better
//  }
//}
//case class Uri(
//    scheme: String,
//    authority: Authority,
//    path: String,
//    rawQuery: String, //actually a sorted map
//    fragment: String)

object Uri {

  private val DEFAULT_PORTS = Map(
    "http" -> 80,
    "https" -> 443,
    "ftp" -> 21,
    "tftp" -> 69,
    "sftp" -> 22,
    "ssh" -> 22,
    "svn+ssh" -> 22,
    "git" -> 22,
    "git+ssh" -> 22,
    "telnet" -> 23,
    "nntp" -> 119,
    "gopher" -> 70,
    "wais" -> 210,
    "ldap" -> 389,
    "prospero" -> 1525,
    "smtp" -> 25,
    "imap" -> 143,
    "imaps" -> 993,
    "pop3" -> 110,
    "pop3s" -> 995,
    "redis" -> 6379,
    "mongo" -> 27017
  )

  /*
   * The regex to split a URI up into its parts for further processing
   * Source: http://tools.ietf.org/html/rfc3986#appendix-B
   */
  val UriParts = """^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?""".r


  /**
   * Returns a [[rl.Uri]] object based on the parsed string. This method is very lenient and just tokenizes the string
   * to the uri parts and creates a [[rl.Uri]] object with those parts. You probably want to call validate on that
   * returned object afterwards
   *
   * @param uriString the [[scala.String]] to tokenize
   *
   * @return the parsed unnormalized [[rl.Uri]]
   */

  private[rl] def tokenize(uriString: String) = {
    val UriParts(_, sch, _, auth, pth, _, qry, _, frag) = uriString
    (sch, auth, pth, qry, frag)
  }

  private val subDelimChars = """[!$&'()*+,;=]""".r
  private val genDelimChars = """[:/?#\[\]@]""".r
  private val hexDigits = """[0123456789abcdefABCDEF]""".r

  val IPv6AddressRegex =
    ("""(?iu)^(((?=(?>.*?::)(?!.*::)))(::)?([0-9A-F]{1,4}::?){0,5}|([0-9A-F]{1,4}:){6})(\2([0-9A-F]{1,4}(::?|$)){0,2}""" +
        """|((25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.|$)){4}|[0-9A-F]{1,4}:[0-9A-F]{1,4})(?<![^:]:)(?<!\.)\z""").r

  trait IPv6AddressParser extends RegexParsers {
    def hexDigit = hexDigits
    def decOctet = """25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d""".r
    def dottedDecOctet = decOctet <~ "."

    private def IPv4Address = dottedDecOctet ~ dottedDecOctet ~ dottedDecOctet ~ decOctet ^^ {
      case a ~ b ~ c ~ d => a + "." + b + "." + c + "." + d
    }


    private def h16_2 = repN(2, hexDigit)
    private def h16_3 = repN(3, hexDigit)
    private def h16_4 = repN(4, hexDigit)
    private def h16_multi = (h16_4 | h16_3 | h16_2) ^^ { _ mkString "" }
    private def h16 = h16_multi | hexDigit

    private def h16Colon = h16 ~ ":" ^^ { case a ~ b => a + b }
    private def h16Colon_2 = h16Colon ~ h16Colon ^^ { case a ~ b => a + b }
    private def h16Colon_3 = repN(3, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_4 = repN(4, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_5 = repN(5, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_6 = repN(6, h16Colon) ^^ { _ mkString "" }
    private def h16Wrap(parser: Parser[String]): Parser[String] = parser ~ h16 ^^ { case a ~ b => a + b }
    private def h16ColonN(max: Int) = max match {
      case 6 => h16Wrap(h16Colon_6) | h16Wrap(h16Colon_5) | h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 5 => h16Wrap(h16Colon_5) | h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 4 => h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 3 => h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 2 => h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 1 => h16Wrap(h16Colon)
    }
    private def h16Colonh16N(max: Int) = h16ColonN(max) | h16
    private def nH16Colon(n: Int) = repN(n, h16Colon) ^^ { _ mkString "" }

    private def flatOpt(parser: => Parser[String]): Parser[String] = opt(parser) ^^ { _ getOrElse  ""}

    private def ls32 = (h16Colon ~ h16 ^^ { case a ~ b => a + b } ) | IPv4Address

    private def ip6_1 = nH16Colon(6) ~ ls32 ^^ { case a ~ b => a + b }
    private def ip6_2 = "::" ~ nH16Colon(5) ~ ls32 ^^ { case a ~ b ~ c => a + b + c }
    private def ip6_3 = flatOpt(h16) ~ "::" ~ nH16Colon(4) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_4 = flatOpt(h16Colonh16N(1)) ~ "::" ~ nH16Colon(3) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_5 = flatOpt(h16Colonh16N(2)) ~ "::" ~ nH16Colon(2) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_6 = flatOpt(h16Colonh16N(3)) ~ "::" ~ nH16Colon(1) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_7 = flatOpt(h16Colonh16N(4)) ~ "::" ~ ls32 ^^ { case a ~ b ~ c  => a + b + c }
    private def ip6_8 = flatOpt(h16Colonh16N(5)) ~ "::" ~ h16 ^^ { case a ~ b ~ c => a + b + c }
    private def ip6_9 = flatOpt(h16Colonh16N(6)) ~ "::" ^^ { case a ~ b => a + b }
    def IPv6Address = ip6_1 | ip6_2 | ip6_3 | ip6_4 | ip6_5 | ip6_6 | ip6_7 | ip6_8 | ip6_9

  }

  /**
   * Implements a parser for the ABNF found in appendix A of RFC-3986
   * Source: http://tools.ietf.org/html/rfc3986#appendix-A
   */
  private[rl] trait UriParser extends IPv6AddressParser {

    def subDelims = subDelimChars
    def genDelims = genDelimChars
    def reserved = genDelims | subDelims
    def alpha = """[a-zA-Z]""".r
    def digit = """\d""".r

    def unreserved = alpha | digit | "-" | "." | "_" | "~"

    def pctEncoded = "%" ~> hexDigit ~ hexDigit ^^ { case a ~ b => a + b }
    def pchar = unreserved | pctEncoded | subDelims | ":" | "@"<commit_msg>All IPv6 address specs pass now<commit_after>package rl

import util.parsing.combinator._
import java.net.IDN

sealed trait URINode

//object UserInfo {
//  def apply(userInfo: String): Option[UserInfo] = {
//    userInfo.toOption map  { uif =>
//      val Array(user, secret) = if (uif.indexOf(":") > -1) (uif.toString split ':') else Array(uif, "")
//      UserInfo(user, secret)
//    }
//  }
//}
//case class UserInfo(user: String, secret: String) extends URINode {
//  override def toString = (user /: secret.toOption) { _ + ":" + _ }
//}
//
//object Authority {
//  def apply(authority: String): Authority = {
//    val `has @` = (authority indexOf '@') > -1
//    val Array(uif, auu) = if (`has @`) authority split '@' else Array("", authority)
//    val au = if (auu.startsWith("@")) auu substring 1 else auu
//
//    val uinf = UserInfo(uif)
//    val `has :` = au.indexOf(':') > -1
//    if (`has :`) {
//      val Array(h, port) = au split ':'
//      new Authority(uinf, h, Some(port.toInt))
//    } else new Authority(uinf, au, None)
//  }
//}
//
//case class Authority(userInfo: Option[UserInfo], host: String, port: Option[Int]) extends URINode {
//
//  override def toString = {
////    ((userInfo map { _.toString }) :\ ((host /: port) { _ + ":" + _ })) { _ + "@" + _ }
//    (userInfo map { _.toString + "@" } getOrElse "") + host + (port map { ":" + _ } getOrElse "") //expresses intent better
//  }
//}
//case class Uri(
//    scheme: String,
//    authority: Authority,
//    path: String,
//    rawQuery: String, //actually a sorted map
//    fragment: String)

object Uri {

  private val DEFAULT_PORTS = Map(
    "http" -> 80,
    "https" -> 443,
    "ftp" -> 21,
    "tftp" -> 69,
    "sftp" -> 22,
    "ssh" -> 22,
    "svn+ssh" -> 22,
    "git" -> 22,
    "git+ssh" -> 22,
    "telnet" -> 23,
    "nntp" -> 119,
    "gopher" -> 70,
    "wais" -> 210,
    "ldap" -> 389,
    "prospero" -> 1525,
    "smtp" -> 25,
    "imap" -> 143,
    "imaps" -> 993,
    "pop3" -> 110,
    "pop3s" -> 995,
    "redis" -> 6379,
    "mongo" -> 27017
  )

  /*
   * The regex to split a URI up into its parts for further processing
   * Source: http://tools.ietf.org/html/rfc3986#appendix-B
   */
  val UriParts = """^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?""".r


  /**
   * Returns a [[rl.Uri]] object based on the parsed string. This method is very lenient and just tokenizes the string
   * to the uri parts and creates a [[rl.Uri]] object with those parts. You probably want to call validate on that
   * returned object afterwards
   *
   * @param uriString the [[scala.String]] to tokenize
   *
   * @return the parsed unnormalized [[rl.Uri]]
   */

  private[rl] def tokenize(uriString: String) = {
    val UriParts(_, sch, _, auth, pth, _, qry, _, frag) = uriString
    (sch, auth, pth, qry, frag)
  }

  private val subDelimChars = """[!$&'()*+,;=]""".r
  private val genDelimChars = """[:/?#\[\]@]""".r
  private val hexDigits = """[0123456789abcdefABCDEF]""".r

  val IPv6AddressRegex =
    ("""(?iu)^(((?=(?>.*?::)(?!.*::)))(::)?([0-9A-F]{1,4}::?){0,5}|([0-9A-F]{1,4}:){6})(\2([0-9A-F]{1,4}(::?|$)){0,2}""" +
        """|((25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.|$)){4}|[0-9A-F]{1,4}:[0-9A-F]{1,4})(?<![^:]:)(?<!\.)\z""").r

  trait IPv6AddressParser extends RegexParsers {
    def hexDigit = hexDigits
    def decOctet = """25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d""".r
    def dottedDecOctet = decOctet <~ "."

    private def IPv4Address = dottedDecOctet ~ dottedDecOctet ~ dottedDecOctet ~ decOctet ^^ {
      case a ~ b ~ c ~ d => a + "." + b + "." + c + "." + d
    }


    private def h16_2 = repN(2, hexDigit)
    private def h16_3 = repN(3, hexDigit)
    private def h16_4 = repN(4, hexDigit)
    private def h16_multi = (h16_4 | h16_3 | h16_2) ^^ { _ mkString "" }
    private def h16 = h16_multi | hexDigit

    private def h16Colon = h16 ~ ":" ^^ { case a ~ b => a + b }
    private def h16Colon_2 = h16Colon ~ h16Colon ^^ { case a ~ b => a + b }
    private def h16Colon_3 = repN(3, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_4 = repN(4, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_5 = repN(5, h16Colon) ^^ { _ mkString "" }
    private def h16Colon_6 = repN(6, h16Colon) ^^ { _ mkString "" }
    private def h16Wrap(parser: Parser[String]): Parser[String] = parser ~ h16 ^^ { case a ~ b => a + b }
    private def h16ColonN(max: Int) = max match {
      case 6 => h16Wrap(h16Colon_6) | h16Wrap(h16Colon_5) | h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 5 => h16Wrap(h16Colon_5) | h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 4 => h16Wrap(h16Colon_4) | h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 3 => h16Wrap(h16Colon_3) | h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 2 => h16Wrap(h16Colon_2) | h16Wrap(h16Colon)
      case 1 => h16Wrap(h16Colon)
    }
    private def h16Colonh16N(max: Int) = h16ColonN(max) | h16
    private def nH16Colon(n: Int) = repN(n, h16Colon) ^^ { _ mkString "" }

    private def flatOpt(parser: => Parser[String]): Parser[String] = opt(parser) ^^ { _ getOrElse  ""}

    private def ls32 = (h16Colon ~ h16 ^^ { case a ~ b => a + b } ) | IPv4Address

    private def ip6_1 = nH16Colon(6) ~ ls32 ^^ { case a ~ b => a + b }
    private def ip6_2 = "::" ~ nH16Colon(5) ~ ls32 ^^ { case a ~ b ~ c => a + b + c }
    private def ip6_3 = flatOpt(h16) ~ "::" ~ nH16Colon(4) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_4 = flatOpt(h16Colonh16N(1)) ~ "::" ~ nH16Colon(3) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_5 = flatOpt(h16Colonh16N(2)) ~ "::" ~ nH16Colon(2) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_6 = flatOpt(h16Colonh16N(3)) ~ "::" ~ nH16Colon(1) ~ ls32 ^^ { case a ~ b ~ c ~ d => a + b + c + d }
    private def ip6_7 = flatOpt(h16Colonh16N(4)) ~ "::" ~ ls32 ^^ { case a ~ b ~ c  => a + b + c }
    private def ip6_8 = flatOpt(h16Colonh16N(5)) ~ "::" ~ h16 ^^ { case a ~ b ~ c => a + b + c }
    private def ip6_9 = flatOpt(h16Colonh16N(6)) ~ "::" ^^ { case a ~ b => a + b }
    def IPv6Address = ip6_1 | ip6_2 | ip6_3 | ip6_4 | ip6_5 | ip6_6 | ip6_7 | ip6_8 | ip6_9

    override def skipWhitespace = false
  }

  /**
   * Implements a parser for the ABNF found in appendix A of RFC-3986
   * Source: http://tools.ietf.org/html/rfc3986#appendix-A
   */
  private[rl] trait UriParser extends IPv6AddressParser {

    def subDelims = subDelimChars
    def genDelims = genDelimChars
    def reserved = genDelims | subDelims
    def alpha = """[a-zA-Z]""".r
    def digit = """\d""".r

    def unreserved = alpha | digit | "-" | "." | "_" | "~"

    def pctEncoded = "%" ~> hexDigit ~ hexDigit ^^ { case a ~ b => a + b }
    def pchar = unreserved | pctEncoded | subDelims | ":" | "@"
{'fraction_non_alphanumeric': 0.20179143606175357, 'fraction_numerical': 0.06371977861928342, 'mean_word_length': 2.7696953060664287, 'pattern_counts': {'":': 28, '<': 9, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5973588', 'n_tokens_mistral': 6167, 'n_tokens_neox': 5622, 'n_words': 1321}