<commit_before>package lila.common

import scala.concurrent.duration._

case class ApiVersion(value: Int) extends AnyVal with IntValue with Ordered[ApiVersion] {
  def compare(other: ApiVersion) = Integer.compare(value, other.value)
  def gt(other: Int)             = value > other
  def gte(other: Int)            = value >= other
}

case class AssetVersion(value: String) extends AnyVal with StringValue

object AssetVersion {
  var current        = random
  def change()       = { current = random }
  private def random = AssetVersion(ornicar.scalalib.Random secureString 6)
}

case class IsMobile(value: Boolean) extends AnyVal with BooleanValue

case class IpAddress(value: String) extends AnyVal with StringValue

object IpAddress {
  // http://stackoverflow.com/questions/106179/regular-expression-to-match-hostname-or-ip-address
  private val ipv4Regex =
    """^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$""".r
  // ipv6 address in standard form (no compression, no leading zeros)
  private val ipv6Regex = """^((0|[1-9a-f][0-9a-f]{0,3}+):){7}(0|[1-9a-f][0-9a-f]{0,3})""".r

  def isv4(a: IpAddress) = ipv4Regex matches a.value
  def isv6(a: IpAddress) = ipv6Regex matches a.value

  def from(str: String): Option[IpAddress] = {
    ipv4Regex.matches(str) || ipv6Regex.matches(str)
  } option IpAddress(str)
}

case class NormalizedEmailAddress(value: String) extends AnyVal with StringValue

case class EmailAddress(value: String) extends AnyVal with StringValue {
  def conceal = value split '@' match {
    case Array(user, domain) => s"${user take 3}*****@${domain}"
    case _                   => value
  }
  def normalize = NormalizedEmailAddress {
    val lower = value.toLowerCase
    lower.split('@') match {
      case Array(name, domain) if EmailAddress.gmailLikeNormalizedDomains(domain) =>
        val normalizedName = name
          .replace(".", "")  // remove all dots
          .takeWhile('+' !=) // skip everything after the first '+'
        if (normalizedName.isEmpty) lower else s"$normalizedName@$domain"
      case _ => lower
    }
  }
  def domain: Option[Domain] = value split '@' match {
    case Array(_, domain) => Domain from domain.toLowerCase
    case _                => none
  }

  def similarTo(other: EmailAddress) = normalize == other.normalize

  def isNoReply  = EmailAddress isNoReply value
  def isSendable = !isNoReply

  // safer logs
  override def toString = "EmailAddress(****)"
}

object EmailAddress {

  private val regex =
    """^[a-zA-Z0-9\.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$""".r

  private val gmailLikeNormalizedDomains =
    Set("gmail.com", "googlemail.com", "protonmail.com", "protonmail.ch", "pm.me")

  private def hasDotAt(str: String)           = str contains ".@"  // mailgun will reject it
  private def hasConsecutiveDots(str: String) = str contains ".."  // mailgun will reject it
  private def startsWithDot(str: String)      = str startsWith "." // mailgun will reject it

  def matches(str: String): Boolean =
    regex.find(str) && !hasDotAt(str) && !hasConsecutiveDots(str) && !startsWithDot(str)

  def from(str: String): Option[EmailAddress] =
    matches(str) option EmailAddress(str)

  private def isNoReply(str: String) = str.startsWith("noreply.") && str.endsWith("@lichess.org")
}

case class Domain private (value: String) extends AnyVal with StringValue {
  // heuristic to remove user controlled subdomain tails:
  // tail.domain.com, tail.domain.co.uk, tail.domain.edu.au, etc.
  def withoutSubdomain: Option[Domain] = value.split('.').toList.reverse match {
    case tld :: sld :: tail :: _ if sld.length <= 3 => Domain from s"$tail.$sld.$tld"
    case tld :: sld :: _                            => Domain from s"$sld.$tld"
    case _                                          => none
  }
  def lower = Domain.Lower(value.toLowerCase)
}
<commit_msg>format<commit_after>package lila.common

import scala.concurrent.duration._

case class ApiVersion(value: Int) extends AnyVal with IntValue with Ordered[ApiVersion] {
  def compare(other: ApiVersion) = Integer.compare(value, other.value)
  def gt(other: Int)             = value > other
  def gte(other: Int)            = value >= other
}

case class AssetVersion(value: String) extends AnyVal with StringValue

object AssetVersion {
  var current        = random
  def change()       = { current = random }
  private def random = AssetVersion(ornicar.scalalib.Random secureString 6)
}

case class IsMobile(value: Boolean) extends AnyVal with BooleanValue

case class IpAddress(value: String) extends AnyVal with StringValue

object IpAddress {
  // http://stackoverflow.com/questions/106179/regular-expression-to-match-hostname-or-ip-address
  private val ipv4Regex =
    """^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$""".r
  // ipv6 address in standard form (no compression, no leading zeros)
  private val ipv6Regex = """^((0|[1-9a-f][0-9a-f]{0,3}+):){7}(0|[1-9a-f][0-9a-f]{0,3})""".r

  def isv4(a: IpAddress) = ipv4Regex matches a.value
  def isv6(a: IpAddress) = ipv6Regex matches a.value

  def from(str: String): Option[IpAddress] = {
    ipv4Regex.matches(str) || ipv6Regex.matches(str)
  } option IpAddress(str)
}

case class NormalizedEmailAddress(value: String) extends AnyVal with StringValue

case class EmailAddress(value: String) extends AnyVal with StringValue {
  def conceal = value split '@' match {
    case Array(user, domain) => s"${user take 3}*****@${domain}"
    case _                   => value
  }
  def normalize = NormalizedEmailAddress {
    val lower = value.toLowerCase
    lower.split('@') match {
      case Array(name, domain) if EmailAddress.gmailLikeNormalizedDomains(domain) =>
        val normalizedName = name
          .replace(".", "")  // remove all dots
          .takeWhile('+' !=) // skip everything after the first '+'
        if (normalizedName.isEmpty) lower else s"$normalizedName@$domain"
      case _ => lower
    }
  }
  def domain: Option[Domain] = value split '@' match {
    case Array(_, domain) => Domain from domain.toLowerCase
    case _                => none
  }

  def similarTo(other: EmailAddress) = normalize == other.normalize

  def isNoReply  = EmailAddress isNoReply value
  def isSendable = !isNoReply

  // safer logs
  override def toString = "EmailAddress(****)"
}

object EmailAddress {

  private val regex =
    """^[a-zA-Z0-9\.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$""".r

  private val gmailLikeNormalizedDomains =
    Set("gmail.com", "googlemail.com", "protonmail.com", "protonmail.ch", "pm.me")

  private def hasDotAt(str: String)           = str contains ".@"  // mailgun will reject it
  private def hasConsecutiveDots(str: String) = str contains ".."  // mailgun will reject it
  private def startsWithDot(str: String)      = str startsWith "." // mailgun will reject it

  def matches(str: String): Boolean =
    regex.find(str) &&
      !hasDotAt(str) &&
      !hasConsecutiveDots(str) &&
      !startsWithDot(str)

  def from(str: String): Option[EmailAddress] =
    matches(str) option EmailAddress(str)

  private def isNoReply(str: String) = str.startsWith("noreply.") && str.endsWith("@lichess.org")
}

case class Domain private (value: String) extends AnyVal with StringValue {
  // heuristic to remove user controlled subdomain tails:
  // tail.domain.com, tail.domain.co.uk, tail.domain.edu.au, etc.
  def withoutSubdomain: Option[Domain] = value.split('.').toList.reverse match {
    case tld :: sld :: tail :: _ if sld.length <= 3 => Domain from s"$tail.$sld.$tld"
    case tld :: sld :: _                            => Domain from s"$sld.$tld"
    case _                                          => none
  }
  def lower = Domain.Lower(value.toLowerCase)
}

{'fraction_non_alphanumeric': 0.1564651395669045, 'fraction_numerical': 0.023031668544248343, 'mean_word_length': 3.2567927543953115, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7131792', 'n_tokens_mistral': 2787, 'n_tokens_neox': 2694, 'n_words': 632}