<commit_before>package lila.common

import chess.Color
import chess.format.{ FEN, Forsyth }
import org.joda.time.{ DateTime, DateTimeZone }
import play.api.data.format.Formats._
import play.api.data.format.{ Formatter, JodaFormats }
import play.api.data.Forms._
import play.api.data.JodaForms._
import play.api.data.validation.{ Constraint, Constraints }
import play.api.data.{ Field, FormError, Mapping }
import play.api.data.{ validation => V }
import scala.util.Try

import lila.common.base.StringUtils

object Form {

  type Options[A] = Iterable[(A, String)]

  type FormLike = {
    def apply(key: String): Field
    def errors: Seq[FormError]
  }

  def options(it: Iterable[Int], pattern: String): Options[Int] =
    it map { d =>
      d -> (pluralize(pattern, d) format d)
    }

  def options(it: Iterable[Int], transformer: Int => Int, pattern: String): Options[Int] =
    it map { d =>
      d -> (pluralize(pattern, transformer(d)) format transformer(d))
    }

  def options(it: Iterable[Int], code: String, pattern: String): Options[String] =
    it map { d =>
      s"$d$code" -> (pluralize(pattern, d) format d)
    }

  def options(it: Iterable[Int], format: Int => String): Options[Int] =
    it map { d =>
      d -> format(d)
    }

  def optionsDouble(it: Iterable[Double], format: Double => String): Options[Double] =
    it map { d =>
      d -> format(d)
    }

  private def mustBeOneOf(choices: Iterable[Any]) = s"Must be one of: ${choices mkString ", "}"

  def numberIn(choices: Options[Int]) =
    number.verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def numberIn(choices: Set[Int]) =
    number.verifying(mustBeOneOf(choices), choices.contains _)

  def numberIn(choices: Seq[Int]) =
    number.verifying(mustBeOneOf(choices), choices.contains _)

  def numberInDouble(choices: Options[Double]) =
    of[Double].verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def id(size: Int, fixed: Option[String])(exists: String => Fu[Boolean]) = {
    val field = text(minLength = size, maxLength = size)
      .verifying("IDs must be made of ASCII letters and numbers", id => """(?i)^[a-z\d]+$""".r matches id)
    fixed match {
      case Some(fixedId) => field.verifying("The ID cannot be changed now", id => id == fixedId)
      case None =>
        import scala.concurrent.duration._
        field.verifying(
          "This ID is already in use",
          id => !exists(id).await(1 second, "tour crud unique ID")
        )
    }
  }

  def trim(m: Mapping[String]) = m.transform[String](_.trim, identity)

  // trims and removes garbage chars before validation
  private def makeCleanTextFormatter(keepSymbols: Boolean): Formatter[String] = new Formatter[String] {
    def bind(key: String, data: Map[String, String]) =
      data
        .get(key)
        .map(String.normalize.apply)
        .map(if (keepSymbols) identity else String.removeMultibyteSymbols)
        .map(_.trim)
        .toRight(Seq(FormError(key, "error.required", Nil)))
    def unbind(key: String, value: String) = Map(key -> String.normalize(value.trim))
  }
  val cleanTextFormatter: Formatter[String]            = makeCleanTextFormatter(keepSymbols = false)
  val cleanTextFormatterWithSymbols: Formatter[String] = makeCleanTextFormatter(keepSymbols = true)

  val garbageCharsConstraint = V.Constraint((s: String) =>
    if (String.hasGarbageChars(s))
      V.Invalid(
        Seq(
          V.ValidationError(
            s"The text contains invalid chars: ${String.distinctGarbageChars(s) mkString " "}"
          )
        )
      )
    else V.Valid
  )

  val cleanText: Mapping[String] = of(cleanTextFormatter) verifying garbageCharsConstraint
  val cleanTextWithSymbols: Mapping[String] =
    of(cleanTextFormatterWithSymbols) verifying garbageCharsConstraint

  def cleanText(minLength: Int = 0, maxLength: Int = Int.MaxValue): Mapping[String] =
    (minLength, maxLength) match {
      case (min, Int.MaxValue) => cleanText.verifying(Constraints minLength min)
      case (0, max)            => cleanText.verifying(Constraints maxLength max)
      case (min, max)          => cleanText.verifying(Constraints minLength min, Constraints maxLength max)
    }

  val cleanNonEmptyText: Mapping[String] = cleanText.verifying(Constraints.nonEmpty)
  def cleanNonEmptyText(minLength: Int = 0, maxLength: Int = Int.MaxValue): Mapping[String] =
    cleanText(minLength, maxLength).verifying(Constraints.nonEmpty)

  object eventName {

    def apply(minLength: Int, maxLength: Int, verifiedUser: Boolean) =
      cleanText.verifying(
        Constraints minLength minLength,
        Constraints maxLength maxLength,
        Constraints.pattern(
          regex = """[\p{L}\p{N}-\s:.,;'°ª\+]+""".r,
          error = "Invalid characters; only letters, numbers, and common punctuation marks are accepted."
        ),
        mustNotContainLichess(verifiedUser)
      )
  }

  object mustNotContainLichess {
    private val regex = "(?iu)l[iıi̇][cс]h[eе]s".r
    def apply(verifiedUser: Boolean) = Constraint[String] { (t: String) =>
      if (regex.find(t) && !verifiedUser)
        V.Invalid(V.ValidationError("Must not contain \"lichess\""))
      else V.Valid
    }
  }

  def stringIn(choices: Options[String]) =
    text.verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def stringIn(choices: Set[String]) =
    text.verifying(mustBeOneOf(choices), choices.contains _)

  def tolerantBoolean = of[Boolean](formatter.tolerantBooleanFormatter)

  def hasKey[A](choices: Options[A], key: A) =
    choices.map(_._1).toList contains key

  def trueish(v: Any) = v == 1 || v == "1" || v == "true" || v == "True" || v == "on" || v == "yes"

  object color {
    val mapping: Mapping[Color] = trim(text)
      .verifying(Color.all.map(_.name).contains _)
      .transform[Color](c => Color.fromWhite(c == "white"), _.name)<commit_msg>Adjust mustNotContainLichess filter<commit_after>package lila.common

import chess.Color
import chess.format.{ FEN, Forsyth }
import org.joda.time.{ DateTime, DateTimeZone }
import play.api.data.format.Formats._
import play.api.data.format.{ Formatter, JodaFormats }
import play.api.data.Forms._
import play.api.data.JodaForms._
import play.api.data.validation.{ Constraint, Constraints }
import play.api.data.{ Field, FormError, Mapping }
import play.api.data.{ validation => V }
import scala.util.Try

import lila.common.base.StringUtils

object Form {

  type Options[A] = Iterable[(A, String)]

  type FormLike = {
    def apply(key: String): Field
    def errors: Seq[FormError]
  }

  def options(it: Iterable[Int], pattern: String): Options[Int] =
    it map { d =>
      d -> (pluralize(pattern, d) format d)
    }

  def options(it: Iterable[Int], transformer: Int => Int, pattern: String): Options[Int] =
    it map { d =>
      d -> (pluralize(pattern, transformer(d)) format transformer(d))
    }

  def options(it: Iterable[Int], code: String, pattern: String): Options[String] =
    it map { d =>
      s"$d$code" -> (pluralize(pattern, d) format d)
    }

  def options(it: Iterable[Int], format: Int => String): Options[Int] =
    it map { d =>
      d -> format(d)
    }

  def optionsDouble(it: Iterable[Double], format: Double => String): Options[Double] =
    it map { d =>
      d -> format(d)
    }

  private def mustBeOneOf(choices: Iterable[Any]) = s"Must be one of: ${choices mkString ", "}"

  def numberIn(choices: Options[Int]) =
    number.verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def numberIn(choices: Set[Int]) =
    number.verifying(mustBeOneOf(choices), choices.contains _)

  def numberIn(choices: Seq[Int]) =
    number.verifying(mustBeOneOf(choices), choices.contains _)

  def numberInDouble(choices: Options[Double]) =
    of[Double].verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def id(size: Int, fixed: Option[String])(exists: String => Fu[Boolean]) = {
    val field = text(minLength = size, maxLength = size)
      .verifying("IDs must be made of ASCII letters and numbers", id => """(?i)^[a-z\d]+$""".r matches id)
    fixed match {
      case Some(fixedId) => field.verifying("The ID cannot be changed now", id => id == fixedId)
      case None =>
        import scala.concurrent.duration._
        field.verifying(
          "This ID is already in use",
          id => !exists(id).await(1 second, "tour crud unique ID")
        )
    }
  }

  def trim(m: Mapping[String]) = m.transform[String](_.trim, identity)

  // trims and removes garbage chars before validation
  private def makeCleanTextFormatter(keepSymbols: Boolean): Formatter[String] = new Formatter[String] {
    def bind(key: String, data: Map[String, String]) =
      data
        .get(key)
        .map(String.normalize.apply)
        .map(if (keepSymbols) identity else String.removeMultibyteSymbols)
        .map(_.trim)
        .toRight(Seq(FormError(key, "error.required", Nil)))
    def unbind(key: String, value: String) = Map(key -> String.normalize(value.trim))
  }
  val cleanTextFormatter: Formatter[String]            = makeCleanTextFormatter(keepSymbols = false)
  val cleanTextFormatterWithSymbols: Formatter[String] = makeCleanTextFormatter(keepSymbols = true)

  val garbageCharsConstraint = V.Constraint((s: String) =>
    if (String.hasGarbageChars(s))
      V.Invalid(
        Seq(
          V.ValidationError(
            s"The text contains invalid chars: ${String.distinctGarbageChars(s) mkString " "}"
          )
        )
      )
    else V.Valid
  )

  val cleanText: Mapping[String] = of(cleanTextFormatter) verifying garbageCharsConstraint
  val cleanTextWithSymbols: Mapping[String] =
    of(cleanTextFormatterWithSymbols) verifying garbageCharsConstraint

  def cleanText(minLength: Int = 0, maxLength: Int = Int.MaxValue): Mapping[String] =
    (minLength, maxLength) match {
      case (min, Int.MaxValue) => cleanText.verifying(Constraints minLength min)
      case (0, max)            => cleanText.verifying(Constraints maxLength max)
      case (min, max)          => cleanText.verifying(Constraints minLength min, Constraints maxLength max)
    }

  val cleanNonEmptyText: Mapping[String] = cleanText.verifying(Constraints.nonEmpty)
  def cleanNonEmptyText(minLength: Int = 0, maxLength: Int = Int.MaxValue): Mapping[String] =
    cleanText(minLength, maxLength).verifying(Constraints.nonEmpty)

  object eventName {

    def apply(minLength: Int, maxLength: Int, verifiedUser: Boolean) =
      cleanText.verifying(
        Constraints minLength minLength,
        Constraints maxLength maxLength,
        Constraints.pattern(
          regex = """[\p{L}\p{N}-\s:.,;'°ª\+]+""".r,
          error = "Invalid characters; only letters, numbers, and common punctuation marks are accepted."
        ),
        mustNotContainLichess(verifiedUser)
      )
  }

  object mustNotContainLichess {
    private val regex = "(?iu)l(?:[iі]|ı̇)[cс][hһ][eе]s".r
    def apply(verifiedUser: Boolean) = Constraint[String] { (t: String) =>
      if (regex.find(t) && !verifiedUser)
        V.Invalid(V.ValidationError("Must not contain \"lichess\""))
      else V.Valid
    }
  }

  def stringIn(choices: Options[String]) =
    text.verifying(mustBeOneOf(choices.map(_._1)), hasKey(choices, _))

  def stringIn(choices: Set[String]) =
    text.verifying(mustBeOneOf(choices), choices.contains _)

  def tolerantBoolean = of[Boolean](formatter.tolerantBooleanFormatter)

  def hasKey[A](choices: Options[A], key: A) =
    choices.map(_._1).toList contains key

  def trueish(v: Any) = v == 1 || v == "1" || v == "true" || v == "True" || v == "on" || v == "yes"

  object color {
    val mapping: Mapping[Color] = trim(text)
      .verifying(Color.all.map(_.name).contains _)
      .transform[Color](c => Color.fromWhite(c == "white"), _.name)
{'fraction_non_alphanumeric': 0.13724991522550017, 'fraction_numerical': 0.00169548999660902, 'mean_word_length': 3.5009538344143456, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3378978', 'n_tokens_mistral': 3863, 'n_tokens_neox': 3748, 'n_words': 887}