<commit_before>package lila.study

import chess.format.pgn.{ Tag, TagType }

private object PgnTags {

  def apply(tags: List[Tag]): List[Tag] = sort(tags filter isRelevant)

  private def isRelevant(tag: Tag) = tag match {
    case Tag(t, _) if !relevantTags(t) => false
    case Tag(_, v) if unknownValues(v) => false
    case _                             => true
  }

  private val unknownValues = Set("", "?", "unknown")

  private val relevantTags: Set[TagType] = {
    import Tag._
    Set(Event, Site, Date, Round, White, Black, TimeControl,
      WhiteElo, BlackElo, WhiteTitle, BlackTitle, WhiteTeam, BlackTeam,
      Tag.Result, Termination, Annotator)
  }

  private def sort(tags: List[Tag]) = tags.sortBy {
    case Tag(Tag.White, _) => 1
    case Tag(Tag.Black, _) => 2
    case Tag(Tag.Date, _) => 3
    case Tag(Tag.Termination, _) => 4
    case Tag(Tag.Site | Tag.Event | Tag.Round | Tag.Annotator, _) => 10
    case _ => 9
  }
}<commit_msg>better sort & filter study tags<commit_after>package lila.study

import chess.format.pgn.{ Tag, TagType }

private object PgnTags {

  def apply(tags: List[Tag]): List[Tag] = sort(tags filter isRelevant)

  private def isRelevant(tag: Tag) =
    relevantTypeSet(tag.name) && !unknownValues(tag.value)

  private val unknownValues = Set("", "?", "unknown")

  private val sortedTypes: List[TagType] = {
    import Tag._
    List(
      White, WhiteElo, WhiteTitle, WhiteTeam,
      Black, BlackElo, BlackTitle, BlackTeam,
      TimeControl,
      Date,
      Result,
      Termination,
      Site, Event, Round, Annotator)
  }

  private val relevantTypeSet: Set[TagType] = sortedTypes.toSet

  private val typePositions: Map[TagType, Int] = sortedTypes.zipWithIndex.toMap

  private def sort(tags: List[Tag]) = tags.sortBy { t =>
    typePositions.getOrElse(t.name, Int.MaxValue)
  }
}
{'fraction_non_alphanumeric': 0.12581344902386118, 'fraction_numerical': 0.003796095444685466, 'mean_word_length': 3.260969976905312, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '349324', 'n_tokens_mistral': 642, 'n_tokens_neox': 606, 'n_words': 160}