<commit_before>package visceljs


import scala.annotation.tailrec


object SearchUtil {
  def search[T](query: String, items: List[(String, T)]): List[T] = {
    val lcql = query.toLowerCase.replaceAll("""\s+""", "")
    if (lcql.isEmpty) items.map(_._2)
    else items
      .view
      .map { item => item -> fuzzyMatch(lcql, item._1.toLowerCase) }
      .filter {_._2 > 0}
      .toList
      .sortBy {-_._2}
      .map {_._1._2}
  }

  def fuzzyMatch(query: String, text: String): Long = {
    @tailrec
    def rec(score: Long, bestScore: Long, qpos: Int, tpos: Int): Long = {
      if (query.length <= qpos) {
        bestScore + score * score
      }
      else {
        if (text.length <= tpos) {
          0
        }
        else {
          if (text(tpos) == query(qpos)) rec(score + 1, bestScore, qpos + 1, tpos + 1)
          else rec(0, score * score + bestScore, qpos, tpos + 1)
        }
      }<commit_msg>normalize search string to better deal with accents<commit_after>package visceljs


import scala.annotation.tailrec
import scala.util.control.NonFatal


object SearchUtil {
  def search[T](query: String, items: List[(String, T)]): List[T] = {
    val lcql = normstr(query.toLowerCase.replaceAll("""\s+""", ""))
    if (lcql.isEmpty) items.map(_._2)
    else items
      .view
      .map { item => item -> fuzzyMatch(lcql, normstr(item._1.toLowerCase)) }
      .filter {_._2 > 0}
      .toList
      .sortBy {-_._2}
      .map {_._1._2}
  }


  def normstr(in: String) =
    try {
      in.asInstanceOf[scalajs.js.Dynamic].normalize("NFD").asInstanceOf[String]
    }
    catch {
      case NonFatal(e) => in
    }

  def fuzzyMatch(query: String, text: String): Long = {
    @tailrec
    def rec(score: Long, bestScore: Long, qpos: Int, tpos: Int): Long = {
      if (query.length <= qpos) {
        bestScore + score * score
      }
      else {
        if (text.length <= tpos) {
          0
        }
        else {
          if (text(tpos) == query(qpos)) rec(score + 1, bestScore, qpos + 1, tpos + 1)
          else rec(0, score * score + bestScore, qpos, tpos + 1)
        }
      }
{'fraction_non_alphanumeric': 0.14881516587677726, 'fraction_numerical': 0.012322274881516588, 'mean_word_length': 2.174436090225564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5443806', 'n_tokens_mistral': 763, 'n_tokens_neox': 739, 'n_words': 182}