<commit_before>package lila.i18n

import java.io._
import scala.concurrent.Future
// import scala.jdk.CollectionConverters._
import play.api.libs.json.{ JsObject, JsString }
import play.api.i18n.Lang

final private[i18n] class JsDump(path: String)(implicit ec: scala.concurrent.ExecutionContext) {

  def apply: Funit =
    Future {
      pathFile.mkdir
      writeRefs
      writeFullJson
    } void

  private val pathFile = new File(path)

  private def dumpFromKey(keys: Set[String], lang: Lang): String =
    keys
      .map { key =>
        """"%s":"%s"""".format(key, escape(Translator.txt.literal(key, Nil, lang)))
      }
      .mkString("{", ",", "}")

  private def writeRefs() = writeFile(
    new File("%s/refs.json".format(pathFile.getCanonicalPath)),
    LangList.all.toList
      .sortBy(_._1.code)
      .map {
        case (lang, name) => s"""["${lang.code}","$name"]"""
      }
      .mkString("[", ",", "]")
  )

  private def writeFullJson() = Registry.langs foreach { lang =>
    ???
  // val code = dumpFromKey(I18nDb.site(defaultLang).keySet.asScala.toSet, lang)
  // writeFile(new File("%s/%s.all.json".format(pathFile.getCanonicalPath, lang.code)), code)
  }

  private def writeFile(file: File, content: String) = {
    val out = new PrintWriter(file)
    try {
      out.print(content)
    } finally {
      out.close
    }<commit_msg>also remove i18n db prefix while doing full translation dump<commit_after>package lila.i18n

import java.io._
import scala.concurrent.Future
import scala.jdk.CollectionConverters._
import play.api.libs.json.{ JsObject, JsString }
import play.api.i18n.Lang

final private[i18n] class JsDump(path: String)(implicit ec: scala.concurrent.ExecutionContext) {

  def apply: Funit =
    Future {
      pathFile.mkdir
      writeRefs
      writeFullJson
    } void

  private val pathFile = new File(path)

  private def dumpFromKey(keys: Set[String], lang: Lang): String =
    keys
      .map { key =>
        """"%s":"%s"""".format(JsDump removeDbPrefix key, escape(Translator.txt.literal(key, Nil, lang)))
      }
      .mkString("{", ",", "}")

  private def writeRefs() = writeFile(
    new File("%s/refs.json".format(pathFile.getCanonicalPath)),
    LangList.all.toList
      .sortBy(_._1.code)
      .map {
        case (lang, name) => s"""["${lang.code}","$name"]"""
      }
      .mkString("[", ",", "]")
  )

  private def writeFullJson() = Registry.langs foreach { lang =>
    val code = dumpFromKey(Registry.default.keySet.asScala.toSet, lang)
    writeFile(new File("%s/%s.all.json".format(pathFile.getCanonicalPath, lang.code)), code)
  }

  private def writeFile(file: File, content: String) = {
    val out = new PrintWriter(file)
    try {
      out.print(content)
    } finally {
      out.close
    }
{'fraction_non_alphanumeric': 0.15465032444124008, 'fraction_numerical': 0.006488824801730353, 'mean_word_length': 3.1294642857142856, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4642856', 'n_tokens_mistral': 965, 'n_tokens_neox': 940, 'n_words': 193}