<commit_before>package com.wcaokaze.japanecraft

import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
import java.util.concurrent.Executors
import kotlin.reflect.KProperty

class JapaneCraftCore(
   private val configLoader: ConfigLoader,
   private val romajiTableConfigLoader: RomajiTableConfigLoader,
   private val dictionaryConfigLoader: DictionaryConfigLoader
) {
   private val executor = Executors.newScheduledThreadPool(3)

   private fun getConfiguration(): Configuration = configLoader.loadConfig()

   private val romajiRegex: Regex by cached(
      { getConfiguration().romajiRegex },
      { Regex(it) }
   )

   private val timeFormatter: DateFormat by cached(
      { getConfiguration().timeFormat },
      { SimpleDateFormat(it) }
   )

   private val chatFormatter: VariableExpander by cached(
      { getConfiguration().chatFormat },
      { VariableExpander(it) }
   )

   data class ChatMessage(
      val username: String,
      val message: String,
      val postDate: Date
   )

   data class ConvertResult(
      val rawMessage: ChatMessage,
      val japaneseMessage: String
   )

   fun interface ConversionFinishCallback {
      fun onConverted(formattedMessage: String)
   }

   fun formatConvertedMessage(
      chatMessage: ChatMessage,
      conversionFinishCallback: ConversionFinishCallback
   ) {
      executor.submit {
         val convertedMessage = formatConvertedMessage(chatMessage)
         conversionFinishCallback.onConverted(convertedMessage)
      }
   }

   fun formatConvertedMessage(chatMessage: ChatMessage): String {
      return try {
         format(convert(chatMessage))
      } catch (e: Exception) {
         chatMessage.message
      }
   }

   fun format(convertResult: ConvertResult): String {
      val variableMap = mapOf(
         "n"                to "\n",
         "$"                to "\$",
         "username"         to convertResult.rawMessage.username,
         "time"             to timeFormatter.format(convertResult.rawMessage.postDate),
         "rawMessage"       to convertResult.rawMessage.message,
         "convertedMessage" to convertResult.japaneseMessage
      )

      return chatFormatter.expand(variableMap)
   }

   fun convert(rawMessage: ChatMessage): ConvertResult {
      if (rawMessage.message.any { it >= 0x80.toChar() }) {
         return ConvertResult(rawMessage, rawMessage.message)
      }

      val japaneseMessage = convertToJapanese(rawMessage.message)

      if (japaneseMessage == rawMessage.message.filter { it != '`' }) {
         return ConvertResult(rawMessage, rawMessage.message)
      }

      return ConvertResult(rawMessage, japaneseMessage)
   }

   private fun convertToJapanese(englishMessage: String): String {
      val configuration = getConfiguration()

      val dictionaryAppliedWords = englishMessage.split('`')
         .asSequence()
         .flatMapIndexed { i, chunk ->
            if (i % 2 == 1) {
               sequenceOf(KanjiConverter.Word(KanjiConverter.Word.Type.ENGLISH, chunk))
            } else {
               splitWords(chunk, configuration.wordSeparators.toCharArray()).map { romaji ->
                  if (romaji.matches(romajiRegex)) {
                     val hiragana = convertToHiragana(romaji)<commit_msg>return englishMessage when JapaneCraftCore.convert fails<commit_after>package com.wcaokaze.japanecraft

import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
import java.util.concurrent.Executors
import kotlin.reflect.KProperty

class JapaneCraftCore(
   private val configLoader: ConfigLoader,
   private val romajiTableConfigLoader: RomajiTableConfigLoader,
   private val dictionaryConfigLoader: DictionaryConfigLoader
) {
   private val executor = Executors.newScheduledThreadPool(3)

   private fun getConfiguration(): Configuration = configLoader.loadConfig()

   private val romajiRegex: Regex by cached(
      { getConfiguration().romajiRegex },
      { Regex(it) }
   )

   private val timeFormatter: DateFormat by cached(
      { getConfiguration().timeFormat },
      { SimpleDateFormat(it) }
   )

   private val chatFormatter: VariableExpander by cached(
      { getConfiguration().chatFormat },
      { VariableExpander(it) }
   )

   data class ChatMessage(
      val username: String,
      val message: String,
      val postDate: Date
   )

   data class ConvertResult(
      val rawMessage: ChatMessage,
      val japaneseMessage: String
   )

   fun interface ConversionFinishCallback {
      fun onConverted(formattedMessage: String)
   }

   fun formatConvertedMessage(
      chatMessage: ChatMessage,
      conversionFinishCallback: ConversionFinishCallback
   ) {
      executor.submit {
         val convertedMessage = formatConvertedMessage(chatMessage)
         conversionFinishCallback.onConverted(convertedMessage)
      }
   }

   fun formatConvertedMessage(chatMessage: ChatMessage): String {
      return try {
         format(convert(chatMessage))
      } catch (e: Exception) {
         chatMessage.message
      }
   }

   fun format(convertResult: ConvertResult): String {
      val variableMap = mapOf(
         "n"                to "\n",
         "$"                to "\$",
         "username"         to convertResult.rawMessage.username,
         "time"             to timeFormatter.format(convertResult.rawMessage.postDate),
         "rawMessage"       to convertResult.rawMessage.message,
         "convertedMessage" to convertResult.japaneseMessage
      )

      return chatFormatter.expand(variableMap)
   }

   fun convert(rawMessage: ChatMessage): ConvertResult {
      try {
         if (rawMessage.message.any { it >= 0x80.toChar() }) {
            return ConvertResult(rawMessage, rawMessage.message)
         }

         val japaneseMessage = convertToJapanese(rawMessage.message)

         if (japaneseMessage == rawMessage.message.filter { it != '`' }) {
            return ConvertResult(rawMessage, rawMessage.message)
         }

         return ConvertResult(rawMessage, japaneseMessage)
      } catch (e: Exception) {
         return ConvertResult(rawMessage, rawMessage.message)
      }
   }

   private fun convertToJapanese(englishMessage: String): String {
      val configuration = getConfiguration()

      val dictionaryAppliedWords = englishMessage.split('`')
         .asSequence()
         .flatMapIndexed { i, chunk ->
            if (i % 2 == 1) {
               sequenceOf(KanjiConverter.Word(KanjiConverter.Word.Type.ENGLISH, chunk))
            } else {
               splitWords(chunk, configuration.wordSeparators.toCharArray()).map { romaji ->
                  if (romaji.matches(romajiRegex)) {
                     val hiragana = convertToHiragana(romaji)
{'fraction_non_alphanumeric': 0.08540925266903915, 'fraction_numerical': 0.0017793594306049821, 'mean_word_length': 2.753478018920423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2895391', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1766, 'n_words': 384}