<commit_before>var JaTrans = (function(){
  var hiraganaTrans = {
    "ふぁ": "fa",
    "ふぉ": "fo",
    "ふぃ": "fi",
    "か": "ka",
    "き": "ki",
    "く": "ku",
    "け": "ke",
    "こ": "ko",
    "た": "ta",
    "ち": "chi",
    "ちゃ": "cha",
    "ちぇ": "che",
    "ちゅ": "chu",
    "ちょ": "cho",
    "つ": "tsu",
    "て": "te",
    "と": "to",
    "さ": "sa",
    "し": "shi",
    "しゃ": "sha",
    "しぇ": "she",
    "しゅ": "shu",
    "しょ": "sho",
    "し": "shi",
    "す": "su",
    "せ": "se",
    "そ": "so",
    "な": "na",
    "に": "ni",
    "ぬ": "nu",
    "ね": "ne",
    "の": "no",
    "は": "ha",
    "ひ": "hi",
    "ふ": "fu",
    "へ": "he",
    "ほ": "ho",
    "ま": "ma",
    "み": "mi",
    "む": "mu",
    "め": "me",
    "も": "mo",
    "や": "ya",
    "ゆ": "yu",
    "よ": "yo",
    "ら": "ra",
    "り": "ri",
    "る": "ru",
    "れ": "re",
    "ろ": "ro",
    "わ": "wa",
    "を": "wo",
    "が": "ga",
    "ぎ": "gi",
    "ぐ": "gu",
    "げ": "ge",
    "ご": "go",
    "ざ": "za",
    "じ": "ji",
    "ず": "zu",
    "ぜ": "ze",
    "ぞ": "zo",
    "だ": "da",
    "ぢ": "di",
    "づ": "du",
    "で": "de",
    "ど": "do",
    "ば": "ba",
    "び": "bi",
    "ぶ": "bu",
    "べ": "be",
    "ぼ": "bo",
    "ぱ": "pa",
    "ぴ": "pi",
    "ぷ": "pu",
    "ぺ": "pe",
    "ぽ": "po",
    "あ": "a",
    "い": "i",
    "う": "u",
    "え": "e",
    "お": "o",
    "ん": "n",
    "ゃ": "ya",
    "ぇ": "ye",
    "ゅ": "yu",
    "ぃ": "yi",
    "ょ": "yo"
  };

  var loneUnTrans = {
    "ku": "く",
    "su": "す",
    "nu": "ぬ",
    "fu": "ふ",
    "hu": "ふ",
    "mu": "む",
    "yu": "ゆ",
    "ru": "る",
    "gu": "ぐ",
    "zu": "ず",
    "du": "づ",
    "bu": "ぶ",
    "pu": "ぷ"
  };

  /**
  * Constructor to create a Japanese translaterator
  */
  function JaTrans() {
    Trans.call(this, hiraganaTrans);
    Trans.prototype.addUnTransPreFunction(unTransPreFunction);
    Trans.prototype.addUnTransPostFunction(loneCharReplace);
  }


  /**
  * Replace the doubled characters with a little "tsu" if different from "n"
  * @param  {String} text The text to be replaced
  * @return {String} The same string but the repeated characters are replaced
  */
  var doubleReplace = function(text){
    return text.replace(/(.)\1+/gi, function(x){
      var repChar = "っ";
      if (x[0] === "n"){
        repChar = "ん";
      }
      var p = new Array(x.length).join(repChar);
      return p + x[0];
    });
  }

  var xya2Jap = function (text){
    return text.replace(/(.)y[aeuio]/gi, function(x){
      var key = x[0] + "i";
      if (key in Trans.prototype.inverseLookup){
        var result =  Trans.prototype.inverseLookup[key];
        key = "y" + x[2];
        if (key in Trans.prototype.inverseLookup){
          result += Trans.prototype.inverseLookup[key];
        }
        return result;
      } else {
        return x;
      }

    });
  }

  var unTransPreFunction = function(text){
    return xya2Jap(doubleReplace(text));
  }



  /**
  * Replace the lone characters with their equivalent + "u"
  * @param  {[String]} text the text to be replaced
  * @return {[String]}      The resulted text
  */
  var loneCharReplace = function(text){<commit_msg>fix xy[aeuio] while untranslating<commit_after>var JaTrans = (function(){
  var hiraganaTrans = {
    "ふぁ": "fa",
    "ふぉ": "fo",
    "ふぃ": "fi",
    "か": "ka",
    "き": "ki",
    "く": "ku",
    "け": "ke",
    "こ": "ko",
    "た": "ta",
    "ち": "chi",
    "ちゃ": "cha",
    "ちぇ": "che",
    "ちゅ": "chu",
    "ちょ": "cho",
    "つ": "tsu",
    "て": "te",
    "と": "to",
    "さ": "sa",
    "し": "shi",
    "しゃ": "sha",
    "しぇ": "she",
    "しゅ": "shu",
    "しょ": "sho",
    "し": "shi",
    "す": "su",
    "せ": "se",
    "そ": "so",
    "な": "na",
    "に": "ni",
    "ぬ": "nu",
    "ね": "ne",
    "の": "no",
    "は": "ha",
    "ひ": "hi",
    "ふ": "fu",
    "へ": "he",
    "ほ": "ho",
    "ま": "ma",
    "み": "mi",
    "む": "mu",
    "め": "me",
    "も": "mo",
    "や": "ya",
    "ゆ": "yu",
    "よ": "yo",
    "ら": "ra",
    "り": "ri",
    "る": "ru",
    "れ": "re",
    "ろ": "ro",
    "わ": "wa",
    "を": "wo",
    "が": "ga",
    "ぎ": "gi",
    "ぐ": "gu",
    "げ": "ge",
    "ご": "go",
    "ざ": "za",
    "じ": "ji",
    "ず": "zu",
    "ぜ": "ze",
    "ぞ": "zo",
    "だ": "da",
    "ぢ": "di",
    "づ": "du",
    "で": "de",
    "ど": "do",
    "ば": "ba",
    "び": "bi",
    "ぶ": "bu",
    "べ": "be",
    "ぼ": "bo",
    "ぱ": "pa",
    "ぴ": "pi",
    "ぷ": "pu",
    "ぺ": "pe",
    "ぽ": "po",
    "あ": "a",
    "い": "i",
    "う": "u",
    "え": "e",
    "お": "o",
    "ん": "n",
    "ゃ": "xya",
    "ぇ": "xye",
    "ゅ": "xyu",
    "ぃ": "xyi",
    "ょ": "xyo"
  };

  var loneUnTrans = {
    "ku": "く",
    "su": "す",
    "nu": "ぬ",
    "fu": "ふ",
    "hu": "ふ",
    "mu": "む",
    "yu": "ゆ",
    "ru": "る",
    "gu": "ぐ",
    "zu": "ず",
    "du": "づ",
    "bu": "ぶ",
    "pu": "ぷ"
  };

  /**
  * Constructor to create a Japanese translaterator
  */
  function JaTrans() {
    Trans.call(this, hiraganaTrans);
    Trans.prototype.addUnTransPreFunction(unTransPreFunction);
    Trans.prototype.addUnTransPostFunction(loneCharReplace);
  }


  /**
  * Replace the doubled characters with a little "tsu" if different from "n"
  * @param  {String} text The text to be replaced
  * @return {String} The same string but the repeated characters are replaced
  */
  var doubleReplace = function(text){
    return text.replace(/(.)\1+/gi, function(x){
      var repChar = "っ";
      if (x[0] === "n"){
        repChar = "ん";
      }
      var p = new Array(x.length).join(repChar);
      return p + x[0];
    });
  }

  var xya2Jap = function (text){
    return text.replace(/(.)y[aeuio]/gi, function(x){
      var key = x[0] + "i";
      if (key in Trans.prototype.inverseLookup){
        var result =  Trans.prototype.inverseLookup[key];
        key = "xy" + x[2];
        if (key in Trans.prototype.inverseLookup){
          result += Trans.prototype.inverseLookup[key];
        }
        return result;
      } else {
        return x;
      }

    });
  }

  var unTransPreFunction = function(text){
    return xya2Jap(doubleReplace(text));
  }



  /**
  * Replace the lone characters with their equivalent + "u"
  * @param  {[String]} text the text to be replaced
  * @return {[String]}      The resulted text
  */
  var loneCharReplace = function(text){
{'fraction_non_alphanumeric': 0.2613016992625842, 'fraction_numerical': 0.0022443090734209684, 'mean_word_length': 2.002406159769009, 'pattern_counts': {'":': 202, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5673976', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2730, 'n_words': 670}