<commit_before>/* jshint node: true */
'use strict';

/**
  * converts yaml file to js file
  * params: 
  *   --yaml-path=<YAML DIRECTORY LOCATION>
  *   --js-path=<JS DIRECTORY LOCATION>
  **/
var yaml = require('js-yaml');
var fs = require('fs');
var YAMLToJSON ={
  convert: function(){
    var pathChecker = require('path');
    var path = this.parsePath(arguments[0]);
    var yamlDir = path.yaml;
    var jsDir = path.js;
    var defaultLocale = path.locale;
    var files = fs.readdirSync(yamlDir);
    if(!fs.existsSync(jsDir)){
      fs.mkdirSync(jsDir);
    }
    this.extractRefDoc(yamlDir,jsDir,defaultLocale);
    for(var i=0;i<files.length;i++){
      var yamlFileURL = yamlDir+"/"+files[i];
      if(pathChecker.extname(files[i])!=".yml" || files[i]==(defaultLocale+".yml")){continue;}
      var locale = files[i].split(".")[0];
      this.comparableDoc = this.toJSON(yamlFileURL,defaultLocale);
      this.doctorMissingElements(this.refDoc);
      this.writeToJS(jsDir,locale,this.comparableDoc);
    }
    console.log("\nYaml files from '"+pathChecker.resolve(yamlDir)+"' has been converted to js in '"+pathChecker.resolve(jsDir)+"'");
  },
  refDoc:null,
  comparableDoc:null,
  extractRefDoc:function(refDir,jsDir,defaultLocale){
    var refURL = refDir+"/"+defaultLocale+".yml";
    this.refDoc = this.toJSON(refURL,defaultLocale);
    this.writeToJS(jsDir,defaultLocale,this.refDoc);
  },
  writeToJS:function(jsDir,locale,content){
    var localeDir = jsDir+"/"+locale;
    if(!fs.existsSync(localeDir)){
      fs.mkdirSync(localeDir);
    }
    var jsFileURL = localeDir+"/translations.js";    
    content = "export default "+JSON.stringify(content)+";";
    fs.writeFileSync(jsFileURL, content);
  },
  toJSON:function(yamlFileURL,locale){
    var file = fs.readFileSync(yamlFileURL,'utf8');
    var doc = yaml.safeLoad(file);
    return doc[locale];
  },
  keyChain:[],
  doctorMissingElements:function(doc){
    for(var key in doc){        
      this.keyChain.push(key);
      var actual = this["refDoc."+this.keyChain.join(".")];
      var comparable = this["comparableDoc."+this.keyChain.join(".")];<commit_msg>convert interpolations<commit_after>/* jshint node: true */
'use strict';

/**
  * converts yaml file to js file
  * params: 
  *   --yaml-path=<YAML DIRECTORY LOCATION>
  *   --js-path=<JS DIRECTORY LOCATION>
  **/
var yaml = require('js-yaml');
var fs = require('fs');
var YAMLToJSON ={
  convert: function(){
    var pathChecker = require('path');
    var path = this.parsePath(arguments[0]);
    var yamlDir = path.yaml;
    var jsDir = path.js;
    var defaultLocale = path.locale;
    var files = fs.readdirSync(yamlDir);
    if(!fs.existsSync(jsDir)){
      fs.mkdirSync(jsDir);
    }
    this.extractRefDoc(yamlDir,jsDir,defaultLocale);
    for(var i=0;i<files.length;i++){
      var yamlFileURL = yamlDir+"/"+files[i];
      if(pathChecker.extname(files[i])!=".yml" || files[i]==(defaultLocale+".yml")){continue;}
      var locale = files[i].split(".")[0];
      this.comparableDoc = this.toJSON(yamlFileURL,defaultLocale);
      this.doctorMissingElements(this.refDoc);
      this.writeToJS(jsDir,locale,this.comparableDoc);
    }
    console.log("\nYaml files from '"+pathChecker.resolve(yamlDir)+"' has been converted to js in '"+pathChecker.resolve(jsDir)+"'");
  },
  refDoc:null,
  comparableDoc:null,
  extractRefDoc:function(refDir,jsDir,defaultLocale){
    var refURL = refDir+"/"+defaultLocale+".yml";
    this.refDoc = this.toJSON(refURL,defaultLocale);
    this.writeToJS(jsDir,defaultLocale,this.refDoc);
  },
  convertInterpolations:function(text){
    return text.replace(/%(\{\w+\})/g, '{$1}');
  },
  writeToJS:function(jsDir,locale,content){
    var localeDir = jsDir+"/"+locale;
    if(!fs.existsSync(localeDir)){
      fs.mkdirSync(localeDir);
    }
    var jsFileURL = localeDir+"/translations.js";    
    content = "export default "+this.convertInterpolations(JSON.stringify(content))+";";
    fs.writeFileSync(jsFileURL, content);
  },
  toJSON:function(yamlFileURL,locale){
    var file = fs.readFileSync(yamlFileURL,'utf8');
    var doc = yaml.safeLoad(file);
    return doc[locale];
  },
  keyChain:[],
  doctorMissingElements:function(doc){
    for(var key in doc){        
      this.keyChain.push(key);
      var actual = this["refDoc."+this.keyChain.join(".")];
      var comparable = this["comparableDoc."+this.keyChain.join(".")];
{'fraction_non_alphanumeric': 0.16991391028545538, 'fraction_numerical': 0.0020389669234254643, 'mean_word_length': 3.9495515695067263, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373880', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1493, 'n_words': 246}