<commit_before>/* ************************************************************************
 *
 *    qooxdoo-compiler - node.js based replacement for the Qooxdoo python
 *    toolchain
 *
 *    https://github.com/qooxdoo/qooxdoo-compiler
 *
 *    Copyright:
 *      2011-2017 Zenesis Limited, http://www.zenesis.com
 *
 *    License:
 *      MIT: https://opensource.org/licenses/MIT
 *
 *      This software is provided under the same licensing terms as Qooxdoo,
 *      please see the LICENSE file in the Qooxdoo project's top-level directory
 *      for details.
 *
 *    Authors:
 *      * <NAME> (<EMAIL>, @johnspackman)
 *
 * ************************************************************************/

var path = require("path");
require("qooxdoo");
var fs = require("fs");
var async = require("async");
var util = require("../util");

const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);

var log = util.createLog("translation");

/**
 * Reads and writes .po files for translation
 */
module.exports = qx.Class.define("qx.tool.compiler.app.Translation", {
  extend: qx.core.Object,

  /**
   * Constructor
   * @param library {Library}
   * @param locale {String}
   */
  construct: function(library, locale) {
    this.base(arguments);
    this.setLibrary(library);
    if (locale)
      this.setLocale(locale);
    this.__translations = {};
    this.__headers = {};
  },

  properties: {
    /** The library that this translation is for */
    library: {
      nullable: false,
      check: "qx.tool.compiler.app.Library"
    },

    /** The locale */
    locale: {
      init: "en",
      nullable: false,
      check :"String"
    }
  },

  members: {
    __translations: null,
    __headers: null,
    __mtime: 0,

    /**
     * Filename for the .po file
     * @returns {string}
     */
    getPoFilename: function() {
      var library = this.getLibrary();
      return library.getRootDir() + "/" + library.getTranslationPath() + "/" + this.getLocale() + ".po";
    },

    /**
     * Reads the .po file, but only if it has not been loaded or has changed
     * 
     * @returns {Promise}|
     */
    checkRead: async function() {
      var t = this;
      if (!this.__mtime)
        return this.read();
      var poFile = this.getPoFilename();
      return new Promise((resolve) => {
          fs.stat(poFile, function (err, stat) {
            if (err)
              return err.code == "ENOENT" ? resolve() : reject(err);
            if (t.__mtime == stat.mtime)
              return resolve();
            return t.read(cb);
          });
        });
    },

    /**
     * Reads the .po file
     */
    read: async function() {
      var t = this;
      if (t.__onRead)
        return t.__onRead;
      
      return t.__onRead = new Promise((resolve, reject) => {
        t.__translations = {};
        t.__headers = {};
        var poFile = this.getPoFilename();
        
        fs.stat(poFile, function(err, stat) {
          if (err) {
            if (err.code == "ENOENT")
              return resolve();
            return reject(err);
          }
          t.__mtime = stat.mtime;
          readFile(poFile, { encoding: "utf8" })
            .then((data) => {
              
              function saveEntry() {
                if (entry) {
                  var key;
                  if (entry.msgctxt)
                    key = entry.msgctxt + ":" + entry.msgid;
                  else
                    key = entry.msgid;
                  t.__translations[key] = entry;
                }
                entry = null;
                lastKey = null;
              }
  
              function set(key, value, append) {
                var index = null;
                var m = key.match(/^([^[]+)\[([0-9]+)\]$/);
                if (m) {
                  key = m[1];
                  index = parseInt(m[2]);
                  if (entry[key] === undefined)
                    entry[key] = [];
                  if (!append || typeof entry[key][index] !== "string")
                    entry[key][index] = value;
                  else
                    entry[key][index] += value;
                } else if (!append || typeof entry[key] !== "string")
                  entry[key] = value;
                else
                  entry[key] += value;
              }
  
              var entry = null;
              var lastKey = null;
              data.split('\n').forEach(function (line, lineNo) {
                line = line.trim();
                if (!line)
                  return saveEntry();
  
                if (!entry)
                  entry = {};
  
                // Comment?
                var m = line.match(/^#([^ ]?) (.*)$/);
                if (m) {
                  var type = m[1];
                  var comment = m[2];
                  var key;
                  if (!entry.comments)
                    entry.comments = {};
                  switch (type) {
                    case "":
                      entry.comments.translator = comment;
                      break;
  
                    case ".":
                      entry.comments.extracted = comment;
                      break;
  
                    case ":":
                      if (!entry.comments.reference) {
                        entry.comments.reference = {};
                      }
                      const ref = entry.comments.reference;
                      (comment.match(/[\w/\.]+:\d+/g) || []).forEach(entry => {
                        const split = entry.split(":");
                        const classname = split[0];
                        const lineNo = parseInt(split[1], 10);
                        if (!ref[classname]) {
                          ref[classname] = [lineNo];
                        } else if (!ref[classname].includes(lineNo)) {
                          ref[classname].push(lineNo);
                        }
                      });
                      break;
  
                    case ",":
                      entry.comments.flags = comment.split(',');
                      break;
  
                    case "|":<commit_msg>comment can be null as well<commit_after>/* ************************************************************************
 *
 *    qooxdoo-compiler - node.js based replacement for the Qooxdoo python
 *    toolchain
 *
 *    https://github.com/qooxdoo/qooxdoo-compiler
 *
 *    Copyright:
 *      2011-2017 Zenesis Limited, http://www.zenesis.com
 *
 *    License:
 *      MIT: https://opensource.org/licenses/MIT
 *
 *      This software is provided under the same licensing terms as Qooxdoo,
 *      please see the LICENSE file in the Qooxdoo project's top-level directory
 *      for details.
 *
 *    Authors:
 *      * <NAME> (<EMAIL>, @johnspackman)
 *
 * ************************************************************************/

var path = require("path");
require("qooxdoo");
var fs = require("fs");
var async = require("async");
var util = require("../util");

const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);

var log = util.createLog("translation");

/**
 * Reads and writes .po files for translation
 */
module.exports = qx.Class.define("qx.tool.compiler.app.Translation", {
  extend: qx.core.Object,

  /**
   * Constructor
   * @param library {Library}
   * @param locale {String}
   */
  construct: function(library, locale) {
    this.base(arguments);
    this.setLibrary(library);
    if (locale)
      this.setLocale(locale);
    this.__translations = {};
    this.__headers = {};
  },

  properties: {
    /** The library that this translation is for */
    library: {
      nullable: false,
      check: "qx.tool.compiler.app.Library"
    },

    /** The locale */
    locale: {
      init: "en",
      nullable: false,
      check :"String"
    }
  },

  members: {
    __translations: null,
    __headers: null,
    __mtime: 0,

    /**
     * Filename for the .po file
     * @returns {string}
     */
    getPoFilename: function() {
      var library = this.getLibrary();
      return library.getRootDir() + "/" + library.getTranslationPath() + "/" + this.getLocale() + ".po";
    },

    /**
     * Reads the .po file, but only if it has not been loaded or has changed
     * 
     * @returns {Promise}|
     */
    checkRead: async function() {
      var t = this;
      if (!this.__mtime)
        return this.read();
      var poFile = this.getPoFilename();
      return new Promise((resolve) => {
          fs.stat(poFile, function (err, stat) {
            if (err)
              return err.code == "ENOENT" ? resolve() : reject(err);
            if (t.__mtime == stat.mtime)
              return resolve();
            return t.read(cb);
          });
        });
    },

    /**
     * Reads the .po file
     */
    read: async function() {
      var t = this;
      if (t.__onRead)
        return t.__onRead;
      
      return t.__onRead = new Promise((resolve, reject) => {
        t.__translations = {};
        t.__headers = {};
        var poFile = this.getPoFilename();
        
        fs.stat(poFile, function(err, stat) {
          if (err) {
            if (err.code == "ENOENT")
              return resolve();
            return reject(err);
          }
          t.__mtime = stat.mtime;
          readFile(poFile, { encoding: "utf8" })
            .then((data) => {
              
              function saveEntry() {
                if (entry) {
                  var key;
                  if (entry.msgctxt)
                    key = entry.msgctxt + ":" + entry.msgid;
                  else
                    key = entry.msgid;
                  t.__translations[key] = entry;
                }
                entry = null;
                lastKey = null;
              }
  
              function set(key, value, append) {
                var index = null;
                var m = key.match(/^([^[]+)\[([0-9]+)\]$/);
                if (m) {
                  key = m[1];
                  index = parseInt(m[2]);
                  if (entry[key] === undefined)
                    entry[key] = [];
                  if (!append || typeof entry[key][index] !== "string")
                    entry[key][index] = value;
                  else
                    entry[key][index] += value;
                } else if (!append || typeof entry[key] !== "string")
                  entry[key] = value;
                else
                  entry[key] += value;
              }
  
              var entry = null;
              var lastKey = null;
              data.split('\n').forEach(function (line, lineNo) {
                line = line.trim();
                if (!line)
                  return saveEntry();
  
                if (!entry)
                  entry = {};
  
                // Comment?
                var m = line.match(/^#([^ ]?) (.*)$/);
                if (m) {
                  var type = m[1];
                  var comment = m[2];
                  var key;
                  if (!entry.comments)
                    entry.comments = {};
                  switch (type) {
                    case "":
                      entry.comments.translator = comment;
                      break;
  
                    case ".":
                      entry.comments.extracted = comment;
                      break;
  
                    case ":":
                      if (!entry.comments.reference) {
                        entry.comments.reference = {};
                      }
                      const ref = entry.comments.reference;
                      (comment && comment.match(/[\w/\.]+:\d+/g) || []).forEach(entry => {
                        const split = entry.split(":");
                        const classname = split[0];
                        const lineNo = parseInt(split[1], 10);
                        if (!ref[classname]) {
                          ref[classname] = [lineNo];
                        } else if (!ref[classname].includes(lineNo)) {
                          ref[classname].push(lineNo);
                        }
                      });
                      break;
  
                    case ",":
                      entry.comments.flags = comment.split(',');
                      break;
  
                    case "|":
{'fraction_non_alphanumeric': 0.15368403988003568, 'fraction_numerical': 0.0032422793223636217, 'mean_word_length': 1.2967237527922562, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3616288', 'n_tokens_mistral': 3422, 'n_tokens_neox': 3235, 'n_words': 756}