<commit_before>'use strict';

var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var through2 = require('through2');
var gettextParser = require('gettext-parser');

// Publish a Node.js require() handler for .handlebars and .hbs files
var extension = function(module, filename) {
  module.exports = fs.readFileSync(filename, 'utf8');
};

require.extensions['.handlebars'] = extension;
require.extensions['.hbs'] = extension;

var replaceText = function(catalog, opts, chunk, enc, callback) {
  var template = _.template(chunk.toString(), {
    interpolate: opts.interpolate || /{{trans\s"([\s\S]+?)"}}/g
  });

  var translatedString = template(catalog)
    .replace(/\n/g, '\\n')
    .replace(/"/g, '\\"');

  var chunkString = '';

  chunkString += 'module.exports = "';
  chunkString += translatedString;
  chunkString += '";';

  callback(null, chunkString);
};

var filterInterpolator = function(file, catalog, opts) {
  if(file.split('.').pop() !== 'hbs') {
    return through2();
  }

  return through2(_.partial(replaceText, catalog, opts));
};

var getCatalog = function(locale, localeDirs) {
  var poParser = function(localeDir) {
    var fp = path.join(localeDir, locale, 'LC_MESSAGES', 'messages.po');
    var hasPo = fs.existsSync(fp);
    var po = hasPo ? fs.readFileSync(fp, {encoding: 'utf8'}) : null;
    var catalog = po ? gettextParser.po.parse(po).translations[''] : {};

    return catalog;
  };

  var catalogParser = function(defaultLang, localeDirs) {
    if (locale === defaultLang) {
      return {};
    }

    var jsonPoArray = _.map(localeDirs, poParser);

    return _.reduce(jsonPoArray, _.defaults);
  };

  var catalog = catalogParser('en', localeDirs);

  return _.transform(catalog, function(acc, msgObject, msgKey) {
    var msgId = msgObject.msgid;
    var msg = (msgObject.msgstr && msgObject.msgstr[0]) || msgId;
    acc[msgId] = msg;
  });
};

var i18n = function(file, opts) {
  var locale = opts.locale;
  var localeDirs = opts.localeDirs;
  var catalog = getCatalog(locale, localeDirs);

  return filterInterpolator(file, catalog, opts);
};

i18n.fast = function(fastOpts) {
  var locale = fastOpts.locale;
  var localeDirs = fastOpts.localeDirs;
  var catalog = getCatalog(locale, localeDirs);

  return function(file, opts) {
    var mergedOpts = {};

    _.extend(mergedOpts, opts, fastOpts);

    return filterInterpolator(file, catalog, mergedOpts);
  };
};

module.exports = i18n;<commit_msg>logic overhaul - remove _.template, replace with while loop for string-escape safety - handle default language by using empty catalog instead of explicit kickout - kick out of translating non-translatable extentions immediately in i18n/fast exports<commit_after>'use strict';

var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var through2 = require('through2');
var gettextParser = require('gettext-parser');

// Publish a Node.js require() handler for .handlebars and .hbs files
var extension = function(module, filename) {
  module.exports = fs.readFileSync(filename, 'utf8');
};
require.extensions['.handlebars'] = extension;
require.extensions['.hbs'] = extension;


var addSlash = {
  '\n': '\\n',
  '"': '\\"'
};

var escapeMatch = function(match) {
  return addSlash[match];
};

var getTranslator = function(catalog, opts) {
  var re = opts.interpolate || /\{\{trans\s*(?:"([^"]+)"|\'([^\']+)\')\s*\}\}/g;

  return function(chunk, enc, callback) {
    var template = chunk.toString();
    var match, msgid, needle, translated;

    translated = template;
    match = re.exec(template);

    while (match) {
      needle = match[0];
      msgid = match[1] || match[2];
      translated = translated.replace(needle, catalog[msgid] || msgid);
      match = re.exec(template);
    }

    var translatedChunk = translated
      .replace(/(\n|")/g, escapeMatch);

    var moduleString = 'module.exports = "' + translatedChunk + '";';

    callback(null, moduleString);
  };
};

var getJSONCatalog = function(locale, localeDirs) {
  var catalogDir = function(localeDir) {
    var fp = path.join(localeDir, locale, 'LC_MESSAGES', 'messages.po');
    var hasPo = fs.existsSync(fp);
    var po = hasPo ? fs.readFileSync(fp, {encoding: 'utf8'}) : null;
    var catalog = po ? gettextParser.po.parse(po).translations[''] : {};

    return catalog;
  };

  var makeMasterCatalog = function(localeDirs) {
    var catalogs = _.map(localeDirs, catalogDir);
    return _.reduce(catalogs, _.defaults);
  };

  var catalog = makeMasterCatalog(localeDirs);

  return _.transform(catalog, function(acc, msgObject, msgKey) {
    var msgId = msgObject.msgid;
    var msg = (msgObject.msgstr && msgObject.msgstr[0]) || msgId;
    acc[msgId] = msg;
  });
};

var acceptedExtensions = [
  '.handlebars',
  '.hbs'
];

var translatable = function(file) {
  var ext = path.extname(file);
  return _.include(acceptedExtensions, ext);
};

var i18n = function(file, opts) {
  if(!translatable(file)) {
    return through2();
  }
  var catalog = getJSONCatalog(opts.locale, opts.localeDirs);
  var translator = getTranslator(catalog, opts);
  return through2(translator);
};

i18n.fast = function(fastOpts) {
  var catalog = getJSONCatalog(fastOpts.locale, fastOpts.localeDirs);

  return function(file, opts) {
    if(!translatable(file)) {
      return through2();
    }
    var mergedOpts = _.extend({}, opts, fastOpts);
    var translator = getTranslator(catalog, mergedOpts);
    return through2(translator);
  };
};

module.exports = i18n;
{'fraction_non_alphanumeric': 0.15781844537053813, 'fraction_numerical': 0.005979344084073202, 'mean_word_length': 3.9909584086799277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4053034', 'n_tokens_mistral': 1802, 'n_tokens_neox': 1765, 'n_words': 421}