<commit_before>
var at = require('../../ast').js;
var tmplAt = require('../../ast').tmpl;

module.exports = function(flow){
  function packKey(key){
    return l10nIndex.keys.hasOwnProperty(key)
      ? '#' + l10nIndex.keys[key].toString(36)
      : '';
  }

  function packBind(name){
    if (name.substr(0, 5) == 'l10n:')
    {
      var packed = packKey(name.substr(5));

      if (packed)
      {
        packed = 'l10n:' + packed;

        fconsole.log(name + ' -> ' + packed);

        return packed;
      }
      else
        fconsole.log('[!] l10n key ' + packed + ' not found (ignored)');
    }

    return name;
  }

  var fconsole = flow.console;

  if (flow.options.l10nPack)
  {
    var l10nIndex = flow.l10n.index;

    if (!l10nIndex)
      throw 'l10n index must be built before compression';

    //
    // pack definitions
    //
    fconsole.start('Pack createDictionary');
    flow.l10n.defList.forEach(function(entry){
      fconsole.log(entry.name);

      var dict = {};
      dict[entry.name] = entry.keys;

      entry.args[2] = ['array', packDictionary(dict, l10nIndex.map).map(function(token){
        return [typeof token == 'number' ? 'num' : 'string', token];
      })];
    });
    fconsole.endl();


    //
    // pack getToken
    //
    fconsole.start('Pack getToken');
    flow.l10n.getTokenList.forEach(function(entry){
      var key = entry.args[0][1];
      var packed = packKey(key);

      if (packed)
      {
        fconsole.log(key + ' -> ' + packed);
        entry.args[0][1] = packed;
      }
      else
        fconsole.log('[!] l10n key ' + key + ' not found (ignored)');

    });
    fconsole.endl();


    //
    // pack packages
    //
    fconsole.start('Pack packages');
    flow.l10n.packages.forEach(function(file){
      fconsole.log(file.jsRef);
      file.jsResourceContent = packDictionary(file.jsResourceContent, l10nIndex.map);
    });
    fconsole.endl();


    //
    // process templates
    //
    fconsole.start('Pack keys in templates');
    flow.files.queue.forEach(function(file){
      if (file.type == 'template')
      {
        fconsole.start(file.relpath);

        tmplAt.walk(file.ast, {
          text: function(token){
            if (token[1])
              token[1] = packBind(token[1]);
          },
          attr: function(token){
            var name = this.tokenName(token);
            if (token[1] && token[0] == 2 && name != 'class' && name != 'style')
              token[1][0] = token[1][0].map(packBind)
          }
        });

        fconsole.endl();
      }<commit_msg>make stable for non-string bindings<commit_after>
var at = require('../../ast').js;
var tmplAt = require('../../ast').tmpl;

module.exports = function(flow){
  function packKey(key){
    return l10nIndex.keys.hasOwnProperty(key)
      ? '#' + l10nIndex.keys[key].toString(36)
      : '';
  }

  function packBind(name){
    if (typeof name == 'string' && name.substr(0, 5) == 'l10n:')
    {
      var packed = packKey(name.substr(5));

      if (packed)
      {
        packed = 'l10n:' + packed;

        fconsole.log(name + ' -> ' + packed);

        return packed;
      }
      else
        fconsole.log('[!] l10n key ' + packed + ' not found (ignored)');
    }

    return name;
  }

  var fconsole = flow.console;

  if (flow.options.l10nPack)
  {
    var l10nIndex = flow.l10n.index;

    if (!l10nIndex)
      throw 'l10n index must be built before compression';

    //
    // pack definitions
    //
    fconsole.start('Pack createDictionary');
    flow.l10n.defList.forEach(function(entry){
      fconsole.log(entry.name);

      var dict = {};
      dict[entry.name] = entry.keys;

      entry.args[2] = ['array', packDictionary(dict, l10nIndex.map).map(function(token){
        return [typeof token == 'number' ? 'num' : 'string', token];
      })];
    });
    fconsole.endl();


    //
    // pack getToken
    //
    fconsole.start('Pack getToken');
    flow.l10n.getTokenList.forEach(function(entry){
      var key = entry.args[0][1];
      var packed = packKey(key);

      if (packed)
      {
        fconsole.log(key + ' -> ' + packed);
        entry.args[0][1] = packed;
      }
      else
        fconsole.log('[!] l10n key ' + key + ' not found (ignored)');

    });
    fconsole.endl();


    //
    // pack packages
    //
    fconsole.start('Pack packages');
    flow.l10n.packages.forEach(function(file){
      fconsole.log(file.jsRef);
      file.jsResourceContent = packDictionary(file.jsResourceContent, l10nIndex.map);
    });
    fconsole.endl();


    //
    // process templates
    //
    fconsole.start('Pack keys in templates');
    flow.files.queue.forEach(function(file){
      if (file.type == 'template')
      {
        fconsole.start(file.relpath);

        tmplAt.walk(file.ast, {
          text: function(token){
            if (token[1])
              token[1] = packBind(token[1]);
          },
          attr: function(token){
            var name = this.tokenName(token);
            if (token[1] && token[0] == 2 && name != 'class' && name != 'style')
              token[1][0] = token[1][0].map(packBind);
          }
        });

        fconsole.endl();
      }
{'fraction_non_alphanumeric': 0.16801853997682503, 'fraction_numerical': 0.020084974893781384, 'mean_word_length': 2.048263684520306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741707', 'n_tokens_mistral': 1767, 'n_tokens_neox': 1686, 'n_words': 311}