<commit_before>/* global window */

import Handlebars from "handlebars";
import {jsdom} from "jsdom";
import I18nliner from "i18nliner";
import Errors from "./errors";

var dom = (function(){
  if (typeof window !== 'undefined') {
    return window.document;
  } else {
    return jsdom().parentWindow.document;
  }
})();

var CallHelpers = I18nliner.CallHelpers
  , AST = Handlebars.AST
  , StringNode = AST.StringNode
  , BooleanNode = AST.BooleanNode
  , HashNode = AST.HashNode
  , SexprNode = AST.SexprNode
  , IdNode = AST.IdNode
  , slice = [].slice
  , TEMP_PLACEHOLDER = /__i18nliner_\d+__/g;

function TempPlaceholderMap() {
  this._map = {};
  this._size = 0;
}
TempPlaceholderMap.prototype.get = function(key) {
  return this._map[key];
};
TempPlaceholderMap.prototype.add = function(node) {
  var key = "__i18nliner_" + this._size + "__";
  this._map[key] = node;
  this._size++;
  return key;
};

function splitCapture(pattern, string) {
  var result = []
    , lastIndex = pattern.lastIndex = 0
    , match
    , s;
  while ((match = pattern.exec(string))) {
    match = match[0];
    s = string.slice(lastIndex, pattern.lastIndex - match.length);
    if (s.length) result.push(s);
    result.push(match);
    lastIndex = pattern.lastIndex;
  }
  s = string.slice(lastIndex);
  if (s.length) result.push(s);
  return result;
}

function stringNode(string) {
  return new StringNode(string);
}

function safeNode(string) {
  var parts = [
    new IdNode([{part: '__i18nliner_safe'}]),
    new StringNode(string)
  ];
  return new SexprNode(parts);
}

function escapeNode(sexpr) {
  var parts = [
    new IdNode([{part: '__i18nliner_escape'}]),
    sexpr
  ];
  return new SexprNode(parts);
}

function concatNode(string, tempMap) {
  var parts = splitCapture(TEMP_PLACEHOLDER, string)
    , partsLen = parts.length
    , part
    , i;
  for (i = 0; i < partsLen; i++) {
    part = parts[i];
    parts[i] = part.match(TEMP_PLACEHOLDER) ?
      escapeNode(tempMap.get(part)) :
      stringNode(part);
  }
  parts.unshift(new IdNode([{part: '__i18nliner_concat'}]));
  return new SexprNode(parts);
}

var PreProcessor = {
  process: function(ast) {
    var statements = ast.statements
      , statementsLen = statements.length
      , statement
      , i;
    for (i = 0; i < statementsLen; i++) {
      statement = statements[i];
      if (statement.type !== 'block') continue;
      // consume anything inside this first (e.g. if we have nested t blocks)
      this.process(statement.program);
      if (statement.inverse)
        this.process(statement.inverse);
      if (statement.mustache.id.string === "t") {
        statements[i] = this.transform(statement);
      }
    }
  },

  transform: function(node) {
    var parts = this.inferParts(node.program.statements)
      , defaultValue = parts.defaultValue
      , hash = parts.hash
      , pairs;
    node = node.mustache;
    if (!node.params.length) {
      node.params.push(this.inferKey(defaultValue));
      hash.push(["i18n_inferred_key", new BooleanNode(true)]);
    }
    node.params.push(new StringNode(defaultValue));
    node.isHelper = 1;
    node.sexpr.isHelper = 1;
    if (hash.length) {
      if (!node.hash)
        node.hash = node.sexpr.hash = new HashNode([]);
      pairs = node.hash.pairs;
      node.hash.pairs = pairs.concat(hash);
    }
    return node;
  },

  inferKey: function(defaultValue) {
    return new StringNode(CallHelpers.inferKey(defaultValue));
  },
<commit_msg>split this into two methods for easier extensibility<commit_after>/* global window */

import Handlebars from "handlebars";
import {jsdom} from "jsdom";
import I18nliner from "i18nliner";
import Errors from "./errors";

var dom = (function(){
  if (typeof window !== 'undefined') {
    return window.document;
  } else {
    return jsdom().parentWindow.document;
  }
})();

var CallHelpers = I18nliner.CallHelpers
  , AST = Handlebars.AST
  , StringNode = AST.StringNode
  , BooleanNode = AST.BooleanNode
  , HashNode = AST.HashNode
  , SexprNode = AST.SexprNode
  , IdNode = AST.IdNode
  , slice = [].slice
  , TEMP_PLACEHOLDER = /__i18nliner_\d+__/g;

function TempPlaceholderMap() {
  this._map = {};
  this._size = 0;
}
TempPlaceholderMap.prototype.get = function(key) {
  return this._map[key];
};
TempPlaceholderMap.prototype.add = function(node) {
  var key = "__i18nliner_" + this._size + "__";
  this._map[key] = node;
  this._size++;
  return key;
};

function splitCapture(pattern, string) {
  var result = []
    , lastIndex = pattern.lastIndex = 0
    , match
    , s;
  while ((match = pattern.exec(string))) {
    match = match[0];
    s = string.slice(lastIndex, pattern.lastIndex - match.length);
    if (s.length) result.push(s);
    result.push(match);
    lastIndex = pattern.lastIndex;
  }
  s = string.slice(lastIndex);
  if (s.length) result.push(s);
  return result;
}

function stringNode(string) {
  return new StringNode(string);
}

function safeNode(string) {
  var parts = [
    new IdNode([{part: '__i18nliner_safe'}]),
    new StringNode(string)
  ];
  return new SexprNode(parts);
}

function escapeNode(sexpr) {
  var parts = [
    new IdNode([{part: '__i18nliner_escape'}]),
    sexpr
  ];
  return new SexprNode(parts);
}

function concatNode(string, tempMap) {
  var parts = splitCapture(TEMP_PLACEHOLDER, string)
    , partsLen = parts.length
    , part
    , i;
  for (i = 0; i < partsLen; i++) {
    part = parts[i];
    parts[i] = part.match(TEMP_PLACEHOLDER) ?
      escapeNode(tempMap.get(part)) :
      stringNode(part);
  }
  parts.unshift(new IdNode([{part: '__i18nliner_concat'}]));
  return new SexprNode(parts);
}

var PreProcessor = {
  process: function(ast) {
    var statements = ast.statements
      , statementsLen = statements.length
      , statement
      , i;
    for (i = 0; i < statementsLen; i++) {
      statement = this.processStatement(statements[i]);
      if (typeof statement !== 'undefined')
        statements[i] = statement;
    }
  },

  processStatement: function(statement) {
    if (statement.type !== 'block') return;

    // consume anything inside this first (e.g. if we have nested t blocks)
    this.process(statement.program);
    if (statement.inverse)
      this.process(statement.inverse);

    if (statement.mustache.id.string !== "t") return;

    return this.transform(statement);
  },

  transform: function(node) {
    var parts = this.inferParts(node.program.statements)
      , defaultValue = parts.defaultValue
      , hash = parts.hash
      , pairs;
    node = node.mustache;
    if (!node.params.length) {
      node.params.push(this.inferKey(defaultValue));
      hash.push(["i18n_inferred_key", new BooleanNode(true)]);
    }
    node.params.push(new StringNode(defaultValue));
    node.isHelper = 1;
    node.sexpr.isHelper = 1;
    if (hash.length) {
      if (!node.hash)
        node.hash = node.sexpr.hash = new HashNode([]);
      pairs = node.hash.pairs;
      node.hash.pairs = pairs.concat(hash);
    }
    return node;
  },

  inferKey: function(defaultValue) {
    return new StringNode(CallHelpers.inferKey(defaultValue));
  },

{'fraction_non_alphanumeric': 0.141229427486285, 'fraction_numerical': 0.007033338022225348, 'mean_word_length': 3.2195845697329375, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6773863', 'n_tokens_mistral': 2453, 'n_tokens_neox': 2380, 'n_words': 508}