<commit_before>/**
 * Compiles a template Tree into JavaScript.
 *
 * @module compiler
 * @requires shared/register_partial
 * @requires shared/register_filter
 * @requires shared/get_partial
 * @requires shared/get_filter
 * @requires shared/map
 * @requires shared/encode
 * @requires utils/type
 * @requires utils/create_object
 */
define(function(require, exports, module) {
  "use strict";

  // Shared.
  var registerPartial = require("./shared/register_partial");
  var registerFilter = require("./shared/register_filter");
  var getPartial = require("./shared/get_partial");
  var getFilter = require("./shared/get_filter");
  var map = require("./shared/map");
  var encode = require("./shared/encode");

  // Utils.
  var type = require("./utils/type");
  var createObject = require("./utils/create_object");

  // Support.
  require("./support/array/map");
  require("./support/array/reduce");

  // Borrowed from Underscore.js template function.
  var escaper = /\\|'|\r|\n|\t|\u2028|\u2029/g;

  // Borrowed from Underscore.js template function.
  var escapes = {
    "'": "'",
    "\\": "\\",
    "\r": "r",
    "\n": "n",
    "\t": "t",
    "\u2028": "u2028",
    "\u2029": "u2029"
  };

  /**
   * Escapes passed values.
   *
   * @private
   * @param {string} value - The value to escape.
   * @returns {string} The value escaped.
   */
  function escapeValue(value) {
    return value.replace(escaper, function(match) {
      return "\\" + escapes[match];
    });
  }

  /**
   * Normalizes properties in the identifier to be looked up via hash-style
   * instead of dot-notation.
   *
   * @private
   * @param {string} identifier - The identifier to normalize.
   * @returns {string} The identifier normalized.
   */
  function normalizeIdentifier(identifier) {
    if (identifier === ".") {
      // '.' might be referencing either the root or a locally scoped variable
      // called '.'. So try for the locally scoped variable first and then
      // default to the root
      return "(data['.'] || data)";
    }

    return "data" + identifier.split(".").map(function(property) {
      return property ? "['" + property + "']" : "";
    }).join("");
  }

  /**
   * Represents a Compiler.
   *
   * @class
   * @memberOf module:compiler
   * @param {Tree} tree - A template [Tree]{@link module:tree.Tree} to compile.
   */
  function Compiler(tree) {
    this.tree = tree;
    this.string = "";

    // Optimization pass will flatten large templates to result in faster
    // rendering.
    var nodes = this.optimize(this.tree.nodes);
    var compiledSource = this.process(nodes);

    // The compiled function body.
    var body = [];

    // If there is a function, concatenate it to the default empty value.
    if (compiledSource) {
      compiledSource = " + " + compiledSource;
    }

    // Include map and its dependencies.
    if (compiledSource.indexOf("map(") > -1) {
      body.push(createObject, type, map);
    }

    // Include encode and its dependencies.
    if (compiledSource.indexOf("encode(") > -1) {
      body.push(type, encode);
    }

    // The compiled function body.
    body = body.concat([
      // Return the evaluated contents.
      "return ''" + compiledSource
    ]).join(";\n");

    // Create the JavaScript function from the source code.
    this.func = new Function("data", "template", body);

    // toString the function to get its raw source and expose.
    this.source = [
      "{",
        "_partials: {},",
        "_filters: {},",
        "getPartial: " + getPartial + ",",
        "getFilter: " + getFilter + ",",
        "registerPartial: " + registerPartial + ",",
        "registerFilter: " + registerFilter + ",",
        "render: function(data) {",
          "return " + this.func + "(data, this)",
        "}",
      "}"
    ].join("\n");
  }

  /**
   * Takes a pass over the full token set to optimize and eliminate potential
   * bugs associated with super large templates.
   *
   * @memberOf module:compiler.Compiler
   * @param {array} nodes
   * @return {array} optimized nodes
   */
  Compiler.prototype.optimize = function(nodes) {
    return nodes.reduce(function(memo, node) {
      var previous = memo[memo.length - 1];

      // Optimize text nodes together.
      if (previous && previous.type === "Text" && node.type === "Text") {
        previous.value += node.value;
      }

      // This is the first item
      else {
        memo.push(node);
      }<commit_msg>Updated compiler to spit out warnings instead of crashing<commit_after>/**
 * Compiles a template Tree into JavaScript.
 *
 * @module compiler
 * @requires shared/register_partial
 * @requires shared/register_filter
 * @requires shared/get_partial
 * @requires shared/get_filter
 * @requires shared/map
 * @requires shared/encode
 * @requires utils/type
 * @requires utils/create_object
 */
define(function(require, exports, module) {
  "use strict";

  // Shared.
  var registerPartial = require("./shared/register_partial");
  var registerFilter = require("./shared/register_filter");
  var getPartial = require("./shared/get_partial");
  var getFilter = require("./shared/get_filter");
  var map = require("./shared/map");
  var encode = require("./shared/encode");

  // Utils.
  var type = require("./utils/type");
  var createObject = require("./utils/create_object");

  // Support.
  require("./support/array/map");
  require("./support/array/reduce");

  // Borrowed from Underscore.js template function.
  var escaper = /\\|'|\r|\n|\t|\u2028|\u2029/g;

  // Borrowed from Underscore.js template function.
  var escapes = {
    "'": "'",
    "\\": "\\",
    "\r": "r",
    "\n": "n",
    "\t": "t",
    "\u2028": "u2028",
    "\u2029": "u2029"
  };

  /**
   * Escapes passed values.
   *
   * @private
   * @param {string} value - The value to escape.
   * @returns {string} The value escaped.
   */
  function escapeValue(value) {
    return value.replace(escaper, function(match) {
      return "\\" + escapes[match];
    });
  }

  /**
   * Normalizes properties in the identifier to be looked up via hash-style
   * instead of dot-notation.
   *
   * @private
   * @param {string} identifier - The identifier to normalize.
   * @returns {string} The identifier normalized.
   */
  function normalizeIdentifier(identifier) {
    if (identifier === ".") {
      // '.' might be referencing either the root or a locally scoped variable
      // called '.'. So try for the locally scoped variable first and then
      // default to the root
      return "(data['.'] || data)";
    }

    return "data" + identifier.split(".").map(function(property) {
      return property ? "['" + property + "']" : "";
    }).join("");
  }

  /**
   * Represents a Compiler.
   *
   * @class
   * @memberOf module:compiler
   * @param {Tree} tree - A template [Tree]{@link module:tree.Tree} to compile.
   */
  function Compiler(tree) {
    this.tree = tree;
    this.string = "";

    // Optimization pass will flatten large templates to result in faster
    // rendering.
    var nodes = this.optimize(this.tree.nodes);
    var compiledSource = this.process(nodes);

    // The compiled function body.
    var body = [];

    // If there is a function, concatenate it to the default empty value.
    if (compiledSource) {
      compiledSource = " + " + compiledSource;
    }

    // Include map and its dependencies.
    if (compiledSource.indexOf("map(") > -1) {
      body.push(createObject, type, map);
    }

    // Include encode and its dependencies.
    if (compiledSource.indexOf("encode(") > -1) {
      body.push(type, encode);
    }

    // The compiled function body.
    body = body.concat([
      // Return the evaluated contents.
      "return ''" + compiledSource
    ]).join(";\n");

    // Create the JavaScript function from the source code.
    this.func = new Function("data", "template", body);

    // toString the function to get its raw source and expose.
    this.source = [
      "{",
        "_partials: {},",
        "_filters: {},",
        "getPartial: " + getPartial + ",",
        "getFilter: " + getFilter + ",",
        "registerPartial: " + registerPartial + ",",
        "registerFilter: " + registerFilter + ",",
        "render: function(data) {",
          "try {",
            "return " + this.func + "(data, this);",
          "} catch (ex) {",
            type, encode,
            "console.error(ex);",
            "return '<pre>' + encode(ex + ex.stack) + '</pre>';",
          "}",
        "}",
      "}"
    ].join("\n");
  }

  /**
   * Takes a pass over the full token set to optimize and eliminate potential
   * bugs associated with super large templates.
   *
   * @memberOf module:compiler.Compiler
   * @param {array} nodes
   * @return {array} optimized nodes
   */
  Compiler.prototype.optimize = function(nodes) {
    return nodes.reduce(function(memo, node) {
      var previous = memo[memo.length - 1];

      // Optimize text nodes together.
      if (previous && previous.type === "Text" && node.type === "Text") {
        previous.value += node.value;
      }

      // This is the first item
      else {
        memo.push(node);
      }
{'fraction_non_alphanumeric': 0.15613058194125995, 'fraction_numerical': 0.005895840157222404, 'mean_word_length': 2.5095785440613025, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747569', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2700, 'n_words': 747}