<commit_before>// Calculate dependency trees and watch them.
//
// Dependency trees are hierarchical representations of HTML/JS/CSS files.
// Each node looks like: [fileObj, children].
// * see fileobj.js for more information about file objects.
//   These file objects don't contain the parsing data. That is kept in a
//   separate dictionary.
// * children are the potential HTML/JS/CSS dependencies of the fileObj.

var _ = require('underscore');
var assert = require('assert');
var fileobj = require('./fileobj');
var oakstache = require('./oakstache/handler');
var path = require('path');
var queryselector = require('./queryselector');
var watcher = require('./watcher');

// Every tree has these dependencies.
var widgetDependencies =
    ['create.mjs', 'widget.mjs', 'rapid.mjs', 'public/widget.js',
     'public/extra.js']
    .map(function(x) {
  return { type: path.extname(x).substr(1), filename: __dirname + '/' + x };
});

/** Builds a tree structure given root file objects and file dictionary. */
function build(roots, files, callback) {
  (function recurse(fileObj, fileObjs, cycles, callback) {
    if (fileObj.filename) {
      if (cycles[fileObj.filename]) {
        return callback([fileObj, []]);
      } else {
        cycles = Object.create(cycles);
        cycles[fileObj.filename] = true;
      }
    }

    if (!fileObjs.length) {
      return callback([fileObj, []]);
    }

    var total = fileObjs.length;
    var childResults = new Array(total);
    fileObjs.forEach(function(childFileObj, i) {
      fileobj.fetch(files, childFileObj, function childCallback(err, data) {
        if (err && childFileObj.type == 'js' &&
            path.basename(childFileObj.filename) == 'widget.js') {
          // No widget.js file in local directory, so it must be referring to
          // redoak's widget. Start this particular child fetch over, with the
          // real path to widget.js.
          childFileObj = [];
          data = { fileObjs: widgetDependencies };
        } else if (err) {
          data = { error: err, fileObjs: [] };
        }

        if (childFileObj.filename) {
          files[childFileObj.filename] = data;
        } else if (childFileObj.data) {
          _.extend(childFileObj, data);
        }

        recurse(childFileObj, data.fileObjs, cycles, function(t) {
          childResults[i] = t;
          if (--total == 0) {
            callback([fileObj, childResults]);
          }
        });
      });
    });
  })([], roots, {}, callback);
}

/**
 * Expands a DOM subtree with its template dependencies.
 *
 * @param templates Reference of templates that can be applied.
 * @param node HTML template element or any element that needs to be expanded.
 * @param uses Use JSON structure from template.
 * @param view Transform unbound variables using render function.
 * @param render The function that will render the template AST with the given
 *               view and the view defined in uses.
 * @return Expanded uses structure. These qsis defines where the children
 *         widget elements are.
 */
function expandDOM(templates, node, uses, view, render) {
  var templateFor = function(use) {
    // Use the last mixin that matches a template.
    return use.mixins.reduce(function(x, y) {
      var index = _(templates).pluck('name').indexOf(y);
      return index >= 0 ? templates[index] : x;
    }, null);
  };

  var callbacks = uses.map(function(use, i) {
    // Map to a template. Use the last mixin to match a template.
    var template = templateFor(use);
    if (!template) {
      // TODO: post error system
      console.error('Cannot find template for use.', use.mixins);
      return null;
    }

    var obj;
    var jsonstr = oakstache.renderStringUnbound(use.obj, view);
    try {
      obj = JSON.parse(jsonstr);
    } catch(e) {
      console.error('Could not parse JSON for use.', jsonstr);
      return null;
    }
    var newView = _.extend(obj, view);

    var data = template.uses.length ? template.expandedData: template.data;
    var html = render(data, newView);

    var div = node.ownerDocument.createElement('div');
    div.innerHTML = html;

    var el = div.firstChild;
    while (el.nodeType != el.ELEMENT_NODE) {
      el = el.nextSibling;
    }

    // Insert into DOM.
    queryselector.inject(el, node, use);
    if (el.parentNode == div) {
      // TODO: post error system
      console.error('Queryselector inject did not work.', use);
      return null;
    }

    // Construct the QS must be done after everything is injected.
    return function() {
      var qsobj = queryselector.to(el, node);
      return _.extend({}, use, qsobj);
    };
  });

  return _(callbacks).compact().map(function(f) { return f(); });
}

/** Find all the templates in the tree. */
function allTemplates(tree) {
  var oakFilter = function(d) { return d.type == 'oak' && d.templates; };
  var fileObjs = tree.flatten();
  return (_.chain(fileObjs).filter(oakFilter).pluck('templates').flatten()
           .value());
}

/** TODO */
function tree(roots, callback) {
  var files = {};
  var firstTree = new Tree(
      files, [[], roots.map(function(x) { return [x, []]; })]);
  firstTree.recalculate(null, function(t) {
    callback(t);
  });
}

/** TODO */
function Tree(files, tree) {
  this.files_ = files;
  this.tree_ = tree;
}

Tree.prototype = {
  expand_: function() {
    var templates = _.flatten(allTemplates(this));
    templates.forEach(function(template) {
      var div = template.document.createElement('div');
      div.innerHTML = template.data;

      var expandedUses = expandDOM(
          templates, div.firstChild, template.uses, {},
          oakstache.renderStringUnbound);
      template.ast = oakstache.parse(template.data, {});
      template.expandedUses = expandedUses;
      template.expandedData = div.innerHTML;
    });
  },

  addRoot: function(fileObj, callback) {
    var files = _.extend({}, this.files_);
    var roots = _.pluck(this.tree_[1], '0');
    roots.push(fileObj);
    build(roots, files, function(t) {<commit_msg>AST should be generated from expanded data.<commit_after>// Calculate dependency trees and watch them.
//
// Dependency trees are hierarchical representations of HTML/JS/CSS files.
// Each node looks like: [fileObj, children].
// * see fileobj.js for more information about file objects.
//   These file objects don't contain the parsing data. That is kept in a
//   separate dictionary.
// * children are the potential HTML/JS/CSS dependencies of the fileObj.

var _ = require('underscore');
var assert = require('assert');
var fileobj = require('./fileobj');
var oakstache = require('./oakstache/handler');
var path = require('path');
var queryselector = require('./queryselector');
var watcher = require('./watcher');

// Every tree has these dependencies.
var widgetDependencies =
    ['create.mjs', 'widget.mjs', 'rapid.mjs', 'public/widget.js',
     'public/extra.js']
    .map(function(x) {
  return { type: path.extname(x).substr(1), filename: __dirname + '/' + x };
});

/** Builds a tree structure given root file objects and file dictionary. */
function build(roots, files, callback) {
  (function recurse(fileObj, fileObjs, cycles, callback) {
    if (fileObj.filename) {
      if (cycles[fileObj.filename]) {
        return callback([fileObj, []]);
      } else {
        cycles = Object.create(cycles);
        cycles[fileObj.filename] = true;
      }
    }

    if (!fileObjs.length) {
      return callback([fileObj, []]);
    }

    var total = fileObjs.length;
    var childResults = new Array(total);
    fileObjs.forEach(function(childFileObj, i) {
      fileobj.fetch(files, childFileObj, function childCallback(err, data) {
        if (err && childFileObj.type == 'js' &&
            path.basename(childFileObj.filename) == 'widget.js') {
          // No widget.js file in local directory, so it must be referring to
          // redoak's widget. Start this particular child fetch over, with the
          // real path to widget.js.
          childFileObj = [];
          data = { fileObjs: widgetDependencies };
        } else if (err) {
          data = { error: err, fileObjs: [] };
        }

        if (childFileObj.filename) {
          files[childFileObj.filename] = data;
        } else if (childFileObj.data) {
          _.extend(childFileObj, data);
        }

        recurse(childFileObj, data.fileObjs, cycles, function(t) {
          childResults[i] = t;
          if (--total == 0) {
            callback([fileObj, childResults]);
          }
        });
      });
    });
  })([], roots, {}, callback);
}

/**
 * Expands a DOM subtree with its template dependencies.
 *
 * @param templates Reference of templates that can be applied.
 * @param node HTML template element or any element that needs to be expanded.
 * @param uses Use JSON structure from template.
 * @param view Transform unbound variables using render function.
 * @param render The function that will render the template AST with the given
 *               view and the view defined in uses.
 * @return Expanded uses structure. These qsis defines where the children
 *         widget elements are.
 */
function expandDOM(templates, node, uses, view, render) {
  var templateFor = function(use) {
    // Use the last mixin that matches a template.
    return use.mixins.reduce(function(x, y) {
      var index = _(templates).pluck('name').indexOf(y);
      return index >= 0 ? templates[index] : x;
    }, null);
  };

  var callbacks = uses.map(function(use, i) {
    // Map to a template. Use the last mixin to match a template.
    var template = templateFor(use);
    if (!template) {
      // TODO: post error system
      console.error('Cannot find template for use.', use.mixins);
      return null;
    }

    var obj;
    var jsonstr = oakstache.renderStringUnbound(use.obj, view);
    try {
      obj = JSON.parse(jsonstr);
    } catch(e) {
      console.error('Could not parse JSON for use.', jsonstr);
      return null;
    }
    var newView = _.extend(obj, view);

    var data = template.uses.length ? template.expandedData: template.data;
    var html = render(data, newView);

    var div = node.ownerDocument.createElement('div');
    div.innerHTML = html;

    var el = div.firstChild;
    while (el.nodeType != el.ELEMENT_NODE) {
      el = el.nextSibling;
    }

    // Insert into DOM.
    queryselector.inject(el, node, use);
    if (el.parentNode == div) {
      // TODO: post error system
      console.error('Queryselector inject did not work.', use);
      return null;
    }

    // Construct the QS must be done after everything is injected.
    return function() {
      var qsobj = queryselector.to(el, node);
      return _.extend({}, use, qsobj);
    };
  });

  return _(callbacks).compact().map(function(f) { return f(); });
}

/** Find all the templates in the tree. */
function allTemplates(tree) {
  var oakFilter = function(d) { return d.type == 'oak' && d.templates; };
  var fileObjs = tree.flatten();
  return (_.chain(fileObjs).filter(oakFilter).pluck('templates').flatten()
           .value());
}

/** TODO */
function tree(roots, callback) {
  var files = {};
  var firstTree = new Tree(
      files, [[], roots.map(function(x) { return [x, []]; })]);
  firstTree.recalculate(null, function(t) {
    callback(t);
  });
}

/** TODO */
function Tree(files, tree) {
  this.files_ = files;
  this.tree_ = tree;
}

Tree.prototype = {
  expand_: function() {
    var templates = _.flatten(allTemplates(this));
    templates.forEach(function(template) {
      var div = template.document.createElement('div');
      div.innerHTML = template.data;

      var expandedUses = expandDOM(
          templates, div.firstChild, template.uses, {},
          oakstache.renderStringUnbound);
      template.expandedUses = expandedUses;
      template.expandedData = div.innerHTML;
      template.ast = oakstache.parse(template.expandedData, {});
    });
  },

  addRoot: function(fileObj, callback) {
    var files = _.extend({}, this.files_);
    var roots = _.pluck(this.tree_[1], '0');
    roots.push(fileObj);
    build(roots, files, function(t) {
{'fraction_non_alphanumeric': 0.1347675568743818, 'fraction_numerical': 0.0008242664029014178, 'mean_word_length': 2.7963078848560703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1635402', 'n_tokens_mistral': 3601, 'n_tokens_neox': 3540, 'n_words': 1052}