<commit_before>var AstNode = require('./ast_node');

module.exports = function AstBuilder () {

  var stack = [new AstNode('None')];

  this.startRule = function (ruleType) {
    stack.push(new AstNode(ruleType));
  };

  this.endRule = function (ruleType) {
    var node = stack.pop();
    var transformedNode = transformNode(node);
    currentNode().add(node.ruleType, transformedNode);
  };

  this.build = function (token) {
    currentNode().add(token.matchedType, token);
  };

  this.getResult = function () {
    return currentNode().getSingle('Feature');
  };

  function currentNode () {
    return stack[stack.length - 1];
  }

  function getLocation (token, column) {
    return !column ? token.location : {line: token.location.line, column: column};
  }

  function getTags (node) {
    var tags = [];
    var tagsNode = node.getSingle('Tags');
    if (!tagsNode) return tags;
    tagsNode.getTokens('TagLine').forEach(function (token) {
      token.matchedItems.forEach(function (tagItem) {
        tags.push({
          location: getLocation(token, tagItem.column),
          name: tagItem.text
        });
      });

      return tags;
    });<commit_msg>Generate tags correctly for JS.<commit_after>var AstNode = require('./ast_node');

module.exports = function AstBuilder () {

  var stack = [new AstNode('None')];

  this.startRule = function (ruleType) {
    stack.push(new AstNode(ruleType));
  };

  this.endRule = function (ruleType) {
    var node = stack.pop();
    var transformedNode = transformNode(node);
    currentNode().add(node.ruleType, transformedNode);
  };

  this.build = function (token) {
    currentNode().add(token.matchedType, token);
  };

  this.getResult = function () {
    return currentNode().getSingle('Feature');
  };

  function currentNode () {
    return stack[stack.length - 1];
  }

  function getLocation (token, column) {
    return !column ? token.location : {line: token.location.line, column: column};
  }

  function getTags (node) {
    var tags = [];
    var tagsNode = node.getSingle('Tags');
    if (!tagsNode) return tags;
    tagsNode.getTokens('TagLine').forEach(function (token) {
      token.matchedItems.forEach(function (tagItem) {
        tags.push({
          location: getLocation(token, tagItem.column),
          name: tagItem.text
        });
      });

    });
    return tags;
{'fraction_non_alphanumeric': 0.14631043256997456, 'fraction_numerical': 0.0008481764206955047, 'mean_word_length': 3.153169014084507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6856680', 'n_tokens_mistral': 715, 'n_tokens_neox': 696, 'n_words': 154}