<commit_before>var traverse = require("./traverse");
var astTypes = require("ast-types");
var recast   = require("recast");
var path     = require("path");
var fs       = require("fs");
var _        = require("lodash");

var n = astTypes.namedTypes;
var b = astTypes.builders;

exports.ensureBlock = function (node) {
  var block = node.body;
  if (block.type === "BlockStatement") return;

  if (!_.isArray(block)) {
    if (!n.Statement.check(block)) block = b.returnStatement(block);
    block = [block];
  }

  node.body = b.blockStatement(block);
};

exports.isPattern = function (node) {
  return node.type === "ArrayPattern" || node.type === "ObjectPattern";
};

exports.isAbsolute = function (loc) {
  if (!loc) return false;
  if (loc[0] === "/") return true; // unix
  if (loc[1] === ":" && loc[2] === "\\") return true; // windows
  return false;
};

exports.removeProperties = function (tree) {
  var clear = function (node) {
    delete node.tokens;
    delete node.range;
    delete node.loc;
  };

  clear(tree);
  traverse(tree, clear);

  return tree;
};

exports.getIds = function (node) {
  var search = [node];
  var ids  = [];

  while (search.length) {
    var id = search.shift();

    if (id.type === "Identifier") {
      ids.push(id.name);
    } else if (id.type === "ArrayPattern") {
      _.each(id.elements, function (elem) {
        search.push(elem);
      });
    } else if (id.type === "ObjectPattern") {
      _.each(id.properties, function (prop) {
        search.push(prop.value);
      });
    } else {
      throw new Error("unknown node " + id.type);
    }
  }

  return ids;
};

exports.errorWithNode = function (node, msg) {
  var line = node.loc.start.line;
  var col  = node.loc.start.column;

  var err = new SyntaxError("Line " + line + ": " + msg);
  err.lineNumber = line;
  err.column = col;
  return err;
};

exports.canCompile = function (filename) {
  var ext = path.extname(filename);
  return _.contains([".js", ".es6", ".jsx"], ext);
};

exports.sourceMapToComment = function (map) {
  var json = JSON.stringify(map);
  var base64 = new Buffer(json).toString("base64");
  return "//# sourceMappingURL=data:application/json;base64," + base64;
};

exports.pushMutatorMap = function (mutatorMap, key, kind, method) {
  var map = mutatorMap[key] = mutatorMap[key] || {};
  if (map[kind]) {
    throw new Error("a " + kind + " already exists for this property");
  } else {
    map[kind] = method;
  }
};

exports.buildDefineProperties = function (mutatorMap, keyNode) {
  var objExpr = b.objectExpression([]);

  _.each(mutatorMap, function (map, key) {
    var mapNode = b.objectExpression([]);

    var propNode = b.property("init", b.identifier(key), mapNode);

    _.each(map, function (node, key) {
      node = _.clone(node);
      if (node.type === "MethodDefinition") node = node.value;
      mapNode.properties.push(b.property("init", b.identifier(key), node));
    });

    objExpr.properties.push(propNode);
  });

  return exports.template("object-define-properties", {
    OBJECT: keyNode,
    PROPS: objExpr
  }, true);
};

exports.template = function (name, nodes, keepExpression) {
  var template = exports.templates[name];
  if (!template) throw new ReferenceError("unknown template " + name);

  template = _.cloneDeep(template);

  if (!_.isEmpty(nodes)) {
    traverse(template, function (node) {
      if (node.type === "Identifier" && _.has(nodes, node.name)) {
        var newNode = nodes[node.name];
        if (_.isString(newNode)) {
          node.name = newNode;
        } else {
          return newNode;
        }
      }
    });
  }

  var node = template.body[0];

  if (!keepExpression && node.type === "ExpressionStatement") {
    return node.expression;
  } else {
    return node;
  }
};

exports.codeFrame = function (lines, lineNumber, colNumber) {
  colNumber = Math.max(colNumber, 0);

  lines = lines.split("\n");
  var start = Math.max(lineNumber - 3, 0);
  var end   = Math.min(lines.length, lineNumber + 3);
  var width = (end + "").length;

  return "\n" + lines.slice(start, end).map(function (line, i) {
    var curr = i + start + 1;

    var gutter = curr === lineNumber ? "> " : "  ";

    var sep = curr + exports.repeat(width + 1);
    gutter += sep + "| ";

    var str = gutter + line;

    if (colNumber && curr === lineNumber) {
      str += "\n";
      str += exports.repeat(gutter.length - 2);
      str += "|" + exports.repeat(colNumber) + "^";
    }

    return str;
  }).join("\n");<commit_msg>add util.isReferenced to check if a node is directly referenced<commit_after>var traverse = require("./traverse");
var astTypes = require("ast-types");
var recast   = require("recast");
var path     = require("path");
var fs       = require("fs");
var _        = require("lodash");

var n = astTypes.namedTypes;
var b = astTypes.builders;

exports.ensureBlock = function (node) {
  var block = node.body;
  if (block.type === "BlockStatement") return;

  if (!_.isArray(block)) {
    if (!n.Statement.check(block)) block = b.returnStatement(block);
    block = [block];
  }

  node.body = b.blockStatement(block);
};

exports.isPattern = function (node) {
  return node.type === "ArrayPattern" || node.type === "ObjectPattern";
};

exports.isAbsolute = function (loc) {
  if (!loc) return false;
  if (loc[0] === "/") return true; // unix
  if (loc[1] === ":" && loc[2] === "\\") return true; // windows
  return false;
};

exports.removeProperties = function (tree) {
  var clear = function (node) {
    delete node.tokens;
    delete node.range;
    delete node.loc;
  };

  clear(tree);
  traverse(tree, clear);

  return tree;
};

exports.getIds = function (node) {
  var search = [node];
  var ids  = [];

  while (search.length) {
    var id = search.shift();

    if (id.type === "Identifier") {
      ids.push(id.name);
    } else if (id.type === "ArrayPattern") {
      _.each(id.elements, function (elem) {
        search.push(elem);
      });
    } else if (id.type === "ObjectPattern") {
      _.each(id.properties, function (prop) {
        search.push(prop.value);
      });
    } else {
      throw new Error("unknown node " + id.type);
    }
  }

  return ids;
};

exports.errorWithNode = function (node, msg) {
  var line = node.loc.start.line;
  var col  = node.loc.start.column;

  var err = new SyntaxError("Line " + line + ": " + msg);
  err.lineNumber = line;
  err.column = col;
  return err;
};

exports.canCompile = function (filename) {
  var ext = path.extname(filename);
  return _.contains([".js", ".es6", ".jsx"], ext);
};

exports.sourceMapToComment = function (map) {
  var json = JSON.stringify(map);
  var base64 = new Buffer(json).toString("base64");
  return "//# sourceMappingURL=data:application/json;base64," + base64;
};

exports.pushMutatorMap = function (mutatorMap, key, kind, method) {
  var map = mutatorMap[key] = mutatorMap[key] || {};
  if (map[kind]) {
    throw new Error("a " + kind + " already exists for this property");
  } else {
    map[kind] = method;
  }
};

exports.buildDefineProperties = function (mutatorMap, keyNode) {
  var objExpr = b.objectExpression([]);

  _.each(mutatorMap, function (map, key) {
    var mapNode = b.objectExpression([]);

    var propNode = b.property("init", b.identifier(key), mapNode);

    _.each(map, function (node, key) {
      node = _.clone(node);
      if (node.type === "MethodDefinition") node = node.value;
      mapNode.properties.push(b.property("init", b.identifier(key), node));
    });

    objExpr.properties.push(propNode);
  });

  return exports.template("object-define-properties", {
    OBJECT: keyNode,
    PROPS: objExpr
  }, true);
};

exports.template = function (name, nodes, keepExpression) {
  var template = exports.templates[name];
  if (!template) throw new ReferenceError("unknown template " + name);

  template = _.cloneDeep(template);

  if (!_.isEmpty(nodes)) {
    traverse(template, function (node) {
      if (node.type === "Identifier" && _.has(nodes, node.name)) {
        var newNode = nodes[node.name];
        if (_.isString(newNode)) {
          node.name = newNode;
        } else {
          return newNode;
        }
      }
    });
  }

  var node = template.body[0];

  if (!keepExpression && node.type === "ExpressionStatement") {
    return node.expression;
  } else {
    return node;
  }
};

exports.isReferenced = function (node, parent) {
  // we're a property key
  if (parent.type === "Property" && parent.key === node) return false;

  var isMemberExpression = parent.type === "MemberExpression";

  // we're in a member expression and we're the computed property so we're referenced
  var isComputedProperty = isMemberExpression && parent.property === node && parent.computed;

  // we're in a member expression and we're the object so we're referenced
  var isObject = isMemberExpression && parent.object === node;

  // we are referenced
  if (!isMemberExpression || isComputedProperty || isObject) return true;

  return false;
};

exports.codeFrame = function (lines, lineNumber, colNumber) {
  colNumber = Math.max(colNumber, 0);

  lines = lines.split("\n");
  var start = Math.max(lineNumber - 3, 0);
  var end   = Math.min(lines.length, lineNumber + 3);
  var width = (end + "").length;

  return "\n" + lines.slice(start, end).map(function (line, i) {
    var curr = i + start + 1;

    var gutter = curr === lineNumber ? "> " : "  ";

    var sep = curr + exports.repeat(width + 1);
    gutter += sep + "| ";

    var str = gutter + line;

    if (colNumber && curr === lineNumber) {
      str += "\n";
      str += exports.repeat(gutter.length - 2);
      str += "|" + exports.repeat(colNumber) + "^";
    }

    return str;
  }).join("\n");
{'fraction_non_alphanumeric': 0.16867594023699123, 'fraction_numerical': 0.004121586810922205, 'mean_word_length': 3.073017205203525, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4963064', 'n_tokens_mistral': 3205, 'n_tokens_neox': 3108, 'n_words': 782}