<commit_before>module.exports = traverse;

var Scope = require("./scope");
var t     = require("../types");
var _     = require("lodash");

function traverse(parent, callbacks, opts) {
  // falsy node
  if (!parent) return;

  // array of nodes
  if (_.isArray(parent)) {
    _.each(parent, function (node) {
      traverse(node, callbacks, opts);
    });
    return;
  }

  // unknown node type to traverse
  var keys = t.VISITOR_KEYS[parent.type];
  if (!keys) return;

  opts = opts || {};
  if (_.isArray(opts)) opts = { blacklist: opts };

  // blacklist these node types from being traversed
  var blacklistTypes = opts.blacklist || [];

  // normalise callbacks
  if (_.isFunction(callbacks)) callbacks = { enter: callbacks };

  _.each(keys, function (key) {
    var nodes = parent[key];
    if (!nodes) return;

    var handle = function (obj, key) {
      var node = obj[key];
      if (!node) return;

      // type is blacklisted
      if (_.contains(blacklistTypes, node.type)) return;

      var result;

      // replace node
      var maybeReplace = function (result) {
        if (result === false) return;
        if (result != null) node = obj[key] = result;
      };

      //
      var opts2 = _.clone(opts);
      if (t.isScope(node)) opts2.scope = new Scope(opts.scope, node);

      // enter
      if (callbacks.enter) {
        result = callbacks.enter(node, parent, opts2);

        // stop iteration
        if (result === false) return;

        maybeReplace(result);
      }

      // traverse node
      traverse(node, callbacks, opts2);

      // exit
      if (callbacks.exit) {
        maybeReplace(callbacks.exit(node, parent, opts2));
      }
    };

    if (_.isArray(nodes)) {
      _.each(nodes, function (node, i) {
        handle(nodes, i);
      });

      parent[key] = _.flatten(parent[key]);
    } else {
      handle(parent, key);
    }
  });
}

traverse.removeProperties = function (tree) {
  var clear = function (node) {
    delete node.extendedRange;
    delete node._scopeIds;
    delete node.tokens;
    delete node.range;
    delete node.start;
    delete node.end;
    delete node.loc;
    delete node.raw;

    clearComments(node.trailingComments);
    clearComments(node.leadingComments);<commit_msg>remove _parent from nodes in traverse.removeProperties<commit_after>module.exports = traverse;

var Scope = require("./scope");
var t     = require("../types");
var _     = require("lodash");

function traverse(parent, callbacks, opts) {
  // falsy node
  if (!parent) return;

  // array of nodes
  if (_.isArray(parent)) {
    _.each(parent, function (node) {
      traverse(node, callbacks, opts);
    });
    return;
  }

  // unknown node type to traverse
  var keys = t.VISITOR_KEYS[parent.type];
  if (!keys) return;

  opts = opts || {};
  if (_.isArray(opts)) opts = { blacklist: opts };

  // blacklist these node types from being traversed
  var blacklistTypes = opts.blacklist || [];

  // normalise callbacks
  if (_.isFunction(callbacks)) callbacks = { enter: callbacks };

  _.each(keys, function (key) {
    var nodes = parent[key];
    if (!nodes) return;

    var handle = function (obj, key) {
      var node = obj[key];
      if (!node) return;

      // type is blacklisted
      if (_.contains(blacklistTypes, node.type)) return;

      var result;

      // replace node
      var maybeReplace = function (result) {
        if (result === false) return;
        if (result != null) node = obj[key] = result;
      };

      //
      var opts2 = _.clone(opts);
      if (t.isScope(node)) opts2.scope = new Scope(opts.scope, node);

      // enter
      if (callbacks.enter) {
        result = callbacks.enter(node, parent, opts2);

        // stop iteration
        if (result === false) return;

        maybeReplace(result);
      }

      // traverse node
      traverse(node, callbacks, opts2);

      // exit
      if (callbacks.exit) {
        maybeReplace(callbacks.exit(node, parent, opts2));
      }
    };

    if (_.isArray(nodes)) {
      _.each(nodes, function (node, i) {
        handle(nodes, i);
      });

      parent[key] = _.flatten(parent[key]);
    } else {
      handle(parent, key);
    }
  });
}

traverse.removeProperties = function (tree) {
  var clear = function (node) {
    delete node.extendedRange;
    delete node._scopeIds;
    delete node._parent;
    delete node.tokens;
    delete node.range;
    delete node.start;
    delete node.end;
    delete node.loc;
    delete node.raw;

    clearComments(node.trailingComments);
    clearComments(node.leadingComments);
{'fraction_non_alphanumeric': 0.13833041958041958, 'fraction_numerical': 0.0021853146853146855, 'mean_word_length': 2.4055059523809526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2943129', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1398, 'n_words': 372}