<commit_before>var nodeChildren = require('./ast-node-children');
var _ = require('lodash');

// Walks an AST, calling fn for each node
// fn(node, parent, locationInParent
// TODO allow fn to specify that a node should be re-processed via return value?
function walk(rootNode, fn) {
    // visit this node
    fn(rootNode, null, null);
    // visit children
    _walk(rootNode, fn);
}

// Visit all children of a node
function _walk(node, fn) {
    var type = node.type;
    var children = nodeChildren[type];
    _.each(children, function(childName) {
        var child = node[childName];
        if(_.isArray(child)) {
            _.each(child, function(child, i) {
                fn(child, node, childName + '.' + i);
            });
        } else {
            fn(child, node, childName);
        }
    });
}<commit_msg>Updated walked to allow callback to prevent children of a node from being walked.  Also added comments explaining future behavior<commit_after>var nodeChildren = require('./ast-node-children');
var _ = require('lodash');

// Walks an AST, calling fn for each node
// fn(node, parent, locationInParent)
// TODO allow fn to specify that a node should be re-processed via return value?
// If fn returns:
//   an object, that object will replace the node in the tree
//   null, node will be removed from the tree
//   false, children will *not* be visited
function walk(rootNode, fn) {
    // visit this node
    fn(rootNode, null, null);
    // visit children
    _walk(rootNode, fn);
}

// Visit all children of a node
function _walk(node, fn) {
    var type = node.type;
    var children = nodeChildren[type];
    _.each(children, function(childName) {
        var child = node[childName];
        var ret;
        if(_.isArray(child)) {
            _.each(child, function(child, i) {
                ret = fn(child, node, childName + '.' + i);
                ret === false || _walk(child, fn);
            });
        } else {
            ret = fn(child, node, childName);
            ret === false || _walk(child, fn);
        }
    });
}

module.exports = walk;
{'fraction_non_alphanumeric': 0.12434241989478718, 'fraction_numerical': 0.0, 'mean_word_length': 2.4351395730706074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4074919', 'n_tokens_mistral': 608, 'n_tokens_neox': 599, 'n_words': 211}