<commit_before>###
# Walker:
# Traverses a JavaScript AST.
#
#     class MyWalker extends Walker
#
#     w = new MyWalker(ast)
#     w.run()
#
###

module.exports = class Walker
  constructor: (@root, @options) ->
    @path = []

  run: ->
    @walk(@root)

  walk: (node) =>
    oldLength = @path.length
    @path.push(node)

    type = node.type
    fn = this[type]

    if fn
      out = fn.call(this, node, { path: @path })
      out = @decorator(node, out)  if @decorator?
    else
      out = @onUnknownNode(node, { path: @path })

    @path.splice(oldLength)
    out
<commit_msg>Add preliminary support for filters.<commit_after>###
# Walker:
# Traverses a JavaScript AST.
#
#     class MyWalker extends Walker
#
#     w = new MyWalker(ast)
#     w.run()
#
###

module.exports = class Walker
  constructor: (@root, @options) ->
    @path = []

  run: ->
    @walk(@root)

  walk: (node) =>
    oldLength = @path.length
    @path.push(node)

    type = node.type

    # check for a filter first
    filters = @filters?[type]
    if filters?
      node = filter(node) for filter in filters

    fn = this[type]

    if fn
      out = fn.call(this, node, { path: @path })
      out = @decorator(node, out) if @decorator?
    else
      out = @onUnknownNode(node, { path: @path })

    @path.splice(oldLength)
    out

{'fraction_non_alphanumeric': 0.14912944738834216, 'fraction_numerical': 0.0, 'mean_word_length': 2.451697127937337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3346295', 'n_tokens_mistral': 476, 'n_tokens_neox': 456, 'n_words': 121}