<commit_before>""" Meager code path measurement tool.
    <NAME>
    http://nedbatchelder.com/blog/200803/python_code_complexity_microtool.html
    MIT License.
"""
from __future__ import with_statement

import _ast
import optparse
import sys
from collections import defaultdict
try:
    from ast import iter_child_nodes
except ImportError:
    def iter_child_nodes(node):
        for name in node._fields:
            field = getattr(node, name)
            if isinstance(field, _ast.AST):
                yield field
            elif isinstance(field, list):
                for item in field:
                    yield item

__version__ = '0.1'


class ASTVisitor(object):
    """Performs a depth-first walk of the AST."""

    def __init__(self):
        self.node = None
        self._cache = {}

    def default(self, node, *args):
        for child in iter_child_nodes(node):
            self.dispatch(child, *args)

    def dispatch(self, node, *args):
        self.node = node
        klass = node.__class__
        meth = self._cache.get(klass)
        if meth is None:
            className = klass.__name__
            meth = getattr(self.visitor, 'visit' + className, self.default)
            self._cache[klass] = meth
        return meth(node, *args)

    def preorder(self, tree, visitor, *args):
        """Do preorder walk of tree using visitor"""
        self.visitor = visitor
        visitor.visit = self.dispatch<commit_msg>fallback on flint.util.iter_child_nodes for Python 2.5<commit_after>""" Meager code path measurement tool.
    <NAME>
    http://nedbatchelder.com/blog/200803/python_code_complexity_microtool.html
    MIT License.
"""
from __future__ import with_statement

import _ast
import optparse
import sys
from collections import defaultdict
try:
    from ast import iter_child_nodes
except ImportError:   # Python 2.5
    from flint.util import iter_child_nodes

__version__ = '0.1'


class ASTVisitor(object):
    """Performs a depth-first walk of the AST."""

    def __init__(self):
        self.node = None
        self._cache = {}

    def default(self, node, *args):
        for child in iter_child_nodes(node):
            self.dispatch(child, *args)

    def dispatch(self, node, *args):
        self.node = node
        klass = node.__class__
        meth = self._cache.get(klass)
        if meth is None:
            className = klass.__name__
            meth = getattr(self.visitor, 'visit' + className, self.default)
            self._cache[klass] = meth
        return meth(node, *args)

    def preorder(self, tree, visitor, *args):
        """Do preorder walk of tree using visitor"""
        self.visitor = visitor
        visitor.visit = self.dispatch
{'fraction_non_alphanumeric': 0.0942350332594235, 'fraction_numerical': 0.007390983000739098, 'mean_word_length': 2.4440203562340965, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462277', 'n_tokens_mistral': 797, 'n_tokens_neox': 775, 'n_words': 231}