<commit_before>from collections import OrderedDict
from jinja2.visitor import NodeVisitor
from jinja2._compat import iteritems


VAR_LOAD_PARAMETER = 'param'
VAR_LOAD_RESOLVE = 'resolve'
VAR_LOAD_ALIAS = 'alias'
VAR_LOAD_UNDEFINED = 'undefined'


def find_symbols(nodes, parent_symbols=None):
    sym = Symbols(parent=parent_symbols)
    visitor = FrameSymbolVisitor(sym)
    for node in nodes:
        visitor.visit(node)
    return sym


def symbols_for_node(node, parent_symbols=None):
    sym = Symbols(parent=parent_symbols)
    sym.analyze_node(node)
    return sym


class Symbols(object):

    def __init__(self, parent=None):
        if parent is None:
            self.level = 0
        else:
            self.level = parent.level + 1
        self.parent = parent
        self.refs = {}
        self.loads = OrderedDict()
        self.stores = set()

    def analyze_node(self, node, **kwargs):
        visitor = RootVisitor(self)
        visitor.visit(node, **kwargs)

    def _define_ref(self, name, load=None):
        ident = 'l_%d_%s' % (self.level, name)
        self.refs[name] = ident
        if load is not None:
            self.loads[ident] = load
        return ident<commit_msg>Turns out order tracking for loads is not useful<commit_after>from jinja2.visitor import NodeVisitor
from jinja2._compat import iteritems


VAR_LOAD_PARAMETER = 'param'
VAR_LOAD_RESOLVE = 'resolve'
VAR_LOAD_ALIAS = 'alias'
VAR_LOAD_UNDEFINED = 'undefined'


def find_symbols(nodes, parent_symbols=None):
    sym = Symbols(parent=parent_symbols)
    visitor = FrameSymbolVisitor(sym)
    for node in nodes:
        visitor.visit(node)
    return sym


def symbols_for_node(node, parent_symbols=None):
    sym = Symbols(parent=parent_symbols)
    sym.analyze_node(node)
    return sym


class Symbols(object):

    def __init__(self, parent=None):
        if parent is None:
            self.level = 0
        else:
            self.level = parent.level + 1
        self.parent = parent
        self.refs = {}
        self.loads = {}
        self.stores = set()

    def analyze_node(self, node, **kwargs):
        visitor = RootVisitor(self)
        visitor.visit(node, **kwargs)

    def _define_ref(self, name, load=None):
        ident = 'l_%d_%s' % (self.level, name)
        self.refs[name] = ident
        if load is not None:
            self.loads[ident] = load
        return ident
{'fraction_non_alphanumeric': 0.09707112970711297, 'fraction_numerical': 0.0033472803347280333, 'mean_word_length': 2.7535321821036107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2316914', 'n_tokens_mistral': 776, 'n_tokens_neox': 763, 'n_words': 184}