<commit_before>from rupypy import consts
from rupypy.objects.codeobject import W_CodeObject


class BaseSymbolTable(object):
    FREEVAR = 0
    CELLVAR = 1

    def __init__(self, parent_symtable=None):
        self.parent_symtable = parent_symtable
        self.subscopes = {}
        self.locals = {}
        self.cells = {}
        self.arguments = []

        self.local_numbers = {}
        self.cell_numbers = {}

    def add_subscope(self, node, symtable):
        self.subscopes[node] = symtable

    def get_subscope(self, node):
        return self.subscopes[node]

    def declare_argument(self, name):
        self.arguments.append(name)
        self.declare_local(name)

    def declare_local(self, name):
        if name not in self.locals:
            self.locals[name] = None

    def is_defined(self, name):
        return self.is_local(name) or self.is_cell(name)

    def is_local(self, name):
        return name in self.locals

    def get_local_num(self, name):
        if name not in self.local_numbers:
            self.local_numbers[name] = len(self.local_numbers)
        return self.local_numbers[name]

    def is_cell(self, name):
        return name in self.cells

    def get_cell_num(self, name):
        if name not in self.cell_numbers:
            self.cell_numbers[name] = len(self.cell_numbers)
        return self.cell_numbers[name]


class SymbolTable(BaseSymbolTable):
    def declare_write(self, name):
        if not self.is_defined(name):
            self.declare_local(name)

    def declare_read(self, name):
        pass

    def upgrade_to_closure(self, name):
        if name in self.locals:
            del self.locals[name]
            self.cells[name] = self.CELLVAR


class BlockSymbolTable(BaseSymbolTable):
    def declare_read(self, name):
        if (name not in self.locals and name not in self.cells and
            self.parent_symtable.is_defined(name)):

            self.cells[name] = self.FREEVAR
            self.parent_symtable.upgrade_to_closure(name)

    def declare_write(self, name):
        if name not in self.locals and name not in self.cells:
            if self.parent_symtable.is_defined(name):
                self.cells[name] = self.FREEVAR
                self.parent_symtable.upgrade_to_closure(name)
            else:
                self.declare_local(name)

    def is_defined(self, name):
        return BaseSymbolTable.is_defined(self, name) or self.parent_symtable.is_defined(name)

    def upgrade_to_closure(self, name):
        if self.is_local(name):
            del self.locals[name]
            self.cells[name] = self.CELLVAR
        elif name not in self.cells:
            self.parent_symtable.upgrade_to_closure(name)<commit_msg>more cleanup<commit_after>from rupypy import consts
from rupypy.objects.codeobject import W_CodeObject


class BaseSymbolTable(object):
    FREEVAR = 0
    CELLVAR = 1

    def __init__(self, parent_symtable=None):
        self.parent_symtable = parent_symtable
        self.subscopes = {}
        self.locals = {}
        self.cells = {}
        self.arguments = []

        self.local_numbers = {}
        self.cell_numbers = {}

    def add_subscope(self, node, symtable):
        self.subscopes[node] = symtable

    def get_subscope(self, node):
        return self.subscopes[node]

    def declare_argument(self, name):
        self.arguments.append(name)
        self.declare_local(name)

    def declare_local(self, name):
        if name not in self.locals:
            self.locals[name] = None

    def is_defined(self, name):
        return self.is_local(name) or self.is_cell(name)

    def is_local(self, name):
        return name in self.locals

    def get_local_num(self, name):
        if name not in self.local_numbers:
            self.local_numbers[name] = len(self.local_numbers)
        return self.local_numbers[name]

    def is_cell(self, name):
        return name in self.cells

    def get_cell_num(self, name):
        if name not in self.cell_numbers:
            self.cell_numbers[name] = len(self.cell_numbers)
        return self.cell_numbers[name]


class SymbolTable(BaseSymbolTable):
    def declare_write(self, name):
        if not self.is_defined(name):
            self.declare_local(name)

    def declare_read(self, name):
        pass

    def upgrade_to_closure(self, name):
        if name in self.locals:
            del self.locals[name]
            self.cells[name] = self.CELLVAR


class BlockSymbolTable(BaseSymbolTable):
    def declare_read(self, name):
        if (name not in self.locals and name not in self.cells and
            self.parent_symtable.is_defined(name)):

            self.cells[name] = self.FREEVAR
            self.parent_symtable.upgrade_to_closure(name)

    def declare_write(self, name):
        if name not in self.locals and name not in self.cells:
            if self.parent_symtable.is_defined(name):
                self.cells[name] = self.FREEVAR
                self.parent_symtable.upgrade_to_closure(name)
            else:
                self.declare_local(name)

    def is_defined(self, name):
        return BaseSymbolTable.is_defined(self, name) or self.parent_symtable.is_defined(name)

    def upgrade_to_closure(self, name):
        if self.is_local(name):
            del self.locals[name]
            self.cells[name] = self.CELLVAR
        elif not self.is_cell(name):
            self.parent_symtable.upgrade_to_closure(name)
{'fraction_non_alphanumeric': 0.09065883648375848, 'fraction_numerical': 0.0007340796476417691, 'mean_word_length': 2.4190715181932245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7182182', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1654, 'n_words': 364}