<commit_before>from __future__ import print_function, division, absolute_import

import collections
import dis
import sys

from numba import ir, controlflow, dataflow, utils
from numba.utils import builtins


class Assigner(object):
    """
    This object keeps track of potential assignment simplifications
    inside a code block.
    For example `$O.1 = x` followed by `y = $0.1` can be simplified
    into `y = x`, but it's not possible anymore if we have `x = z`
    in-between those two instructions.

    NOTE: this is not only an optimization, but is actually necessary
    due to certain limitations of Numba - such as only accepting the
    returning of an array passed as function argument.
    """

    def __init__(self):
        # { destination variable name -> source Var object }
        self.dest_to_src = {}
        # Basically a reverse mapping of dest_to_src:
        # { source variable name -> all destination names in dest_to_src }
        self.src_invalidate = collections.defaultdict(list)
        self.unused_dests = set()

    def assign(self, srcvar, destvar):
        """
        Assign *srcvar* to *destvar*. Return either *srcvar* or a possible
        simplified assignment source (earlier assigned to *srcvar*).
        """
        srcname = srcvar.name
        destname = destvar.name
        if destname in self.src_invalidate:
            # destvar will change, invalidate all previously known simplifications
            for d in self.src_invalidate.pop(destname):
                self.dest_to_src.pop(d)
        if srcname in self.dest_to_src:
            srcvar = self.dest_to_src[srcname]
        if destvar.is_temp:
            self.dest_to_src[destname] = srcvar
            self.src_invalidate[srcname].append(destname)
            self.unused_dests.add(destname)
        return srcvar

    def get_assignment_source(self, destname):
        """
        Get a possible assignment source (a ir.Var instance) to replace
        *destname*, otherwise None.
        """
        if destname in self.dest_to_src:
            return self.dest_to_src[destname]
        self.unused_dests.discard(destname)
        return None


class Interpreter(object):
    """A bytecode interpreter that builds up the IR.
    """

    def __init__(self, bytecode):
        self.bytecode = bytecode
        self.scopes = []
        self.loc = ir.Loc(filename=bytecode.filename, line=1)
        self.argspec = bytecode.argspec
        # Control flow analysis
        self.cfa = controlflow.ControlFlowAnalysis(bytecode)
        self.cfa.run()
        # Data flow analysis
        self.dfa = dataflow.DataFlowAnalysis(self.cfa)
        self.dfa.run()

        global_scope = ir.Scope(parent=None, loc=self.loc)
        self._fill_global_scope(global_scope)
        self.scopes.append(global_scope)

        # { inst offset : ir.Block }
        self.blocks = {}
        # { name: value } of global variables used by the bytecode
        self.used_globals = {}
        # { name: [definitions] } of local variables
        self.definitions = collections.defaultdict(list)
        # { ir.Block: { variable names (potentially) alive at start of block } }
        self.block_entry_vars = collections.defaultdict(set)

        # Temp states during interpretation
        self.current_block = None
        self.current_block_offset = None
        self.syntax_blocks = []
        self.dfainfo = None

    def _fill_global_scope(self, scope):
        """TODO
        """
        pass

    def get_used_globals(self):
        """
        Return a dictionary of global variables used by the bytecode.
        """
        return self.used_globals

    def get_block_entry_vars(self, block):
        """
        Return a set of variable names possibly alive at the beginning of
        the block.
        """
        return self.block_entry_vars[block]

    def interpret(self):
        firstblk = min(self.cfa.blocks.keys())
        self.loc = ir.Loc(filename=self.bytecode.filename,
                          line=self.bytecode[firstblk].lineno)
        self.scopes.append(ir.Scope(parent=self.current_scope, loc=self.loc))
        # Interpret loop
        for inst, kws in self._iter_inst():
            self._dispatch(inst, kws)
        # Analysis on generated IR
        self._insert_var_dels()
        self._compute_block_entry_vars()

    def _compute_block_entry_vars(self):
        """
        Compute the live variables at the beginning of each block.
        This must be done after insertion of ir.Dels.
        """
        # Scan for variable additions and deletions in each block
        block_var_adds = {}
        block_var_dels = {}
        cfg = self.cfa.graph

        for ir_block in self.blocks.values():
            adds = block_var_adds[ir_block] = set()
            dels = block_var_dels[ir_block] = set()
            for stmt in ir_block.body:
                if isinstance(stmt, ir.Assign):
                    name = stmt.target.name
                    adds.add(name)
                    dels.discard(name)
                elif isinstance(stmt, ir.Del):
                    name = stmt.value
                    adds.discard(name)
                    dels.add(name)

        # Propagate defined variables from every block to its successors.
        # (note the entry block automatically gets an empty set)
        changed = True
        while changed:
            # We iterate until the result stabilizes.  This is necessary<commit_msg>Improve comment<commit_after>from __future__ import print_function, division, absolute_import

import collections
import dis
import sys

from numba import ir, controlflow, dataflow, utils
from numba.utils import builtins


class Assigner(object):
    """
    This object keeps track of potential assignment simplifications
    inside a code block.
    For example `$O.1 = x` followed by `y = $0.1` can be simplified
    into `y = x`, but it's not possible anymore if we have `x = z`
    in-between those two instructions.

    NOTE: this is not only an optimization, but is actually necessary
    due to certain limitations of Numba - such as only accepting the
    returning of an array passed as function argument.
    """

    def __init__(self):
        # { destination variable name -> source Var object }
        self.dest_to_src = {}
        # Basically a reverse mapping of dest_to_src:
        # { source variable name -> all destination names in dest_to_src }
        self.src_invalidate = collections.defaultdict(list)
        self.unused_dests = set()

    def assign(self, srcvar, destvar):
        """
        Assign *srcvar* to *destvar*. Return either *srcvar* or a possible
        simplified assignment source (earlier assigned to *srcvar*).
        """
        srcname = srcvar.name
        destname = destvar.name
        if destname in self.src_invalidate:
            # destvar will change, invalidate all previously known simplifications
            for d in self.src_invalidate.pop(destname):
                self.dest_to_src.pop(d)
        if srcname in self.dest_to_src:
            srcvar = self.dest_to_src[srcname]
        if destvar.is_temp:
            self.dest_to_src[destname] = srcvar
            self.src_invalidate[srcname].append(destname)
            self.unused_dests.add(destname)
        return srcvar

    def get_assignment_source(self, destname):
        """
        Get a possible assignment source (a ir.Var instance) to replace
        *destname*, otherwise None.
        """
        if destname in self.dest_to_src:
            return self.dest_to_src[destname]
        self.unused_dests.discard(destname)
        return None


class Interpreter(object):
    """A bytecode interpreter that builds up the IR.
    """

    def __init__(self, bytecode):
        self.bytecode = bytecode
        self.scopes = []
        self.loc = ir.Loc(filename=bytecode.filename, line=1)
        self.argspec = bytecode.argspec
        # Control flow analysis
        self.cfa = controlflow.ControlFlowAnalysis(bytecode)
        self.cfa.run()
        # Data flow analysis
        self.dfa = dataflow.DataFlowAnalysis(self.cfa)
        self.dfa.run()

        global_scope = ir.Scope(parent=None, loc=self.loc)
        self._fill_global_scope(global_scope)
        self.scopes.append(global_scope)

        # { inst offset : ir.Block }
        self.blocks = {}
        # { name: value } of global variables used by the bytecode
        self.used_globals = {}
        # { name: [definitions] } of local variables
        self.definitions = collections.defaultdict(list)
        # { ir.Block: { variable names (potentially) alive at start of block } }
        self.block_entry_vars = collections.defaultdict(set)

        # Temp states during interpretation
        self.current_block = None
        self.current_block_offset = None
        self.syntax_blocks = []
        self.dfainfo = None

    def _fill_global_scope(self, scope):
        """TODO
        """
        pass

    def get_used_globals(self):
        """
        Return a dictionary of global variables used by the bytecode.
        """
        return self.used_globals

    def get_block_entry_vars(self, block):
        """
        Return a set of variable names possibly alive at the beginning of
        the block.
        """
        return self.block_entry_vars[block]

    def interpret(self):
        firstblk = min(self.cfa.blocks.keys())
        self.loc = ir.Loc(filename=self.bytecode.filename,
                          line=self.bytecode[firstblk].lineno)
        self.scopes.append(ir.Scope(parent=self.current_scope, loc=self.loc))
        # Interpret loop
        for inst, kws in self._iter_inst():
            self._dispatch(inst, kws)
        # Post-processing and analysis on generated IR
        self._insert_var_dels()
        self._compute_block_entry_vars()

    def _compute_block_entry_vars(self):
        """
        Compute the live variables at the beginning of each block.
        This must be done after insertion of ir.Dels.
        """
        # Scan for variable additions and deletions in each block
        block_var_adds = {}
        block_var_dels = {}
        cfg = self.cfa.graph

        for ir_block in self.blocks.values():
            adds = block_var_adds[ir_block] = set()
            dels = block_var_dels[ir_block] = set()
            for stmt in ir_block.body:
                if isinstance(stmt, ir.Assign):
                    name = stmt.target.name
                    adds.add(name)
                    dels.discard(name)
                elif isinstance(stmt, ir.Del):
                    name = stmt.value
                    adds.discard(name)
                    dels.add(name)

        # Propagate defined variables from every block to its successors.
        # (note the entry block automatically gets an empty set)
        changed = True
        while changed:
            # We iterate until the result stabilizes.  This is necessary
{'fraction_non_alphanumeric': 0.08149297317028655, 'fraction_numerical': 0.0007300602299689724, 'mean_word_length': 2.13921512460613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1417276', 'n_tokens_mistral': 3070, 'n_tokens_neox': 3012, 'n_words': 878}