<commit_before>from __future__ import print_function, division, absolute_import
import sys

from numba import ir, ir_utils, types, rewrites, config, analysis
from numba import array_analysis, postproc
from numba.ir_utils import *
from numba.analysis import *
from numba.controlflow import CFGraph
from numba.typing import npydecl
from numba.types.functions import Function
import numpy as np
import numba.parfor
# circular dependency: import numba.npyufunc.dufunc.DUFunc

class LoopNest(object):
    '''The LoopNest class holds information of a single loop including
    the index variable (of a non-negative integer value), and the
    range variable, e.g. range(r) is 0 to r-1 with step size 1.
    '''
    def __init__(self, index_variable, range_variable, correlation=-1):
        self.index_variable = index_variable
        self.range_variable = range_variable
        self.correlation = correlation

    def __repr__(self):
        return ("LoopNest(index_variable=%s, " % self.index_variable
                + "range_variable=%s, " % self.range_variable
                + "correlation=%d)" % self.correlation)


class Parfor(ir.Expr, ir.Stmt):
    def __init__(self, loop_nests, init_block, loop_body, loc, array_analysis, index_var):
        super(Parfor, self).__init__(
            op   = 'parfor',
            loc  = loc
        )

        #self.input_info  = input_info
        #self.output_info = output_info
        self.loop_nests = loop_nests
        self.init_block = init_block
        self.loop_body = loop_body
        self.array_analysis = array_analysis
        self.index_var = index_var

    def __repr__(self):
        return repr(self.loop_nests) + repr(self.loop_body)

    def list_vars(self):
        """list variables used (read/written) in this parfor by
        traversing the body and combining block uses.
        """
        all_uses = []
        for l,b in self.loop_body.items():
            for stmt in b.body:
                all_uses += stmt.list_vars()

        for loop in self.loop_nests:
            all_uses.append(loop.index_variable)
            all_uses.append(loop.range_variable)

        for stmt in self.init_block.body:
            all_uses += stmt.list_vars()

        return all_uses

    def dump(self,  file=None):
        file = file or sys.stdout
        print(("begin parfor").center(20,'-'), file=file)
        for loopnest in self.loop_nests:
            print(loopnest, file=file)
        print("init block:", file=file)
        self.init_block.dump()
        for offset, block in sorted(self.loop_body.items()):
            print('label %s:' % (offset,), file=file)
            block.dump(file)
        print(("end parfor").center(20,'-'), file=file)


class ParforPass(object):
    """ParforPass class is responsible for converting Numpy
    calls in Numba intermediate representation to Parfors, which
    will lower into either sequential or parallel loops during lowering
    stage.
    """
    def __init__(self, func_ir, typemap, calltypes):
        self.func_ir = func_ir
        self.typemap = typemap
        self.calltypes = calltypes
        self.array_analysis = array_analysis.ArrayAnalysis(func_ir, typemap,
            calltypes)
        ir_utils._max_label = max(func_ir.blocks.keys())

    def run(self):
        """run parfor conversion pass: replace Numpy calls
        with Parfors when possible and optimize the IR."""

        self.array_analysis.run()
        for (key, block) in self.func_ir.blocks.items():
            new_body = []
            for instr in block.body:
                if isinstance(instr, ir.Assign):
                    expr = instr.value
                    lhs = instr.target
                    if lhs.name in self.array_analysis.array_shape_classes:
                        lhs_shapes = self.array_analysis.array_shape_classes[lhs.name]
                        if -1 in lhs_shapes:
                            # avoid parfor generation if any shape is unknown
                            new_body.append(instr)
                            continue
                        if self._is_supported_npycall(expr):
                            instr = self._numpy_to_parfor(lhs, expr)
                        if isinstance(expr, ir.Expr) and expr.op == 'arrayexpr':
                            instr = self._arrayexpr_to_parfor(lhs, expr)
                new_body.append(instr)
            block.body = new_body

        # remove Del statements for easier optimization
        remove_dels(self.func_ir.blocks)
        dprint_func_ir(self.func_ir, "after parfor pass")
        # get copies in to blocks and out from blocks
        in_cps, out_cps = copy_propagate(self.func_ir.blocks)
        # table mapping variable names to ir.Var objects to help replacement
        name_var_table = get_name_var_table(self.func_ir.blocks)
        apply_copy_propagate(self.func_ir.blocks, in_cps, name_var_table)
        # remove dead code to enable fusion
        remove_dead(self.func_ir.blocks)
        # reorder statements to maximize fusion
        maximize_fusion(self.func_ir.blocks)
        fuse_parfors(self.func_ir.blocks)
        # remove dead code after fusion to remove extra arrays and variables
        remove_dead(self.func_ir.blocks)
        # push function call variables inside parfors so gufunc function
        # wouldn't need function variables as argument
        push_call_vars(self.func_ir.blocks, {}, {})
        # run post processor again to generate Del nodes
        # post_proc = postproc.PostProcessor(self.func_ir)
        # post_proc.run()
        # after optimization, some size variables are not available anymore
        remove_dead_class_sizes(self.func_ir.blocks, self.array_analysis)
        dprint_func_ir(self.func_ir, "after optimization")
        # lower_parfor_sequential(self.func_ir, self.typemap, self.calltypes)
        return

    def _arrayexpr_to_parfor(self, lhs, arrayexpr):
        """generate parfor from arrayexpr node, which is essentially a<commit_msg>insert Del nodes after optimization<commit_after>from __future__ import print_function, division, absolute_import
import sys

from numba import ir, ir_utils, types, rewrites, config, analysis
from numba import array_analysis, postproc
from numba.ir_utils import *
from numba.analysis import *
from numba.controlflow import CFGraph
from numba.typing import npydecl
from numba.types.functions import Function
import numpy as np
import numba.parfor
# circular dependency: import numba.npyufunc.dufunc.DUFunc

class LoopNest(object):
    '''The LoopNest class holds information of a single loop including
    the index variable (of a non-negative integer value), and the
    range variable, e.g. range(r) is 0 to r-1 with step size 1.
    '''
    def __init__(self, index_variable, range_variable, correlation=-1):
        self.index_variable = index_variable
        self.range_variable = range_variable
        self.correlation = correlation

    def __repr__(self):
        return ("LoopNest(index_variable=%s, " % self.index_variable
                + "range_variable=%s, " % self.range_variable
                + "correlation=%d)" % self.correlation)


class Parfor(ir.Expr, ir.Stmt):
    def __init__(self, loop_nests, init_block, loop_body, loc, array_analysis, index_var):
        super(Parfor, self).__init__(
            op   = 'parfor',
            loc  = loc
        )

        #self.input_info  = input_info
        #self.output_info = output_info
        self.loop_nests = loop_nests
        self.init_block = init_block
        self.loop_body = loop_body
        self.array_analysis = array_analysis
        self.index_var = index_var

    def __repr__(self):
        return repr(self.loop_nests) + repr(self.loop_body)

    def list_vars(self):
        """list variables used (read/written) in this parfor by
        traversing the body and combining block uses.
        """
        all_uses = []
        for l,b in self.loop_body.items():
            for stmt in b.body:
                all_uses += stmt.list_vars()

        for loop in self.loop_nests:
            all_uses.append(loop.index_variable)
            all_uses.append(loop.range_variable)

        for stmt in self.init_block.body:
            all_uses += stmt.list_vars()

        return all_uses

    def dump(self,  file=None):
        file = file or sys.stdout
        print(("begin parfor").center(20,'-'), file=file)
        for loopnest in self.loop_nests:
            print(loopnest, file=file)
        print("init block:", file=file)
        self.init_block.dump()
        for offset, block in sorted(self.loop_body.items()):
            print('label %s:' % (offset,), file=file)
            block.dump(file)
        print(("end parfor").center(20,'-'), file=file)


class ParforPass(object):
    """ParforPass class is responsible for converting Numpy
    calls in Numba intermediate representation to Parfors, which
    will lower into either sequential or parallel loops during lowering
    stage.
    """
    def __init__(self, func_ir, typemap, calltypes):
        self.func_ir = func_ir
        self.typemap = typemap
        self.calltypes = calltypes
        self.array_analysis = array_analysis.ArrayAnalysis(func_ir, typemap,
            calltypes)
        ir_utils._max_label = max(func_ir.blocks.keys())

    def run(self):
        """run parfor conversion pass: replace Numpy calls
        with Parfors when possible and optimize the IR."""

        self.array_analysis.run()
        for (key, block) in self.func_ir.blocks.items():
            new_body = []
            for instr in block.body:
                if isinstance(instr, ir.Assign):
                    expr = instr.value
                    lhs = instr.target
                    if lhs.name in self.array_analysis.array_shape_classes:
                        lhs_shapes = self.array_analysis.array_shape_classes[lhs.name]
                        if -1 in lhs_shapes:
                            # avoid parfor generation if any shape is unknown
                            new_body.append(instr)
                            continue
                        if self._is_supported_npycall(expr):
                            instr = self._numpy_to_parfor(lhs, expr)
                        if isinstance(expr, ir.Expr) and expr.op == 'arrayexpr':
                            instr = self._arrayexpr_to_parfor(lhs, expr)
                new_body.append(instr)
            block.body = new_body

        # remove Del statements for easier optimization
        remove_dels(self.func_ir.blocks)
        dprint_func_ir(self.func_ir, "after parfor pass")
        # get copies in to blocks and out from blocks
        in_cps, out_cps = copy_propagate(self.func_ir.blocks)
        # table mapping variable names to ir.Var objects to help replacement
        name_var_table = get_name_var_table(self.func_ir.blocks)
        apply_copy_propagate(self.func_ir.blocks, in_cps, name_var_table)
        # remove dead code to enable fusion
        remove_dead(self.func_ir.blocks)
        # reorder statements to maximize fusion
        maximize_fusion(self.func_ir.blocks)
        fuse_parfors(self.func_ir.blocks)
        # remove dead code after fusion to remove extra arrays and variables
        remove_dead(self.func_ir.blocks)
        # push function call variables inside parfors so gufunc function
        # wouldn't need function variables as argument
        push_call_vars(self.func_ir.blocks, {}, {})
        # after optimization, some size variables are not available anymore
        remove_dead_class_sizes(self.func_ir.blocks, self.array_analysis)
        dprint_func_ir(self.func_ir, "after optimization")
        # run post processor again to generate Del nodes
        post_proc = postproc.PostProcessor(self.func_ir)
        post_proc.run()
        # lower_parfor_sequential(self.func_ir, self.typemap, self.calltypes)
        return

    def _arrayexpr_to_parfor(self, lhs, arrayexpr):
        """generate parfor from arrayexpr node, which is essentially a
{'fraction_non_alphanumeric': 0.0813373253493014, 'fraction_numerical': 0.0014970059880239522, 'mean_word_length': 2.3580005585032113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '297662', 'n_tokens_mistral': 3473, 'n_tokens_neox': 3378, 'n_words': 911}