<commit_before>from collections import namedtuple, defaultdict
import ast
import inspect
import textwrap
import operator
import warnings
from functools import partial

from llvmlite.llvmpy.core import Constant, Type, Builder

from numba.core import (typing, utils, types, ir, debuginfo, funcdesc,
                        generators, config, ir_utils, cgutils, removerefctpass,
                        targetconfig)
from numba.core.errors import (LoweringError, new_error_context, TypingError,
                               LiteralTypingError, UnsupportedError,
                               NumbaWarning)
from numba.core.funcdesc import default_mangler
from numba.core.environment import Environment
from numba.core.analysis import compute_use_defs


_VarArgItem = namedtuple("_VarArgItem", ("vararg", "index"))


class BaseLower(object):
    """
    Lower IR to LLVM
    """

    def __init__(self, context, library, fndesc, func_ir, metadata=None):
        self.library = library
        self.fndesc = fndesc
        self.blocks = utils.SortedMap(func_ir.blocks.items())
        self.func_ir = func_ir
        self.call_conv = context.call_conv
        self.generator_info = func_ir.generator_info
        self.metadata = metadata
        self.flags = targetconfig.ConfigStack.top_or_none()

        # Initialize LLVM
        self.module = self.library.create_ir_module(self.fndesc.unique_name)

        # Python execution environment (will be available to the compiled
        # function).
        self.env = Environment.from_fndesc(self.fndesc)

        # Internal states
        self.blkmap = {}
        self.pending_phis = {}
        self.varmap = {}
        self.firstblk = min(self.blocks.keys())
        self.loc = -1

        # Specializes the target context as seen inside the Lowerer
        # This adds:
        #  - environment: the python execution environment
        self.context = context.subtarget(environment=self.env,
                                         fndesc=self.fndesc)

        # Debuginfo
        dibuildercls = (self.context.DIBuilder
                        if self.context.enable_debuginfo
                        else debuginfo.DummyDIBuilder)

        # debuginfo def location
        self.defn_loc = self._compute_def_location()

        self.debuginfo = dibuildercls(module=self.module,
                                      filepath=func_ir.loc.filename,
                                      cgctx=context)

        # Subclass initialization
        self.init()

    def init(self):
        pass

    def init_pyapi(self):
        """
        Init the Python API and Environment Manager for the function being
        lowered.
        """
        if self.pyapi is not None:
            return
        self.pyapi = self.context.get_python_api(self.builder)

        # Store environment argument for later use
        self.env_manager = self.context.get_env_manager(self.builder)
        self.env_body = self.env_manager.env_body
        self.envarg = self.env_manager.env_ptr

    def _compute_def_location(self):
        # Debuginfo requires source to be accurate. Find it and warn if not
        # found. If it's not found, use the func_ir line + 1, this assumes that
        # the function definition is decorated with a 1 line jit decorator.
        defn_loc = self.func_ir.loc.with_lineno(self.func_ir.loc.line + 1)
        if self.context.enable_debuginfo:
            fn = self.func_ir.func_id.func
            try:
                raw_source_str, _ = inspect.getsourcelines(fn)
            except OSError:
                msg = ("Could not find source for function: "
                       f"{self.func_ir.func_id.func}")
                warnings.warn(NumbaWarning(msg))
            else:
                # Parse the source and find the line with `def <func>` in it, it
                # is assumed that if the compilation has made it this far that
                # the source is at least legal and has valid syntax.

                # join the source as a block and dedent it
                source_str = textwrap.dedent(''.join(raw_source_str))
                src_ast = ast.parse(source_str)
                # pull the definition out of the AST, only if it seems valid
                # i.e. one thing in the body
                if len(src_ast.body) == 1:
                    pydef = src_ast.body.pop()
                    # -1 as lines start at 1 and this is an offset.
                    pydef_offset = pydef.lineno - 1

                    func_ir_loc = self.func_ir.loc
                    defn_line = func_ir_loc.line + pydef_offset
                    defn_loc = func_ir_loc.with_lineno(defn_line)
        return defn_loc

    def pre_lower(self):
        """
        Called before lowering all blocks.
        """
        # A given Lower object can be used for several LL functions
        # (for generators) and it's important to use a new API and
        # EnvironmentManager.
        self.pyapi = None
        self.debuginfo.mark_subprogram(function=self.builder.function,
                                       fndesc=self.fndesc,
                                       line=self.func_ir.loc.line)

    def post_lower(self):
        """
        Called after all blocks are lowered
        """
        self.debuginfo.finalize()

    def pre_block(self, block):
        """
        Called before lowering a block.
        """

    def post_block(self, block):
        """
        Called after lowering a block.
        """

    def return_exception(self, exc_class, exc_args=None, loc=None):
        """Propagate exception to the caller.
        """<commit_msg>Fix subprogram marker line number<commit_after>from collections import namedtuple, defaultdict
import ast
import inspect
import textwrap
import operator
import warnings
from functools import partial

from llvmlite.llvmpy.core import Constant, Type, Builder

from numba.core import (typing, utils, types, ir, debuginfo, funcdesc,
                        generators, config, ir_utils, cgutils, removerefctpass,
                        targetconfig)
from numba.core.errors import (LoweringError, new_error_context, TypingError,
                               LiteralTypingError, UnsupportedError,
                               NumbaWarning)
from numba.core.funcdesc import default_mangler
from numba.core.environment import Environment
from numba.core.analysis import compute_use_defs


_VarArgItem = namedtuple("_VarArgItem", ("vararg", "index"))


class BaseLower(object):
    """
    Lower IR to LLVM
    """

    def __init__(self, context, library, fndesc, func_ir, metadata=None):
        self.library = library
        self.fndesc = fndesc
        self.blocks = utils.SortedMap(func_ir.blocks.items())
        self.func_ir = func_ir
        self.call_conv = context.call_conv
        self.generator_info = func_ir.generator_info
        self.metadata = metadata
        self.flags = targetconfig.ConfigStack.top_or_none()

        # Initialize LLVM
        self.module = self.library.create_ir_module(self.fndesc.unique_name)

        # Python execution environment (will be available to the compiled
        # function).
        self.env = Environment.from_fndesc(self.fndesc)

        # Internal states
        self.blkmap = {}
        self.pending_phis = {}
        self.varmap = {}
        self.firstblk = min(self.blocks.keys())
        self.loc = -1

        # Specializes the target context as seen inside the Lowerer
        # This adds:
        #  - environment: the python execution environment
        self.context = context.subtarget(environment=self.env,
                                         fndesc=self.fndesc)

        # Debuginfo
        dibuildercls = (self.context.DIBuilder
                        if self.context.enable_debuginfo
                        else debuginfo.DummyDIBuilder)

        # debuginfo def location
        self.defn_loc = self._compute_def_location()

        self.debuginfo = dibuildercls(module=self.module,
                                      filepath=func_ir.loc.filename,
                                      cgctx=context)

        # Subclass initialization
        self.init()

    def init(self):
        pass

    def init_pyapi(self):
        """
        Init the Python API and Environment Manager for the function being
        lowered.
        """
        if self.pyapi is not None:
            return
        self.pyapi = self.context.get_python_api(self.builder)

        # Store environment argument for later use
        self.env_manager = self.context.get_env_manager(self.builder)
        self.env_body = self.env_manager.env_body
        self.envarg = self.env_manager.env_ptr

    def _compute_def_location(self):
        # Debuginfo requires source to be accurate. Find it and warn if not
        # found. If it's not found, use the func_ir line + 1, this assumes that
        # the function definition is decorated with a 1 line jit decorator.
        defn_loc = self.func_ir.loc.with_lineno(self.func_ir.loc.line + 1)
        if self.context.enable_debuginfo:
            fn = self.func_ir.func_id.func
            try:
                raw_source_str, _ = inspect.getsourcelines(fn)
            except OSError:
                msg = ("Could not find source for function: "
                       f"{self.func_ir.func_id.func}")
                warnings.warn(NumbaWarning(msg))
            else:
                # Parse the source and find the line with `def <func>` in it, it
                # is assumed that if the compilation has made it this far that
                # the source is at least legal and has valid syntax.

                # join the source as a block and dedent it
                source_str = textwrap.dedent(''.join(raw_source_str))
                src_ast = ast.parse(source_str)
                # pull the definition out of the AST, only if it seems valid
                # i.e. one thing in the body
                if len(src_ast.body) == 1:
                    pydef = src_ast.body.pop()
                    # -1 as lines start at 1 and this is an offset.
                    pydef_offset = pydef.lineno - 1

                    func_ir_loc = self.func_ir.loc
                    defn_line = func_ir_loc.line + pydef_offset
                    defn_loc = func_ir_loc.with_lineno(defn_line)
        return defn_loc

    def pre_lower(self):
        """
        Called before lowering all blocks.
        """
        # A given Lower object can be used for several LL functions
        # (for generators) and it's important to use a new API and
        # EnvironmentManager.
        self.pyapi = None
        self.debuginfo.mark_subprogram(function=self.builder.function,
                                       fndesc=self.fndesc,
                                       line=self.defn_loc.line)

    def post_lower(self):
        """
        Called after all blocks are lowered
        """
        self.debuginfo.finalize()

    def pre_block(self, block):
        """
        Called before lowering a block.
        """

    def post_block(self, block):
        """
        Called after lowering a block.
        """

    def return_exception(self, exc_class, exc_args=None, loc=None):
        """Propagate exception to the caller.
        """
{'fraction_non_alphanumeric': 0.07461627184810576, 'fraction_numerical': 0.001419572353828409, 'mean_word_length': 1.8321608040201005, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965362', 'n_tokens_mistral': 3127, 'n_tokens_neox': 3016, 'n_words': 840}