<commit_before>"""
Compiling extension classes works as follows:

    * Create an extension Numba/minivect type holding a symtab
    * Capture attribute types in the symtab ...

        * ... from the class attributes:

            @jit
            class Foo(object):
                attr = double

        * ... from __init__

            @jit
            class Foo(object):
                def __init__(self, attr):
                    self.attr = double(attr)

    * Type infer all methods
    * Compile all extension methods

        * Process signatures such as @void(double)
        * Infer native attributes through type inference on __init__
        * Path the extension type with a native attributes struct
        * Infer types for all other methods
        * Update the ext_type with a vtab type
        * Compile all methods

    * Create descriptors that wrap the native attributes
    * Create an extension type:

      {
        PyObject_HEAD
        ...
        virtual function table (func **)
        native attributes
      }

    The virtual function table (vtab) is a ctypes structure set as
    attribute of the extension types. Objects have a direct pointer
    for efficiency.

See also extension_types.pyx
"""


import types
import ctypes
import logging
import warnings
import inspect

import numba
from numba import pipeline, error, symtab
from numba import typesystem
from numba.exttypes import extension_types
from numba.minivect import minitypes

logger = logging.getLogger(__name__)

#------------------------------------------------------------------------
# Parse method signatures
#------------------------------------------------------------------------

def validate_method(py_func, sig, is_static):
    assert isinstance(py_func, types.FunctionType)
    nargs = py_func.__code__.co_argcount - 1 + is_static
    if len(sig.args) != nargs:
        raise error.NumbaError(
            "Expected %d argument types in function "
            "%s (don't include 'self')" % (nargs, py_func.__name__))

def get_signature(ext_type, is_class, is_static, sig):
    """
    Create a signature given the user-specified signature. E.g.

        class Foo(object):
            @void()                 # becomes: void(ext_type(Foo))
            def method(self): ...
    """
    if is_static:
        leading_arg_types = ()
    elif is_class:
        leading_arg_types = (numba.object_,)
    else:
        leading_arg_types = (ext_type,)

    argtypes = leading_arg_types + sig.args
    restype = sig.return_type
    return minitypes.FunctionType(return_type=restype, args=argtypes)

def get_classmethod_func(func):
    """
    Get the Python function the classmethod or staticmethod is wrapping.

    In Python2.6 classmethod and staticmethod don't have the '__func__'
    attribute.
    """
    if isinstance(func, classmethod):
        return func.__get__(object()).__func__
    else:
        assert isinstance(func, staticmethod)
        return func.__get__(object())

def process_signature(ext_type, method, default_signature,
                      is_static=False, is_class=False):
    """
    Verify a method signature.

    Returns a Method object and the resolved signature.
    """
    while True:
        if isinstance(method, types.FunctionType):
            # Process function
            if default_signature is None:
                # TODO: support deferred type inference and autojit methods
                raise error.NumbaError(
                    "Method '%s' does not have signature" % (method.__name__,))
            validate_method(method, default_signature or numba.object_(),
                            is_static)
            if default_signature is None:
                default_signature = minitypes.FunctionType(return_type=None,
                                                           args=[])
            sig = get_signature(ext_type, is_class, is_static, default_signature)
            method = Method(method, is_class, is_static)
            return method, sig.return_type, sig.args

        elif isinstance(method, minitypes.Function):
            # @double(...)
            # def func(self, ...): ...
            default_signature = method.signature
            method = method.py_func

        else:
            # Process staticmethod and classmethod
            if isinstance(method, staticmethod):
                is_static = True
            elif isinstance(method, classmethod):
                is_class = True
            else:
                return None, None, None

            method = get_classmethod_func(method)

class Method(object):
    """
    py_func: the python 'def' function
    """

    def __init__(self, py_func, is_class, is_static):
        self.py_func = py_func
        py_func.live_objects = []
        self.is_class = is_class
        self.is_static = is_static
        self.name = py_func.__name__

    def result(self, py_func):
        if self.is_class:
            return classmethod(py_func)
        elif self.is_static:
            return staticmethod(py_func)
        else:
            return py_func

#------------------------------------------------------------------------
# Populate Extension Type with Methods
#------------------------------------------------------------------------

def process_method_signatures(class_dict, ext_type):
    """
    Process all method signatures:

        * Verify signatures
        * Populate ext_type with method signatures (ExtMethodType)
    """
    for method_name, method in class_dict.iteritems():
        default_signature = None
        if (method_name == '__init__' and
                isinstance(method, types.FunctionType)):
            if inspect.getargspec(method).args:
                warnings.warn(
                    "Constructor for class '%s' has no signature, "
                    "assuming arguments have type 'object'" %
                                        ext_type.py_class.__name__)
            argtypes = [numba.object_] * (method.__code__.co_argcount - 1)
            default_signature = numba.void(*argtypes)

        method, restype, argtypes = process_signature(ext_type, method,
                                                      default_signature)
        if method is None:
            continue

        signature = typesystem.ExtMethodType(
                    return_type=restype, args=argtypes, name=method.name,
                    is_class=method.is_class, is_static=method.is_static)
        ext_type.add_method(method_name, signature)
        class_dict[method_name] = method

def _type_infer_method(env, ext_type, method, method_name, class_dict, flags):
    if method_name not in ext_type.methoddict:
        return

    signature = ext_type.get_signature(method_name)
    restype, argtypes = signature.return_type, signature.args

    class_dict[method_name] = method
    func_signature, symtab, ast = pipeline.infer_types2(
                        env, method.py_func, restype, argtypes, **flags)
    ext_type.add_method(method_name, func_signature)

def _type_infer_init_method(env, class_dict, ext_type, flags):
    initfunc = class_dict.get('__init__', None)
    if initfunc is None:
        return
<commit_msg>Make method signature processing and error issuing a bit more pluggable<commit_after>"""
Compiling extension classes works as follows:

    * Create an extension Numba/minivect type holding a symtab
    * Capture attribute types in the symtab ...

        * ... from the class attributes:

            @jit
            class Foo(object):
                attr = double

        * ... from __init__

            @jit
            class Foo(object):
                def __init__(self, attr):
                    self.attr = double(attr)

    * Type infer all methods
    * Compile all extension methods

        * Process signatures such as @void(double)
        * Infer native attributes through type inference on __init__
        * Path the extension type with a native attributes struct
        * Infer types for all other methods
        * Update the ext_type with a vtab type
        * Compile all methods

    * Create descriptors that wrap the native attributes
    * Create an extension type:

      {
        PyObject_HEAD
        ...
        virtual function table (func **)
        native attributes
      }

    The virtual function table (vtab) is a ctypes structure set as
    attribute of the extension types. Objects have a direct pointer
    for efficiency.

See also extension_types.pyx
"""


import types
import ctypes
import logging
import warnings
import inspect

import numba
from numba import *
from numba import pipeline, error, symtab
from numba import typesystem
from numba.exttypes import extension_types
from numba.minivect import minitypes

logger = logging.getLogger(__name__)

#------------------------------------------------------------------------
# Parse method signatures
#------------------------------------------------------------------------

class Method(object):
    """
    py_func: the python 'def' function
    """

    def __init__(self, py_func, name, signature, is_class, is_static):
        self.py_func = py_func
        # Name of this function, py_func.__name__ is inherently unreliable
        self.name = name
        self.signature = signature
        py_func.live_objects = []
        self.is_class = is_class
        self.is_static = is_static

    def result(self, py_func):
        if self.is_class:
            return classmethod(py_func)
        elif self.is_static:
            return staticmethod(py_func)
        else:
            return py_func

def validate_method(py_func, sig, is_static):
    assert isinstance(py_func, types.FunctionType)

    nargs = py_func.__code__.co_argcount - 1 + is_static
    if len(sig.args) != nargs:
        raise error.NumbaError(
            "Expected %d argument types in function "
            "%s (don't include 'self')" % (nargs, py_func.__name__))

def get_classmethod_func(func):
    """
    Get the Python function the classmethod or staticmethod is wrapping.

    In Python2.6 classmethod and staticmethod don't have the '__func__'
    attribute.
    """
    if isinstance(func, classmethod):
        return func.__get__(object()).__func__
    else:
        assert isinstance(func, staticmethod)
        return func.__get__(object())

class MethodMaker(object):
    """
    Creates Methods from python functions and validates user-declared
    signatures.
    """

    def __init__(self, ext_type):
        self.ext_type = ext_type

    def no_signature(self, method):
        "Called when no signature is found for the method"

    def default_signature(self, method, method_name):
        "Retrieve the default method signature for the given method"

    def make_method_type(self, method):
        "Create a method type for the given Method and declared signature"


class JitMethodMaker(MethodMaker):

    def no_signature(self, method):
        raise error.NumbaError(
            "Method '%s' does not have signature" % (method.__name__,))

    def validate_init_method(self, init_method):
        if inspect.getargspec(init_method).args:
            warnings.warn(
                "Constructor for class '%s' has no signature, "
                "assuming arguments have type 'object'" %
                self.ext_type.py_class.__name__)

    def default_signature(self, method, method_name):
        if (method_name == '__init__' and
                isinstance(method, types.FunctionType)):
            self.validate_init_method(method)

            argtypes = [numba.object_] * (method.__code__.co_argcount - 1)
            default_signature = numba.void(*argtypes)
            return default_signature
        else:
            return None

    def make_method_type(self, method):
        restype = method.signature.return_type
        argtypes = method.signature.args
        signature = typesystem.ExtMethodType(
                    return_type=restype, args=argtypes, name=method.name,
                    is_class=method.is_class, is_static=method.is_static)
        return signature


class MethodSignatureProcessor(object):
    """
    Processes signatures of extension types.
    """

    def __init__(self, class_dict, ext_type, method_maker):
        self.class_dict = class_dict
        self.ext_type = ext_type
        self.method_maker = method_maker

    def get_signature(self, is_class, is_static, sig):
        """
        Create a signature given the user-specified signature. E.g.

            class Foo(object):
                @void()                 # becomes: void(ext_type(Foo))
                def method(self): ...
        """
        if is_static:
            leading_arg_types = ()
        elif is_class:
            leading_arg_types = (numba.object_,)
        else:
            leading_arg_types = (self.ext_type,)

        argtypes = leading_arg_types + sig.args
        restype = sig.return_type
        return minitypes.FunctionType(return_type=restype, args=argtypes)

    def process_signature(self, method, method_name, default_signature,
                          is_static=False, is_class=False):
        """
        Verify a method signature.

        Returns a Method object and the resolved signature.
        """
        while True:
            if isinstance(method, types.FunctionType):
                # Process function
                if default_signature is None:
                    self.method_maker.no_signature(method)

                validate_method(method, default_signature or object_(),
                                is_static)
                if default_signature is None:
                    default_signature = minitypes.FunctionType(return_type=None,
                                                               args=[])
                sig = self.get_signature(is_class, is_static, default_signature)
                method = Method(method, method_name, sig, is_class, is_static)
                return method

            elif isinstance(method, minitypes.Function):
                # @double(...)
                # def func(self, ...): ...
                default_signature = method.signature
                method = method.py_func

            else:
                # Process staticmethod and classmethod
                if isinstance(method, staticmethod):
                    is_static = True
                elif isinstance(method, classmethod):
                    is_class = True
                else:
                    return None

                method = get_classmethod_func(method)

    def get_method_signatures(self):
        """
        Return [Method] for each decorated method in the class
        """
        methods = []

        for method_name, method in self.class_dict.iteritems():
            default_signature = self.method_maker.default_signature(method,
                                                                    method_name)

            method = self.process_signature(method, method_name,
                                            default_signature)
            if method is None:
                continue

            method_type = self.method_maker.make_method_type(method)
            methods.append((method, method_type))

        return methods


#------------------------------------------------------------------------
# Populate Extension Type with Methods
#------------------------------------------------------------------------

def process_method_signatures(class_dict, ext_type):
    """
    Process all method signatures:

        * Verify signatures
        * Populate ext_type with method signatures (ExtMethodType)
    """
    method_maker = JitMethodMaker(ext_type)
    processor = MethodSignatureProcessor(class_dict, ext_type, method_maker)
    for method, method_type in processor.get_method_signatures():
        ext_type.add_method(method.name, method_type)
        class_dict[method.name] = method

def _type_infer_method(env, ext_type, method, method_name, class_dict, flags):
    if method_name not in ext_type.methoddict:
        return

    signature = ext_type.get_signature(method_name)
    restype, argtypes = signature.return_type, signature.args

    class_dict[method_name] = method
    func_signature, symtab, ast = pipeline.infer_types2(
                        env, method.py_func, restype, argtypes, **flags)
    ext_type.add_method(method_name, func_signature)

def _type_infer_init_method(env, class_dict, ext_type, flags):
    initfunc = class_dict.get('__init__', None)
    if initfunc is None:
        return

{'fraction_non_alphanumeric': 0.10716871832005793, 'fraction_numerical': 0.0006034274680183442, 'mean_word_length': 1.9136779184247539, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4615288', 'n_tokens_mistral': 4308, 'n_tokens_neox': 4156, 'n_words': 1192}