<commit_before># For reference:
#    typedef struct {
#    PyObject_HEAD                   // indices (skipping the head)
#    char *data;                     // 0
#    int nd;                         // 1
#    int *dimensions, *strides;      // 2, 3
#    PyObject *base;                 // 4
#    PyArray_Descr *descr;           // 5
#    int flags;                      // 6
#    } PyArrayObject;

from numba import *
from numba import typesystem
from numba.llvm_types import _head_len, _int32
import llvm.core as lc

const_int = lambda X: lc.Constant.int(_int32, X)

def set_metadata(tbaa, instr, type):
    if type is not None:
        metadata = tbaa.get_metadata(type)
        instr.set_metadata("tbaa", metadata)

def make_property(type=None, invariant=True):
    """
    type: The type to be used for TBAA annotation
    """
    def decorator(access_func):
        def load(self):
            instr = self.builder.load(access_func(self))
            set_metadata(self.tbaa, instr, type)
            return instr

        def store(self, value):
            ptr = access_func(self)
            instr = self.builder.store(value, ptr)
            set_metadata(self.tbaa, instr, type)

        return property(load, store)

    return decorator

class PyArrayAccessor(object):
    """
    Convenient access to a the native fields of a NumPy array.

    builder: llvmpy IRBuilder
    pyarray_ptr: pointer to the numpy array
    tbaa: metadata.TBAAMetadata instance
    """

    def __init__(self, builder, pyarray_ptr, tbaa, dtype):
        self.builder = builder
        self.pyarray_ptr = pyarray_ptr
        self.tbaa = tbaa
        self.dtype = dtype

    def _get_element(self, idx):
        indices = map(const_int, [0, _head_len + idx])
        ptr = self.builder.gep(self.pyarray_ptr, indices)
        return ptr

    def get_data(self):
        instr = self.builder.load(self._get_element(0))
        set_metadata(self.tbaa, instr, self.dtype.pointer())
        return instr

    def set_data(self, value):
        instr = self.builder.store(value, self._get_element(0))
        set_metadata(self.tbaa, instr, self.dtype.pointer())

    data = property(get_data, set_data, "The array.data attribute")

    def typed_data(self, context):
        data = self.data
        ltype = self.dtype.pointer().to_llvm(context)
        return self.builder.bitcast(data, ltype)<commit_msg>Make tbaa optional in PyArrayAccessor<commit_after># For reference:
#    typedef struct {
#    PyObject_HEAD                   // indices (skipping the head)
#    char *data;                     // 0
#    int nd;                         // 1
#    int *dimensions, *strides;      // 2, 3
#    PyObject *base;                 // 4
#    PyArray_Descr *descr;           // 5
#    int flags;                      // 6
#    } PyArrayObject;

from numba import *
from numba import typesystem
from numba.llvm_types import _head_len, _int32
import llvm.core as lc

const_int = lambda X: lc.Constant.int(_int32, X)

def set_metadata(tbaa, instr, type):
    if type is not None:
        metadata = tbaa.get_metadata(type)
        instr.set_metadata("tbaa", metadata)

def make_property(type=None, invariant=True):
    """
    type: The type to be used for TBAA annotation
    """
    def decorator(access_func):
        def load(self):
            instr = self.builder.load(access_func(self))
            if self.tbaa:
                set_metadata(self.tbaa, instr, type)
            return instr

        def store(self, value):
            ptr = access_func(self)
            instr = self.builder.store(value, ptr)
            if self.tbaa:
                set_metadata(self.tbaa, instr, type)

        return property(load, store)

    return decorator

class PyArrayAccessor(object):
    """
    Convenient access to a the native fields of a NumPy array.

    builder: llvmpy IRBuilder
    pyarray_ptr: pointer to the numpy array
    tbaa: metadata.TBAAMetadata instance
    """

    def __init__(self, builder, pyarray_ptr, tbaa=None, dtype=None):
        self.builder = builder
        self.pyarray_ptr = pyarray_ptr
        self.tbaa = tbaa # this maybe None
        self.dtype = dtype

    def _get_element(self, idx):
        indices = map(const_int, [0, _head_len + idx])
        ptr = self.builder.gep(self.pyarray_ptr, indices)
        return ptr

    def get_data(self):
        instr = self.builder.load(self._get_element(0))
        if self.tbaa:
            set_metadata(self.tbaa, instr, self.dtype.pointer())
        return instr

    def set_data(self, value):
        instr = self.builder.store(value, self._get_element(0))
        if self.tbaa:
            set_metadata(self.tbaa, instr, self.dtype.pointer())

    data = property(get_data, set_data, "The array.data attribute")

    def typed_data(self, context):
        data = self.data
        ltype = self.dtype.pointer().to_llvm(context)
        return self.builder.bitcast(data, ltype)
{'fraction_non_alphanumeric': 0.10020242914979757, 'fraction_numerical': 0.005668016194331984, 'mean_word_length': 2.161228406909789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415205', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1494, 'n_words': 394}