<commit_before>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/

#include <Python.h>
#include <dlfcn.h>
#include <bh.h>

#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <numpy/arrayobject.h>

//The NumPy API changed in version 1.7
#if(NPY_API_VERSION >= 0x00000007)
    #define BH_PyArrayObject PyArrayObject_fields
#else
    #define BH_PyArrayObject PyArrayObject
    #define NPY_ARRAY_OWNDATA NPY_OWNDATA
#endif

//Forward declaration
static PyObject *BhArray_data_bhc2np(PyObject *self, PyObject *args);
static PyTypeObject BhArrayType;

#define BhArray_CheckExact(op) (((PyObject*)(op))->ob_type == &BhArrayType)
PyObject *ndarray = NULL; //The ndarray Python module
PyObject *ufunc = NULL; //The ufunc Python module
PyObject *bohrium = NULL; //The Bohrium Python module
PyObject *array_create = NULL; //The array_create Python module

typedef struct
{
    BH_PyArrayObject base;
    PyObject *bhc_ary;
    PyObject *array_priority;
    int mmap_allocated;
}BhArray;

//Help function to retrieve the Bohrium-C data pointer
//Return -1 on error
static int get_bhc_data_pointer(PyObject *ary, int force_allocation, int nullify, void **out_data)
{
    if(((BhArray*)ary)->mmap_allocated == 0)
    {
        PyErr_SetString(PyExc_TypeError, "The array data wasn't allocated through mmap(). "
                  "Typically, this is because the base array was created from a template, "
                  "which is not support by Bohrium");
        return -1;
    }
    PyObject *data = PyObject_CallMethod(ndarray, "get_bhc_data_pointer",
                                         "Oii", ary, force_allocation, nullify);
    if(data == NULL)
        return -1;
    if(!PyInt_Check(data))
    {
        PyErr_SetString(PyExc_TypeError, "get_bhc_data_pointer(ary) should "
                "return a Python integer that represents a memory address");
        Py_DECREF(data);
        return -1;
    }
    void *d = PyLong_AsVoidPtr(data);
    Py_DECREF(data);
    if(force_allocation && d == NULL)
    {
        PyErr_SetString(PyExc_TypeError, "get_bhc_data_pointer(ary, allocate=True) "
                                         "shouldn't return a NULL pointer");
        return -1;
    }
    *out_data = d;
    return 0;
}

//Help function for unprotect memory
//Return -1 on error
static int _munprotect(void *data, bh_intp size)
{
    if(mprotect(data, size, PROT_WRITE) != 0)
    {
        int errsv = errno;//mprotect() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not (un-)mprotect a "
                     "data region. Returned error code by mprotect(): %s.\n"
                     ,strerror(errsv));
        return -1;
    }
    return 0;
}

//Help function for memory un-map
//Return -1 on error
static int _munmap(void *addr, bh_intp size)
{
    if(munmap(addr, size) == -1)
    {
        int errsv = errno;//munmmap() sets the errno.
        PyErr_Format(PyExc_RuntimeError, "The Array Data Protection "
                     "could not mummap the data region: %p (size: %ld). "
                     "Returned error code by mmap: %s.", addr, size,
                     strerror(errsv));
        return -1;
    }
    return 0;
}

//Help function for memory re-map
//Return -1 on error
static int _mremap_data(void *dst, void *src, bh_intp size)
{
#if MREMAP_FIXED
    if(mremap(src, size, size, MREMAP_FIXED|MREMAP_MAYMOVE, dst) == MAP_FAILED)
    {
        int errsv = errno;//mremap() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not mremap a "
                     "data region. Returned error code by mremap(): %s.\n"
                     ,strerror(errsv));
        return -1;
    }
    return 0;
#else
    //Systems that doesn't support mremap will use memcpy, which introduces a
    //race-condition if another thread access the 'dst' memory before memcpy finishes.
    if(_munprotect(dst, size) != 0)
        return NULL;
    memcpy(dst, src, size);
    return _munmap(src, size);
#endif
}

//Callback function called by the signal library
void mem_access_callback(unsigned long id, uintptr_t addr)
{
    PyObject *ary = (PyObject *) id;
//    printf("mem_access_callback() - ary: %p, addr: %p\n", ary, (void*) addr);

    PyGILState_STATE GIL = PyGILState_Ensure();
    PyErr_WarnEx(NULL,"Encountering an operation not supported by Bohrium. "
                      "It will be handled by the original NumPy.",1);

    if(BhArray_data_bhc2np(ary, NULL) == NULL)
        PyErr_Print();
    PyGILState_Release(GIL);
}

//Help function for protecting the memory of the NumPy part of 'ary'
//Return -1 on error
static int _mprotect_np_part(BhArray *ary)
{
    //Finally we memory protect the NumPy data
    if(mprotect(ary->base.data, PyArray_NBYTES((PyArrayObject*)ary), PROT_NONE) == -1)
    {
        int errsv = errno;//mprotect() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not protect a data"
                     "data region. Returned error code by mprotect: %s.\n",
                     strerror(errsv));
        return -1;
    }
    attach_signal((signed long)ary, (uintptr_t) ary->base.data,
                  PyArray_NBYTES((PyArrayObject*)ary), mem_access_callback);
    return 0;
}

//Help function for allocate protected memory for the NumPy part of 'ary'
//This function only allocates if the 'ary' is a new base array and avoids multiple
//allocations by checking and setting the ary->mmap_allocated
//Return -1 on error
static int _protected_malloc(BhArray *ary)
{
    if(ary->mmap_allocated || !PyArray_CHKFLAGS((PyArrayObject*)ary, NPY_ARRAY_OWNDATA))
        return 0;
    ary->mmap_allocated = 1;

    //Allocate page-size aligned memory.
    //The MAP_PRIVATE and MAP_ANONYMOUS flags is not 100% portable. See:
    //<http://stackoverflow.com/questions/4779188/how-to-use-mmap-to-allocate-a-memory-in-heap>
    void *addr = mmap(0, PyArray_NBYTES((PyArrayObject*)ary), PROT_NONE,
                      MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
    if(addr == MAP_FAILED)
    {
        int errsv = errno;//mmap() sets the errno.
        PyErr_Format(PyExc_RuntimeError, "The Array Data Protection "
                     "could not mmap a data region. "
                     "Returned error code by mmap: %s.", strerror(errsv));
        return -1;
    }

    //Lets free the NumPy allocated memory and use the mprotect'ed memory instead
    free(ary->base.data);
    ary->base.data = addr;

    attach_signal((signed long)ary, (uintptr_t) ary->base.data,
                  PyArray_NBYTES((PyArrayObject*)ary), mem_access_callback);
    return 0;
}

static PyObject *
BhArray_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    PyObject *ret = PyArray_Type.tp_new(type, args, kwds);
    if(_protected_malloc((BhArray *) ret) != 0)
        return NULL;
    return ret;
}

static PyObject *
BhArray_finalize(PyObject *self, PyObject *args)
{
    int e = PyObject_IsInstance(self, (PyObject*) &BhArrayType);
    if(e == -1)
    {
        return NULL;
    }
    else if (e == 0)
    {
        Py_RETURN_NONE;
    }
    ((BhArray*)self)->bhc_ary = Py_None;
    Py_INCREF(Py_None);
    //The __array_priority__ should be greater than 0.0 to give Bohrium precedence
    ((BhArray*)self)->array_priority = PyFloat_FromDouble(2.0);

    Py_RETURN_NONE;
}

static PyObject *
BhArray_alloc(PyTypeObject *type, Py_ssize_t nitems)
{
    PyObject *obj;
    obj = (PyObject *)malloc(type->tp_basicsize);
    PyObject_Init(obj, type);
    ((BhArray*)obj)->mmap_allocated = 0;
    return obj;
}

static void
BhArray_dealloc(BhArray* self)
{
    assert(BhArray_CheckExact(self));

    if(self->bhc_ary == NULL)
        goto finish;

    if(self->bhc_ary == Py_None)
    {
        Py_DECREF(Py_None);
        goto finish;
    }
    PyObject *r = PyObject_CallMethod(ndarray, "del_bhc_obj", "O", self->bhc_ary);
    if(r == NULL)
    {
        PyErr_Print();
        goto finish;
    }
finish:
    if(!PyArray_CHKFLAGS((PyArrayObject*)self, NPY_ARRAY_OWNDATA))
    {
        BhArrayType.tp_base->tp_dealloc((PyObject*)self);
        return;//The array doesn't own the array data
    }

    assert(!PyDataType_FLAGCHK(PyArray_DESCR((PyArrayObject*)self), NPY_ITEM_REFCOUNT));

    if(self->mmap_allocated)
    {
        if(_munmap(PyArray_DATA((PyArrayObject*)self),
                   PyArray_NBYTES((PyArrayObject*)self)) == -1)
            PyErr_Print();

        detach_signal((signed long)self, mem_access_callback);
        self->base.data = NULL;
    }
    BhArrayType.tp_base->tp_dealloc((PyObject*)self);
}

static PyObject *
BhArray_data_bhc2np(PyObject *self, PyObject *args)
{
    assert(args == NULL);
    assert(BhArray_CheckExact(self));

    //We move the whole array (i.e. the base array) from Bohrium to NumPy
    PyObject *base = PyObject_CallMethod(ndarray, "get_base", "O", self);

    //Note that we always detach the signal before returning
    detach_signal((signed long)base, mem_access_callback);

    if(base == NULL)
        return NULL;
    assert(BhArray_CheckExact(base));

/* TODO: handle the case where bhc_ary is None by unprotecting the memory.
    //Check if we need to do anything
    if(((BhArray*)base)->bhc_ary == Py_None)
    {
        Py_DECREF(base);
        Py_RETURN_NONE;
    }
*/
    if(!PyArray_CHKFLAGS((PyArrayObject*)base, NPY_ARRAY_OWNDATA))
    {
        PyErr_Format(PyExc_ValueError,"The base array doesn't own its data");
        return NULL;
    }

    //Calling get_bhc_data_pointer(base, allocate=False, nullify=True)
    void *d = NULL;
    if(get_bhc_data_pointer(base, 0, 1, &d) == -1)
        return NULL;
    if(d != NULL)
    {
        if(_mremap_data(PyArray_DATA((PyArrayObject*)base), d,
                        PyArray_NBYTES((PyArrayObject*)base)) != 0)
            return NULL;
            Py_DECREF(base);
    }
    else
    {
        if(_munprotect(PyArray_DATA((PyArrayObject*)base),
                       PyArray_NBYTES((PyArrayObject*)base)) != 0)
            return NULL;
            Py_DECREF(base);
    }

    //Lets delete the current bhc_ary
    if(PyObject_CallMethod(ndarray, "del_bhc", "O", self) == NULL)
        return NULL;
    Py_RETURN_NONE;
}

static PyObject *
BhArray_data_np2bhc(PyObject *self, PyObject *args)
{
    assert(args == NULL);
    assert(BhArray_CheckExact(self));

    //We move the whole array (i.e. the base array) from Bohrium to NumPy
    PyObject *base = PyObject_CallMethod(ndarray, "get_base", "O", self);
    if(base == NULL)
        return NULL;
    assert(BhArray_CheckExact(base));

    if(!PyArray_CHKFLAGS((PyArrayObject*)base, NPY_ARRAY_OWNDATA))
    {
        PyErr_Format(PyExc_ValueError,"The base array doesn't own its data");
        return NULL;
    }

    //Make sure that bhc_ary exist
    if(((BhArray*)base)->bhc_ary == Py_None)
    {
        PyObject *err = PyObject_CallMethod(ndarray, "new_bhc_base", "O", base);
        if(err == NULL)
            return NULL;
        Py_DECREF(err);
    }

    //Calling get_bhc_data_pointer(base, allocate=True, nullify=False)
    void *d = NULL;
    if(get_bhc_data_pointer(base, 1, 0, &d) == -1)
        return NULL;
    Py_DECREF(base);
    if(d != NULL)
    {
        memcpy(d, PyArray_DATA((PyArrayObject*)base), PyArray_NBYTES((PyArrayObject*)base));
        if(_mprotect_np_part((BhArray*)base) != 0)
            return NULL;
    }
    Py_RETURN_NONE;
}

static PyObject *
BhArray_data_fill(PyObject *self, PyObject *args)
{
    PyObject *np_ary;
    if(!PyArg_ParseTuple(args, "O:ndarray", &np_ary))
        return NULL;

    if(!PyArray_Check(np_ary))
    {
        PyErr_SetString(PyExc_TypeError, "must be a NumPy array");
        return NULL;
    }

    if(!PyArray_ISCARRAY_RO((PyArrayObject*)np_ary))<commit_msg>bhpy: now we protect memory on BhArray_finalize() as well<commit_after>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/

#include <Python.h>
#include <dlfcn.h>
#include <bh.h>

#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <numpy/arrayobject.h>

//The NumPy API changed in version 1.7
#if(NPY_API_VERSION >= 0x00000007)
    #define BH_PyArrayObject PyArrayObject_fields
#else
    #define BH_PyArrayObject PyArrayObject
    #define NPY_ARRAY_OWNDATA NPY_OWNDATA
#endif

//Forward declaration
static PyObject *BhArray_data_bhc2np(PyObject *self, PyObject *args);
static PyTypeObject BhArrayType;

#define BhArray_CheckExact(op) (((PyObject*)(op))->ob_type == &BhArrayType)
PyObject *ndarray = NULL; //The ndarray Python module
PyObject *ufunc = NULL; //The ufunc Python module
PyObject *bohrium = NULL; //The Bohrium Python module
PyObject *array_create = NULL; //The array_create Python module

typedef struct
{
    BH_PyArrayObject base;
    PyObject *bhc_ary;
    PyObject *array_priority;
    int mmap_allocated;
}BhArray;

//Help function to retrieve the Bohrium-C data pointer
//Return -1 on error
static int get_bhc_data_pointer(PyObject *ary, int force_allocation, int nullify, void **out_data)
{
    if(((BhArray*)ary)->mmap_allocated == 0)
    {
        PyErr_SetString(PyExc_TypeError, "The array data wasn't allocated through mmap(). "
                  "Typically, this is because the base array was created from a template, "
                  "which is not support by Bohrium");
        return -1;
    }
    PyObject *data = PyObject_CallMethod(ndarray, "get_bhc_data_pointer",
                                         "Oii", ary, force_allocation, nullify);
    if(data == NULL)
        return -1;
    if(!PyInt_Check(data))
    {
        PyErr_SetString(PyExc_TypeError, "get_bhc_data_pointer(ary) should "
                "return a Python integer that represents a memory address");
        Py_DECREF(data);
        return -1;
    }
    void *d = PyLong_AsVoidPtr(data);
    Py_DECREF(data);
    if(force_allocation && d == NULL)
    {
        PyErr_SetString(PyExc_TypeError, "get_bhc_data_pointer(ary, allocate=True) "
                                         "shouldn't return a NULL pointer");
        return -1;
    }
    *out_data = d;
    return 0;
}

//Help function for unprotect memory
//Return -1 on error
static int _munprotect(void *data, bh_intp size)
{
    if(mprotect(data, size, PROT_WRITE) != 0)
    {
        int errsv = errno;//mprotect() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not (un-)mprotect a "
                     "data region. Returned error code by mprotect(): %s.\n"
                     ,strerror(errsv));
        return -1;
    }
    return 0;
}

//Help function for memory un-map
//Return -1 on error
static int _munmap(void *addr, bh_intp size)
{
    if(munmap(addr, size) == -1)
    {
        int errsv = errno;//munmmap() sets the errno.
        PyErr_Format(PyExc_RuntimeError, "The Array Data Protection "
                     "could not mummap the data region: %p (size: %ld). "
                     "Returned error code by mmap: %s.", addr, size,
                     strerror(errsv));
        return -1;
    }
    return 0;
}

//Help function for memory re-map
//Return -1 on error
static int _mremap_data(void *dst, void *src, bh_intp size)
{
#if MREMAP_FIXED
    if(mremap(src, size, size, MREMAP_FIXED|MREMAP_MAYMOVE, dst) == MAP_FAILED)
    {
        int errsv = errno;//mremap() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not mremap a "
                     "data region. Returned error code by mremap(): %s.\n"
                     ,strerror(errsv));
        return -1;
    }
    return 0;
#else
    //Systems that doesn't support mremap will use memcpy, which introduces a
    //race-condition if another thread access the 'dst' memory before memcpy finishes.
    if(_munprotect(dst, size) != 0)
        return NULL;
    memcpy(dst, src, size);
    return _munmap(src, size);
#endif
}

//Callback function called by the signal library
void mem_access_callback(unsigned long id, uintptr_t addr)
{
    PyObject *ary = (PyObject *) id;
//    printf("mem_access_callback() - ary: %p, addr: %p\n", ary, (void*) addr);

    PyGILState_STATE GIL = PyGILState_Ensure();
    PyErr_WarnEx(NULL,"Encountering an operation not supported by Bohrium. "
                      "It will be handled by the original NumPy.",1);

    if(BhArray_data_bhc2np(ary, NULL) == NULL)
        PyErr_Print();
    PyGILState_Release(GIL);
}

//Help function for protecting the memory of the NumPy part of 'ary'
//Return -1 on error
static int _mprotect_np_part(BhArray *ary)
{
    //Finally we memory protect the NumPy data
    if(mprotect(ary->base.data, PyArray_NBYTES((PyArrayObject*)ary), PROT_NONE) == -1)
    {
        int errsv = errno;//mprotect() sets the errno.
        PyErr_Format(PyExc_RuntimeError,"Error - could not protect a data"
                     "data region. Returned error code by mprotect: %s.\n",
                     strerror(errsv));
        return -1;
    }
    attach_signal((signed long)ary, (uintptr_t) ary->base.data,
                  PyArray_NBYTES((PyArrayObject*)ary), mem_access_callback);
    return 0;
}

//Help function for allocate protected memory for the NumPy part of 'ary'
//This function only allocates if the 'ary' is a new base array and avoids multiple
//allocations by checking and setting the ary->mmap_allocated
//Return -1 on error
static int _protected_malloc(BhArray *ary)
{
    if(ary->mmap_allocated || !PyArray_CHKFLAGS((PyArrayObject*)ary, NPY_ARRAY_OWNDATA))
        return 0;
    ary->mmap_allocated = 1;

    //Allocate page-size aligned memory.
    //The MAP_PRIVATE and MAP_ANONYMOUS flags is not 100% portable. See:
    //<http://stackoverflow.com/questions/4779188/how-to-use-mmap-to-allocate-a-memory-in-heap>
    void *addr = mmap(0, PyArray_NBYTES((PyArrayObject*)ary), PROT_NONE,
                      MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
    if(addr == MAP_FAILED)
    {
        int errsv = errno;//mmap() sets the errno.
        PyErr_Format(PyExc_RuntimeError, "The Array Data Protection "
                     "could not mmap a data region. "
                     "Returned error code by mmap: %s.", strerror(errsv));
        return -1;
    }

    //Lets free the NumPy allocated memory and use the mprotect'ed memory instead
    free(ary->base.data);
    ary->base.data = addr;

    attach_signal((signed long)ary, (uintptr_t) ary->base.data,
                  PyArray_NBYTES((PyArrayObject*)ary), mem_access_callback);
    return 0;
}

static PyObject *
BhArray_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    PyObject *ret = PyArray_Type.tp_new(type, args, kwds);
    if(_protected_malloc((BhArray *) ret) != 0)
        return NULL;
    return ret;
}

static PyObject *
BhArray_finalize(PyObject *self, PyObject *args)
{
    int e = PyObject_IsInstance(self, (PyObject*) &BhArrayType);
    if(e == -1)
    {
        return NULL;
    }
    else if (e == 0)
    {
        Py_RETURN_NONE;
    }
    ((BhArray*)self)->bhc_ary = Py_None;
    Py_INCREF(Py_None);
    //The __array_priority__ should be greater than 0.0 to give Bohrium precedence
    ((BhArray*)self)->array_priority = PyFloat_FromDouble(2.0);

    if(_protected_malloc((BhArray *) self) != 0)
        return NULL;

    Py_RETURN_NONE;
}

static PyObject *
BhArray_alloc(PyTypeObject *type, Py_ssize_t nitems)
{
    PyObject *obj;
    obj = (PyObject *)malloc(type->tp_basicsize);
    PyObject_Init(obj, type);
    ((BhArray*)obj)->mmap_allocated = 0;
    return obj;
}

static void
BhArray_dealloc(BhArray* self)
{
    assert(BhArray_CheckExact(self));

    if(self->bhc_ary == NULL)
        goto finish;

    if(self->bhc_ary == Py_None)
    {
        Py_DECREF(Py_None);
        goto finish;
    }
    PyObject *r = PyObject_CallMethod(ndarray, "del_bhc_obj", "O", self->bhc_ary);
    if(r == NULL)
    {
        PyErr_Print();
        goto finish;
    }
finish:
    if(!PyArray_CHKFLAGS((PyArrayObject*)self, NPY_ARRAY_OWNDATA))
    {
        BhArrayType.tp_base->tp_dealloc((PyObject*)self);
        return;//The array doesn't own the array data
    }

    assert(!PyDataType_FLAGCHK(PyArray_DESCR((PyArrayObject*)self), NPY_ITEM_REFCOUNT));

    if(self->mmap_allocated)
    {
        if(_munmap(PyArray_DATA((PyArrayObject*)self),
                   PyArray_NBYTES((PyArrayObject*)self)) == -1)
            PyErr_Print();

        detach_signal((signed long)self, mem_access_callback);
        self->base.data = NULL;
    }
    BhArrayType.tp_base->tp_dealloc((PyObject*)self);
}

static PyObject *
BhArray_data_bhc2np(PyObject *self, PyObject *args)
{
    assert(args == NULL);
    assert(BhArray_CheckExact(self));

    //We move the whole array (i.e. the base array) from Bohrium to NumPy
    PyObject *base = PyObject_CallMethod(ndarray, "get_base", "O", self);

    //Note that we always detach the signal before returning
    detach_signal((signed long)base, mem_access_callback);

    if(base == NULL)
        return NULL;
    assert(BhArray_CheckExact(base));

/* TODO: handle the case where bhc_ary is None by unprotecting the memory.
    //Check if we need to do anything
    if(((BhArray*)base)->bhc_ary == Py_None)
    {
        Py_DECREF(base);
        Py_RETURN_NONE;
    }
*/
    if(!PyArray_CHKFLAGS((PyArrayObject*)base, NPY_ARRAY_OWNDATA))
    {
        PyErr_Format(PyExc_ValueError,"The base array doesn't own its data");
        return NULL;
    }

    //Calling get_bhc_data_pointer(base, allocate=False, nullify=True)
    void *d = NULL;
    if(get_bhc_data_pointer(base, 0, 1, &d) == -1)
        return NULL;
    if(d != NULL)
    {
        if(_mremap_data(PyArray_DATA((PyArrayObject*)base), d,
                        PyArray_NBYTES((PyArrayObject*)base)) != 0)
            return NULL;
            Py_DECREF(base);
    }
    else
    {
        if(_munprotect(PyArray_DATA((PyArrayObject*)base),
                       PyArray_NBYTES((PyArrayObject*)base)) != 0)
            return NULL;
            Py_DECREF(base);
    }

    //Lets delete the current bhc_ary
    if(PyObject_CallMethod(ndarray, "del_bhc", "O", self) == NULL)
        return NULL;
    Py_RETURN_NONE;
}

static PyObject *
BhArray_data_np2bhc(PyObject *self, PyObject *args)
{
    assert(args == NULL);
    assert(BhArray_CheckExact(self));

    //We move the whole array (i.e. the base array) from Bohrium to NumPy
    PyObject *base = PyObject_CallMethod(ndarray, "get_base", "O", self);
    if(base == NULL)
        return NULL;
    assert(BhArray_CheckExact(base));

    if(!PyArray_CHKFLAGS((PyArrayObject*)base, NPY_ARRAY_OWNDATA))
    {
        PyErr_Format(PyExc_ValueError,"The base array doesn't own its data");
        return NULL;
    }

    //Make sure that bhc_ary exist
    if(((BhArray*)base)->bhc_ary == Py_None)
    {
        PyObject *err = PyObject_CallMethod(ndarray, "new_bhc_base", "O", base);
        if(err == NULL)
            return NULL;
        Py_DECREF(err);
    }

    //Calling get_bhc_data_pointer(base, allocate=True, nullify=False)
    void *d = NULL;
    if(get_bhc_data_pointer(base, 1, 0, &d) == -1)
        return NULL;
    Py_DECREF(base);
    if(d != NULL)
    {
        detach_signal((signed long)base, mem_access_callback);
        if(_munprotect(PyArray_DATA((PyArrayObject*)base),
                       PyArray_NBYTES((PyArrayObject*)base)) != 0)
            return NULL;
        memcpy(d, PyArray_DATA((PyArrayObject*)base), PyArray_NBYTES((PyArrayObject*)base));
    }
    if(_mprotect_np_part((BhArray*)base) != 0)
        return NULL;
    Py_RETURN_NONE;
}

static PyObject *
BhArray_data_fill(PyObject *self, PyObject *args)
{
    PyObject *np_ary;
    if(!PyArg_ParseTuple(args, "O:ndarray", &np_ary))
        return NULL;

    if(!PyArray_Check(np_ary))
    {
        PyErr_SetString(PyExc_TypeError, "must be a NumPy array");
        return NULL;
    }

    if(!PyArray_ISCARRAY_RO((PyArrayObject*)np_ary))
{'fraction_non_alphanumeric': 0.1120162932790224, 'fraction_numerical': 0.006868735274150393, 'mean_word_length': 2.862717877525837, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5594441', 'n_tokens_mistral': 8474, 'n_tokens_neox': 8300, 'n_words': 2150}