<commit_before>#------------------------------------------------------------------------------

# Python 3 buffer interface (PEP 3118)
cdef extern from "Python.h":
    ctypedef struct Py_buffer:
        void *obj
        void *buf
        Py_ssize_t len
        Py_ssize_t itemsize
        bint readonly
        char *format
        #int ndim
        #Py_ssize_t *shape
        #Py_ssize_t *strides
        #Py_ssize_t *suboffsets
    cdef enum:
        PyBUF_SIMPLE
        PyBUF_WRITABLE
        PyBUF_FORMAT
        PyBUF_ANY_CONTIGUOUS
        PyBUF_ND
        PyBUF_STRIDES
    int  PyObject_CheckBuffer(object)
    int  PyObject_GetBuffer(object, Py_buffer *, int) except -1
    void PyBuffer_Release(Py_buffer *)
    int  PyBuffer_FillInfo(Py_buffer *, object,
                           void *, Py_ssize_t,
                           bint, int) except -1

# Python 2 buffer interface (legacy)
cdef extern from "Python.h":
    ctypedef void const_void "const void"
    int PyObject_CheckReadBuffer(object)
    int PyObject_AsReadBuffer (object, const_void **, Py_ssize_t *) except -1
    int PyObject_AsWriteBuffer(object, void **, Py_ssize_t *) except -1
    object PyBuffer_FromObject(object, Py_ssize_t, Py_ssize_t)
    object PyBuffer_FromReadWriteObject(object, Py_ssize_t, Py_ssize_t)

#------------------------------------------------------------------------------

cdef extern from *:
    enum: PYPY "PyMPI_RUNTIME_PYPY"

cdef type array_array
cdef type numpy_array
if PYPY: from array   import array   as array_array
if PYPY: from numpypy import ndarray as numpy_array

cdef int \
PyPy_GetBuffer(object obj, Py_buffer *view, int flags) \
except -1:
    cdef Py_ssize_t addr = 0
    cdef Py_ssize_t size = 0
    cdef bint readonly = 0
    if isinstance(obj, bytes):
        addr = <Py_ssize_t> PyBytes_AsString(obj)
        size = PyBytes_Size(obj)
        readonly = 1
    #elif isinstance(obj, bytearray):
    #    addr = <Py_ssize_t> PyByteArray_AsString(obj)
    #    size = PyByteArray_Size(obj)
    #    readonly = 0
    elif isinstance(obj, array_array):
        addr, size = obj.buffer_info()
        size *= obj.itemsize
        readonly = 0
    elif isinstance(obj, numpy_array):
        addr, readonly = obj.__array_interface__['data']
        size =  obj.size
        size *= obj.itemsize
    else:
        if (flags & PyBUF_WRITABLE) == PyBUF_WRITABLE:
            readonly = 0
            PyObject_AsWriteBuffer(obj, <void**>&addr, &size)
        else:
            readonly = 1
            PyObject_AsReadBuffer(obj, <const_void**>&addr, &size)
    PyBuffer_FillInfo(view, obj, <void*>addr, size, readonly, flags)
    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT: view.format = b"B"
    return 0
<commit_msg>PyPy: Rework buffer interface emulation<commit_after>#------------------------------------------------------------------------------

# Python 3 buffer interface (PEP 3118)
cdef extern from "Python.h":
    ctypedef struct Py_buffer:
        void *obj
        void *buf
        Py_ssize_t len
        Py_ssize_t itemsize
        bint readonly
        char *format
        #int ndim
        #Py_ssize_t *shape
        #Py_ssize_t *strides
        #Py_ssize_t *suboffsets
    cdef enum:
        PyBUF_SIMPLE
        PyBUF_WRITABLE
        PyBUF_FORMAT
        PyBUF_ANY_CONTIGUOUS
        PyBUF_ND
        PyBUF_STRIDES
    int  PyObject_CheckBuffer(object)
    int  PyObject_GetBuffer(object, Py_buffer *, int) except -1
    void PyBuffer_Release(Py_buffer *)
    int  PyBuffer_FillInfo(Py_buffer *, object,
                           void *, Py_ssize_t,
                           bint, int) except -1

# Python 2 buffer interface (legacy)
cdef extern from "Python.h":
    ctypedef void const_void "const void"
    int PyObject_CheckReadBuffer(object)
    int PyObject_AsReadBuffer (object, const_void **, Py_ssize_t *) except -1
    int PyObject_AsWriteBuffer(object, void **, Py_ssize_t *) except -1
    object PyBuffer_FromObject(object, Py_ssize_t, Py_ssize_t)
    object PyBuffer_FromReadWriteObject(object, Py_ssize_t, Py_ssize_t)

#------------------------------------------------------------------------------

cdef extern from *:
    enum: PYPY "PyMPI_RUNTIME_PYPY"

cdef type array_array
cdef type numpy_array
if PYPY: from array   import array   as array_array
if PYPY: from numpypy import ndarray as numpy_array

cdef int \
PyPy_GetBuffer(object obj, Py_buffer *view, int flags) \
except -1:
    cdef object addr
    cdef void *buf = NULL
    cdef Py_ssize_t size = 0
    cdef bint readonly = 0
    if isinstance(obj, bytes):
        buf = <void*> PyBytes_AsString(obj)
        size = PyBytes_Size(obj)
        readonly = 1
    #elif isinstance(obj, bytearray):
    #    buf = <void*> PyByteArray_AsString(obj)
    #    size = PyByteArray_Size(obj)
    #    readonly = 0
    elif isinstance(obj, array_array):
        addr, size = obj.buffer_info()
        buf = PyLong_AsVoidPtr(addr)
        size *= obj.itemsize
        readonly = 0
    elif isinstance(obj, numpy_array):
        addr, readonly = obj.__array_interface__['data']
        buf = PyLong_AsVoidPtr(addr)
        size = obj.nbytes
    else:
        if (flags & PyBUF_WRITABLE) == PyBUF_WRITABLE:
            readonly = 0
            PyObject_AsWriteBuffer(obj, &buf, &size)
        else:
            readonly = 1
            PyObject_AsReadBuffer(obj, <const_void**>&buf, &size)
    PyBuffer_FillInfo(view, obj, buf, size, readonly, flags)
    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT: view.format = b"B"
    return 0

{'fraction_non_alphanumeric': 0.1316931982633864, 'fraction_numerical': 0.007054992764109986, 'mean_word_length': 2.114929577464789, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7017827', 'n_tokens_mistral': 1773, 'n_tokens_neox': 1750, 'n_words': 445}