<commit_before># NumPy static imports for Cython
#
# If any of the PyArray_* functions are called, import_array must be
# called first.
#
# This also defines backwards-compatibility buffer acquisition
# code for use in Python 2.x (or Python <= 2.5 when NumPy starts
# implementing PEP-3118 directly).
#
# Because of laziness, the format string of the buffer is statically
# allocated. Increase the size if this is not enough, or submit a
# patch to do this properly.
#
# Author: <NAME>
#

DEF _buffer_format_string_len = 255

cimport cpython.buffer as pybuf
from cpython.ref cimport Py_INCREF
from cpython.mem cimport PyObject_Malloc, PyObject_Free
from cpython.object cimport PyObject, PyTypeObject
from cpython.type cimport type
cimport libc.stdio as stdio
<commit_msg>Add a marker to the generated C code if the NumPy declarations came from Cython/Includes/numpy/. See https://github.com/cython/cython/issues/3573<commit_after># NumPy static imports for Cython
#
# If any of the PyArray_* functions are called, import_array must be
# called first.
#
# This also defines backwards-compatibility buffer acquisition
# code for use in Python 2.x (or Python <= 2.5 when NumPy starts
# implementing PEP-3118 directly).
#
# Because of laziness, the format string of the buffer is statically
# allocated. Increase the size if this is not enough, or submit a
# patch to do this properly.
#
# Author: <NAME>
#

DEF _buffer_format_string_len = 255

cimport cpython.buffer as pybuf
from cpython.ref cimport Py_INCREF
from cpython.mem cimport PyObject_Malloc, PyObject_Free
from cpython.object cimport PyObject, PyTypeObject
from cpython.type cimport type
cimport libc.stdio as stdio


cdef extern from *:
    # Leave a marker that the NumPy declarations came from Cython and not from NumPy itself.
    # See https://github.com/cython/cython/issues/3573
    """
    /* Using NumPy API declarations from "Cython/Includes/numpy/" */
    """


{'fraction_non_alphanumeric': 0.06839378238341969, 'fraction_numerical': 0.014507772020725389, 'mean_word_length': 4.378830083565459, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867808', 'n_tokens_mistral': 595, 'n_tokens_neox': 570, 'n_words': 240}