<commit_before>/////////////// BufferStructDeclare.proto ///////////////

/* structs for buffer access */

typedef struct {
  Py_ssize_t shape, strides, suboffsets;
} __Pyx_Buf_DimInfo;

typedef struct {
  size_t refcount;
  Py_buffer pybuffer;
} __Pyx_Buffer;

typedef struct {
  __Pyx_Buffer *rcbuffer;
  char *data;
  __Pyx_Buf_DimInfo diminfo[{{max_dims}}];
} __Pyx_LocalBuf_ND;

/////////////// BufferIndexError.proto ///////////////
static void __Pyx_RaiseBufferIndexError(int axis); /*proto*/

/////////////// BufferIndexError ///////////////
static void __Pyx_RaiseBufferIndexError(int axis) {
  PyErr_Format(PyExc_IndexError,
     "Out of bounds on buffer access (axis %d)", axis);
}

/////////////// BufferIndexErrorNogil.proto ///////////////
//@requires: BufferIndexError

static void __Pyx_RaiseBufferIndexErrorNogil(int axis); /*proto*/

/////////////// BufferIndexErrorNogil ///////////////
static void __Pyx_RaiseBufferIndexErrorNogil(int axis) {
    #ifdef WITH_THREAD
    PyGILState_STATE gilstate = PyGILState_Ensure();
    #endif
    __Pyx_RaiseBufferIndexError(axis);
    #ifdef WITH_THREAD
    PyGILState_Release(gilstate);
    #endif
}

/////////////// BufferFallbackError.proto ///////////////
static void __Pyx_RaiseBufferFallbackError(void); /*proto*/

/////////////// BufferFallbackError ///////////////
static void __Pyx_RaiseBufferFallbackError(void) {
  PyErr_Format(PyExc_ValueError,
     "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!");
}

/////////////// BufferFormatStructs.proto ///////////////

#define IS_UNSIGNED(type) (((type) -1) > 0)

/* Run-time type information about structs used with buffers */
struct __Pyx_StructField_;

#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0)

typedef struct {
  const char* name; /* for error messages only */
  struct __Pyx_StructField_* fields;
  size_t size;     /* sizeof(type) */
  size_t arraysize[8]; /* length of array in each dimension */
  int ndim;
  char typegroup; /* _R_eal, _C_omplex, Signed _I_nt, _U_nsigned int, _S_truct, _P_ointer, _O_bject */
  char is_unsigned;
  int flags;
} __Pyx_TypeInfo;

typedef struct __Pyx_StructField_ {
  __Pyx_TypeInfo* type;
  const char* name;
  size_t offset;
} __Pyx_StructField;

typedef struct {
  __Pyx_StructField* field;
  size_t parent_offset;
} __Pyx_BufFmt_StackElem;

typedef struct {
  __Pyx_StructField root;
  __Pyx_BufFmt_StackElem* head;
  size_t fmt_offset;
  size_t new_count, enc_count;
  size_t struct_alignment;
  int is_complex;
  char enc_type;
  char new_packmode;
  char enc_packmode;
  char is_valid_array;
} __Pyx_BufFmt_Context;

/////////////// GetAndReleaseBuffer.proto ///////////////
#if PY_MAJOR_VERSION < 3
    static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
    static void __Pyx_ReleaseBuffer(Py_buffer *view);
#else
    #define __Pyx_GetBuffer PyObject_GetBuffer
    #define __Pyx_ReleaseBuffer PyBuffer_Release
#endif

/////////////// GetAndReleaseBuffer ///////////////
#if PY_MAJOR_VERSION < 3
static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
    PyObject *getbuffer_cobj;

  #if PY_VERSION_HEX >= 0x02060000
    if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags);
  #endif

    {{for type_ptr, getbuffer, releasebuffer in types}}
      {{if getbuffer}}
        if (PyObject_TypeCheck(obj, {{type_ptr}})) return {{getbuffer}}(obj, view, flags);
      {{endif}}
    {{endfor}}

  #if PY_VERSION_HEX < 0x02060000
    if (obj->ob_type->tp_dict &&
        (getbuffer_cobj = PyMapping_GetItemString(obj->ob_type->tp_dict,
                                             "__pyx_getbuffer"))) {
        getbufferproc func;

      #if PY_VERSION_HEX >= 0x02070000 && !(PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 0)
        func = (getbufferproc) PyCapsule_GetPointer(getbuffer_cobj, "getbuffer(obj, view, flags)");
      #else
        func = (getbufferproc) PyCObject_AsVoidPtr(getbuffer_cobj);
      #endif
        Py_DECREF(getbuffer_cobj);
        if (!func)
            goto fail;

        return func(obj, view, flags);
    } else {
        PyErr_Clear();
    }
  #endif

    PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);

#if PY_VERSION_HEX < 0x02060000
fail:
#endif

    return -1;
}

static void __Pyx_ReleaseBuffer(Py_buffer *view) {
    PyObject *obj = view->obj;
    PyObject *releasebuffer_cobj;
<commit_msg>Fix unused variable warning in Python >= 2.6.<commit_after>/////////////// BufferStructDeclare.proto ///////////////

/* structs for buffer access */

typedef struct {
  Py_ssize_t shape, strides, suboffsets;
} __Pyx_Buf_DimInfo;

typedef struct {
  size_t refcount;
  Py_buffer pybuffer;
} __Pyx_Buffer;

typedef struct {
  __Pyx_Buffer *rcbuffer;
  char *data;
  __Pyx_Buf_DimInfo diminfo[{{max_dims}}];
} __Pyx_LocalBuf_ND;

/////////////// BufferIndexError.proto ///////////////
static void __Pyx_RaiseBufferIndexError(int axis); /*proto*/

/////////////// BufferIndexError ///////////////
static void __Pyx_RaiseBufferIndexError(int axis) {
  PyErr_Format(PyExc_IndexError,
     "Out of bounds on buffer access (axis %d)", axis);
}

/////////////// BufferIndexErrorNogil.proto ///////////////
//@requires: BufferIndexError

static void __Pyx_RaiseBufferIndexErrorNogil(int axis); /*proto*/

/////////////// BufferIndexErrorNogil ///////////////
static void __Pyx_RaiseBufferIndexErrorNogil(int axis) {
    #ifdef WITH_THREAD
    PyGILState_STATE gilstate = PyGILState_Ensure();
    #endif
    __Pyx_RaiseBufferIndexError(axis);
    #ifdef WITH_THREAD
    PyGILState_Release(gilstate);
    #endif
}

/////////////// BufferFallbackError.proto ///////////////
static void __Pyx_RaiseBufferFallbackError(void); /*proto*/

/////////////// BufferFallbackError ///////////////
static void __Pyx_RaiseBufferFallbackError(void) {
  PyErr_Format(PyExc_ValueError,
     "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!");
}

/////////////// BufferFormatStructs.proto ///////////////

#define IS_UNSIGNED(type) (((type) -1) > 0)

/* Run-time type information about structs used with buffers */
struct __Pyx_StructField_;

#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0)

typedef struct {
  const char* name; /* for error messages only */
  struct __Pyx_StructField_* fields;
  size_t size;     /* sizeof(type) */
  size_t arraysize[8]; /* length of array in each dimension */
  int ndim;
  char typegroup; /* _R_eal, _C_omplex, Signed _I_nt, _U_nsigned int, _S_truct, _P_ointer, _O_bject */
  char is_unsigned;
  int flags;
} __Pyx_TypeInfo;

typedef struct __Pyx_StructField_ {
  __Pyx_TypeInfo* type;
  const char* name;
  size_t offset;
} __Pyx_StructField;

typedef struct {
  __Pyx_StructField* field;
  size_t parent_offset;
} __Pyx_BufFmt_StackElem;

typedef struct {
  __Pyx_StructField root;
  __Pyx_BufFmt_StackElem* head;
  size_t fmt_offset;
  size_t new_count, enc_count;
  size_t struct_alignment;
  int is_complex;
  char enc_type;
  char new_packmode;
  char enc_packmode;
  char is_valid_array;
} __Pyx_BufFmt_Context;

/////////////// GetAndReleaseBuffer.proto ///////////////
#if PY_MAJOR_VERSION < 3
    static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
    static void __Pyx_ReleaseBuffer(Py_buffer *view);
#else
    #define __Pyx_GetBuffer PyObject_GetBuffer
    #define __Pyx_ReleaseBuffer PyBuffer_Release
#endif

/////////////// GetAndReleaseBuffer ///////////////
#if PY_MAJOR_VERSION < 3
static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
    CYTHON_UNUSED PyObject *getbuffer_cobj;

  #if PY_VERSION_HEX >= 0x02060000
    if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags);
  #endif

    {{for type_ptr, getbuffer, releasebuffer in types}}
      {{if getbuffer}}
        if (PyObject_TypeCheck(obj, {{type_ptr}})) return {{getbuffer}}(obj, view, flags);
      {{endif}}
    {{endfor}}

  #if PY_VERSION_HEX < 0x02060000
    if (obj->ob_type->tp_dict &&
        (getbuffer_cobj = PyMapping_GetItemString(obj->ob_type->tp_dict,
                                             "__pyx_getbuffer"))) {
        getbufferproc func;

      #if PY_VERSION_HEX >= 0x02070000 && !(PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 0)
        func = (getbufferproc) PyCapsule_GetPointer(getbuffer_cobj, "getbuffer(obj, view, flags)");
      #else
        func = (getbufferproc) PyCObject_AsVoidPtr(getbuffer_cobj);
      #endif
        Py_DECREF(getbuffer_cobj);
        if (!func)
            goto fail;

        return func(obj, view, flags);
    } else {
        PyErr_Clear();
    }
  #endif

    PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);

#if PY_VERSION_HEX < 0x02060000
fail:
#endif

    return -1;
}

static void __Pyx_ReleaseBuffer(Py_buffer *view) {
    PyObject *obj = view->obj;
    CYTHON_UNUSED PyObject *releasebuffer_cobj;

{'fraction_non_alphanumeric': 0.15336463223787167, 'fraction_numerical': 0.0111781801922647, 'mean_word_length': 3.0374548736462095, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2936165', 'n_tokens_mistral': 2999, 'n_tokens_neox': 2979, 'n_words': 668}