<commit_before>#ifndef BOTTLENECK_H
#define BOTTLENECK_H

#include <Python.h>
#define NPY_NO_DEPRECATED_API NPY_1_11_API_VERSION
#include <numpy/arrayobject.h>
#include <bn_config.h>

/* THREADS=1 releases the GIL but increases function call
 * overhead. THREADS=0 does not release the GIL but keeps
 * function call overhead low. Curly brackets are for C89
 * support. */
#define THREADS 1
#if THREADS
    #define BN_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS {
    #define BN_END_ALLOW_THREADS   ;} Py_END_ALLOW_THREADS
#else
    #define BN_BEGIN_ALLOW_THREADS {
    #define BN_END_ALLOW_THREADS }
#endif

/* for ease of dtype templating */
#define NPY_float64 NPY_FLOAT64
#define NPY_float32 NPY_FLOAT32
#define NPY_int64   NPY_INT64
#define NPY_int32   NPY_INT32
#define NPY_intp    NPY_INTP
#define NPY_MAX_int64 NPY_MAX_INT64
#define NPY_MAX_int32 NPY_MAX_INT32
#define NPY_MIN_int64 NPY_MIN_INT64
#define NPY_MIN_int32 NPY_MIN_INT32

#if PY_MAJOR_VERSION >= 3
    #define PyString_FromString PyBytes_FromString
    #define PyInt_AsLong PyLong_AsLong
    #define PyString_InternFromString PyUnicode_InternFromString
#endif

#define VARKEY METH_VARARGS | METH_KEYWORDS
#define error_converting(x) (((x) == -1) && PyErr_Occurred())

#define VALUE_ERR(text)   PyErr_SetString(PyExc_ValueError, text)
#define TYPE_ERR(text)    PyErr_SetString(PyExc_TypeError, text)
#define MEMORY_ERR(text)  PyErr_SetString(PyExc_MemoryError, text)
#define RUNTIME_ERR(text) PyErr_SetString(PyExc_RuntimeError, text)

/* `inline` copied from NumPy. */
#if defined(_MSC_VER)
        #define BN_INLINE __inline
#elif defined(__GNUC__)
	#if defined(__STRICT_ANSI__)
		#define BN_INLINE __inline__
	#else
		#define BN_INLINE inline
	#endif
#else
        #define BN_INLINE
#endif

/*
 * NAN and INFINITY like macros (same behavior as glibc for NAN, same as C99
 * for INFINITY). Copied from NumPy.
 */
BN_INLINE static float __bn_inff(void)
{
    const union { npy_uint32 __i; float __f;} __bint = {0x7f800000UL};
    return __bint.__f;
}

BN_INLINE static float __bn_nanf(void)
{
    const union { npy_uint32 __i; float __f;} __bint = {0x7fc00000UL};
    return __bint.__f;
}

#define BN_INFINITYF __bn_inff()
#define BN_NANF      __bn_nanf()
#define BN_INFINITY ((npy_double)BN_INFINITYF)
#define BN_NAN      ((npy_double)BN_NANF)

#define C_CONTIGUOUS(a) PyArray_CHKFLAGS(a, NPY_ARRAY_C_CONTIGUOUS)
#define F_CONTIGUOUS(a) PyArray_CHKFLAGS(a, NPY_ARRAY_F_CONTIGUOUS)
#define IS_CONTIGUOUS(a) (C_CONTIGUOUS(a) || F_CONTIGUOUS(a))

/* WIRTH ----------------------------------------------------------------- */<commit_msg>PERF: Add BN_OPT_3 to allow enabling -O3 at the function level<commit_after>#ifndef BOTTLENECK_H
#define BOTTLENECK_H

#include <Python.h>
#define NPY_NO_DEPRECATED_API NPY_1_11_API_VERSION
#include <numpy/arrayobject.h>
#include <bn_config.h>

/* THREADS=1 releases the GIL but increases function call
 * overhead. THREADS=0 does not release the GIL but keeps
 * function call overhead low. Curly brackets are for C89
 * support. */
#define THREADS 1
#if THREADS
    #define BN_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS {
    #define BN_END_ALLOW_THREADS   ;} Py_END_ALLOW_THREADS
#else
    #define BN_BEGIN_ALLOW_THREADS {
    #define BN_END_ALLOW_THREADS }
#endif

/* for ease of dtype templating */
#define NPY_float64 NPY_FLOAT64
#define NPY_float32 NPY_FLOAT32
#define NPY_int64   NPY_INT64
#define NPY_int32   NPY_INT32
#define NPY_intp    NPY_INTP
#define NPY_MAX_int64 NPY_MAX_INT64
#define NPY_MAX_int32 NPY_MAX_INT32
#define NPY_MIN_int64 NPY_MIN_INT64
#define NPY_MIN_int32 NPY_MIN_INT32

#if PY_MAJOR_VERSION >= 3
    #define PyString_FromString PyBytes_FromString
    #define PyInt_AsLong PyLong_AsLong
    #define PyString_InternFromString PyUnicode_InternFromString
#endif

#define VARKEY METH_VARARGS | METH_KEYWORDS
#define error_converting(x) (((x) == -1) && PyErr_Occurred())

#define VALUE_ERR(text)   PyErr_SetString(PyExc_ValueError, text)
#define TYPE_ERR(text)    PyErr_SetString(PyExc_TypeError, text)
#define MEMORY_ERR(text)  PyErr_SetString(PyExc_MemoryError, text)
#define RUNTIME_ERR(text) PyErr_SetString(PyExc_RuntimeError, text)

/* `inline` and `opt_3` copied from NumPy. */
#if defined(_MSC_VER)
        #define BN_INLINE __inline
#elif defined(__GNUC__)
	#if defined(__STRICT_ANSI__)
		#define BN_INLINE __inline__
	#else
		#define BN_INLINE inline
	#endif
#else
        #define BN_INLINE
#endif

#if HAVE_ATTRIBUTE_OPTIMIZE_OPT_3
    #define BN_OPT_3 __attribute__((optimize("O3")))
#else
    #define BN_OPT_3
#endif

/*
 * NAN and INFINITY like macros (same behavior as glibc for NAN, same as C99
 * for INFINITY). Copied from NumPy.
 */
BN_INLINE static float __bn_inff(void)
{
    const union { npy_uint32 __i; float __f;} __bint = {0x7f800000UL};
    return __bint.__f;
}

BN_INLINE static float __bn_nanf(void)
{
    const union { npy_uint32 __i; float __f;} __bint = {0x7fc00000UL};
    return __bint.__f;
}

#define BN_INFINITYF __bn_inff()
#define BN_NANF      __bn_nanf()
#define BN_INFINITY ((npy_double)BN_INFINITYF)
#define BN_NAN      ((npy_double)BN_NANF)

#define C_CONTIGUOUS(a) PyArray_CHKFLAGS(a, NPY_ARRAY_C_CONTIGUOUS)
#define F_CONTIGUOUS(a) PyArray_CHKFLAGS(a, NPY_ARRAY_F_CONTIGUOUS)
#define IS_CONTIGUOUS(a) (C_CONTIGUOUS(a) || F_CONTIGUOUS(a))

/* WIRTH ----------------------------------------------------------------- */
{'fraction_non_alphanumeric': 0.10595657821488216, 'fraction_numerical': 0.024679903507144184, 'mean_word_length': 4.73404255319149, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279248', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2082, 'n_words': 469}