<commit_before>import string

import numpy

import cupy
from cupy.core._reduction import create_reduction_func
from cupy.core._kernel import create_ufunc
from cupy.core._scalar import get_typename
from cupy.core._ufuncs import elementwise_copy
from cupy import util

from cupy.core._dtype cimport get_dtype
from cupy.core cimport _kernel
from cupy.core.core cimport _ndarray_init
from cupy.core.core cimport compile_with_cache
from cupy.core.core cimport ndarray
from cupy.cuda cimport memory

if cupy.cuda.cub_enabled:
    from cupy.cuda import cub
else:
    cub = None


# ndarray members


cdef ndarray _ndarray_conj(ndarray self):
    if self.dtype.kind == 'c':
        return _conjugate(self)
    else:
        return self


cdef ndarray _ndarray_real_getter(ndarray self):
    if self.dtype.kind == 'c':
        view = ndarray(
            shape=self._shape, dtype=get_dtype(self.dtype.char.lower()),
            memptr=self.data, strides=self._strides)
        view.base = self.base if self.base is not None else self
        return view
    return self


cdef ndarray _ndarray_real_setter(ndarray self, value):
    if self.dtype.kind == 'c':
        _real_setter(value, self)
    else:
        elementwise_copy(value, self)


cdef ndarray _ndarray_imag_getter(ndarray self):
    cdef memory.MemoryPointer memptr
    if self.dtype.kind == 'c':
        dtype = get_dtype(self.dtype.char.lower())
        memptr = self.data
        # Make the memory pointer point to the first imaginary element.
        # Note that even if the array doesn't have a valid memory (e.g. 0-size
        # array), the resulting array should be a view of the original array,
        # aligning with NumPy behavior.
        if memptr.ptr != 0:
            memptr = memptr + self.dtype.itemsize // 2
        view = ndarray(
            shape=self._shape, dtype=dtype, memptr=memptr,
            strides=self._strides)
        view.base = self.base if self.base is not None else self
        return view
    new_array = ndarray(self.shape, dtype=self.dtype)
    new_array.fill(0)
    return new_array


cdef ndarray _ndarray_imag_setter(ndarray self, value):
    if self.dtype.kind == 'c':
        _imag_setter(value, self)
    else:
        raise TypeError('cupy.ndarray does not have imaginary part to set')


cdef ndarray _ndarray_prod(ndarray self, axis, dtype, out, keepdims):
    if cupy.cuda.cub_enabled:
        # result will be None if the reduction is not compatible with CUB
        result = cub.cub_reduction(self, cub.CUPY_CUB_PROD, axis, dtype, out,
                                   keepdims)
        if result is not None:
            return result
    if dtype is None:
        return _prod_auto_dtype(self, axis, dtype, out, keepdims)
    else:
        return _prod_keep_dtype(self, axis, dtype, out, keepdims)


cdef ndarray _ndarray_sum(ndarray self, axis, dtype, out, keepdims):
    #if cupy.cuda.cub_enabled:
    #    # result will be None if the reduction is not compatible with CUB
    #    result = cub.cub_reduction(self, cub.CUPY_CUB_SUM, axis, dtype, out,
    #                               keepdims)
    #    if result is not None:
    #        return result
    if dtype is None:
        return _sum_auto_dtype(self, axis, dtype, out, keepdims)
    else:
        return _sum_keep_dtype(self, axis, dtype, out, keepdims)


cdef ndarray _ndarray_cumsum(ndarray self, axis, dtype, out):
    return cupy.cumsum(self, axis, dtype, out)


cdef ndarray _ndarray_cumprod(ndarray self, axis, dtype, out):
    return cupy.cumprod(self, axis, dtype, out)


cdef ndarray _ndarray_clip(ndarray self, a_min, a_max, out):
    if a_min is None and a_max is None:
        raise ValueError('array_clip: must set either max or min')<commit_msg>bring back CUB device sum<commit_after>import string

import numpy

import cupy
from cupy.core._reduction import create_reduction_func
from cupy.core._kernel import create_ufunc
from cupy.core._scalar import get_typename
from cupy.core._ufuncs import elementwise_copy
from cupy import util

from cupy.core._dtype cimport get_dtype
from cupy.core cimport _kernel
from cupy.core.core cimport _ndarray_init
from cupy.core.core cimport compile_with_cache
from cupy.core.core cimport ndarray
from cupy.cuda cimport memory

if cupy.cuda.cub_enabled:
    from cupy.cuda import cub
else:
    cub = None


# ndarray members


cdef ndarray _ndarray_conj(ndarray self):
    if self.dtype.kind == 'c':
        return _conjugate(self)
    else:
        return self


cdef ndarray _ndarray_real_getter(ndarray self):
    if self.dtype.kind == 'c':
        view = ndarray(
            shape=self._shape, dtype=get_dtype(self.dtype.char.lower()),
            memptr=self.data, strides=self._strides)
        view.base = self.base if self.base is not None else self
        return view
    return self


cdef ndarray _ndarray_real_setter(ndarray self, value):
    if self.dtype.kind == 'c':
        _real_setter(value, self)
    else:
        elementwise_copy(value, self)


cdef ndarray _ndarray_imag_getter(ndarray self):
    cdef memory.MemoryPointer memptr
    if self.dtype.kind == 'c':
        dtype = get_dtype(self.dtype.char.lower())
        memptr = self.data
        # Make the memory pointer point to the first imaginary element.
        # Note that even if the array doesn't have a valid memory (e.g. 0-size
        # array), the resulting array should be a view of the original array,
        # aligning with NumPy behavior.
        if memptr.ptr != 0:
            memptr = memptr + self.dtype.itemsize // 2
        view = ndarray(
            shape=self._shape, dtype=dtype, memptr=memptr,
            strides=self._strides)
        view.base = self.base if self.base is not None else self
        return view
    new_array = ndarray(self.shape, dtype=self.dtype)
    new_array.fill(0)
    return new_array


cdef ndarray _ndarray_imag_setter(ndarray self, value):
    if self.dtype.kind == 'c':
        _imag_setter(value, self)
    else:
        raise TypeError('cupy.ndarray does not have imaginary part to set')


cdef ndarray _ndarray_prod(ndarray self, axis, dtype, out, keepdims):
    if cupy.cuda.cub_enabled:
        # result will be None if the reduction is not compatible with CUB
        result = cub.cub_reduction(self, cub.CUPY_CUB_PROD, axis, dtype, out,
                                   keepdims)
        if result is not None:
            return result
    if dtype is None:
        return _prod_auto_dtype(self, axis, dtype, out, keepdims)
    else:
        return _prod_keep_dtype(self, axis, dtype, out, keepdims)


cdef ndarray _ndarray_sum(ndarray self, axis, dtype, out, keepdims):
    if cupy.cuda.cub_enabled:
        # result will be None if the reduction is not compatible with CUB
        result = cub.cub_reduction(self, cub.CUPY_CUB_SUM, axis, dtype, out,
                                   keepdims)
        if result is not None:
            return result
    if dtype is None:
        return _sum_auto_dtype(self, axis, dtype, out, keepdims)
    else:
        return _sum_keep_dtype(self, axis, dtype, out, keepdims)


cdef ndarray _ndarray_cumsum(ndarray self, axis, dtype, out):
    return cupy.cumsum(self, axis, dtype, out)


cdef ndarray _ndarray_cumprod(ndarray self, axis, dtype, out):
    return cupy.cumprod(self, axis, dtype, out)


cdef ndarray _ndarray_clip(ndarray self, a_min, a_max, out):
    if a_min is None and a_max is None:
        raise ValueError('array_clip: must set either max or min')
{'fraction_non_alphanumeric': 0.07887700534759358, 'fraction_numerical': 0.0010695187165775401, 'mean_word_length': 2.795535261288686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4795398', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2414, 'n_words': 739}