<commit_before>/******************************************************************************
 * Copyright (c) 2011, <NAME>.  All rights reserved.
 * Copyright (c) 2011-2013, NVIDIA CORPORATION.  All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the NVIDIA CORPORATION nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 ******************************************************************************/

/**
 * \file
 * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock.
 */

#pragma once

#include "../../util_arch.cuh"
#include "../../warp/warp_scan.cuh"
#include "../../util_namespace.cuh"

/// Optional outer namespace(s)
CUB_NS_PREFIX

/// CUB namespace
namespace cub {

/**
 * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock.
 */
template <
    typename            T,
    int                 BLOCK_THREADS>
struct BlockScanWarpScans
{
    /// Constants
    enum
    {
        /// Number of active warps
        WARPS = (BLOCK_THREADS + CUB_PTX_WARP_THREADS - 1) / CUB_PTX_WARP_THREADS,
    };

    ///  WarpScan utility type
    typedef WarpScan<T, WARPS, CUB_PTX_WARP_THREADS> WarpScan;

    /// Shared memory storage layout type
    struct _TempStorage
    {
        typename WarpScan::TempStorage      warp_scan;                  ///< Buffer for warp-synchronous scan
        T                                   warp_aggregates[WARPS];     ///< Shared totals from each warp-synchronous scan
        T                                   block_prefix;               ///< Shared prefix for the entire threadblock
    };


    /// Alias wrapper allowing storage to be unioned
    struct TempStorage : Uninitialized<_TempStorage> {};


    // Thread fields
    _TempStorage &temp_storage;
    int linear_tid;
    int warp_id;
    int lane_id;


    /// Constructor
    __device__ __forceinline__ BlockScanWarpScans(
        TempStorage &temp_storage,
        int linear_tid)
    :
        temp_storage(temp_storage.Alias()),
        linear_tid(linear_tid),
        warp_id((BLOCK_THREADS <= CUB_PTX_WARP_THREADS) ?
            0 :
            linear_tid / CUB_PTX_WARP_THREADS),
        lane_id((BLOCK_THREADS <= CUB_PTX_WARP_THREADS) ?
            linear_tid :
            linear_tid % CUB_PTX_WARP_THREADS)
    {}


    /// Update the calling thread's partial reduction with the warp-wide aggregates from preceding warps.
    template <typename ScanOp, int ITERATION>
    __device__ __forceinline__ void ApplyWarpAggregates(
        T                   &partial,           ///< [out] The calling thread's partial reduction
        ScanOp              scan_op,            ///< [in] Binary scan operator
        T                   &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
        bool                lane_valid,         ///< [in] Whether or not the partial belonging to the current thread is valid
        Int2Type<ITERATION> iteration)          ///< [in] Predecessor warp to aggregate from
    {
        if (warp_id == ITERATION)
        {
            partial = (lane_valid) ?
                scan_op(block_aggregate, partial) :
                block_aggregate;
        }

        block_aggregate = scan_op(block_aggregate, temp_storage.warp_aggregates[ITERATION]);

        ApplyWarpAggregates(partial, scan_op, block_aggregate, lane_valid, Int2Type<ITERATION + 1>());
    }

    /// Update the calling thread's partial reduction with the warp-wide aggregates from preceding warps.
    template <typename ScanOp>
    __device__ __forceinline__ void ApplyWarpAggregates(
        T                   &partial,           ///< [out] The calling thread's partial reduction
        ScanOp              scan_op,            ///< [in] Binary scan operator
        T                   &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
        bool                lane_valid,         ///< [in] Whether or not the partial belonging to the current thread is valid
        Int2Type<WARPS>     iteration)          ///< [in] Predecessor warp to aggregate from
    {}

    /// Update the calling thread's partial reduction with the warp-wide aggregates from preceding warps.  Also returns block-wide aggregate in <em>thread</em><sub>0</sub>.
    template <typename ScanOp>
    __device__ __forceinline__ void ApplyWarpAggregates(
        T               &partial,           ///< [out] The calling thread's partial reduction
        ScanOp          scan_op,            ///< [in] Binary scan operator
        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub>s only]</b> Warp-wide aggregate reduction of input items
        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
        bool            lane_valid = true)  ///< [in] Whether or not the partial belonging to the current thread is valid
    {
        // Share lane aggregates
        temp_storage.warp_aggregates[warp_id] = warp_aggregate;

        __syncthreads();

        block_aggregate = temp_storage.warp_aggregates[0];

        // Use template unrolling to aggregate the warp-wide aggregates from preceding warps
        ApplyWarpAggregates(partial, scan_op, block_aggregate, lane_valid, Int2Type<1>());
    }


    /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
    template <typename ScanOp>
    __device__ __forceinline__ void ExclusiveScan(
        T               input,              ///< [in] Calling thread's input items
        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
        const T         &identity,          ///< [in] Identity value
        ScanOp          scan_op,            ///< [in] Binary scan operator
        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
    {
        T warp_aggregate;
        WarpScan(temp_storage.warp_scan, warp_id, lane_id).ExclusiveScan(input, output, identity, scan_op, warp_aggregate);

        // Update outputs and block_aggregate with warp-wide aggregates
        ApplyWarpAggregates(output, scan_op, warp_aggregate, block_aggregate);
    }

<commit_msg>Reduce register pressure in block scan (fix perf regression in device radix sort)<commit_after>/******************************************************************************
 * Copyright (c) 2011, <NAME>.  All rights reserved.
 * Copyright (c) 2011-2013, NVIDIA CORPORATION.  All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the NVIDIA CORPORATION nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 ******************************************************************************/

/**
 * \file
 * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock.
 */

#pragma once

#include "../../util_arch.cuh"
#include "../../warp/warp_scan.cuh"
#include "../../util_namespace.cuh"

/// Optional outer namespace(s)
CUB_NS_PREFIX

/// CUB namespace
namespace cub {

/**
 * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock.
 */
template <
    typename            T,
    int                 BLOCK_THREADS>
struct BlockScanWarpScans
{
    /// Constants
    enum
    {
        /// Number of active warps
        WARPS = (BLOCK_THREADS + CUB_PTX_WARP_THREADS - 1) / CUB_PTX_WARP_THREADS,
    };

    ///  WarpScan utility type
    typedef WarpScan<T, WARPS, CUB_PTX_WARP_THREADS> WarpScan;

    /// Shared memory storage layout type
    struct _TempStorage
    {
        typename WarpScan::TempStorage      warp_scan;                  ///< Buffer for warp-synchronous scan
        T                                   warp_aggregates[WARPS];     ///< Shared totals from each warp-synchronous scan
        T                                   block_prefix;               ///< Shared prefix for the entire threadblock
    };


    /// Alias wrapper allowing storage to be unioned
    struct TempStorage : Uninitialized<_TempStorage> {};


    // Thread fields
    _TempStorage &temp_storage;
    int linear_tid;
    int warp_id;
    int lane_id;


    /// Constructor
    __device__ __forceinline__ BlockScanWarpScans(
        TempStorage &temp_storage,
        int linear_tid)
    :
        temp_storage(temp_storage.Alias()),
        linear_tid(linear_tid),
        warp_id((BLOCK_THREADS <= CUB_PTX_WARP_THREADS) ?
            0 :
            linear_tid / CUB_PTX_WARP_THREADS),
        lane_id((BLOCK_THREADS <= CUB_PTX_WARP_THREADS) ?
            linear_tid :
            linear_tid % CUB_PTX_WARP_THREADS)
    {}


    /// Update the calling thread's partial reduction with the warp-wide aggregates from preceding warps.  Also returns block-wide aggregate in <em>thread</em><sub>0</sub>.
    template <typename ScanOp>
    __device__ __forceinline__ void ApplyWarpAggregates(
        T               &partial,           ///< [out] The calling thread's partial reduction
        ScanOp          scan_op,            ///< [in] Binary scan operator
        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub>s only]</b> Warp-wide aggregate reduction of input items
        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
        bool            lane_valid = true)  ///< [in] Whether or not the partial belonging to the current thread is valid
    {
        // Share lane aggregates
        temp_storage.warp_aggregates[warp_id] = warp_aggregate;

        __syncthreads();

        block_aggregate = temp_storage.warp_aggregates[0];

        #pragma unroll
        for (int WARP = 1; WARP < WARPS; WARP++)
        {
            T inclusive = scan_op(block_aggregate, partial);
            if (warp_id == WARP)
            {
                partial = (lane_valid) ?
                    inclusive :
                    block_aggregate;
            }

            block_aggregate = scan_op(block_aggregate, temp_storage.warp_aggregates[WARP]);
        }
    }


    /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
    template <typename ScanOp>
    __device__ __forceinline__ void ExclusiveScan(
        T               input,              ///< [in] Calling thread's input items
        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
        const T         &identity,          ///< [in] Identity value
        ScanOp          scan_op,            ///< [in] Binary scan operator
        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
    {
        T warp_aggregate;
        WarpScan(temp_storage.warp_scan, warp_id, lane_id).ExclusiveScan(input, output, identity, scan_op, warp_aggregate);

        // Update outputs and block_aggregate with warp-wide aggregates
        ApplyWarpAggregates(output, scan_op, warp_aggregate, block_aggregate);
    }


{'fraction_non_alphanumeric': 0.09633448689879229, 'fraction_numerical': 0.002895684723497422, 'mean_word_length': 2.421116211645325, 'pattern_counts': {'":': 0, '<': 82, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6641320', 'n_tokens_mistral': 3955, 'n_tokens_neox': 3633, 'n_words': 1296}