/****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the NVIDIA CORPORATION nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * ******************************************************************************/ /** * \file * cub::GridQueue is a descriptor utility for dynamic queue management. */ #pragma once #include "../config.cuh" #include "../util_debug.cuh" /// Optional outer namespace(s) CUB_NS_PREFIX /// CUB namespace namespace cub { /** * \addtogroup GridModule * @{ */ /** * \brief GridQueue is a descriptor utility for dynamic queue management. * * \par Overview * GridQueue descriptors provides abstractions for "filling" or * "draining" globally-shared vectors. * * \par * A "filling" GridQueue works by atomically-adding to a zero-initialized counter, * returning a unique offset for the calling thread to write its items. * The GridQueue maintains the total "fill-size". The fill counter must be reset * using GridQueue::ResetFill by the host or kernel instance prior to the kernel instance that * will be filling. * * \par * Similarly, a "draining" GridQueue works by works by atomically-incrementing a * zero-initialized counter, returning a unique offset for the calling thread to * read its items. Threads can safely drain until the array's logical fill-size is * exceeded. The drain counter must be reset using GridQueue::ResetDrain or * GridQueue::FillAndResetDrain by the host or kernel instance prior to the kernel instance that * will be filling. (For dynamic work distribution of existing data, the corresponding fill-size * is simply the number of elements in the array.) * * \par * Iterative work management can be implemented simply with a pair of flip-flopping * work buffers, each with an associated set of fill and drain GridQueue descriptors. * * \tparam OffsetT Signed integer type for global offsets */ template class GridQueue { private: /// Counter indices enum { FILL = 0, DRAIN = 1, }; /// Pair of counters OffsetT *d_counters; public: /// Returns the device allocation size in bytes needed to construct a GridQueue instance __host__ __device__ __forceinline__ static size_t AllocationSize() { return sizeof(OffsetT) * 2; } /// Constructs an invalid GridQueue descriptor __host__ __device__ __forceinline__ GridQueue() : d_counters(NULL) {} /// Constructs a GridQueue descriptor around the device storage allocation __host__ __device__ __forceinline__ GridQueue( void *d_storage) ///< Device allocation to back the GridQueue. Must be at least as big as AllocationSize(). : d_counters((OffsetT*) d_storage) {} /// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for draining in the next kernel instance. To be called by the host or by a kernel prior to that which will be draining. __host__ __device__ __forceinline__ cudaError_t FillAndResetDrain( OffsetT fill_size, cudaStream_t stream = 0) { cudaError_t result = cudaErrorUnknown; if (CUB_IS_DEVICE_CODE) { #if CUB_INCLUDE_DEVICE_CODE (void)stream; d_counters[FILL] = fill_size; d_counters[DRAIN] = 0; result = cudaSuccess; #endif } else { #if CUB_INCLUDE_HOST_CODE OffsetT counters[2]; counters[FILL] = fill_size; counters[DRAIN] = 0; result = CubDebug(cudaMemcpyAsync(d_counters, counters, sizeof(OffsetT) * 2, cudaMemcpyHostToDevice, stream)); #endif } return result; } /// This operation resets the drain so that it may advance to meet the existing fill-size. To be called by the host or by a kernel prior to that which will be draining. __host__ __device__ __forceinline__ cudaError_t ResetDrain(cudaStream_t stream = 0) { cudaError_t result = cudaErrorUnknown; if (CUB_IS_DEVICE_CODE) { #if CUB_INCLUDE_DEVICE_CODE (void)stream; d_counters[DRAIN] = 0; result = cudaSuccess; #endif } else { #if CUB_INCLUDE_HOST_CODE result = CubDebug(cudaMemsetAsync(d_counters + DRAIN, 0, sizeof(OffsetT), stream)); #endif } return result; } /// This operation resets the fill counter. To be called by the host or by a kernel prior to that which will be filling. __host__ __device__ __forceinline__ cudaError_t ResetFill(cudaStream_t stream = 0) { cudaError_t result = cudaErrorUnknown; if (CUB_IS_DEVICE_CODE) { #if CUB_INCLUDE_DEVICE_CODE (void)stream; d_counters[FILL] = 0; result = cudaSuccess; #endif } else { #if CUB_INCLUDE_HOST_CODE result = CubDebug(cudaMemsetAsync(d_counters + FILL, 0, sizeof(OffsetT), stream)); #endif } return result; } /// Returns the fill-size established by the parent or by the previous kernel. __host__ __device__ __forceinline__ cudaError_t FillSize( OffsetT &fill_size, cudaStream_t stream = 0) { cudaError_t result = cudaErrorUnknown; if (CUB_IS_DEVICE_CODE) { #if CUB_INCLUDE_DEVICE_CODE (void)stream; fill_size = d_counters[FILL]; result = cudaSuccess; #endif } else { #if CUB_INCLUDE_HOST_CODE result = CubDebug(cudaMemcpyAsync(&fill_size, d_counters + FILL, sizeof(OffsetT), cudaMemcpyDeviceToHost, stream)); #endif } return result; } /// Drain \p num_items from the queue. Returns offset from which to read items. To be called from CUDA kernel. __device__ __forceinline__ OffsetT Drain(OffsetT num_items) { return atomicAdd(d_counters + DRAIN, num_items); } /// Fill \p num_items into the queue. Returns offset from which to write items. To be called from CUDA kernel. __device__ __forceinline__ OffsetT Fill(OffsetT num_items) { return atomicAdd(d_counters + FILL, num_items); } }; #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document /** * Reset grid queue (call with 1 block of 1 thread) */ template __global__ void FillAndResetDrainKernel( GridQueue grid_queue, OffsetT num_items) { grid_queue.FillAndResetDrain(num_items); } #endif // DOXYGEN_SHOULD_SKIP_THIS /** @} */ // end group GridModule } // CUB namespace CUB_NS_POSTFIX // Optional outer namespace(s)