/home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-hipcub/checkouts/docs-5.3.3/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp Source File

/home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-hipcub/checkouts/docs-5.3.3/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp Source File#

hipCUB: /home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-hipcub/checkouts/docs-5.3.3/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp Source File
warp_scan.hpp
1 /******************************************************************************
2  * Copyright (c) 2010-2011, Duane Merrill. All rights reserved.
3  * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
4  * Modifications Copyright (c) 2017-2020, Advanced Micro Devices, Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are met:
8  * * Redistributions of source code must retain the above copyright
9  * notice, this list of conditions and the following disclaimer.
10  * * Redistributions in binary form must reproduce the above copyright
11  * notice, this list of conditions and the following disclaimer in the
12  * documentation and/or other materials provided with the distribution.
13  * * Neither the name of the NVIDIA CORPORATION nor the
14  * names of its contributors may be used to endorse or promote products
15  * derived from this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20  * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
21  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  *
28  ******************************************************************************/
29 
30 #ifndef HIPCUB_ROCPRIM_WARP_WARP_SCAN_HPP_
31 #define HIPCUB_ROCPRIM_WARP_WARP_SCAN_HPP_
32 
33 #include "../../../config.hpp"
34 
35 #include "../util_ptx.hpp"
36 #include "../thread/thread_operators.hpp"
37 
38 #include <rocprim/warp/warp_scan.hpp>
39 
40 BEGIN_HIPCUB_NAMESPACE
41 
42 template<
43  typename T,
44  int LOGICAL_WARP_THREADS = HIPCUB_DEVICE_WARP_THREADS,
45  int ARCH = HIPCUB_ARCH>
46 class WarpScan : private ::rocprim::warp_scan<T, LOGICAL_WARP_THREADS>
47 {
48  static_assert(LOGICAL_WARP_THREADS > 0, "LOGICAL_WARP_THREADS must be greater than 0");
49  using base_type = typename ::rocprim::warp_scan<T, LOGICAL_WARP_THREADS>;
50 
51  typename base_type::storage_type &temp_storage_;
52 
53 public:
54  using TempStorage = typename base_type::storage_type;
55 
56  HIPCUB_DEVICE inline
57  WarpScan(TempStorage& temp_storage) : temp_storage_(temp_storage)
58  {
59  }
60 
61  HIPCUB_DEVICE inline
62  void InclusiveSum(T input, T& inclusive_output)
63  {
64  base_type::inclusive_scan(input, inclusive_output, temp_storage_);
65  }
66 
67  HIPCUB_DEVICE inline
68  void InclusiveSum(T input, T& inclusive_output, T& warp_aggregate)
69  {
70  base_type::inclusive_scan(input, inclusive_output, warp_aggregate, temp_storage_);
71  }
72 
73  HIPCUB_DEVICE inline
74  void ExclusiveSum(T input, T& exclusive_output)
75  {
76  base_type::exclusive_scan(input, exclusive_output, T(0), temp_storage_);
77  }
78 
79  HIPCUB_DEVICE inline
80  void ExclusiveSum(T input, T& exclusive_output, T& warp_aggregate)
81  {
82  base_type::exclusive_scan(input, exclusive_output, T(0), warp_aggregate, temp_storage_);
83  }
84 
85  template<typename ScanOp>
86  HIPCUB_DEVICE inline
87  void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op)
88  {
89  base_type::inclusive_scan(input, inclusive_output, temp_storage_, scan_op);
90  }
91 
92  template<typename ScanOp>
93  HIPCUB_DEVICE inline
94  void InclusiveScan(T input, T& inclusive_output, ScanOp scan_op, T& warp_aggregate)
95  {
96  base_type::inclusive_scan(
97  input, inclusive_output, warp_aggregate,
98  temp_storage_, scan_op
99  );
100  }
101 
102  template<typename ScanOp>
103  HIPCUB_DEVICE inline
104  void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op)
105  {
106  base_type::inclusive_scan(input, exclusive_output, temp_storage_, scan_op);
107  base_type::to_exclusive(exclusive_output, exclusive_output, temp_storage_);
108  }
109 
110  template<typename ScanOp>
111  HIPCUB_DEVICE inline
112  void ExclusiveScan(T input, T& exclusive_output, T initial_value, ScanOp scan_op)
113  {
114  base_type::exclusive_scan(
115  input, exclusive_output, initial_value,
116  temp_storage_, scan_op
117  );
118  }
119 
120  template<typename ScanOp>
121  HIPCUB_DEVICE inline
122  void ExclusiveScan(T input, T& exclusive_output, ScanOp scan_op, T& warp_aggregate)
123  {
124  base_type::inclusive_scan(
125  input, exclusive_output, warp_aggregate, temp_storage_, scan_op
126  );
127  base_type::to_exclusive(exclusive_output, exclusive_output, temp_storage_);
128  }
129 
130  template<typename ScanOp>
131  HIPCUB_DEVICE inline
132  void ExclusiveScan(T input, T& exclusive_output, T initial_value, ScanOp scan_op, T& warp_aggregate)
133  {
134  base_type::exclusive_scan(
135  input, exclusive_output, initial_value, warp_aggregate,
136  temp_storage_, scan_op
137  );
138  }
139 
140  template<typename ScanOp>
141  HIPCUB_DEVICE inline
142  void Scan(T input, T& inclusive_output, T& exclusive_output, ScanOp scan_op)
143  {
144  base_type::inclusive_scan(input, inclusive_output, temp_storage_, scan_op);
145  base_type::to_exclusive(inclusive_output, exclusive_output, temp_storage_);
146  }
147 
148  template<typename ScanOp>
149  HIPCUB_DEVICE inline
150  void Scan(T input, T& inclusive_output, T& exclusive_output, T initial_value, ScanOp scan_op)
151  {
152  base_type::scan(
153  input, inclusive_output, exclusive_output, initial_value,
154  temp_storage_, scan_op
155  );
156  // In CUB documentation it's unclear if inclusive_output should include initial_value,
157  // however,the implementation includes initial_value in inclusive_output in WarpScan::Scan().
158  // In rocPRIM it's not included, and this is a fix to match CUB implementation.
159  // After confirmation from CUB's developers we will most probably change rocPRIM too.
160  inclusive_output = scan_op(initial_value, inclusive_output);
161  }
162 
163  HIPCUB_DEVICE inline
164  T Broadcast(T input, unsigned int src_lane)
165  {
166  return base_type::broadcast(input, src_lane, temp_storage_);
167  }
168 };
169 
170 END_HIPCUB_NAMESPACE
171 
172 #endif // HIPCUB_ROCPRIM_WARP_WARP_SCAN_HPP_
hipcub::WarpScan
Definition: warp_scan.hpp:47