6 #if defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) || \
7 defined(__gfx9_4_generic__)
10 #if defined(__gfx942__) || defined(__gfx950__) || defined(__gfx9_4_generic__)
13 #if defined(__gfx1010__) || defined(__gfx1011__) || defined(__gfx1012__) || \
14 defined(__gfx1013__) || defined(__gfx10_1_generic__)
17 #if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || \
18 defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || \
19 defined(__gfx10_3_generic__)
22 #if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || \
23 defined(__gfx1103__) || defined(__gfx1150__) || defined(__gfx1151__) || \
24 defined(__gfx1152__) || defined(__gfx1153__) || defined(__gfx11_generic__)
27 #if defined(__gfx1200__) || defined(__gfx1201__) || defined(__gfx12_generic__)
31 #include "hip/hip_version.h"
32 #ifndef CK_TILE_DONT_USE_HIP_RUNTIME_HEADERS
33 #include "hip/hip_runtime.h"
34 #include "hip/hip_fp16.h"
38 #define CK_TILE_HOST inline __host__
39 #define CK_TILE_DEVICE inline __device__
40 #define CK_TILE_HOST_DEVICE inline __host__ __device__
41 #define CK_TILE_DEVICE_EXTERN __device__
42 #if __clang_major__ < 22
43 #define CK_TILE_HOST_DEVICE_EXTERN __host__ __device__
45 #define CK_TILE_HOST_DEVICE_EXTERN
48 #define CK_TILE_HOST inline
49 #define CK_TILE_DEVICE inline
50 #define CK_TILE_HOST_DEVICE inline
51 #define CK_TILE_DEVICE_EXTERN
52 #define CK_TILE_HOST_DEVICE_EXTERN
59 #define CK_TILE_GENERIC_ADDR __attribute__((address_space(0)))
60 #define CK_TILE_GLOBAL_ADDR __attribute__((address_space(1)))
61 #define CK_TILE_LDS_ADDR __attribute__((address_space(3)))
62 #define CK_TILE_BUF_RES_ADDR __attribute__((address_space(8)))
64 #define CK_TILE_GENERIC_ADDR
65 #define CK_TILE_GLOBAL_ADDR
66 #define CK_TILE_LDS_ADDR
67 #define CK_TILE_BUF_RES_ADDR
69 #ifndef CK_TILE_USE_CUSTOM_DATA_TYPE
70 #define CK_TILE_USE_CUSTOM_DATA_TYPE 0
73 #define CK_TILE_FLOAT_TO_BFLOAT16_STANDARD 0
74 #define CK_TILE_FLOAT_TO_BFLOAT16_TRUNCATE_WITH_NAN 1
75 #define CK_TILE_FLOAT_TO_BFLOAT16_TRUNCATE 2
76 #define CK_TILE_FLOAT_TO_BFLOAT16_STANDARD_ASM 3
77 #define CK_TILE_FLOAT_TO_BFLOAT16_RTA_ASM 4
79 #ifndef CK_TILE_FLOAT_TO_BFLOAT16_DEFAULT
80 #define CK_TILE_FLOAT_TO_BFLOAT16_DEFAULT CK_TILE_FLOAT_TO_BFLOAT16_TRUNCATE
83 #define CK_TILE_FLOAT_TO_FP8_STANDARD 0
84 #define CK_TILE_FLOAT_TO_FP8_STOCHASTIC 1
86 #ifndef CK_TILE_FLOAT_TO_FP8_DEFAULT
87 #define CK_TILE_FLOAT_TO_FP8_DEFAULT CK_TILE_FLOAT_TO_FP8_STANDARD
92 #define CK_TILE_STATICALLY_INDEXED_ARRAY_USE_ARRAY 0
93 #define CK_TILE_STATICALLY_INDEXED_ARRAY_USE_TUPLE 1
94 #ifndef CK_TILE_STATICALLY_INDEXED_ARRAY_DEFAULT
95 #define CK_TILE_STATICALLY_INDEXED_ARRAY_DEFAULT CK_TILE_STATICALLY_INDEXED_ARRAY_USE_TUPLE
98 #define CK_TILE_THREAD_BUFFER_USE_ARRAY 0
99 #define CK_TILE_THREAD_BUFFER_USE_TUPLE 1
100 #ifndef CK_TILE_THREAD_BUFFER_DEFAULT
101 #define CK_TILE_THREAD_BUFFER_DEFAULT CK_TILE_THREAD_BUFFER_USE_ARRAY
104 #ifndef CK_TILE_TUPLE_CTOR_WITH_INITIALIZER_LIST
105 #if CK_TILE_THREAD_BUFFER_DEFAULT == CK_TILE_THREAD_BUFFER_USE_TUPLE
108 #define CK_TILE_TUPLE_CTOR_WITH_INITIALIZER_LIST 1
110 #define CK_TILE_TUPLE_CTOR_WITH_INITIALIZER_LIST 0
114 #ifndef CK_TILE_USE_LAUNCH_BOUNDS
115 #define CK_TILE_USE_LAUNCH_BOUNDS 1
118 #ifndef CK_TILE_TIME_KERNEL
119 #define CK_TILE_TIME_KERNEL 1
122 #define CK_TILE_MAX_THREAD_PER_BLOCK 256
123 #define CK_TILE_MIN_BLOCK_PER_CU 2
125 #ifndef CK_TILE_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK
126 #define CK_TILE_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 0
129 #ifndef CK_TILE_EXPERIMENTAL_USE_BUFFER_STORE_OOB_CHECK_OFFSET_TRICK
130 #define CK_TILE_EXPERIMENTAL_USE_BUFFER_STORE_OOB_CHECK_OFFSET_TRICK 1
133 #ifndef CK_TILE_EXPERIMENTAL_USE_BUFFER_ATOMIC_ADD_OOB_CHECK_OFFSET_TRICK
134 #define CK_TILE_EXPERIMENTAL_USE_BUFFER_ATOMIC_ADD_OOB_CHECK_OFFSET_TRICK 1
137 #ifndef CK_TILE_EXPERIMENTAL_USE_BUFFER_ATOMIC_MAX_OOB_CHECK_OFFSET_TRICK
138 #define CK_TILE_EXPERIMENTAL_USE_BUFFER_ATOMIC_MAX_OOB_CHECK_OFFSET_TRICK 1
141 #ifndef CK_TILE_USE_AMD_LDS_DIRECT_LOAD_INLINE_ASM
142 #define CK_TILE_USE_AMD_LDS_DIRECT_LOAD_INLINE_ASM 1
145 #ifndef CK_TILE_USE_AMD_BUFFER_LOAD
146 #define CK_TILE_USE_AMD_BUFFER_LOAD 1
149 #ifndef CK_TILE_USE_AMD_BUFFER_STORE
150 #define CK_TILE_USE_AMD_BUFFER_STORE 1
153 #ifndef CK_TILE_USE_AMD_BUFFER_ATOMIC_ADD_INTEGER
154 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_ADD_INTEGER 1
157 #ifndef CK_TILE_USE_PK4_LAYOUT_SHUFFLE
158 #define CK_TILE_USE_PK4_LAYOUT_SHUFFLE 1
162 #ifndef __HIP_DEVICE_COMPILE__
163 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 1
164 #elif defined(__gfx9__) || defined(__gfx12__)
165 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 1
167 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 0
170 #if(defined(__gfx90a__) || defined(__gfx94__))
171 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_MAX_FLOAT64 1
173 #define CK_TILE_USE_AMD_BUFFER_ATOMIC_MAX_FLOAT64 0
176 #ifndef CK_TILE_EXPERIMENTAL_USE_MEMCPY_FOR_VECTOR_ACCESS
177 #define CK_TILE_EXPERIMENTAL_USE_MEMCPY_FOR_VECTOR_ACCESS 0
180 #ifndef CK_TILE_WORKAROUND_SWDEV_XXXXXX_INT8_DS_WRITE_ISSUE
181 #define CK_TILE_WORKAROUND_SWDEV_XXXXXX_INT8_DS_WRITE_ISSUE 1
184 #ifndef CK_TILE_WORKAROUND_ROCM_6_1_SCRATCH_MEMORY_ISSUE
185 #if HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 1 && HIP_VERSION_PATCH >= 40091
186 #define CK_TILE_WORKAROUND_ROCM_6_1_SCRATCH_MEMORY_ISSUE 1
188 #define CK_TILE_WORKAROUND_ROCM_6_1_SCRATCH_MEMORY_ISSUE 0
193 #ifndef CK_TILE_WORKAROUND_ROCM_6_2_SCRATCH_MEMORY_ISSUE
194 #if(HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 2 && HIP_VERSION_PATCH >= 41133) || \
195 (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 3 && HIP_VERSION_PATCH >= 42131) || \
196 (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR > 3)
197 #define CK_TILE_WORKAROUND_ROCM_6_2_SCRATCH_MEMORY_ISSUE 1
199 #define CK_TILE_WORKAROUND_ROCM_6_2_SCRATCH_MEMORY_ISSUE 0
204 #ifndef CK_TILE_USE_LLVM_BUILTIN_BF16
205 #if(HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 5 && HIP_VERSION_PATCH >= 50421) || \
206 (HIP_VERSION_MAJOR >= 7)
207 #define CK_TILE_USE_LLVM_BUILTIN_BF16 1
209 #define CK_TILE_USE_LLVM_BUILTIN_BF16 0
213 #ifndef CK_TILE_DEBUG_LOG
214 #define CK_TILE_DEBUG_LOG 0
217 #ifndef __HIP_DEVICE_COMPILE__
218 #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0xffffffff
219 #elif defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || \
221 #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x00020000
222 #elif defined(__gfx101__) || defined(__gfx103__)
223 #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x31014000
224 #elif defined(__gfx11__) || defined(__gfx12__)
225 #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x31004000
228 #ifndef CK_TILE_EXPERIMENTAL_BLOCK_SYNC_LDS_WITHOUT_SYNC_VMEM
229 #define CK_TILE_EXPERIMENTAL_BLOCK_SYNC_LDS_WITHOUT_SYNC_VMEM 1
232 #ifndef CK_TILE_USE_SUBDWORD_TILE_CAST
233 #define CK_TILE_USE_SUBDWORD_TILE_CAST 0
236 #ifndef CK_TILE_USE_PK_FP16_TILE_CAST
237 #define CK_TILE_USE_PK_FP16_TILE_CAST 0
241 #ifndef CK_TILE_FMHA_FWD_FAST_EXP2
242 #define CK_TILE_FMHA_FWD_FAST_EXP2 0
245 #ifndef CK_TILE_FMHA_FLOAT_TO_FLOAT16_RTN
246 #define CK_TILE_FMHA_FLOAT_TO_FLOAT16_RTN 0
249 #ifndef CK_TILE_BUFFER_LOAD_RAW_BF16_WA
250 #define CK_TILE_BUFFER_LOAD_RAW_BF16_WA 1
254 #ifndef CK_TILE_WORKAROUND_SWDEV_383542
255 #define CK_TILE_WORKAROUND_SWDEV_383542 1
258 #ifndef CK_TILE_REFERENCE_MOE_SORTING_MOCK_ID
259 #define CK_TILE_REFERENCE_MOE_SORTING_MOCK_ID 1
262 #ifndef CK_TILE_USE_OCP_FP8
263 #if defined(__HIP_DEVICE_COMPILE__)
264 #if defined(__gfx950__) || defined(__gfx12__)
265 #define CK_TILE_USE_OCP_FP8 1
267 #define CK_TILE_USE_OCP_FP8 0
270 #define CK_TILE_USE_OCP_FP8 0
274 #ifndef CK_TILE_USE_BUFFER_ADDRESSING_BUILTIN
275 #if __clang_major__ >= 20 && !(defined(__gfx103__) || defined(__gfx11__) || defined(__gfx12__))
276 #define CK_TILE_USE_BUFFER_ADDRESSING_BUILTIN 1
278 #define CK_TILE_USE_BUFFER_ADDRESSING_BUILTIN 0
282 #ifndef CK_TILE_WA_ISSUE_2028
283 #define CK_TILE_WA_ISSUE_2028 0
288 #ifndef CK_TILE_ENC_SUPPORT_Y_TO_R
289 #define CK_TILE_ENC_SUPPORT_Y_TO_R 0
294 #define CK_TILE_UNSUPPORTED_IMPL(MSG)
296 #define CK_TILE_UNSUPPORTED_IMPL(MSG) __attribute__((deprecated(MSG)))
330 #if defined(__HIP_DEVICE_COMPILE__) && __HIP_DEVICE_COMPILE__
339 #if defined(__gfx908__)
345 #if defined(__gfx90a__)
351 #if defined(__gfx942__)
357 #if defined(__gfx950__)
364 #if defined(__gfx1010__)
370 #if defined(__gfx1030__)
376 #if defined(__gfx1031__)
382 #if defined(__gfx1032__)
388 #if defined(__gfx1034__)
394 #if defined(__gfx1035__)
400 #if defined(__gfx1036__)
406 #if defined(__gfx10_3_generic__)
413 #if defined(__gfx1100__)
419 #if defined(__gfx1101__)
425 #if defined(__gfx1102__)
431 #if defined(__gfx1103__)
437 #if defined(__gfx1150__)
443 #if defined(__gfx1151__)
449 #if defined(__gfx1152__)
455 #if defined(__gfx11_generic__)
462 #if defined(__gfx1200__)
468 #if defined(__gfx1201__)
474 #if defined(__gfx12_generic__)
489 template <
typename T,
typename... Ts>
495 "All search list values must be convertible to the search value type");
496 static_assert(
sizeof...(Ts) >= 1,
"At least one value must be provided to search in");
498 return (
static_cast<uint32_t>(search ==
static_cast<T
>(searchList)) + ...);
501 #define CK_TILE_COMPILER_TARGETS_LIST \
502 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX908, \
503 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX90A, \
504 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX942, \
505 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX950, \
506 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1010, \
507 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1030, \
508 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1031, \
509 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1032, \
510 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1034, \
511 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1035, \
512 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1036, \
513 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX10_3_GENERIC, \
514 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1100, \
515 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1101, \
516 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1102, \
517 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1103, \
518 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1150, \
519 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1151, \
520 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1152, \
521 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX11_GENERIC, \
522 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1200, \
523 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX1201, \
524 amdgcn_compiler_target_state::CK_TILE_ARCH_GFX12_GENERIC
529 "Only one target architecture can be defined during device compile");
534 "No device target architecture can be defined during host compile");
#define CK_TILE_COMPILER_TARGETS_LIST
Definition: config.hpp:501
#define CK_TILE_HOST_DEVICE
Definition: config.hpp:50
Definition: amdgcn_mma.hpp:10
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1697
unsigned int uint32_t
Definition: stdint.h:126
Defines compiler states for supported AMDGCN devices.
Definition: config.hpp:328
static constexpr bool CK_TILE_ARCH_GFX90A
Definition: config.hpp:348
static constexpr bool CK_TILE_ARCH_GFX1032
Definition: config.hpp:385
static constexpr bool CK_TILE_ARCH_GFX908
Definition: config.hpp:342
static constexpr bool CK_TILE_ARCH_GFX11_GENERIC
Definition: config.hpp:458
static constexpr bool CK_TILE_ARCH_GFX1030
Definition: config.hpp:373
static constexpr bool CK_TILE_ARCH_GFX1036
Definition: config.hpp:403
static constexpr bool CK_TILE_ARCH_GFX1200
Definition: config.hpp:465
static constexpr bool CK_TILE_ARCH_GFX1035
Definition: config.hpp:397
static constexpr bool CK_TILE_ARCH_GFX1034
Definition: config.hpp:391
static constexpr bool CK_TILE_ARCH_GFX1152
Definition: config.hpp:452
static constexpr bool CK_TILE_ARCH_GFX1010
Definition: config.hpp:367
static constexpr bool CK_TILE_ARCH_GFX1031
Definition: config.hpp:379
static constexpr bool CK_TILE_ARCH_GFX1103
Definition: config.hpp:434
static constexpr bool CK_TILE_HOST_COMPILE
Definition: config.hpp:335
static constexpr bool CK_TILE_ARCH_GFX1100
Definition: config.hpp:416
static constexpr bool CK_TILE_ARCH_GFX1201
Definition: config.hpp:471
static constexpr bool CK_TILE_ARCH_GFX10_3_GENERIC
Definition: config.hpp:409
static constexpr bool CK_TILE_ARCH_GFX1101
Definition: config.hpp:422
static constexpr bool CK_TILE_ARCH_GFX942
Definition: config.hpp:354
static constexpr bool CK_TILE_DEVICE_COMPILE
Definition: config.hpp:334
static constexpr bool CK_TILE_ARCH_GFX1102
Definition: config.hpp:428
static constexpr bool CK_TILE_ARCH_GFX12_GENERIC
Definition: config.hpp:477
static constexpr bool CK_TILE_ARCH_GFX950
Definition: config.hpp:360
static constexpr bool CK_TILE_ARCH_GFX1150
Definition: config.hpp:440
static constexpr bool CK_TILE_ARCH_GFX1151
Definition: config.hpp:446