21 namespace tensor_operation {
24 template <
typename ALayout,
29 typename AScaleDataType,
31 typename BScaleDataType,
34 typename GemmAccDataType,
35 typename CShuffleDataType,
36 typename AElementwiseOperation,
37 typename BElementwiseOperation,
38 typename CElementwiseOperation,
51 typename ABlockTransferThreadClusterLengths_AK0_M_AK1,
52 typename ABlockTransferThreadClusterArrangeOrder,
53 typename ABlockTransferSrcAccessOrder,
54 index_t ABlockTransferSrcVectorDim,
55 index_t ABlockTransferSrcScalarPerVector,
56 index_t ABlockTransferDstScalarPerVector_AK1,
58 typename BBlockTransferThreadClusterLengths_BK0_N_BK1,
59 typename BBlockTransferThreadClusterArrangeOrder,
60 typename BBlockTransferSrcAccessOrder,
61 index_t BBlockTransferSrcVectorDim,
62 index_t BBlockTransferSrcScalarPerVector,
63 index_t BBlockTransferDstScalarPerVector_BK1,
65 index_t CShuffleMXdlPerWavePerShuffle,
66 index_t CShuffleNXdlPerWavePerShuffle,
67 typename CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock,
68 typename CDEShuffleBlockTransferScalarPerVectors,
72 bool NSwizzle =
false,
73 bool IsInputGemm =
true,
74 bool MulRoutedWeight =
true,
76 typename ComputeTypeA = ADataType,
77 typename ComputeTypeB = BDataType>
89 AElementwiseOperation,
90 BElementwiseOperation,
91 CElementwiseOperation>
97 template <index_t NXdlPerWave_>
111 AElementwiseOperation,
112 BElementwiseOperation,
113 CElementwiseOperation,
126 ABlockTransferThreadClusterLengths_AK0_M_AK1,
127 ABlockTransferThreadClusterArrangeOrder,
128 ABlockTransferSrcAccessOrder,
129 ABlockTransferSrcVectorDim,
130 ABlockTransferSrcScalarPerVector,
131 ABlockTransferDstScalarPerVector_AK1,
134 BBlockTransferThreadClusterLengths_BK0_N_BK1,
135 BBlockTransferThreadClusterArrangeOrder,
136 BBlockTransferSrcAccessOrder,
137 BBlockTransferSrcVectorDim,
138 BBlockTransferSrcScalarPerVector,
139 BBlockTransferDstScalarPerVector_BK1,
142 CShuffleMXdlPerWavePerShuffle,
143 CShuffleNXdlPerWavePerShuffle,
144 CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock,
145 CDEShuffleBlockTransferScalarPerVectors,
168 template <
typename Gr
idwiseGemm>
169 float RunImp(
const typename GridwiseGemm::Argument& arg,
172 if(stream_config.log_level_ > 0)
177 if(!GridwiseGemm::CheckValidity(arg))
179 throw std::runtime_error(
"wrong! GridwiseGemm has invalid setting");
183 std::tie(gdx, gdy, gdz) = GridwiseGemm::CalculateGridSize(arg.M, arg.N);
187 index_t k_grain = arg.KBatch * KPerBlock;
188 index_t K_split = (arg.K + k_grain - 1) / k_grain * KPerBlock;
190 const bool has_main_k_block_loop = GridwiseGemm::CalculateHasMainKBlockLoop(K_split);
192 const auto RunKernel = [&](
const auto& kernel) {
193 if(stream_config.flush_cache)
196 std::array<std::size_t, NumDTensor> DsSize;
200 const auto a_grid_desc_ak0_m_ak1 = GridwiseGemm::MakeAGridDescriptor_AK0_M_AK1(
201 arg_.M, arg_.MPadded, arg_.K, arg_.KPadded, arg_.StrideA, arg_.AK0);
202 const auto b_grid_desc_bk0_n_bk1 = GridwiseGemm::MakeBGridDescriptor_BK0_N_BK1(
203 arg_.K, arg_.KPadded, arg_.N, arg_.NPadded, arg_.StrideB, arg_.BK0);
206 a_grid_desc_ak0_m_ak1.GetElementSpaceSize() *
sizeof(ADataType);
208 b_grid_desc_bk0_n_bk1.GetElementSpaceSize() *
sizeof(BDataType);
210 const auto ds_grid_desc_m_n = GridwiseGemm::MakeDsGridDescriptor_M_N(
211 arg_.M, arg_.MPadded, arg_.N, arg_.NPadded, arg_.StrideDs);
215 DsSize[i] = ds_grid_desc_m_n[i].GetElementSpaceSize() *
sizeof(DDataType);
220 stream_config.rotating_count,
224 rotating_mem.Print();
226 auto run_flush_cache = [&]() {
233 hipGetErrorString(hipMemsetAsync(arg_.p_c_grid,
235 arg_.M * arg_.N *
sizeof(CDataType),
236 stream_config.stream_id_));
239 ave_time = ck::utility::launch_and_time_kernel_with_preprocess<false>(
251 hipGetErrorString(hipMemsetAsync(arg.p_c_grid,
253 arg.M * arg.N *
sizeof(CDataType),
254 stream_config.stream_id_));
257 stream_config, kernel, dim3(gdx, gdy, gdz), dim3(BlockSize), 0, arg);
262 constexpr
index_t minimum_occupancy =
265 MPerBlock * NPerBlock * KPerBlock *
sizeof(ADataType) <= 128 * 128 * 64 * 2)
270 constexpr
auto MemoryDataOp =
272 if(has_main_k_block_loop)
286 if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) ==
TailNumber::Odd)
307 throw std::runtime_error(
"todo: only v1 & v3 support now");
323 if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) ==
TailNumber::Odd)
353 return Run(*
dynamic_cast<const Argument*
>(p_arg), stream_config);
370 if(!ck::is_xdl_wmma_supported<ComputeTypeA, ComputeTypeB, MPerXDL, NPerXDL>())
386 if(arg.N % NPerBlock != 0 || arg.K % KPerBlock != 0)
416 const void* p_sorted_expert_ids,
417 const void* p_max_token_id,
419 const void* p_a_scale,
421 const void* p_b_scale,
422 std::array<const void*, NumDTensor> p_ds,
433 std::array<index_t, NumDTensor> StrideDs,
436 AElementwiseOperation a_element_op,
437 BElementwiseOperation b_element_op,
438 CElementwiseOperation c_element_op)
441 static_cast<const index_t*
>(p_sorted_expert_ids),
442 static_cast<const index_t*
>(p_max_token_id),
443 static_cast<const ADataType*
>(p_a),
444 static_cast<const AScaleDataType*
>(p_a_scale),
445 static_cast<const BDataType*
>(p_b),
446 static_cast<const BScaleDataType*
>(p_b_scale),
448 static_cast<CDataType*
>(p_c),
470 const void* p_a_scale,
472 const void* p_b_scale,
473 std::array<const void*, NumDTensor> p_ds,
482 std::array<ck::index_t, NumDTensor> StrideDs,
485 AElementwiseOperation a_element_op,
486 BElementwiseOperation b_element_op,
487 CElementwiseOperation c_element_op)
override
489 return std::make_unique<Argument>(
nullptr,
492 static_cast<const ADataType*
>(p_a),
493 static_cast<const AScaleDataType*
>(p_a_scale),
494 static_cast<const BDataType*
>(p_b),
495 static_cast<const BScaleDataType*
>(p_b_scale),
497 static_cast<CDataType*
>(p_c),
518 return std::make_unique<Invoker>(
Invoker{});
524 auto str = std::stringstream();
526 std::map<BlockGemmPipelineScheduler, std::string> BlkGemmPipelineSchedulerToString{
530 std::map<BlockGemmPipelineVersion, std::string> BlkGemmPipelineVersionToString{
538 str <<
"DeviceMoeGEmmMx"
541 << std::string(ALayout::name)[0]
542 << std::string(BLayout::name)[0]
543 << std::string(CLayout::name)[0]
548 << MPerBlock<<
"x"<<NPerBlock<<
"x"<<KPerBlock <<
", "
550 << MPerXDL<<
"x"<<NPerXDL <<
", "
552 << MXdlPerWave<<
"x" << NXdlPerWave<<
", "
554 << ABlockTransferSrcScalarPerVector<<
"x"<<BBlockTransferSrcScalarPerVector<<
", "
555 <<
"BlkGemmPipelineScheduler: "
556 << BlkGemmPipelineSchedulerToString[BlkGemmPipeSched] <<
", "
557 <<
"BlkGemmPipelineVersion: "
558 << BlkGemmPipelineVersionToString[BlkGemmPipelineVer] <<
", "
559 <<
"BlkGemmPipelinePrefetchStages: "
560 << GridwiseGemm64::BlockwiseGemmPipe::PrefetchStages;
#define INVOKER_RUN3_IMPL
Definition: device_base.hpp:114
#define GET_NXDL_PER_WAVE_IMPL
Definition: device_base.hpp:81
float launch_and_time_kernel(const StreamConfig &stream_config, F kernel, dim3 grid_dim, dim3 block_dim, std::size_t lds_byte, Args... args)
Definition: kernel_launch.hpp:14
__host__ constexpr __device__ T max(T x)
Definition: math.hpp:84
std::string getGemmSpecializationString(const GemmSpecialization &s)
Definition: gemm_specialization.hpp:32
GemmSpecialization
Definition: gemm_specialization.hpp:11
void flush_icache()
Definition: flush_cache.hpp:361
__global__ void kernel_moe_mxgemm(typename GridwiseGemm::Argument karg)
Definition: gridwise_moe_mx_gemm_bns.hpp:48
typename tuple_element< I, TTuple >::type tuple_element_t
Definition: tuple.hpp:208
BlockGemmPipelineVersion
Definition: blkgemmpipe_scheduler.hpp:12
constexpr Tuple< Args &... > tie(Args &... args) noexcept
Definition: tuple.hpp:218
constexpr __device__ index_t get_warp_size()
Definition: get_id.hpp:10
BlockGemmPipelineScheduler
Definition: blkgemmpipe_scheduler.hpp:25
remove_cv_t< remove_reference_t< T > > remove_cvref_t
Definition: type.hpp:297
int32_t index_t
Definition: ck.hpp:299
bool is_bf16_atomic_supported()
Definition: device_prop.hpp:108
Definition: stream_config.hpp:10
Definition: gridwise_moe_mx_gemm_bns.hpp:654
Definition: gridwise_moe_mx_gemm_bns.hpp:179
static constexpr __host__ bool CheckValidity(const Argument &karg)
Definition: gridwise_moe_mx_gemm_bns.hpp:1111
Definition: functional2.hpp:33
Definition: device_base.hpp:197
Definition: device_base.hpp:208
Definition: device_gemm_multiple_d.hpp:167
Definition: device_moe_mx_gemm_bns.hpp:167
INVOKER_RUN3_IMPL float Run(const BaseArgument *p_arg, const StreamConfig &stream_config=StreamConfig{}) override
Definition: device_moe_mx_gemm_bns.hpp:350
float RunImp(const typename GridwiseGemm::Argument &arg, const StreamConfig &stream_config=StreamConfig{})
Definition: device_moe_mx_gemm_bns.hpp:169
Definition: device_moe_mx_gemm_bns.hpp:92
static constexpr index_t BPackedSize
Definition: device_moe_mx_gemm_bns.hpp:161
std::string GetTypeString() const override
Definition: device_moe_mx_gemm_bns.hpp:522
static auto MakeInvoker()
Definition: device_moe_mx_gemm_bns.hpp:466
static constexpr index_t APackedSize
Definition: device_moe_mx_gemm_bns.hpp:160
typename GridwiseGemm64::Argument Argument
Definition: device_moe_mx_gemm_bns.hpp:158
std::unique_ptr< BaseArgument > MakeArgumentPointer(const void *p_a, const void *p_a_scale, const void *p_b, const void *p_b_scale, std::array< const void *, NumDTensor > p_ds, void *p_c, index_t M, index_t N, index_t K, index_t StrideA, index_t StrideScaleA, index_t StrideB, index_t StrideScaleB, std::array< ck::index_t, NumDTensor > StrideDs, index_t StrideC, index_t KBatch, AElementwiseOperation a_element_op, BElementwiseOperation b_element_op, CElementwiseOperation c_element_op) override
Definition: device_moe_mx_gemm_bns.hpp:469
static auto MakeArgument(const void *p_sorted_token_ids, const void *p_sorted_expert_ids, const void *p_max_token_id, const void *p_a, const void *p_a_scale, const void *p_b, const void *p_b_scale, std::array< const void *, NumDTensor > p_ds, void *p_c, index_t NumTokens, index_t TopK, index_t M, index_t N, index_t K, index_t StrideA, index_t StrideScaleA, index_t StrideB, index_t StrideScaleB, std::array< index_t, NumDTensor > StrideDs, index_t StrideC, index_t KBatch, AElementwiseOperation a_element_op, BElementwiseOperation b_element_op, CElementwiseOperation c_element_op)
Definition: device_moe_mx_gemm_bns.hpp:415
static constexpr auto NXdlPerWave32
Definition: device_moe_mx_gemm_bns.hpp:95
static bool IsSupportedArgument(const Argument &arg)
Definition: device_moe_mx_gemm_bns.hpp:363
int GetPreShuffleParameters() override
Definition: device_moe_mx_gemm_bns.hpp:163
bool IsSupportedArgument(const BaseArgument *p_arg) override
Definition: device_moe_mx_gemm_bns.hpp:410
static constexpr GET_NXDL_PER_WAVE_IMPL auto NXdlPerWave64
Definition: device_moe_mx_gemm_bns.hpp:94
std::unique_ptr< BaseInvoker > MakeInvokerPointer() override
Definition: device_moe_mx_gemm_bns.hpp:516
static constexpr index_t NumDTensor
Definition: device_moe_mx_gemm_bns.hpp:96
static constexpr bool IsValidCompilationParameter()
Definition: device_moe_mx_gemm_bns.hpp:357
Definition: flush_cache.hpp:165