/home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-composable-kernel/checkouts/develop/include/ck/tensor_operation/gpu/device/impl/device_moe_gemm_blockscale.hpp Source File

/home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-composable-kernel/checkouts/develop/include/ck/tensor_operation/gpu/device/impl/device_moe_gemm_blockscale.hpp Source File#

Composable Kernel: /home/docs/checkouts/readthedocs.org/user_builds/advanced-micro-devices-composable-kernel/checkouts/develop/include/ck/tensor_operation/gpu/device/impl/device_moe_gemm_blockscale.hpp Source File
Go to the documentation of this file.
 // Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 // SPDX-License-Identifier: MIT
  
 #pragma once
  
 #include <iostream>
 #include <sstream>
 #include <hip/hip_runtime.h>
  
 #include "ck/utility/common_header.hpp"
 #include "ck/tensor_description/tensor_descriptor.hpp"
 #include "ck/tensor_description/tensor_descriptor_helper.hpp"
 #include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
 #include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_ab_scale.hpp"
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/grid/gridwise_moe_gemm_blockscale.hpp"
 #include "ck/host_utility/device_prop.hpp"
 #include "ck/host_utility/kernel_launch.hpp"
 #include "ck/host_utility/flush_cache.hpp"
  
 namespace ck {
 namespace tensor_operation {
 namespace device {
  
 template <typename ALayout,
           typename BLayout,
           typename DsLayout,
           typename CLayout,
           typename ADataType,
           typename AScaleDataType,
           typename BDataType,
           typename BScaleDataType,
           typename DsDataType,
           typename CDataType,
           typename GemmAccDataType,
           typename CShuffleDataType,
           typename AElementwiseOperation,
           typename BElementwiseOperation,
           typename CElementwiseOperation,
           GemmSpecialization GemmSpec,
           index_t BlockSize,
           index_t ScaleBlockM,
           index_t ScaleBlockN,
           index_t ScaleBlockK,
           index_t MPerBlock,
           index_t NPerBlock,
           index_t KPerBlock,
           index_t AK1,
           index_t BK1,
           index_t MPerXDL,
           index_t NPerXDL,
           index_t MXdlPerWave,
           index_t NXdlPerWave,
           typename ABlockTransferThreadClusterLengths_AK0_M_AK1,
           typename ABlockTransferThreadClusterArrangeOrder,
           typename ABlockTransferSrcAccessOrder,
           index_t ABlockTransferSrcVectorDim,
           index_t ABlockTransferSrcScalarPerVector,
           index_t ABlockTransferDstScalarPerVector_AK1,
           bool ABlockLdsExtraM,
           typename BBlockTransferThreadClusterLengths_BK0_N_BK1,
           typename BBlockTransferThreadClusterArrangeOrder,
           typename BBlockTransferSrcAccessOrder,
           index_t BBlockTransferSrcVectorDim,
           index_t BBlockTransferSrcScalarPerVector,
           index_t BBlockTransferDstScalarPerVector_BK1,
           bool BBlockLdsExtraN,
           index_t CShuffleMXdlPerWavePerShuffle,
           index_t CShuffleNXdlPerWavePerShuffle,
           typename CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock,
           typename CDEShuffleBlockTransferScalarPerVectors,
           BlockGemmPipelineScheduler BlkGemmPipeSched = BlockGemmPipelineScheduler::Intrawave,
           BlockGemmPipelineVersion BlkGemmPipelineVer = BlockGemmPipelineVersion::v1,
           index_t ActivationOP                        = 0,
           bool NSwizzle                               = false,
           bool IsInputGemm                            = true,
           bool IsSplitK                               = false,
           bool MulRoutedWeight                        = false,
           typename IndexType                          = index_t,
           typename ComputeTypeA                       = CDataType,
           typename ComputeTypeB                       = ComputeTypeA,
           typename LDSTypeA                           = ComputeTypeA,
           typename LDSTypeB                           = ComputeTypeB>
 struct DeviceMoeGemmBlockScale
     : public DeviceGemmMultipleD_BlockScale_BPreshuffle<ALayout,
                                                         BLayout,
                                                         DsLayout,
                                                         CLayout,
                                                         ADataType,
                                                         AScaleDataType,
                                                         BDataType,
                                                         BScaleDataType,
                                                         DsDataType,
                                                         CDataType,
                                                         ScaleBlockM,
                                                         ScaleBlockN,
                                                         ScaleBlockK,
                                                         AElementwiseOperation,
                                                         BElementwiseOperation,
                                                         CElementwiseOperation>
 {
     GET_NXDL_PER_WAVE_IMPL
     static constexpr auto NXdlPerWave64 = GetNXdlPerWave<true>();
     static constexpr auto NXdlPerWave32 = GetNXdlPerWave<false>();
     static constexpr index_t NumDTensor = DsDataType::Size();
     template <index_t NXdlPerWave_>
     using GridwiseGemmBase = GridwiseMoeGemmBlockScale<
         ALayout,
         BLayout,
         DsLayout,
         CLayout,
         ADataType,
         BDataType,
         GemmAccDataType,
         CShuffleDataType,
         DsDataType,
         CDataType,
         AElementwiseOperation,
         BElementwiseOperation,
         CElementwiseOperation,
         GemmSpec,
         BlockSize,
         ScaleBlockM,
         ScaleBlockN,
         ScaleBlockK,
         MPerBlock,
         NPerBlock,
         KPerBlock,
         AK1,
         BK1,
         MPerXDL,
         NPerXDL,
         MXdlPerWave,
         NXdlPerWave_,
         ABlockTransferThreadClusterLengths_AK0_M_AK1,
         ABlockTransferThreadClusterArrangeOrder,
         ABlockTransferSrcAccessOrder,
         ABlockTransferSrcVectorDim,
         ABlockTransferSrcScalarPerVector,
         ABlockTransferDstScalarPerVector_AK1,
         false,
         ABlockLdsExtraM,
         BBlockTransferThreadClusterLengths_BK0_N_BK1,
         BBlockTransferThreadClusterArrangeOrder,
         BBlockTransferSrcAccessOrder,
         BBlockTransferSrcVectorDim,
         BBlockTransferSrcScalarPerVector,
         BBlockTransferDstScalarPerVector_BK1,
         false,
         BBlockLdsExtraN,
         CShuffleMXdlPerWavePerShuffle,
         math::min(CShuffleNXdlPerWavePerShuffle, NXdlPerWave_),
         CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock,
         CDEShuffleBlockTransferScalarPerVectors,
         BlkGemmPipeSched,
         BlkGemmPipelineVer,
         ActivationOP,
         NSwizzle,
         IsInputGemm,
         IsSplitK,
         MulRoutedWeight,
         IndexType,
         ComputeTypeA,
         ComputeTypeB,
         LDSTypeA,
         LDSTypeB>;
     using GridwiseGemm64 = GridwiseGemmBase<math::max(NXdlPerWave64, 1)>;
     using GridwiseGemm32 = GridwiseGemmBase<NXdlPerWave32>;
  
     using Argument = typename GridwiseGemm64::Argument;
  
     static constexpr index_t APackedSize = []() {
         if constexpr(is_same_v<remove_cvref_t<ADataType>, pk_i4_t>)
             return 2;
         else
             return 1;
     }();
  
     static constexpr index_t BPackedSize = []() {
         if constexpr(is_same_v<remove_cvref_t<BDataType>, pk_i4_t>)
             return 2;
         else
             return 1;
     }();
  
     int GetPreShuffleParameters() override { return NPerXDL; }
  
     // Invoker
     struct Invoker : public BaseInvoker
     {
         template <typename GridwiseGemm>
         float RunImp(const typename GridwiseGemm::Argument& arg,
                      const StreamConfig& stream_config = StreamConfig{})
         {
             if(stream_config.log_level_ > 0)
             {
                 arg.Print();
             }
  
             if(!GridwiseGemm::CheckValidity(arg))
             {
                 throw std::runtime_error("wrong! GridwiseGemm has invalid setting");
             }
  
             index_t gdx, gdy, gdz;
             std::tie(gdx, gdy, gdz) = GridwiseGemm::CalculateGridSize(
                 arg.M, arg.N * (IsInputGemm && IsSplitK ? 2 : 1), arg.K, arg.KBatch);
  
             float ave_time = 0;
  
             index_t K_split = arg.KBatch == 1 ? arg.K : arg.KBatch * KPerBlock;
  
             const bool has_main_k_block_loop = GridwiseGemm::CalculateHasMainKBlockLoop(K_split);
             const auto RunKernel             = [&](const auto& kernel) {
                 if(stream_config.flush_cache)
                 {
  
                     std::array<std::size_t, NumDTensor> DsSize;
  
                     auto arg_ = arg;
  
                     const auto a_grid_desc_ak0_m_ak1 = GridwiseGemm::MakeAGridDescriptor_AK0_M_AK1(
                         arg_.M, arg_.MPadded, arg_.K, arg_.KPadded, arg_.StrideA, arg_.AK0);
                     const auto b_grid_desc_bk0_n_bk1 = GridwiseGemm::MakeBGridDescriptor_BK0_N_BK1(
                         arg_.K, arg_.KPadded, arg_.N, arg_.NPadded, arg_.StrideB, arg_.BK0);
  
                     auto size_a_buffer = a_grid_desc_ak0_m_ak1.GetElementSpaceSize() *
                                          sizeof(ADataType) / APackedSize;
                     auto size_b_buffer = b_grid_desc_bk0_n_bk1.GetElementSpaceSize() *
                                          sizeof(BDataType) / BPackedSize;
  
                     const auto ds_grid_desc_m_n = GridwiseGemm::MakeDsGridDescriptor_M_N(
                         arg_.M, arg_.MPadded, arg_.N, arg_.NPadded, arg_.StrideDs);
  
                     static_for<0, NumDTensor, 1>{}([&](auto i) {
                         using DDataType = remove_cvref_t<tuple_element_t<i.value, DsDataType>>;
                         DsSize[i] = ds_grid_desc_m_n[i].GetElementSpaceSize() * sizeof(DDataType);
                     });
                     ck::utility::RotatingMemWrapperMultiD<typename GridwiseGemm::Argument,
                                                                       DsDataType>
                         rotating_mem(arg_,
                                      stream_config.rotating_count,
                                      size_a_buffer,
                                      size_b_buffer,
                                      DsSize);
                     rotating_mem.Print();
  
                     auto run_flush_cache = [&]() {
                         // flush icache
                         ck::utility::flush_icache();
                         // rotating mem
                         rotating_mem.Next();
                         // clear c mem
                         // if(arg_.KBatch > 1)
                         //     hipGetErrorString(hipMemsetAsync(arg_.p_c_grid,
                         //                                      0,
                         //                                      arg_.M * arg_.N * sizeof(CDataType)
                         //                                      * (IsInputGemm && IsSplitK ? 2 : 1),
                         //                                      stream_config.stream_id_));
                     };
  
                     ave_time = ck::utility::launch_and_time_kernel_with_preprocess<false>(
                         stream_config,
                         run_flush_cache,
                         kernel,
                         dim3(gdx, gdy, gdz),
                         dim3(BlockSize),
                         0,
                         arg_);
                 }
                 else
                 {
                     // if(arg.KBatch > 1)
                     //     hipGetErrorString(hipMemsetAsync(arg.p_c_grid,
                     //                                      0,
                     //                                      arg.M * arg.N * sizeof(CDataType) *
                     //                                      (IsInputGemm && IsSplitK ? 2 : 1),
                     //                                      stream_config.stream_id_));
  
                     ave_time = launch_and_time_kernel(
                         stream_config, kernel, dim3(gdx, gdy, gdz), dim3(BlockSize), 0, arg);
                 }
             };
  
             constexpr auto estimated_reg_a = MPerBlock * KPerBlock * sizeof(ADataType) / BlockSize /
                                              4 * (1 + GridwiseGemm::NWave);
             constexpr auto estimated_reg_b = NPerBlock * KPerBlock * sizeof(BDataType) / BlockSize /
                                              4 * (2) * (IsInputGemm ? 2 : 1);
             constexpr auto estimated_reg_c = MPerBlock * NPerBlock * sizeof(GemmAccDataType) /
                                              BlockSize / 4 * (IsInputGemm ? 2 : 1);
             constexpr auto estimated_reg_total =
                 estimated_reg_a + estimated_reg_b + estimated_reg_c;
  
             constexpr index_t minimum_occupancy = (estimated_reg_total >= 256) ? 1 : 2;
  
             constexpr auto MemoryDataOp = (IsInputGemm && !IsSplitK)
                                               ? InMemoryDataOperationEnum::Set
                                               : InMemoryDataOperationEnum::AtomicAdd;
  
             if(has_main_k_block_loop)
             {
                 // Tail number always full
                 if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v1)
                 {
                     {
                         if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) == TailNumber::Odd)
                         {
                             const auto kernel = kernel_moe_gemm<GridwiseGemm,
                                                                 true,
                                                                 MemoryDataOp,
                                                                 minimum_occupancy,
                                                                 TailNumber::Odd>;
                             RunKernel(kernel);
                         }
                         else
                         {
                             const auto kernel = kernel_moe_gemm<GridwiseGemm,
                                                                 true,
                                                                 MemoryDataOp,
                                                                 minimum_occupancy,
                                                                 TailNumber::Even>;
                             RunKernel(kernel);
                         }
                     }
                 }
                 else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v2 ||
                                   BlkGemmPipelineVer == BlockGemmPipelineVersion::v3)
                 {
                     if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) == TailNumber::Odd)
                     {
                         const auto kernel = kernel_moe_gemm_2lds<GridwiseGemm,
                                                                  true,
                                                                  MemoryDataOp,
                                                                  minimum_occupancy,
                                                                  TailNumber::Odd>;
                         RunKernel(kernel);
                     }
                     else
                     {
                         const auto kernel = kernel_moe_gemm_2lds<GridwiseGemm,
                                                                  true,
                                                                  MemoryDataOp,
                                                                  minimum_occupancy,
                                                                  TailNumber::Even>;
                         RunKernel(kernel);
                     }
                 }
                 else
                 {
                     throw std::runtime_error("todo: only v1 & v2 support now");
                 }
             }
 #if 1
             else
             {
                 // Tail number always 1
                 if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v1)
                 {
                     if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) == TailNumber::Odd)
                     {
                         const auto kernel = kernel_moe_gemm<GridwiseGemm,
                                                             false,
                                                             MemoryDataOp,
                                                             minimum_occupancy,
                                                             TailNumber::Odd>;
                         RunKernel(kernel);
                     }
                     else
                     {
                         const auto kernel = kernel_moe_gemm<GridwiseGemm,
                                                             false,
                                                             MemoryDataOp,
                                                             minimum_occupancy,
                                                             TailNumber::Even>;
                         RunKernel(kernel);
                     }
                 }
                 else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v2 ||
                                   BlkGemmPipelineVer == BlockGemmPipelineVersion::v3)
                 {
                     if(GridwiseGemm::CalculateKBlockLoopTailNum(K_split) == TailNumber::Odd)
                     {
                         const auto kernel = kernel_moe_gemm_2lds<GridwiseGemm,
                                                                  false,
                                                                  MemoryDataOp,
                                                                  minimum_occupancy,
                                                                  TailNumber::Odd>;
                         RunKernel(kernel);
                     }
                     else
                     {
                         const auto kernel = kernel_moe_gemm_2lds<GridwiseGemm,
                                                                  false,
                                                                  MemoryDataOp,
                                                                  minimum_occupancy,
                                                                  TailNumber::Even>;
                         RunKernel(kernel);
                     }
                 }
             }
 #endif
  
             return ave_time;
         }
  
         INVOKER_RUN3_IMPL
  
         // polymorphic
         float Run(const BaseArgument* p_arg,
                   const StreamConfig& stream_config = StreamConfig{}) override
         {
             return Run(*dynamic_cast<const Argument*>(p_arg), stream_config);
         }
     };
  
     static constexpr bool IsValidCompilationParameter()
     {
         // TODO: properly implement this check
         return true;
     }
  
     static bool IsSupportedArgument(const Argument& arg)
     {
         // only impl kbatch 1 for fp32
         if(arg.KBatch > 1 && !std::is_same_v<CDataType, float>)
         {
             return false;
         }
         if(!ck::is_xdl_wmma_supported<ComputeTypeA, ComputeTypeB, MPerXDL, NPerXDL>())
         {
             return false;
         }
         if(!is_bf16_atomic_supported() && std::is_same_v<CDataType, ck::bhalf_t> && arg.KBatch > 1)
         {
             return false;
         }
  
         if((arg.K % AK1 != 0 || arg.K % BK1 != 0) && !(GemmSpec == GemmSpecialization::MKPadding ||
                                                        GemmSpec == GemmSpecialization::NKPadding ||
                                                        GemmSpec == GemmSpecialization::MNKPadding ||
                                                        GemmSpec == GemmSpecialization::KPadding))
         {
             return false;
         }
         if(arg.N % NPerBlock != 0 || arg.K % KPerBlock != 0)
         {
             return false;
         }
         if(arg.KBatch > 1 && arg.K % (KPerBlock * arg.KBatch) != 0)
         {
             // Not support Kpadding with KBatch > 1
             return false;
         }
  
         if(get_warp_size() == 64)
         {
             if constexpr(NXdlPerWave64 > 0)
             {
                 return GridwiseGemm64::CheckValidity(arg);
             }
         }
         else
         {
             if constexpr(NXdlPerWave32 > 0)
             {
                 return GridwiseGemm32::CheckValidity(
                     reinterpret_cast<const typename GridwiseGemm32::Argument&>(arg));
             }
         }
         return false;
     }
  
     // polymorphic
     bool IsSupportedArgument(const BaseArgument* p_arg) override
     {
         return IsSupportedArgument(*dynamic_cast<const Argument*>(p_arg));
     }
  
     static auto MakeArgument(const void* p_sorted_token_ids,
                              const void* p_sorted_expert_ids,
                              const void* p_max_token_id,
                              const void* p_a,
                              const void* p_b,
                              std::array<const void*, NumDTensor> p_ds,
                              void* p_c,
                              index_t NumTokens,
                              index_t TopK,
                              index_t M,
                              index_t N,
                              index_t K,
                              index_t StrideA,
                              index_t StrideB,
                              std::array<index_t, NumDTensor> StrideDs,
                              index_t StrideC,
                              const void* p_a_scale,
                              const void* p_b_scale,
                              index_t KBatch,
                              AElementwiseOperation a_element_op,
                              BElementwiseOperation b_element_op,
                              CElementwiseOperation c_element_op)
     {
         return Argument{static_cast<const index_t*>(p_sorted_token_ids),
                         static_cast<const index_t*>(p_sorted_expert_ids),
                         static_cast<const index_t*>(p_max_token_id),
                         static_cast<const ADataType*>(p_a),
                         static_cast<const BDataType*>(p_b),
                         p_ds,
                         static_cast<CDataType*>(p_c),
                         NumTokens,
                         TopK,
                         M,
                         N,
                         K,
                         StrideA,
                         StrideB,
                         StrideDs,
                         StrideC,
                         static_cast<const AScaleDataType*>(p_a_scale),
                         static_cast<const BScaleDataType*>(p_b_scale),
                         KBatch,
                         a_element_op,
                         b_element_op,
                         c_element_op};
     }
  
     static auto MakeInvoker() { return Invoker{}; }
  
     // polymorphic
     std::unique_ptr<BaseArgument> MakeArgumentPointer(const void* p_a,
                                                       const void* p_b,
                                                       std::array<const void*, NumDTensor> p_ds,
                                                       void* p_c,
                                                       index_t M,
                                                       index_t N,
                                                       index_t K,
                                                       index_t StrideA,
                                                       index_t StrideB,
                                                       std::array<ck::index_t, NumDTensor> StrideDs,
                                                       index_t StrideC,
                                                       const void* p_a_scale,
                                                       const void* p_b_scale,
                                                       //   index_t KBatch,
                                                       AElementwiseOperation a_element_op,
                                                       BElementwiseOperation b_element_op,
                                                       CElementwiseOperation c_element_op) override
     {
         return std::make_unique<Argument>(nullptr,
                                           nullptr,
                                           nullptr,
                                           static_cast<const ADataType*>(p_a),
                                           static_cast<const BDataType*>(p_b),
                                           p_ds,
                                           static_cast<CDataType*>(p_c),
                                           M, // randoms set, no use
                                           0,
                                           M,
                                           N,
                                           K,
                                           StrideA,
                                           StrideB,
                                           StrideDs,
                                           StrideC,
                                           static_cast<const AScaleDataType*>(p_a_scale),
                                           static_cast<const BScaleDataType*>(p_b_scale),
                                           1, // KBatch,
                                           a_element_op,
                                           b_element_op,
                                           c_element_op);
     }
  
     // polymorphic
     std::unique_ptr<BaseInvoker> MakeInvokerPointer() override
     {
         return std::make_unique<Invoker>(Invoker{});
     }
  
     // polymorphic
     std::string GetTypeString() const override
     {
         auto str = std::stringstream();
  
         std::map<BlockGemmPipelineScheduler, std::string> BlkGemmPipelineSchedulerToString{
             {BlockGemmPipelineScheduler::Intrawave, "Intrawave"},
             {BlockGemmPipelineScheduler::Interwave, "Interwave"}};
  
         std::map<BlockGemmPipelineVersion, std::string> BlkGemmPipelineVersionToString{
             {BlockGemmPipelineVersion::v1, "v1"},
             {BlockGemmPipelineVersion::v2, "v2"},
             {BlockGemmPipelineVersion::v3, "v3"}};
  
         // clang-format off
         str << "DeviceMoeGEmm"
             << "<"
             << getGemmSpecializationString(GemmSpec) << ", "
             << std::string(ALayout::name)[0]
             << std::string(BLayout::name)[0]
             << std::string(CLayout::name)[0]
             << ">"
             << " BlkSize: "
             << BlockSize << ", "
             << "BlkTile: "
             << MPerBlock<<"x"<<NPerBlock<<"x"<<KPerBlock << ", "
             << "WaveTile: "
             << MPerXDL<<"x"<<NPerXDL << ", "
             << "WaveMap: "
             << MXdlPerWave<<"x" << NXdlPerWave<<", "
             << "VmemReadVec: "
             << ABlockTransferSrcScalarPerVector<<"x"<<BBlockTransferSrcScalarPerVector<<", "
             << "BlkGemmPipelineScheduler: "
             << BlkGemmPipelineSchedulerToString[BlkGemmPipeSched] << ", "
             << "BlkGemmPipelineVersion: "
             << BlkGemmPipelineVersionToString[BlkGemmPipelineVer] << ", "
             << "BlkGemmPipelinePrefetchStages: "
             << GridwiseGemm64::BlockwiseGemmPipe::PrefetchStages;
         // clang-format on
  
         return str.str();
     }
 };
  
 } // namespace device
 } // namespace tensor_operation
 } // namespace ck