diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index fb30e02ab2..4548bc9702 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -90,6 +90,25 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +# Descriptors whose base is a K-slice into a bank row (a descriptor advanced +# along K): row-major A and K-major B read from the exact base (issue #454). +- id: sgemm_tcu_wg-a-offset + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_RMAJOR_A -DA_PAD_K=1 -DA_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +- id: sgemm_tcu_wg-b-offset + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_KMAJOR_B -DB_PAD_K=1 -DB_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 + - id: sgemm_tcu_wg_sp-1 via: blackbox drivers: diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index ac9c211604..8d2c53496c 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -129,6 +129,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // alongside fetch_base_r and slot_step_k_r. logic slot_row_major_r; logic [LDM_W-1:0] slot_ldm_words_r; + logic [BANK_ROW_WORDS_LOG2-1:0] slot_desc_a_inrow_r; logic slot_is_sparse_r; wire [`UP(K_STEPS_W)-1:0] req_step_k_trunc = `UP(K_STEPS_W)'(req_step_k); @@ -177,6 +178,9 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // per-row word stride (matches WGMMA SS descriptor). Stride=0 keeps // the canonical block-major layout; non-zero selects the row-major path. wire [LDM_W-1:0] desc_a_ldm_words = req_desc_a[31:TCU_SMEM_DESC_OFF_BITS]; + // The base's 32-bit-word offset inside its bank row; the row-major row + // offsets start from it, as SimX reads A from the exact base. + wire [BANK_ROW_WORDS_LOG2-1:0] desc_a_inrow = BANK_ROW_WORDS_LOG2'(req_desc_a[TCU_SMEM_DESC_OFF_BITS-1:0] >> 2); if (`VX_CFG_XLEN > 32) begin : g_desc_a_upper_unused `UNUSED_VAR (req_desc_a[`VX_CFG_XLEN-1:32]) end @@ -190,6 +194,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [LDM_W-1:0] effective_ldm_words = is_first_compute_uop ? desc_a_ldm_words : slot_ldm_words_r; wire effective_row_major = (effective_ldm_words != '0); + wire [BANK_ROW_WORDS_LOG2-1:0] effective_inrow = + is_first_compute_uop ? desc_a_inrow : slot_desc_a_inrow_r; // Block-major stripe base (unchanged): one fetch covers M_STEPS A-blocks. wire [BANK_ADDR_WIDTH-1:0] stripe_base = @@ -240,7 +246,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [3:0] row_k_words_alloc = req_is_sparse ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [ROW_OFF_W-1:0] row_word_off_init = - ROW_OFF_W'(req_step_k_trunc) * ROW_OFF_W'(row_k_words_alloc); + ROW_OFF_W'(req_step_k_trunc) * ROW_OFF_W'(row_k_words_alloc) + + ROW_OFF_W'(effective_inrow); logic [ROW_OFF_W-1:0] row_word_off_req_r; logic [ROW_OFF_W-1:0] row_word_off_rsp_r; @@ -277,6 +284,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( fetch_base_r <= '0; slot_row_major_r <= 1'b0; slot_ldm_words_r <= '0; + slot_desc_a_inrow_r <= '0; slot_is_sparse_r <= 1'b0; refetched_for_first_compute_r <= 1'b0; end else begin @@ -299,6 +307,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( if (req_valid && is_first_compute_uop) begin slot_desc_a_row_base_r <= desc_a_row_base; slot_ldm_words_r <= desc_a_ldm_words; + slot_desc_a_inrow_r <= desc_a_inrow; slot_row_major_r <= (desc_a_ldm_words != '0); end diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 2b7bdb38cb..8071304bf5 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -173,6 +173,9 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // desc_b's upper bits encode the per-row word stride (WGMMA // SS-descriptor `ldm`). Non-zero stride selects the K-major fetch path. wire [LDM_W-1:0] desc_b_ldm_words = req_desc_b[31:TCU_SMEM_DESC_OFF_BITS]; + // The base's 32-bit-word offset inside its bank row; K-major row offsets + // start from it, as SimX reads B from the exact base. + wire [BANK_ROW_WORDS_LOG2-1:0] desc_b_inrow = BANK_ROW_WORDS_LOG2'(req_desc_b[TCU_SMEM_DESC_OFF_BITS-1:0] >> 2); if (`VX_CFG_XLEN > 32) begin : g_desc_b_upper_unused `UNUSED_VAR (req_desc_b[`VX_CFG_XLEN-1:32]) end @@ -271,7 +274,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // K-major address generation // ----------------------------------------------------------------------- // For row r (= km_req_ctr_r) of the current (step_k, step_n) block: - // word_off = (step_n × TCU_TC_N + r) × ldm_words + step_k × k_words + // word_off = inrow + (step_n × TCU_TC_N + r) × ldm_words + step_k × k_words // bank_row = base + (word_off >> log2(BANK_ROW_WORDS)) // lane = word_off & (BANK_ROW_WORDS - 1) // k_words is fedpK for dense and tcK for sparse @@ -285,7 +288,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [KM_OFF_W-1:0] km_word_off_init = KM_OFF_W'(req_step_n) * KM_OFF_W'(TCU_TC_N) * KM_OFF_W'(desc_b_ldm_words) - + KM_OFF_W'(req_step_k) * KM_OFF_W'(km_k_words_alloc); + + KM_OFF_W'(req_step_k) * KM_OFF_W'(km_k_words_alloc) + + KM_OFF_W'(desc_b_inrow); logic [KM_OFF_W-1:0] km_word_off_req_r; logic [KM_OFF_W-1:0] km_word_off_rsp_r; diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 5a61e3992b..73e6c88a44 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -20,6 +20,21 @@ #define KUNROLL 1 #endif +// Row-major A (WGMMA_RMAJOR_A) and K-major B (WGMMA_KMAJOR_B): rows padded by +// *_PAD_K K-slices, each operand starting *_OFF_K K-slices into its row. +#ifndef A_PAD_K +#define A_PAD_K 0 +#endif +#ifndef A_OFF_K +#define A_OFF_K 0 +#endif +#ifndef B_PAD_K +#define B_PAD_K 0 +#endif +#ifndef B_OFF_K +#define B_OFF_K 0 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index cbff902568..9f8aa6998d 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -7,6 +7,15 @@ namespace vt = vortex::tensor; using ctx = vt::wgmma_context; +// Row-major A / K-major B rows padded by *_PAD_K K-slices (tileK elements), +// each operand starting *_OFF_K K-slices into its row: a descriptor advanced +// along K, whose base need not be bank-row aligned. +[[maybe_unused]] static constexpr uint32_t lda = ctx::tileK * (1 + A_PAD_K); +[[maybe_unused]] static constexpr uint32_t a_off = ctx::tileK * A_OFF_K; +[[maybe_unused]] static constexpr uint32_t ldb = ctx::tileK * (1 + B_PAD_K); +[[maybe_unused]] static constexpr uint32_t b_off = ctx::tileK * B_OFF_K; +static_assert(A_OFF_K <= A_PAD_K && B_OFF_K <= B_PAD_K, "an offset row must fit its padded stride"); + __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); auto pB = reinterpret_cast(arg->B_addr); @@ -25,16 +34,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t tile_row = blockIdx.y * cta_M; uint32_t tile_col = blockIdx.x * ctx::xtileN; - // Shared memory layout: A [cta_M × tileK] then B [tileK × per_warp_N] + // Shared memory layout: A [cta_M × lda] then B [tileK × per_warp_N] auto smem = reinterpret_cast(__local_mem()); auto A_smem = smem; - auto B_smem = smem + cta_M * ctx::tileK; + auto B_smem = smem + cta_M * lda; ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); #ifdef WGMMA_RMAJOR_A - constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements + constexpr uint32_t a_ldm = lda; // row-major A: ldm = row stride in elements #else constexpr uint32_t a_ldm = 0; // block-major A #endif @@ -58,33 +67,42 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t r = idx / ctx::tileK; uint32_t c = idx % ctx::tileK; #ifdef WGMMA_RMAJOR_A - A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; + A_smem[r * lda + a_off + c] = pA[(tile_row + r) * K + (k + c)]; #else A_smem[ctx::a_blockmajor_idx(r, c)] = pA[(tile_row + r) * K + (k + c)]; #endif } - // Cooperatively load B into smem (block-major): - // B_smem[(k_blk*N_STEPS + n_blk)*BW + n_in*tcK + k_in] + // Cooperatively load B into smem: + // Default (block-major): B_smem[(k_blk*N_STEPS + n_blk)*BW + n_in*tcK + k_in] + // WGMMA_KMAJOR_B: B_smem[c * ldb + b_off + r] — N-outer, K-inner. uint32_t b_size = ctx::tileK * ctx::xtileN; for (uint32_t i = 0; i < b_size; i += num_threads) { uint32_t idx = i + tid; uint32_t r = idx / ctx::xtileN; uint32_t c = idx % ctx::xtileN; + #ifdef WGMMA_KMAJOR_B + B_smem[c * ldb + b_off + r] = pB[(k + r) * N + (tile_col + c)]; + #else B_smem[ctx::b_blockmajor_idx(r, c)] = pB[(k + r) * N + (tile_col + c)]; + #endif } __syncthreads(); // Each warp's A slice — base differs by layout: // block-major: warp_rank * a_warp_elems (per a_blockmajor_idx packing). - // row-major: warp_rank * xtileM * tileK (per kernel's m*tileK + k). + // row-major: warp_rank * xtileM * lda + a_off (per kernel's m*lda + k). #ifdef WGMMA_RMAJOR_A - auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; + auto A_warp = A_smem + warp_rank * ctx::xtileM * lda + a_off; #else auto A_warp = A_smem + warp_rank * ctx::a_warp_elems; #endif + #ifdef WGMMA_KMAJOR_B + auto desc_b = vt::vx_make_smem_desc(B_smem + b_off, ldb * sizeof(ctx::input_t)); + #else auto desc_b = vt::vx_make_smem_desc(B_smem, 0); // block-major: stride field unused + #endif #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only). diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 9fe0860e2b..77d094ee23 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -708,7 +708,14 @@ int main(int argc, char *argv[]) { auto time_start = std::chrono::high_resolution_clock::now(); std::cout << "start device" << std::endl; - uint32_t smem_size = (cta_M * wg_cfg::tileK + wg_cfg::tileK * per_warp_N) * sizeof(itype_t); + // A rows (and K-major B rows) are padded by *_PAD_K K-slices; see common.h. + uint32_t a_smem = cta_M * wg_cfg::tileK * (1 + A_PAD_K); +#ifdef WGMMA_KMAJOR_B + uint32_t b_smem = per_warp_N * wg_cfg::tileK * (1 + B_PAD_K); +#else + uint32_t b_smem = wg_cfg::tileK * per_warp_N; +#endif + uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t); vx_event_h launch_ev = nullptr; { vx_launch_info_t li = {};