Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions ci/testcases/tensor_wg.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,25 @@ tests:
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE
-DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1

# Descriptors whose base is a K-slice into a bank row (a descriptor advanced
# along K): row-major A and K-major B read from the exact base (issue #454).
- id: sgemm_tcu_wg-a-offset
via: blackbox
drivers:
- simx
- rtlsim
app: sgemm_tcu_wg
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE
-DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_RMAJOR_A -DA_PAD_K=1 -DA_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1
- id: sgemm_tcu_wg-b-offset
via: blackbox
drivers:
- simx
- rtlsim
app: sgemm_tcu_wg
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE
-DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_KMAJOR_B -DB_PAD_K=1 -DB_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1

- id: sgemm_tcu_wg_sp-1
via: blackbox
drivers:
Expand Down
11 changes: 10 additions & 1 deletion hw/rtl/tcu/VX_tcu_abuf.sv
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
// alongside fetch_base_r and slot_step_k_r.
logic slot_row_major_r;
logic [LDM_W-1:0] slot_ldm_words_r;
logic [BANK_ROW_WORDS_LOG2-1:0] slot_desc_a_inrow_r;
logic slot_is_sparse_r;

wire [`UP(K_STEPS_W)-1:0] req_step_k_trunc = `UP(K_STEPS_W)'(req_step_k);
Expand Down Expand Up @@ -177,6 +178,9 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
// per-row word stride (matches WGMMA SS descriptor). Stride=0 keeps
// the canonical block-major layout; non-zero selects the row-major path.
wire [LDM_W-1:0] desc_a_ldm_words = req_desc_a[31:TCU_SMEM_DESC_OFF_BITS];
// The base's 32-bit-word offset inside its bank row; the row-major row
// offsets start from it, as SimX reads A from the exact base.
wire [BANK_ROW_WORDS_LOG2-1:0] desc_a_inrow = BANK_ROW_WORDS_LOG2'(req_desc_a[TCU_SMEM_DESC_OFF_BITS-1:0] >> 2);
if (`VX_CFG_XLEN > 32) begin : g_desc_a_upper_unused
`UNUSED_VAR (req_desc_a[`VX_CFG_XLEN-1:32])
end
Expand All @@ -190,6 +194,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
wire [LDM_W-1:0] effective_ldm_words =
is_first_compute_uop ? desc_a_ldm_words : slot_ldm_words_r;
wire effective_row_major = (effective_ldm_words != '0);
wire [BANK_ROW_WORDS_LOG2-1:0] effective_inrow =
is_first_compute_uop ? desc_a_inrow : slot_desc_a_inrow_r;

// Block-major stripe base (unchanged): one fetch covers M_STEPS A-blocks.
wire [BANK_ADDR_WIDTH-1:0] stripe_base =
Expand Down Expand Up @@ -240,7 +246,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
wire [3:0] row_k_words_alloc = req_is_sparse ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K);

wire [ROW_OFF_W-1:0] row_word_off_init =
ROW_OFF_W'(req_step_k_trunc) * ROW_OFF_W'(row_k_words_alloc);
ROW_OFF_W'(req_step_k_trunc) * ROW_OFF_W'(row_k_words_alloc)
+ ROW_OFF_W'(effective_inrow);

logic [ROW_OFF_W-1:0] row_word_off_req_r;
logic [ROW_OFF_W-1:0] row_word_off_rsp_r;
Expand Down Expand Up @@ -277,6 +284,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
fetch_base_r <= '0;
slot_row_major_r <= 1'b0;
slot_ldm_words_r <= '0;
slot_desc_a_inrow_r <= '0;
slot_is_sparse_r <= 1'b0;
refetched_for_first_compute_r <= 1'b0;
end else begin
Expand All @@ -299,6 +307,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
if (req_valid && is_first_compute_uop) begin
slot_desc_a_row_base_r <= desc_a_row_base;
slot_ldm_words_r <= desc_a_ldm_words;
slot_desc_a_inrow_r <= desc_a_inrow;
slot_row_major_r <= (desc_a_ldm_words != '0);
end

Expand Down
8 changes: 6 additions & 2 deletions hw/rtl/tcu/VX_tcu_bbuf.sv
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,9 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
// desc_b's upper bits encode the per-row word stride (WGMMA
// SS-descriptor `ldm`). Non-zero stride selects the K-major fetch path.
wire [LDM_W-1:0] desc_b_ldm_words = req_desc_b[31:TCU_SMEM_DESC_OFF_BITS];
// The base's 32-bit-word offset inside its bank row; K-major row offsets
// start from it, as SimX reads B from the exact base.
wire [BANK_ROW_WORDS_LOG2-1:0] desc_b_inrow = BANK_ROW_WORDS_LOG2'(req_desc_b[TCU_SMEM_DESC_OFF_BITS-1:0] >> 2);
if (`VX_CFG_XLEN > 32) begin : g_desc_b_upper_unused
`UNUSED_VAR (req_desc_b[`VX_CFG_XLEN-1:32])
end
Expand Down Expand Up @@ -271,7 +274,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
// K-major address generation
// -----------------------------------------------------------------------
// For row r (= km_req_ctr_r) of the current (step_k, step_n) block:
// word_off = (step_n × TCU_TC_N + r) × ldm_words + step_k × k_words
// word_off = inrow + (step_n × TCU_TC_N + r) × ldm_words + step_k × k_words
// bank_row = base + (word_off >> log2(BANK_ROW_WORDS))
// lane = word_off & (BANK_ROW_WORDS - 1)
// k_words is fedpK for dense and tcK for sparse
Expand All @@ -285,7 +288,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(

wire [KM_OFF_W-1:0] km_word_off_init =
KM_OFF_W'(req_step_n) * KM_OFF_W'(TCU_TC_N) * KM_OFF_W'(desc_b_ldm_words)
+ KM_OFF_W'(req_step_k) * KM_OFF_W'(km_k_words_alloc);
+ KM_OFF_W'(req_step_k) * KM_OFF_W'(km_k_words_alloc)
+ KM_OFF_W'(desc_b_inrow);

logic [KM_OFF_W-1:0] km_word_off_req_r;
logic [KM_OFF_W-1:0] km_word_off_rsp_r;
Expand Down
15 changes: 15 additions & 0 deletions tests/regression/sgemm_tcu_wg/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,21 @@
#define KUNROLL 1
#endif

// Row-major A (WGMMA_RMAJOR_A) and K-major B (WGMMA_KMAJOR_B): rows padded by
// *_PAD_K K-slices, each operand starting *_OFF_K K-slices into its row.
#ifndef A_PAD_K
#define A_PAD_K 0
#endif
#ifndef A_OFF_K
#define A_OFF_K 0
#endif
#ifndef B_PAD_K
#define B_PAD_K 0
#endif
#ifndef B_OFF_K
#define B_OFF_K 0
#endif

typedef struct {
uint32_t M, N, K;
uint64_t A_addr;
Expand Down
34 changes: 26 additions & 8 deletions tests/regression/sgemm_tcu_wg/kernel.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ namespace vt = vortex::tensor;

using ctx = vt::wgmma_context<VX_CFG_NUM_THREADS, vt::ITYPE, vt::OTYPE, false, WGMMA_NRC, vt::ldst_matrix_enabled>;

// Row-major A / K-major B rows padded by *_PAD_K K-slices (tileK elements),
// each operand starting *_OFF_K K-slices into its row: a descriptor advanced
// along K, whose base need not be bank-row aligned.
[[maybe_unused]] static constexpr uint32_t lda = ctx::tileK * (1 + A_PAD_K);
[[maybe_unused]] static constexpr uint32_t a_off = ctx::tileK * A_OFF_K;
[[maybe_unused]] static constexpr uint32_t ldb = ctx::tileK * (1 + B_PAD_K);
[[maybe_unused]] static constexpr uint32_t b_off = ctx::tileK * B_OFF_K;
static_assert(A_OFF_K <= A_PAD_K && B_OFF_K <= B_PAD_K, "an offset row must fit its padded stride");

__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) {
auto pA = reinterpret_cast<ctx::input_t *>(arg->A_addr);
auto pB = reinterpret_cast<ctx::input_t *>(arg->B_addr);
Expand All @@ -25,16 +34,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) {
uint32_t tile_row = blockIdx.y * cta_M;
uint32_t tile_col = blockIdx.x * ctx::xtileN;

// Shared memory layout: A [cta_M × tileK] then B [tileK × per_warp_N]
// Shared memory layout: A [cta_M × lda] then B [tileK × per_warp_N]
auto smem = reinterpret_cast<ctx::input_t *>(__local_mem());
auto A_smem = smem;
auto B_smem = smem + cta_M * ctx::tileK;
auto B_smem = smem + cta_M * lda;

ctx::fragment_acc fragC;
ctx::fill_fragment(fragC, 0);

#ifdef WGMMA_RMAJOR_A
constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements
constexpr uint32_t a_ldm = lda; // row-major A: ldm = row stride in elements
#else
constexpr uint32_t a_ldm = 0; // block-major A
#endif
Expand All @@ -58,33 +67,42 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) {
uint32_t r = idx / ctx::tileK;
uint32_t c = idx % ctx::tileK;
#ifdef WGMMA_RMAJOR_A
A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)];
A_smem[r * lda + a_off + c] = pA[(tile_row + r) * K + (k + c)];
#else
A_smem[ctx::a_blockmajor_idx(r, c)] = pA[(tile_row + r) * K + (k + c)];
#endif
}

// Cooperatively load B into smem (block-major):
// B_smem[(k_blk*N_STEPS + n_blk)*BW + n_in*tcK + k_in]
// Cooperatively load B into smem:
// Default (block-major): B_smem[(k_blk*N_STEPS + n_blk)*BW + n_in*tcK + k_in]
// WGMMA_KMAJOR_B: B_smem[c * ldb + b_off + r] — N-outer, K-inner.
uint32_t b_size = ctx::tileK * ctx::xtileN;
for (uint32_t i = 0; i < b_size; i += num_threads) {
uint32_t idx = i + tid;
uint32_t r = idx / ctx::xtileN;
uint32_t c = idx % ctx::xtileN;
#ifdef WGMMA_KMAJOR_B
B_smem[c * ldb + b_off + r] = pB[(k + r) * N + (tile_col + c)];
#else
B_smem[ctx::b_blockmajor_idx(r, c)] = pB[(k + r) * N + (tile_col + c)];
#endif
}

__syncthreads();

// Each warp's A slice — base differs by layout:
// block-major: warp_rank * a_warp_elems (per a_blockmajor_idx packing).
// row-major: warp_rank * xtileM * tileK (per kernel's m*tileK + k).
// row-major: warp_rank * xtileM * lda + a_off (per kernel's m*lda + k).
#ifdef WGMMA_RMAJOR_A
auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK;
auto A_warp = A_smem + warp_rank * ctx::xtileM * lda + a_off;
#else
auto A_warp = A_smem + warp_rank * ctx::a_warp_elems;
#endif
#ifdef WGMMA_KMAJOR_B
auto desc_b = vt::vx_make_smem_desc(B_smem + b_off, ldb * sizeof(ctx::input_t));
#else
auto desc_b = vt::vx_make_smem_desc(B_smem, 0); // block-major: stride field unused
#endif

#if defined(WGMMA_RS) && (WGMMA_NRC <= 16)
// RS: A from registers, B from smem (NRC <= 16 only).
Expand Down
9 changes: 8 additions & 1 deletion tests/regression/sgemm_tcu_wg/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,14 @@ int main(int argc, char *argv[]) {
auto time_start = std::chrono::high_resolution_clock::now();

std::cout << "start device" << std::endl;
uint32_t smem_size = (cta_M * wg_cfg::tileK + wg_cfg::tileK * per_warp_N) * sizeof(itype_t);
// A rows (and K-major B rows) are padded by *_PAD_K K-slices; see common.h.
uint32_t a_smem = cta_M * wg_cfg::tileK * (1 + A_PAD_K);
#ifdef WGMMA_KMAJOR_B
uint32_t b_smem = per_warp_N * wg_cfg::tileK * (1 + B_PAD_K);
#else
uint32_t b_smem = wg_cfg::tileK * per_warp_N;
#endif
uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t);
vx_event_h launch_ev = nullptr;
{
vx_launch_info_t li = {};
Expand Down
Loading