From 9ea22ba1e1f0e9e055951ffc7bf2d74c8522659b Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 19 Nov 2026 03:37:46 -0800 Subject: [PATCH 1/3] tcu: define and enforce the WGMMA shared-memory descriptor alignment contract #475 made row-major A and K-major B honour a base inside a bank row, but the tile buffers still mishandle some descriptors silently, while SimX reads the exact byte address, so the two disagreed with no error: - a row-major/K-major row is fetched one LMEM access per row, and words of its K-slice past the bank row are dropped (abuf/bbuf write guards), so a base or stride that is not a multiple of the slice reads stale data; - the in-row offset is the byte offset >> 2, so a 2-byte (fp16/bf16) base is truncated; - a block-major operand (stride 0) uses only the bank-row address, so an unaligned base is rounded down. The rule, defined once in tensor_cfg.h (smem_desc_check) and shared by SimX and the kernel library: for a strided operand the base is a multiple of 4*slice bytes and the stride a multiple of slice words, where slice is fedpK words dense, TC_K (A) or 2*TC_K (B) sparse; a block-major base is LMEM bank-row aligned. The slice divides a bank row in every legal configuration (static_assert in SimX). The rule is slightly stricter than the RTL needs in some geometries, so that legality does not depend on bank geometry; any multiple of one tile row satisfies it everywhere. SimX now aborts with a message naming the rule (both descriptor decode paths share decode_descs), wgmma_sync asserts both descriptors in debug kernel builds, vx_make_smem_desc asserts the documented 4-byte stride multiple, and the WGMMA design doc gains the alignment table. sgemm_tcu_wg gains an A_OFF_ELEMS knob (default 0, no codegen change): a 2-byte and a 4-byte misaligned fp16 base now abort in SimX, a one-slice offset passes. The legal CI cases still pass (a-offset, b-offset, sgemm_tcu_wg-2, flash_attn_tcu_wg-fp16, sgemm_tcu_wg_sp-fedp2k-ss, sgemm_tcu_wg_dxa-1, SimX XLEN 32). The RTL gains simulation-only asserts for the block-major part of the rule in the A and B tile buffers, whose fetch drops the in-row offset, so a kernel that breaks it fails loudly on rtlsim as it does on SimX instead of reading the wrong rows. No synthesized logic changes. The rtlsim cases sgemm_tcu_wg-9, -b-offset, sgemm_tcu_wg_sp-8, sgemm_tcu_wg_dxa-lmem128k and sgemm_tcu_wg_dxa_mcast-pad pass with them. Fixes #454. Co-Authored-By: Claude Opus 5.5 --- docs/designs/tensor_core_wgmma_engine.md | 20 ++++++++++- hw/rtl/tcu/VX_tcu_abuf.sv | 7 ++++ hw/rtl/tcu/VX_tcu_bbuf.sv | 7 ++++ sim/simx/tcu/tcu_unit.cpp | 44 +++++++++++++++++++----- sw/common/tensor_cfg.h | 35 +++++++++++++++++++ sw/kernel/include/vx_tensor.h | 21 +++++++++++ tests/regression/sgemm_tcu_wg/common.h | 5 +++ tests/regression/sgemm_tcu_wg/kernel.cpp | 4 +-- 8 files changed, 132 insertions(+), 11 deletions(-) diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index 2a83ba4b64..0bae3ef6c4 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -315,6 +315,24 @@ stride folds into the packing, so building a descriptor costs one OR. Row-major is the layout of a DXA-loaded slab. The buffers' output muxes are layout-agnostic; only the fetch differs. +**Alignment.** Each LMEM access returns one bank row +(`NUM_THREADS × XLEN/8` bytes), and the buffers take a row's words from that +one access only, so a descriptor must keep every fetch inside a bank row: + +| layout | rule | +|---|---| +| block-major | the offset is a multiple of a bank row | +| row-major | the offset (in bytes) and the stride (in words) are multiples of the operand's **K-slice**: `fedpK` words, or for sparse `tcK` words of A and `2·tcK` of B | + +The K-slice is the part of a row one micro-op consumes. It divides a bank +row in every configuration, so an aligned slice never straddles two. One +tile row (`tileK` elements, `tileK/2` for sparse A) is a multiple of the +K-slice everywhere, so a descriptor stepped along K by `tileK` stays legal. +`smem_desc_check` in [`tensor_cfg.h`](../../sw/common/tensor_cfg.h) states the +rule once: SimX aborts on a descriptor that breaks it rather than read the +exact address the hardware cannot, and `wgmma_sync` asserts it in debug +kernel builds. + ### 6.2 A buffer — `VX_tcu_abuf` One per block. It holds one **k-stripe**: the `M_STEPS` A blocks of the @@ -525,7 +543,7 @@ constant on both sides. | API | purpose | |---|---| | `wmma_context`, `wgmma_context` | compile-time tile geometry and the instruction emitters for a given pair of formats | -| `vx_make_smem_desc(ptr, leading_bytes)` | build a shared-memory descriptor; a zero stride selects block-major | +| `vx_make_smem_desc(ptr, leading_bytes)` | build a shared-memory descriptor; a zero stride selects block-major; `ptr` and `leading_bytes` follow the alignment rule of §6.1 | | `load_sp_metadata`, `load_mx_metadata` | emit the `TCU_LD` instructions for a fragment | | `a_blockmajor_idx`, `b_blockmajor_idx`, `a_sp_blockmajor_idx` | where element `(r, c)` lives in a block-major slab | diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index 8d2c53496c..8eed2496f5 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -502,6 +502,13 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end `endif + // A block-major A operand must start an LMEM bank row: the fetch drops the + // offset inside the row. + `RUNTIME_ASSERT (~(req_valid && req_a_is_smem && is_first_compute_uop && (desc_a_ldm_words == '0) + && (req_desc_a[BANK_ROW_WORDS_LOG2+1:0] != '0)), + ("%t: *** %s: block-major A base 0x%0h is not bank-row aligned", $time, INSTANCE_ID, + req_desc_a[TCU_SMEM_DESC_OFF_BITS-1:0])) + endmodule `endif // VX_CFG_TCU_WGMMA_ENABLE diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 8071304bf5..9b43879ca6 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -707,6 +707,13 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end `endif + // A block-major B operand must start an LMEM bank row: the fetch drops the + // offset inside the row. + `RUNTIME_ASSERT (~(req_valid && ~req_wants_kmajor + && (req_desc_b[BANK_ROW_WORDS_LOG2+1:0] != '0)), + ("%t: *** %s: block-major B base 0x%0h is not bank-row aligned", $time, INSTANCE_ID, + req_desc_b[TCU_SMEM_DESC_OFF_BITS-1:0])) + endmodule `endif // VX_CFG_TCU_WGMMA_ENABLE diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index be70301739..4d1bb8e359 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -64,7 +64,12 @@ static constexpr uint64_t kLineMask = ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); // WGMMA tile-buffer geometry in 32-bit words. static constexpr uint32_t kLmemBanks = VX_CFG_LMEM_NUM_BANKS; static constexpr uint32_t kBankRowWords = kLmemBanks * (VX_CFG_XLEN / 32); -static constexpr uint32_t kBBlockWords = kFedpWords * cfg::tcN; +#ifdef VX_CFG_TCU_WGMMA_ENABLE +// A slice-aligned row-major K-slice then never straddles two bank rows. +static_assert((kBankRowWords % kFedpWords) == 0 && (kBankRowWords % (2 * cfg::tcK)) == 0, + "a WGMMA K-slice must divide an LMEM bank row"); +#endif +static constexpr uint32_t kBBlockWords = kFedpWords * cfg::tcN; static constexpr uint32_t kBBlockWordsSp = cfg::tcK * cfg::tcN * 2; // One bank row holds kBSubBlocks consecutive dense B blocks. static constexpr uint32_t kBSubBlocks = @@ -856,7 +861,7 @@ class TcuUnit::Impl { wgmma_desc_[wid][0] = trace->src_data.at(0).at(0).u32; } wgmma_desc_[wid][1] = trace->src_data.at(1).at(0).u32; - this->decode_descs(wid, tpuArgs.fmt_s, tpuArgs.is_a_smem); + this->decode_descs(wid, tpuArgs.fmt_s, tpuArgs.is_a_smem, is_sparse); } auto& req = reqs.at(b); @@ -970,14 +975,41 @@ class TcuUnit::Impl { return ready_mask; } + // The tile buffers cannot fetch a descriptor that breaks the alignment rule, + // so SimX must not read it from its exact address either. + static void check_desc(uint32_t desc, bool is_b, bool is_sparse) { + uint32_t slice_words = vt::wgmma_slice_words(is_b, is_sparse, cfg::tcK, kFedpWords); + auto fault = vt::smem_desc_check(desc, slice_words, kBankRowWords * 4); + if (fault == vt::smem_desc_fault::none) { + return; + } + std::cerr << "*** WGMMA " << (is_b ? "B" : "A") << " descriptor 0x" << std::hex << desc << std::dec + << " (offset " << vt::smem_desc_offset(desc) << " B, stride " << vt::smem_desc_stride(desc) << " B): "; + switch (fault) { + case vt::smem_desc_fault::block_base: + std::cerr << "a block-major base must be aligned to an LMEM bank row (" << (kBankRowWords * 4) << " B)"; + break; + case vt::smem_desc_fault::row_base: + std::cerr << "a row-major base must be a multiple of the " << (slice_words * 4) << "-byte K-slice"; + break; + default: + std::cerr << "a row-major stride must be a multiple of the " << (slice_words * 4) << "-byte K-slice"; + break; + } + std::cerr << std::endl; + std::abort(); + } + // Decode a warp's WGMMA descriptors into LMEM base and row stride. - void decode_descs(uint32_t wid, uint32_t fmt_s, bool is_a_smem) { + void decode_descs(uint32_t wid, uint32_t fmt_s, bool is_a_smem, bool is_sparse) { uint32_t e_bits = elem_bits(fmt_s); uint32_t a_desc = wgmma_desc_[wid][0]; uint32_t b_desc = wgmma_desc_[wid][1]; if (is_a_smem) { + check_desc(a_desc, false, is_sparse); lmem_desc_[wid][0] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(a_desc), vt::smem_desc_stride(a_desc) * 8 / e_bits, false}; } + check_desc(b_desc, true, is_sparse); lmem_desc_[wid][1] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(b_desc), vt::smem_desc_stride(b_desc) * 8 / e_bits, false}; } @@ -1199,16 +1231,12 @@ class TcuUnit::Impl { uint32_t ratio = elem_ratio(fmt_s); uint32_t k_words = is_sparse ? cfg::tcK : kFedpWords; - uint32_t e_bits = elem_bits(fmt_s); // Decode smem descriptors (B always from smem, A optionally). if (is_setup_uop) { wgmma_desc_[wid][0] = a_desc; wgmma_desc_[wid][1] = b_desc; - if (is_a_smem) { - lmem_desc_[wid][0] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(a_desc), vt::smem_desc_stride(a_desc) * 8 / e_bits, false}; - } - lmem_desc_[wid][1] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(b_desc), vt::smem_desc_stride(b_desc) * 8 / e_bits, false}; + this->decode_descs(wid, fmt_s, is_a_smem, is_sparse); uint32_t nrc = (cd_nregs == 0) ? 8 : (cd_nregs == 1) ? 16 : 32; cur_xtile_n_ = (nrc * VX_CFG_NUM_THREADS) / wg_cfg::xtileM; return; diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 79bf70a181..06d8576cc5 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -174,6 +174,41 @@ inline constexpr uint32_t smem_desc_stride(uint32_t desc) { return (desc >> smem_desc_offset_bits) << smem_desc_stride_lg; } +// 32-bit words of its K dimension one row of a row-major operand supplies to +// a WGMMA micro-op: a sparse A row is compressed, a sparse B row carries both +// candidates of each K step. +inline constexpr uint32_t wgmma_slice_words(bool is_b, bool is_sparse, + uint32_t tc_k, uint32_t fedp_k) { + return is_sparse ? (is_b ? 2 * tc_k : tc_k) : fedp_k; +} + +// What the TCU tile buffers can fetch. A row-major operand (stride != 0) is +// read one row per LMEM access, so each row's K-slice must lie in one bank +// row: the base and the stride must be multiples of the slice. A block-major +// operand (stride == 0) is read in whole bank rows, so its base must start one. +enum class smem_desc_fault { + none, + block_base, // block-major base not aligned to an LMEM bank row + row_base, // row-major base not a multiple of the K-slice + row_stride // row-major stride not a multiple of the K-slice +}; + +inline constexpr smem_desc_fault smem_desc_check(uint32_t desc, uint32_t slice_words, + uint32_t bank_row_bytes) { + uint32_t offset = smem_desc_offset(desc); + uint32_t stride_words = smem_desc_stride(desc) >> smem_desc_stride_lg; + if (stride_words == 0) { + return (offset % bank_row_bytes) ? smem_desc_fault::block_base : smem_desc_fault::none; + } + if (offset % (slice_words * 4)) { + return smem_desc_fault::row_base; + } + if (stride_words % slice_words) { + return smem_desc_fault::row_stride; + } + return smem_desc_fault::none; +} + // The TCU accumulates and returns fp32 (int32 for integer inputs); it has no // other output format. inline constexpr bool tcu_output_valid(uint32_t it, uint32_t ot) { diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 52092ef52d..a5fd89466c 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -30,7 +30,15 @@ struct smem_matrix_desc { }; // Build a smem descriptor from a pointer and a row stride in bytes, a multiple of 4. +// The TCU fetches only aligned operands (smem_desc_check), checked by wgmma_sync: +// row-major (leading_bytes != 0): ptr and leading_bytes are multiples of the +// operand's K-slice, fedpK words (sparse: tcK words for A, 2*tcK for B). +// A multiple of one tile row (tileK elements, tileK/2 for sparse A) holds +// in every configuration, so stepping a descriptor along K by tileK does. +// block-major (leading_bytes == 0): ptr starts an LMEM bank row, +// NUM_THREADS * XLEN/8 bytes. static inline __attribute__((always_inline)) smem_matrix_desc vx_make_smem_desc(const void* ptr, uint32_t leading_bytes) { + assert((leading_bytes % 4) == 0 && "smem descriptor stride must be a multiple of 4 bytes"); size_t lmem_base = csr_read_nv(VX_CSR_LOCAL_MEM_BASE); uint32_t offset = static_cast(static_cast(reinterpret_cast(ptr)) - lmem_base); return {smem_desc_pack(offset, leading_bytes)}; @@ -910,6 +918,14 @@ struct wgmma_context { | ((a_is_smem ? 1 : 0) << 3); } + // The TCU requires one LMEM bank per thread. + static constexpr uint32_t smem_bank_row_bytes = NT * (__riscv_xlen / 8); + + static bool smem_desc_valid(smem_matrix_desc desc, bool is_b) { + uint32_t slice_words = wgmma_slice_words(is_b, is_sparse, tcK, fedpK); + return smem_desc_check(desc.value, slice_words, smem_bank_row_bytes) == smem_desc_fault::none; + } + public: // Types using input_t = typename ctx_c::input_t; @@ -1223,6 +1239,11 @@ struct wgmma_context { } static_assert(b_is_smem, "B must be smem_matrix_desc (SR mode is not supported)"); + if constexpr (a_is_smem) { + assert(smem_desc_valid(op_a, false) && "WGMMA A descriptor breaks the TCU alignment rule (vx_make_smem_desc)"); + } + assert(smem_desc_valid(op_b, true) && "WGMMA B descriptor breaks the TCU alignment rule (vx_make_smem_desc)"); + constexpr int flags = wgmma_flags(); // --- SS path: both from smem --- diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 73e6c88a44..2fbf8a5fd2 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -28,6 +28,11 @@ #ifndef A_OFF_K #define A_OFF_K 0 #endif +// Extra A offset in elements: one that is not a multiple of the K-slice builds +// a descriptor the TCU cannot fetch, which SimX must reject. +#ifndef A_OFF_ELEMS +#define A_OFF_ELEMS 0 +#endif #ifndef B_PAD_K #define B_PAD_K 0 #endif diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index 9f8aa6998d..b138c8376d 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -11,10 +11,10 @@ using ctx = vt::wgmma_context(arg->A_addr); From 40a753b3ea42ba446169e1737196eab191045e36 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 19 Nov 2026 03:37:46 -0800 Subject: [PATCH 2/3] cta: align each CTA's LMEM slot to a bank row when an LMEM DMA agent is present The KMU and the SimX dispatcher pitch the per-CTA LMEM slots at the CTA's LMEM size rounded up to MEM_BLOCK_SIZE (64 B), so a CTA's __local_mem() base was only 64 B aligned. The LMEM DMA agents move whole bank rows (LMEM_NUM_BANKS x XLEN/8 bytes): a block-major WGMMA operand must start a bank row, and DXA multicast steps receivers by whole rows. Where a bank row is wider than 64 B (more than 16 banks at XLEN 32, 8 at XLEN 64), the second co-resident CTA's operands started mid-row: the RTL rounded them down and computed wrong results, and SimX now rejects the descriptor. The slot alignment is now max(MEM_BLOCK_SIZE, bank row) when DXA or WGMMA is enabled (LMEM_DMA_EN), and MEM_BLOCK_SIZE otherwise, so every default and narrow configuration keeps its pitch. One constant per side: RTL LMEM_SLOT_ALIGN in VX_gpu_pkg (used by the KMU rounding), and sw/common/lmem_slot.h for the SimX dispatcher, the SimX DXA multicast stride and the runtime occupancy estimate. Derived from existing config defines, so the resolved config and the perf baselines are unchanged. sgemm_tcu_wg gains an SMEM_PAD_BYTES knob (default 0) and a CI case on SimX and rtlsim, sgemm_tcu_wg-lmem-pad: 32 threads, two CTAs per core, 64 B pad. It aborted on the descriptor check before and passes now; on rtlsim it passes with the same instruction count, and with the KMU rounding reverted to MEM_BLOCK_SIZE it fails the tile buffer's bank-row assert. sgemm_tcu_wg-2/-10, sgemm_tcu_wg_dxa_mcast-1/-3 and cta_cluster-1 still pass on SimX. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/tensor_wg.yaml | 16 +++++++++ docs/designs/cta_dispatch_architecture.md | 15 ++++++++- hw/rtl/VX_gpu_pkg.sv | 6 ++++ hw/rtl/VX_kmu.sv | 7 ++-- hw/rtl/core/VX_cta_dispatch.sv | 7 ++-- hw/rtl/dxa/VX_dxa_smem_wr.sv | 6 ++-- sim/simx/cta_dispatcher.cpp | 16 ++++----- sim/simx/cta_dispatcher.h | 2 +- sim/simx/dxa/dxa_core.cpp | 5 +-- sw/common/lmem_slot.h | 41 +++++++++++++++++++++++ sw/runtime/common/legacy_utils.cpp | 8 ++--- tests/regression/sgemm_tcu_wg/common.h | 5 +++ tests/regression/sgemm_tcu_wg/main.cpp | 2 +- 13 files changed, 110 insertions(+), 26 deletions(-) create mode 100644 sw/common/lmem_slot.h diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index d3a13b5657..b427a2c0e4 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -117,6 +117,22 @@ tests: app: sgemm_tcu_wg configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_KMAJOR_B -DB_PAD_K=1 -DB_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +# Two co-resident CTAs whose LMEM footprint is not a bank-row multiple (128 B +# at 32 threads): the second CTA's block-major operands must still start a bank +# row, so the dispatcher's slot pitch keeps the bank-row alignment. +- id: sgemm_tcu_wg-lmem-pad + via: blackbox + drivers: + - simx + - rtlsim + xlen: + - 32 + touches: [hw/rtl/VX_kmu.sv, hw/rtl/VX_gpu_pkg.sv, sim/simx/cta_dispatcher.cpp, sw/common/lmem_slot.h, + sim/simx/tcu, sw/kernel/include/vx_tensor.h] + app: sgemm_tcu_wg + args: -m 128 -n 128 -k 32 + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DSMEM_PAD_BYTES=64 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-1 via: blackbox diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 698dd2c116..7af2097ebf 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -184,7 +184,7 @@ envelope plus a `kind`-discriminated union: | `PC`, `entry` | program image PC and kernel entry PC | | `param` | kargs pointer | | `ctx_id` | 8-bit launch identity, bumped every `start` | -| `aligned_lmem_size` | per-CTA LMEM footprint, rounded to `MEM_BLOCK_SIZE` | +| `aligned_lmem_size` | per-CTA LMEM footprint, rounded to `LMEM_SLOT_ALIGN` | `args.compute` ([`:735-744`](../../hw/rtl/VX_gpu_pkg.sv#L735)) carries `grid_dim[3]`, `block_idx[3]`, `block_dim[3]`, `block_size`, `warp_step[3]`, @@ -376,6 +376,19 @@ receiver-side translation. The stride must be `MEM_BLOCK_SIZE`-aligned (the KMU rounds it) because LMEM is block-addressed; a ragged stride would target the wrong block. +When the core has an LMEM DMA agent (`VX_CFG_EXT_DXA_ENABLE` or +`VX_CFG_TCU_WGMMA_ENABLE`) the KMU rounds the stride to a whole LMEM bank row +(`LMEM_NUM_BANKS × XLEN/8` bytes) instead, when that is larger: the agents +move whole bank rows, so DXA multicast steps receivers by whole rows and a +block-major WGMMA operand at `__local_mem()` must start one +([tensor_core_wgmma_engine.md §6.1](tensor_core_wgmma_engine.md)). This is +`LMEM_SLOT_ALIGN` in `VX_gpu_pkg`, mirrored for SimX and the runtime by +[`lmem_slot.h`](../../sw/common/lmem_slot.h). The bank row exceeds +`MEM_BLOCK_SIZE` only when `LMEM_NUM_BANKS` x XLEN/8 does: with the default +64-byte block, more than 16 LMEM banks at XLEN 32 or more than 8 at XLEN 64 +(one bank per thread by default), so narrower cores keep the `MEM_BLOCK_SIZE` +pitch. + `get_cluster_rank()` is derived, never sent: `CTA_ID % CTA_CLUSTER_SIZE` ([`vx_spawn2.h:133`](../../sw/kernel/include/vx_spawn2.h#L133)). diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 86b3976a16..afb37c1a4e 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -2013,6 +2013,12 @@ package VX_gpu_pkg; localparam LMEM_DMA_IN_TAG_MAX = `MAX(DXA_LMEM_OUT_TAG_W, TCU_LMEM_TAG_W); localparam LMEM_DMA_TAG_WIDTH = LMEM_DMA_IN_TAG_MAX + `ARB_SEL_BITS(LMEM_DMA_INPUTS, 1); + // Alignment of every CTA's LMEM slot, and so of its LMEM base. A DMA agent + // moves whole bank rows: a block-major WGMMA operand must start one, and + // DXA multicast steps receivers by whole rows. + localparam LMEM_SLOT_ALIGN = LMEM_DMA_EN ? `MAX(`VX_CFG_MEM_BLOCK_SIZE, LMEM_DMA_DATA_SIZE) + : `VX_CFG_MEM_BLOCK_SIZE; + ////////////////////////// Icache Parameters ////////////////////////////// // Word size in bytes diff --git a/hw/rtl/VX_kmu.sv b/hw/rtl/VX_kmu.sv index 5d12aafe7c..cae8f136fe 100644 --- a/hw/rtl/VX_kmu.sv +++ b/hw/rtl/VX_kmu.sv @@ -84,14 +84,15 @@ module VX_kmu import VX_gpu_pkg::*; import VX_trace_pkg::*; #( // cluster_size : K = cluster_dim product, bounded by NUM_WARPS (cluster // members are co-resident on one core), so NW_WIDTH+1 bits // suffice; each dim is sliced to that width. - // aligned_lmem_size : per-CTA LMEM footprint rounded up to MEM_BLOCK_SIZE. + // aligned_lmem_size : per-CTA LMEM footprint rounded up to LMEM_SLOT_ALIGN. // lmem_slots : the CTA slots that footprint fits (lmem_slots_fit). // Registered so the broadcast path is flop-driven. All three settle one // cycle after their DCR write, the same cycle, and DCRs are written before // `start`, so all are settled before the first CTA is dispatched. + `STATIC_ASSERT(`IS_POW2(LMEM_SLOT_ALIGN), ("LMEM_SLOT_ALIGN must be a power of two")) wire [`VX_CFG_LMEM_LOG_SIZE:0] aligned_lmem_size_w = - ((`VX_CFG_LMEM_LOG_SIZE+1)'(dcr_lmem_size) + (`VX_CFG_LMEM_LOG_SIZE+1)'(`VX_CFG_MEM_BLOCK_SIZE - 1)) - & ~((`VX_CFG_LMEM_LOG_SIZE+1)'(`VX_CFG_MEM_BLOCK_SIZE - 1)); + ((`VX_CFG_LMEM_LOG_SIZE+1)'(dcr_lmem_size) + (`VX_CFG_LMEM_LOG_SIZE+1)'(LMEM_SLOT_ALIGN - 1)) + & ~((`VX_CFG_LMEM_LOG_SIZE+1)'(LMEM_SLOT_ALIGN - 1)); reg [NW_WIDTH:0] cluster_size_r; reg [`VX_CFG_LMEM_LOG_SIZE:0] aligned_lmem_size_r; reg [`VX_CFG_NUM_WARPS-1:0] lmem_slots_r; diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index a281342e34..98cd9ccc7b 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -476,9 +476,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( localparam LMEM_LOG = `VX_CFG_LMEM_LOG_SIZE; - // Per-CTA LMEM footprint, block-aligned to MEM_BLOCK_SIZE by the KMU (DXA - // multicast resolves receiver addresses as issuer_addr + r*smem_stride; - // a non-aligned stride would target the wrong block). + // Per-CTA LMEM footprint, aligned to LMEM_SLOT_ALIGN by the KMU, so every + // slot base is too (DXA multicast resolves receiver addresses as + // issuer_addr + r*smem_stride; a non-aligned stride would target the wrong + // block). wire [LMEM_LOG:0] stride = kmu_req.aligned_lmem_size; wire is_first_of_cluster = eff_is_first; diff --git a/hw/rtl/dxa/VX_dxa_smem_wr.sv b/hw/rtl/dxa/VX_dxa_smem_wr.sv index d535ecfe7c..3922811fa5 100644 --- a/hw/rtl/dxa/VX_dxa_smem_wr.sv +++ b/hw/rtl/dxa/VX_dxa_smem_wr.sv @@ -699,9 +699,9 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Base word addr per beat: km_word_addr in K-major (per-elem scatter), // fb_word_addr_r in row-major (streamed). Replay adds the per-receiver - // offset accumulated in beat_offset_r (the dispatcher guarantees smem_stride - // is word-aligned — a multiple of MEM_BLOCK_SIZE ≥ SMEM_WORD_SIZE — so the - // byte→word shift in stride_words is lossless). + // offset accumulated in beat_offset_r (smem_stride is the CTA slot pitch, a + // multiple of LMEM_SLOT_ALIGN ≥ SMEM_WORD_SIZE, so the byte→word shift in + // stride_words is lossless). wire [SMEM_ADDR_WIDTH-1:0] base_word_addr = scatter ? km_word_addr : fb_word_addr_r; wire [SMEM_ADDR_WIDTH-1:0] replay_addr = base_word_addr + beat_offset_r; // Only the low SMEM_ADDR_WIDTH+SMEM_OFF_W bits of smem_stride are used. diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 2fbe69a9db..3bd887829a 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -13,6 +13,7 @@ #include #include "cta_dispatcher.h" +#include #include "core.h" #include #include @@ -142,14 +143,13 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w return 0; const kmu_req_t& next = bus_in.peek(); - // Fixed-stride admission. Round the per-CTA LMEM size up to a - // MEM_BLOCK_SIZE multiple — this uniform stride is the slot pitch. - // Multicast resolves receiver destinations as `issuer_base + r * stride` - // and the LMEM model is block-addressed (byteen-masked), so a non-aligned - // stride would truncate the destination; the descriptor handler rounds - // its stride the same way to stay consistent. - uint32_t stride = (next.lmem_size + VX_CFG_MEM_BLOCK_SIZE - 1u) - & ~uint32_t(VX_CFG_MEM_BLOCK_SIZE - 1u); + // Fixed-stride admission. Round the per-CTA LMEM size up to the slot + // alignment — this uniform stride is the slot pitch. Multicast resolves + // receiver destinations as `issuer_base + r * stride` and the LMEM model + // is block-addressed (byteen-masked), so a non-aligned stride would + // truncate the destination; the descriptor handler rounds its stride the + // same way to stay consistent. + uint32_t stride = lmem_slot_pitch(next.lmem_size); uint32_t max_slots = usable_slots(stride); uint32_t k = next.cluster_dim[0] diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 0c37defe5e..395f5c32fb 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -102,7 +102,7 @@ class CtaDispatcher : public SimObject { mutable bool lmem_overrun_reported_; // Fixed-stride slot allocation. Every resident CTA gets LMEM base - // slot × stride, where stride = align(lmem_size, MEM_BLOCK_SIZE) is uniform + // slot × stride, where stride = lmem_slot_pitch(lmem_size) is uniform // within a kernel. The occupancy bound floor(capacity/stride) (capped at // NUM_WARPS) replaces the byte ring's free_size accounting, so no wrap or // padding arithmetic is needed. A CTA alone in its cluster takes the lowest diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index bc908e448e..1e7f62ec38 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -13,6 +13,7 @@ #include "dxa_core.h" #include "dxa_meta.h" +#include "lmem_slot.h" #include #include #include @@ -202,14 +203,14 @@ class DxaCore::Impl { case VX_DCR_DXA_DESC_TILESIZE4_OFF: d.tile_sizes[4] = uint16_t(value & 0xffff); break; case VX_DCR_DXA_DESC_CFILL_OFF: d.cfill = value; break; case VX_DCR_DXA_DESC_SMEM_STRIDE_OFF: - // Round up to the LMEM word size so multicast destination addresses + // Round up to the CTA slot pitch so multicast destination addresses // `issuer_addr + r * smem_stride` stay block-aligned. The LMEM model // treats requests as block-aligned with a byteen mask; a non-aligned // stride would truncate the address and land writes in the wrong // block. The dispatcher applies the same rounding to per-CTA LMEM // allocations so the host-visible "same offset in receiver LMEM" // contract holds end-to-end. - d.smem_stride = (value + kLmemWordSize - 1u) & ~uint32_t(kLmemWordSize - 1u); + d.smem_stride = lmem_slot_pitch(value); break; default: break; } diff --git a/sw/common/lmem_slot.h b/sw/common/lmem_slot.h new file mode 100644 index 0000000000..797778f051 --- /dev/null +++ b/sw/common/lmem_slot.h @@ -0,0 +1,41 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The pitch of the equal per-CTA LMEM slots the CTA dispatcher partitions LMEM +// into. + +#pragma once + +#include +#include + +namespace vortex { + +// Alignment of every CTA's LMEM slot, and so of its __local_mem() base. An +// LMEM DMA agent (DXA, the WGMMA tile buffers) moves whole bank rows: a +// block-major WGMMA operand must start one, and DXA multicast steps receivers +// by whole rows. +inline constexpr uint32_t lmem_slot_align() { + constexpr uint32_t bank_row = VX_CFG_LMEM_NUM_BANKS * (VX_CFG_XLEN / 8); + constexpr bool has_dma = VX_CFG_EXT_DXA_ENABLED || VX_CFG_TCU_WGMMA_ENABLED; + return (has_dma && bank_row > VX_CFG_MEM_BLOCK_SIZE) ? bank_row : VX_CFG_MEM_BLOCK_SIZE; +} + +static_assert((lmem_slot_align() & (lmem_slot_align() - 1)) == 0, + "the LMEM slot alignment must be a power of two"); + +inline constexpr uint32_t lmem_slot_pitch(uint32_t lmem_size) { + return (lmem_size + lmem_slot_align() - 1u) & ~(lmem_slot_align() - 1u); +} + +} // namespace vortex diff --git a/sw/runtime/common/legacy_utils.cpp b/sw/runtime/common/legacy_utils.cpp index 4d771fe65e..3254accb40 100644 --- a/sw/runtime/common/legacy_utils.cpp +++ b/sw/runtime/common/legacy_utils.cpp @@ -12,6 +12,7 @@ // limitations under the License. #include +#include #include #include @@ -176,10 +177,9 @@ int vx_check_occupancy(vx_device_h hdevice, uint32_t block_size, uint32_t max_lo max_localmem, (unsigned long)local_mem_size); return -1; } - // LMEM-limited co-residency uses the same MEM_BLOCK-aligned stride as the - // CTA dispatcher; report when LMEM (not threads) caps occupancy. - uint32_t stride = (max_localmem + VX_CFG_MEM_BLOCK_SIZE - 1u) - & ~uint32_t(VX_CFG_MEM_BLOCK_SIZE - 1u); + // LMEM-limited co-residency uses the same slot pitch as the CTA + // dispatcher; report when LMEM (not threads) caps occupancy. + uint32_t stride = vortex::lmem_slot_pitch(max_localmem); int blocks_by_lmem = (int)(local_mem_size / stride); if (blocks_by_lmem < blocks_per_core) blocks_per_core = blocks_by_lmem; diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 2fbf8a5fd2..e7bf4971c7 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -39,6 +39,11 @@ #ifndef B_OFF_K #define B_OFF_K 0 #endif +// Extra per-CTA LMEM in bytes: a footprint that is not a multiple of an LMEM +// bank row still gives every co-resident CTA a bank-row aligned base. +#ifndef SMEM_PAD_BYTES +#define SMEM_PAD_BYTES 0 +#endif typedef struct { uint32_t M, N, K; diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 77d094ee23..94d1ba084c 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -715,7 +715,7 @@ int main(int argc, char *argv[]) { #else uint32_t b_smem = wg_cfg::tileK * per_warp_N; #endif - uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t); + uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t) + SMEM_PAD_BYTES; vx_event_h launch_ev = nullptr; { vx_launch_info_t li = {}; From cdf4a2f4a223ccd2562ef316d886c7931ad121be Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 18 Nov 2026 23:41:25 -0800 Subject: [PATCH 3/3] dxa: round the multicast SMEM stride up to the LMEM slot pitch in the RTL The host programs the multicast stride as the per-CTA LMEM size, and the receivers sit one CTA slot pitch apart. SimX rounded the stride up to the slot pitch at DCR write, but the RTL used the value as given (truncated to bank-row words), so the two agreed only when the host happened to pass an already-aligned size. With a 400 B footprint (448 B pitch) the RTL stepped receivers by 400 B and every receiver CTA computed garbage, while SimX passed. VX_dxa_setup now rounds the stride to LMEM_SLOT_ALIGN when the descriptor is latched, the same rounding the KMU applies to the slot pitch. The adder sits on the latch from the descriptor RAM's registered output, off the per-beat path. Beacon (dxa): depth unchanged (15, mean-50 9.4, 40 registers at 8+ levels), +11 LUT. dxa.h and the multicast design doc define the stride as the per-CTA LMEM size. sgemm_tcu_wg_dxa_mcast gains a runtime -p LMEM pad (default 0) and now passes its LMEM size as the stride; new CI case sgemm_tcu_wg_dxa_mcast-pad (SimX and rtlsim, -p 16) fails on rtlsim before this change and passes after, with SimX's instruction count. sgemm_tcu_wg_dxa_mcast-1/-2 and cta_cluster-1 still pass. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/dxa.yaml | 11 +++++++++++ docs/designs/dxa_async_copy_multicast.md | 4 +++- hw/rtl/dxa/VX_dxa_setup.sv | 7 ++++++- hw/rtl/dxa/VX_dxa_smem_wr.sv | 8 +++++--- sw/runtime/include/dxa.h | 4 +++- tests/regression/cta_cluster/main.cpp | 7 +++---- .../regression/sgemm_tcu_wg_dxa_mcast/main.cpp | 18 +++++++++--------- 7 files changed, 40 insertions(+), 19 deletions(-) diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 164ca7f94a..636c4b0a03 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -306,6 +306,17 @@ tests: app: sgemm_tcu_wg_dxa_mcast configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI args: -m 128 -n 64 -k 32 +# A per-CTA LMEM size that is not a slot-pitch multiple, passed as the multicast +# stride: the device rounds it to the pitch the receivers sit at. +- id: sgemm_tcu_wg_dxa_mcast-pad + touches: [hw/rtl/dxa, sim/simx/dxa, hw/rtl/VX_kmu.sv, sim/simx/cta_dispatcher.cpp, sw/common/lmem_slot.h] + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_wg_dxa_mcast + configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI + args: -p 16 - id: sgemm_tcu_wg_dxa_mcast-mc touches: [hw/rtl/VX_kmu.sv, hw/rtl/interfaces/VX_kmu_bus_if.sv, hw/rtl/core/VX_kmu_bus_arb.sv, hw/rtl/core/VX_cta_dispatch.sv, diff --git a/docs/designs/dxa_async_copy_multicast.md b/docs/designs/dxa_async_copy_multicast.md index fcb0c831eb..ed3a613e83 100644 --- a/docs/designs/dxa_async_copy_multicast.md +++ b/docs/designs/dxa_async_copy_multicast.md @@ -47,7 +47,9 @@ There is no software→global path; DXA is GMEM-read / LMEM-write only. ([`VX_dxa_pkg.sv:54`](../../hw/rtl/dxa/VX_dxa_pkg.sv#L54)): `base_addr` + `meta` + packed tile sizes (`tile01/tile23/tile4`) + `cfill` + `size0..4` (OOB bounds) + `stride0..3` (byte row strides) + - `smem_stride` (per-CTA LMEM stride for multicast). Programmed by the + `smem_stride` (per-CTA LMEM stride for multicast: the host writes the + kernel's per-CTA LMEM size, and the setup stage rounds it up to the CTA + slot pitch, `LMEM_SLOT_ALIGN`, as the KMU rounds the slots). Programmed by the host over a DCR block `[dcr_dxa]` at 0x100, 16 descriptors ([`VX_types.toml:87-123`](../../VX_types.toml#L87)). - **META subfields** `[dxa_desc_meta]`: `DIM` (3b), `ELEMSZ` (2b), diff --git a/hw/rtl/dxa/VX_dxa_setup.sv b/hw/rtl/dxa/VX_dxa_setup.sv index 440c827920..bcb3db0454 100644 --- a/hw/rtl/dxa/VX_dxa_setup.sv +++ b/hw/rtl/dxa/VX_dxa_setup.sv @@ -127,6 +127,11 @@ module VX_dxa_setup import VX_gpu_pkg::*, VX_dxa_pkg::*; ( wire [31:0] dec_tile3 = rank_ge4 ? 32'(desc_data.tile23[31:16]) : 32'd1; wire [31:0] dec_tile4 = rank_ge5 ? desc_data.tile4 : 32'd1; + // The host programs the multicast stride as the per-CTA LMEM size; the + // receivers sit one slot pitch apart, so round it as the KMU rounds the pitch. + wire [31:0] desc_smem_pitch = (desc_data.smem_stride + 32'(LMEM_SLOT_ALIGN - 1)) + & ~32'(LMEM_SLOT_ALIGN - 1); + wire [31:0] dec_stride0 = rank_ge2 ? desc_data.stride0 : 32'd0; wire [31:0] dec_stride1 = rank_ge3 ? desc_data.stride1 : 32'd0; wire [31:0] dec_stride2 = rank_ge4 ? desc_data.stride2 : 32'd0; @@ -465,7 +470,7 @@ module VX_dxa_setup import VX_gpu_pkg::*, VX_dxa_pkg::*; ( `endif s_is_multicast <= ($countones(req_data.cta_mask) > 1); s_cta_mask <= req_data.cta_mask; - s_smem_stride <= desc_data.smem_stride; + s_smem_stride <= desc_smem_pitch; s_initial_smem_base <= req_data.smem_addr; s_cfill <= desc_data.cfill; s_dest_kmajor <= dec_dest_kmajor; diff --git a/hw/rtl/dxa/VX_dxa_smem_wr.sv b/hw/rtl/dxa/VX_dxa_smem_wr.sv index 3922811fa5..c39682b77b 100644 --- a/hw/rtl/dxa/VX_dxa_smem_wr.sv +++ b/hw/rtl/dxa/VX_dxa_smem_wr.sv @@ -699,9 +699,11 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Base word addr per beat: km_word_addr in K-major (per-elem scatter), // fb_word_addr_r in row-major (streamed). Replay adds the per-receiver - // offset accumulated in beat_offset_r (smem_stride is the CTA slot pitch, a - // multiple of LMEM_SLOT_ALIGN ≥ SMEM_WORD_SIZE, so the byte→word shift in - // stride_words is lossless). + // offset accumulated in beat_offset_r (VX_dxa_setup rounds smem_stride to + // the CTA slot pitch, a multiple of LMEM_SLOT_ALIGN, itself a multiple of + // SMEM_WORD_SIZE, so the byte→word shift in stride_words is lossless). + `STATIC_ASSERT((LMEM_SLOT_ALIGN % SMEM_WORD_SIZE) == 0, + ("the CTA slot pitch must be a whole number of SMEM words")) wire [SMEM_ADDR_WIDTH-1:0] base_word_addr = scatter ? km_word_addr : fb_word_addr_r; wire [SMEM_ADDR_WIDTH-1:0] replay_addr = base_word_addr + beat_offset_r; // Only the low SMEM_ADDR_WIDTH+SMEM_OFF_W bits of smem_stride are used. diff --git a/sw/runtime/include/dxa.h b/sw/runtime/include/dxa.h index ee77ba8a0d..07edf1c7db 100644 --- a/sw/runtime/include/dxa.h +++ b/sw/runtime/include/dxa.h @@ -255,7 +255,9 @@ inline int program_5d( } // Program the multicast SMEM stride for an existing DXA descriptor. -// smem_stride_bytes: byte offset between consecutive CTAs' SMEM bases +// smem_stride_bytes: the per-CTA LMEM size the kernel is launched with +// (lmem_size). The device rounds it up to the CTA slot pitch, the distance +// between consecutive CTAs' SMEM bases, as it rounds each CTA's LMEM slot. // Bar stride is always 1 (hardcoded in hardware). // Multicast is active when cta_mask has more than one bit set. inline int set_multicast( diff --git a/tests/regression/cta_cluster/main.cpp b/tests/regression/cta_cluster/main.cpp index e2c364a645..e001733664 100644 --- a/tests/regression/cta_cluster/main.cpp +++ b/tests/regression/cta_cluster/main.cpp @@ -241,10 +241,9 @@ int main(int argc, char* argv[]) { RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); // Default: one launch where one cluster fills one core (= num_warps - // single-warp CTAs). The cta_dispatcher rounds each CTA's LMEM up to - // VX_CFG_MEM_BLOCK_SIZE so consecutive CTAs land at block-aligned - // offsets; the DXA multicast descriptor's smem_stride must match that, - // else peer CTAs receive the scale at the wrong SMEM offset. + // single-warp CTAs). The device rounds each CTA's LMEM slot, and the DXA + // multicast descriptor's smem_stride, up to the same slot alignment, so a + // stride of the per-CTA LMEM size lands peer CTAs at their slot offsets. if (launch_specs.empty()) { const uint32_t K = (uint32_t)num_warps; launch_specs.push_back({K, num_clusters, VX_CFG_MEM_BLOCK_SIZE}); diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index c72114c769..49702e8046 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -217,6 +217,9 @@ const char *kernel_file = "kernel.vxbin"; uint32_t xm = 64; uint32_t xn = 64; uint32_t xk = 64; +// Extra per-CTA LMEM bytes: a footprint that is not a multiple of the CTA slot +// pitch, passed as the multicast stride, must still reach each receiver. +uint32_t smem_pad = 0; // `warps` (= warps per CTA = WGMMA group size) is derived at runtime from // VX_CAPS_ISSUE_WIDTH after the device opens. WGMMA requires CTA-warp count // to match the hardware's TCU BLOCK_SIZE (= ISSUE_WIDTH), so it's not a @@ -236,16 +239,17 @@ constexpr uint32_t kDescB = 1; static void show_usage() { std::cout << "Vortex SGEMM TCU WGMMA+DXA Test." << std::endl; - std::cout << "Usage: [-m M] [-n N] [-k K] [-h help]" << std::endl; + std::cout << "Usage: [-m M] [-n N] [-k K] [-p smem_pad_bytes] [-h help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "m:n:k:h")) != -1) { + while ((c = getopt(argc, argv, "m:n:k:p:h")) != -1) { switch (c) { case 'm': xm = atoi(optarg); break; case 'n': xn = atoi(optarg); break; case 'k': xk = atoi(optarg); break; + case 'p': smem_pad = atoi(optarg); break; case 'h': show_usage(); exit(0); break; default: show_usage(); exit(-1); } @@ -346,7 +350,7 @@ int main(int argc, char *argv[]) { uint32_t block_dim[2] = {warps * (uint32_t)NT, 1}; // SMEM: A tile [cta_M x tileK] + B tile [tileK x tileN] - uint32_t smem_size = (cta_M * cfg::tileK + cfg::tileK * cfg::xtileN) * sizeof(itype_t); + uint32_t smem_size = (cta_M * cfg::tileK + cfg::tileK * cfg::xtileN) * sizeof(itype_t) + smem_pad; std::cout << "input type: " << vt::ITYPE::name << ", output type: " << vt::OTYPE::name << std::endl; std::cout << "WGMMA tile: M=" << cfg::xtileM << ", N=" << cfg::xtileN << ", K=" << cfg::tileK << std::endl; @@ -433,12 +437,8 @@ int main(int argc, char *argv[]) { // Multicast attribute on B descriptor: smem_stride per receiver. Each // co-resident CTA has its own LMEM region; the dispatcher allocates them - // sequentially with stride = per-CTA SMEM size. - const uint32_t b_tile_bytes = cfg::xtileN * cfg::tileK * sizeof(itype_t); - const uint32_t smem_stride = (cta_M * cfg::tileK + cfg::xtileN * cfg::tileK) - * sizeof(itype_t); - RT_CHECK(vortex::dxa::set_multicast(device, kDescB, smem_stride)); - (void)b_tile_bytes; + // sequentially, one per-CTA SMEM size (rounded to the slot pitch) apart. + RT_CHECK(vortex::dxa::set_multicast(device, kDescB, smem_size)); std::cout << "load kernel module" << std::endl; RT_CHECK(vx_module_load_file(device, kernel_file, &module_));