diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 164ca7f94a..636c4b0a03 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -306,6 +306,17 @@ tests: app: sgemm_tcu_wg_dxa_mcast configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI args: -m 128 -n 64 -k 32 +# A per-CTA LMEM size that is not a slot-pitch multiple, passed as the multicast +# stride: the device rounds it to the pitch the receivers sit at. +- id: sgemm_tcu_wg_dxa_mcast-pad + touches: [hw/rtl/dxa, sim/simx/dxa, hw/rtl/VX_kmu.sv, sim/simx/cta_dispatcher.cpp, sw/common/lmem_slot.h] + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_wg_dxa_mcast + configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI + args: -p 16 - id: sgemm_tcu_wg_dxa_mcast-mc touches: [hw/rtl/VX_kmu.sv, hw/rtl/interfaces/VX_kmu_bus_if.sv, hw/rtl/core/VX_kmu_bus_arb.sv, hw/rtl/core/VX_cta_dispatch.sv, diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index d3a13b5657..b427a2c0e4 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -117,6 +117,22 @@ tests: app: sgemm_tcu_wg configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DWGMMA_KMAJOR_B -DB_PAD_K=1 -DB_OFF_K=1 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +# Two co-resident CTAs whose LMEM footprint is not a bank-row multiple (128 B +# at 32 threads): the second CTA's block-major operands must still start a bank +# row, so the dispatcher's slot pitch keeps the bank-row alignment. +- id: sgemm_tcu_wg-lmem-pad + via: blackbox + drivers: + - simx + - rtlsim + xlen: + - 32 + touches: [hw/rtl/VX_kmu.sv, hw/rtl/VX_gpu_pkg.sv, sim/simx/cta_dispatcher.cpp, sw/common/lmem_slot.h, + sim/simx/tcu, sw/kernel/include/vx_tensor.h] + app: sgemm_tcu_wg + args: -m 128 -n 128 -k 32 + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DSMEM_PAD_BYTES=64 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-1 via: blackbox diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 698dd2c116..7af2097ebf 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -184,7 +184,7 @@ envelope plus a `kind`-discriminated union: | `PC`, `entry` | program image PC and kernel entry PC | | `param` | kargs pointer | | `ctx_id` | 8-bit launch identity, bumped every `start` | -| `aligned_lmem_size` | per-CTA LMEM footprint, rounded to `MEM_BLOCK_SIZE` | +| `aligned_lmem_size` | per-CTA LMEM footprint, rounded to `LMEM_SLOT_ALIGN` | `args.compute` ([`:735-744`](../../hw/rtl/VX_gpu_pkg.sv#L735)) carries `grid_dim[3]`, `block_idx[3]`, `block_dim[3]`, `block_size`, `warp_step[3]`, @@ -376,6 +376,19 @@ receiver-side translation. The stride must be `MEM_BLOCK_SIZE`-aligned (the KMU rounds it) because LMEM is block-addressed; a ragged stride would target the wrong block. +When the core has an LMEM DMA agent (`VX_CFG_EXT_DXA_ENABLE` or +`VX_CFG_TCU_WGMMA_ENABLE`) the KMU rounds the stride to a whole LMEM bank row +(`LMEM_NUM_BANKS × XLEN/8` bytes) instead, when that is larger: the agents +move whole bank rows, so DXA multicast steps receivers by whole rows and a +block-major WGMMA operand at `__local_mem()` must start one +([tensor_core_wgmma_engine.md §6.1](tensor_core_wgmma_engine.md)). This is +`LMEM_SLOT_ALIGN` in `VX_gpu_pkg`, mirrored for SimX and the runtime by +[`lmem_slot.h`](../../sw/common/lmem_slot.h). The bank row exceeds +`MEM_BLOCK_SIZE` only when `LMEM_NUM_BANKS` x XLEN/8 does: with the default +64-byte block, more than 16 LMEM banks at XLEN 32 or more than 8 at XLEN 64 +(one bank per thread by default), so narrower cores keep the `MEM_BLOCK_SIZE` +pitch. + `get_cluster_rank()` is derived, never sent: `CTA_ID % CTA_CLUSTER_SIZE` ([`vx_spawn2.h:133`](../../sw/kernel/include/vx_spawn2.h#L133)). diff --git a/docs/designs/dxa_async_copy_multicast.md b/docs/designs/dxa_async_copy_multicast.md index fcb0c831eb..ed3a613e83 100644 --- a/docs/designs/dxa_async_copy_multicast.md +++ b/docs/designs/dxa_async_copy_multicast.md @@ -47,7 +47,9 @@ There is no software→global path; DXA is GMEM-read / LMEM-write only. ([`VX_dxa_pkg.sv:54`](../../hw/rtl/dxa/VX_dxa_pkg.sv#L54)): `base_addr` + `meta` + packed tile sizes (`tile01/tile23/tile4`) + `cfill` + `size0..4` (OOB bounds) + `stride0..3` (byte row strides) + - `smem_stride` (per-CTA LMEM stride for multicast). Programmed by the + `smem_stride` (per-CTA LMEM stride for multicast: the host writes the + kernel's per-CTA LMEM size, and the setup stage rounds it up to the CTA + slot pitch, `LMEM_SLOT_ALIGN`, as the KMU rounds the slots). Programmed by the host over a DCR block `[dcr_dxa]` at 0x100, 16 descriptors ([`VX_types.toml:87-123`](../../VX_types.toml#L87)). - **META subfields** `[dxa_desc_meta]`: `DIM` (3b), `ELEMSZ` (2b), diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index 2a83ba4b64..0bae3ef6c4 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -315,6 +315,24 @@ stride folds into the packing, so building a descriptor costs one OR. Row-major is the layout of a DXA-loaded slab. The buffers' output muxes are layout-agnostic; only the fetch differs. +**Alignment.** Each LMEM access returns one bank row +(`NUM_THREADS × XLEN/8` bytes), and the buffers take a row's words from that +one access only, so a descriptor must keep every fetch inside a bank row: + +| layout | rule | +|---|---| +| block-major | the offset is a multiple of a bank row | +| row-major | the offset (in bytes) and the stride (in words) are multiples of the operand's **K-slice**: `fedpK` words, or for sparse `tcK` words of A and `2·tcK` of B | + +The K-slice is the part of a row one micro-op consumes. It divides a bank +row in every configuration, so an aligned slice never straddles two. One +tile row (`tileK` elements, `tileK/2` for sparse A) is a multiple of the +K-slice everywhere, so a descriptor stepped along K by `tileK` stays legal. +`smem_desc_check` in [`tensor_cfg.h`](../../sw/common/tensor_cfg.h) states the +rule once: SimX aborts on a descriptor that breaks it rather than read the +exact address the hardware cannot, and `wgmma_sync` asserts it in debug +kernel builds. + ### 6.2 A buffer — `VX_tcu_abuf` One per block. It holds one **k-stripe**: the `M_STEPS` A blocks of the @@ -525,7 +543,7 @@ constant on both sides. | API | purpose | |---|---| | `wmma_context`, `wgmma_context` | compile-time tile geometry and the instruction emitters for a given pair of formats | -| `vx_make_smem_desc(ptr, leading_bytes)` | build a shared-memory descriptor; a zero stride selects block-major | +| `vx_make_smem_desc(ptr, leading_bytes)` | build a shared-memory descriptor; a zero stride selects block-major; `ptr` and `leading_bytes` follow the alignment rule of §6.1 | | `load_sp_metadata`, `load_mx_metadata` | emit the `TCU_LD` instructions for a fragment | | `a_blockmajor_idx`, `b_blockmajor_idx`, `a_sp_blockmajor_idx` | where element `(r, c)` lives in a block-major slab | diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 86b3976a16..afb37c1a4e 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -2013,6 +2013,12 @@ package VX_gpu_pkg; localparam LMEM_DMA_IN_TAG_MAX = `MAX(DXA_LMEM_OUT_TAG_W, TCU_LMEM_TAG_W); localparam LMEM_DMA_TAG_WIDTH = LMEM_DMA_IN_TAG_MAX + `ARB_SEL_BITS(LMEM_DMA_INPUTS, 1); + // Alignment of every CTA's LMEM slot, and so of its LMEM base. A DMA agent + // moves whole bank rows: a block-major WGMMA operand must start one, and + // DXA multicast steps receivers by whole rows. + localparam LMEM_SLOT_ALIGN = LMEM_DMA_EN ? `MAX(`VX_CFG_MEM_BLOCK_SIZE, LMEM_DMA_DATA_SIZE) + : `VX_CFG_MEM_BLOCK_SIZE; + ////////////////////////// Icache Parameters ////////////////////////////// // Word size in bytes diff --git a/hw/rtl/VX_kmu.sv b/hw/rtl/VX_kmu.sv index 5d12aafe7c..cae8f136fe 100644 --- a/hw/rtl/VX_kmu.sv +++ b/hw/rtl/VX_kmu.sv @@ -84,14 +84,15 @@ module VX_kmu import VX_gpu_pkg::*; import VX_trace_pkg::*; #( // cluster_size : K = cluster_dim product, bounded by NUM_WARPS (cluster // members are co-resident on one core), so NW_WIDTH+1 bits // suffice; each dim is sliced to that width. - // aligned_lmem_size : per-CTA LMEM footprint rounded up to MEM_BLOCK_SIZE. + // aligned_lmem_size : per-CTA LMEM footprint rounded up to LMEM_SLOT_ALIGN. // lmem_slots : the CTA slots that footprint fits (lmem_slots_fit). // Registered so the broadcast path is flop-driven. All three settle one // cycle after their DCR write, the same cycle, and DCRs are written before // `start`, so all are settled before the first CTA is dispatched. + `STATIC_ASSERT(`IS_POW2(LMEM_SLOT_ALIGN), ("LMEM_SLOT_ALIGN must be a power of two")) wire [`VX_CFG_LMEM_LOG_SIZE:0] aligned_lmem_size_w = - ((`VX_CFG_LMEM_LOG_SIZE+1)'(dcr_lmem_size) + (`VX_CFG_LMEM_LOG_SIZE+1)'(`VX_CFG_MEM_BLOCK_SIZE - 1)) - & ~((`VX_CFG_LMEM_LOG_SIZE+1)'(`VX_CFG_MEM_BLOCK_SIZE - 1)); + ((`VX_CFG_LMEM_LOG_SIZE+1)'(dcr_lmem_size) + (`VX_CFG_LMEM_LOG_SIZE+1)'(LMEM_SLOT_ALIGN - 1)) + & ~((`VX_CFG_LMEM_LOG_SIZE+1)'(LMEM_SLOT_ALIGN - 1)); reg [NW_WIDTH:0] cluster_size_r; reg [`VX_CFG_LMEM_LOG_SIZE:0] aligned_lmem_size_r; reg [`VX_CFG_NUM_WARPS-1:0] lmem_slots_r; diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index a281342e34..98cd9ccc7b 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -476,9 +476,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( localparam LMEM_LOG = `VX_CFG_LMEM_LOG_SIZE; - // Per-CTA LMEM footprint, block-aligned to MEM_BLOCK_SIZE by the KMU (DXA - // multicast resolves receiver addresses as issuer_addr + r*smem_stride; - // a non-aligned stride would target the wrong block). + // Per-CTA LMEM footprint, aligned to LMEM_SLOT_ALIGN by the KMU, so every + // slot base is too (DXA multicast resolves receiver addresses as + // issuer_addr + r*smem_stride; a non-aligned stride would target the wrong + // block). wire [LMEM_LOG:0] stride = kmu_req.aligned_lmem_size; wire is_first_of_cluster = eff_is_first; diff --git a/hw/rtl/dxa/VX_dxa_setup.sv b/hw/rtl/dxa/VX_dxa_setup.sv index 440c827920..bcb3db0454 100644 --- a/hw/rtl/dxa/VX_dxa_setup.sv +++ b/hw/rtl/dxa/VX_dxa_setup.sv @@ -127,6 +127,11 @@ module VX_dxa_setup import VX_gpu_pkg::*, VX_dxa_pkg::*; ( wire [31:0] dec_tile3 = rank_ge4 ? 32'(desc_data.tile23[31:16]) : 32'd1; wire [31:0] dec_tile4 = rank_ge5 ? desc_data.tile4 : 32'd1; + // The host programs the multicast stride as the per-CTA LMEM size; the + // receivers sit one slot pitch apart, so round it as the KMU rounds the pitch. + wire [31:0] desc_smem_pitch = (desc_data.smem_stride + 32'(LMEM_SLOT_ALIGN - 1)) + & ~32'(LMEM_SLOT_ALIGN - 1); + wire [31:0] dec_stride0 = rank_ge2 ? desc_data.stride0 : 32'd0; wire [31:0] dec_stride1 = rank_ge3 ? desc_data.stride1 : 32'd0; wire [31:0] dec_stride2 = rank_ge4 ? desc_data.stride2 : 32'd0; @@ -465,7 +470,7 @@ module VX_dxa_setup import VX_gpu_pkg::*, VX_dxa_pkg::*; ( `endif s_is_multicast <= ($countones(req_data.cta_mask) > 1); s_cta_mask <= req_data.cta_mask; - s_smem_stride <= desc_data.smem_stride; + s_smem_stride <= desc_smem_pitch; s_initial_smem_base <= req_data.smem_addr; s_cfill <= desc_data.cfill; s_dest_kmajor <= dec_dest_kmajor; diff --git a/hw/rtl/dxa/VX_dxa_smem_wr.sv b/hw/rtl/dxa/VX_dxa_smem_wr.sv index d535ecfe7c..c39682b77b 100644 --- a/hw/rtl/dxa/VX_dxa_smem_wr.sv +++ b/hw/rtl/dxa/VX_dxa_smem_wr.sv @@ -699,9 +699,11 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Base word addr per beat: km_word_addr in K-major (per-elem scatter), // fb_word_addr_r in row-major (streamed). Replay adds the per-receiver - // offset accumulated in beat_offset_r (the dispatcher guarantees smem_stride - // is word-aligned — a multiple of MEM_BLOCK_SIZE ≥ SMEM_WORD_SIZE — so the - // byte→word shift in stride_words is lossless). + // offset accumulated in beat_offset_r (VX_dxa_setup rounds smem_stride to + // the CTA slot pitch, a multiple of LMEM_SLOT_ALIGN, itself a multiple of + // SMEM_WORD_SIZE, so the byte→word shift in stride_words is lossless). + `STATIC_ASSERT((LMEM_SLOT_ALIGN % SMEM_WORD_SIZE) == 0, + ("the CTA slot pitch must be a whole number of SMEM words")) wire [SMEM_ADDR_WIDTH-1:0] base_word_addr = scatter ? km_word_addr : fb_word_addr_r; wire [SMEM_ADDR_WIDTH-1:0] replay_addr = base_word_addr + beat_offset_r; // Only the low SMEM_ADDR_WIDTH+SMEM_OFF_W bits of smem_stride are used. diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index 8d2c53496c..8eed2496f5 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -502,6 +502,13 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end `endif + // A block-major A operand must start an LMEM bank row: the fetch drops the + // offset inside the row. + `RUNTIME_ASSERT (~(req_valid && req_a_is_smem && is_first_compute_uop && (desc_a_ldm_words == '0) + && (req_desc_a[BANK_ROW_WORDS_LOG2+1:0] != '0)), + ("%t: *** %s: block-major A base 0x%0h is not bank-row aligned", $time, INSTANCE_ID, + req_desc_a[TCU_SMEM_DESC_OFF_BITS-1:0])) + endmodule `endif // VX_CFG_TCU_WGMMA_ENABLE diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 8071304bf5..9b43879ca6 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -707,6 +707,13 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end `endif + // A block-major B operand must start an LMEM bank row: the fetch drops the + // offset inside the row. + `RUNTIME_ASSERT (~(req_valid && ~req_wants_kmajor + && (req_desc_b[BANK_ROW_WORDS_LOG2+1:0] != '0)), + ("%t: *** %s: block-major B base 0x%0h is not bank-row aligned", $time, INSTANCE_ID, + req_desc_b[TCU_SMEM_DESC_OFF_BITS-1:0])) + endmodule `endif // VX_CFG_TCU_WGMMA_ENABLE diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index 2fbe69a9db..3bd887829a 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -13,6 +13,7 @@ #include #include "cta_dispatcher.h" +#include #include "core.h" #include #include @@ -142,14 +143,13 @@ uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_w return 0; const kmu_req_t& next = bus_in.peek(); - // Fixed-stride admission. Round the per-CTA LMEM size up to a - // MEM_BLOCK_SIZE multiple — this uniform stride is the slot pitch. - // Multicast resolves receiver destinations as `issuer_base + r * stride` - // and the LMEM model is block-addressed (byteen-masked), so a non-aligned - // stride would truncate the destination; the descriptor handler rounds - // its stride the same way to stay consistent. - uint32_t stride = (next.lmem_size + VX_CFG_MEM_BLOCK_SIZE - 1u) - & ~uint32_t(VX_CFG_MEM_BLOCK_SIZE - 1u); + // Fixed-stride admission. Round the per-CTA LMEM size up to the slot + // alignment — this uniform stride is the slot pitch. Multicast resolves + // receiver destinations as `issuer_base + r * stride` and the LMEM model + // is block-addressed (byteen-masked), so a non-aligned stride would + // truncate the destination; the descriptor handler rounds its stride the + // same way to stay consistent. + uint32_t stride = lmem_slot_pitch(next.lmem_size); uint32_t max_slots = usable_slots(stride); uint32_t k = next.cluster_dim[0] diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 0c37defe5e..395f5c32fb 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -102,7 +102,7 @@ class CtaDispatcher : public SimObject { mutable bool lmem_overrun_reported_; // Fixed-stride slot allocation. Every resident CTA gets LMEM base - // slot × stride, where stride = align(lmem_size, MEM_BLOCK_SIZE) is uniform + // slot × stride, where stride = lmem_slot_pitch(lmem_size) is uniform // within a kernel. The occupancy bound floor(capacity/stride) (capped at // NUM_WARPS) replaces the byte ring's free_size accounting, so no wrap or // padding arithmetic is needed. A CTA alone in its cluster takes the lowest diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index bc908e448e..1e7f62ec38 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -13,6 +13,7 @@ #include "dxa_core.h" #include "dxa_meta.h" +#include "lmem_slot.h" #include #include #include @@ -202,14 +203,14 @@ class DxaCore::Impl { case VX_DCR_DXA_DESC_TILESIZE4_OFF: d.tile_sizes[4] = uint16_t(value & 0xffff); break; case VX_DCR_DXA_DESC_CFILL_OFF: d.cfill = value; break; case VX_DCR_DXA_DESC_SMEM_STRIDE_OFF: - // Round up to the LMEM word size so multicast destination addresses + // Round up to the CTA slot pitch so multicast destination addresses // `issuer_addr + r * smem_stride` stay block-aligned. The LMEM model // treats requests as block-aligned with a byteen mask; a non-aligned // stride would truncate the address and land writes in the wrong // block. The dispatcher applies the same rounding to per-CTA LMEM // allocations so the host-visible "same offset in receiver LMEM" // contract holds end-to-end. - d.smem_stride = (value + kLmemWordSize - 1u) & ~uint32_t(kLmemWordSize - 1u); + d.smem_stride = lmem_slot_pitch(value); break; default: break; } diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index be70301739..4d1bb8e359 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -64,7 +64,12 @@ static constexpr uint64_t kLineMask = ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); // WGMMA tile-buffer geometry in 32-bit words. static constexpr uint32_t kLmemBanks = VX_CFG_LMEM_NUM_BANKS; static constexpr uint32_t kBankRowWords = kLmemBanks * (VX_CFG_XLEN / 32); -static constexpr uint32_t kBBlockWords = kFedpWords * cfg::tcN; +#ifdef VX_CFG_TCU_WGMMA_ENABLE +// A slice-aligned row-major K-slice then never straddles two bank rows. +static_assert((kBankRowWords % kFedpWords) == 0 && (kBankRowWords % (2 * cfg::tcK)) == 0, + "a WGMMA K-slice must divide an LMEM bank row"); +#endif +static constexpr uint32_t kBBlockWords = kFedpWords * cfg::tcN; static constexpr uint32_t kBBlockWordsSp = cfg::tcK * cfg::tcN * 2; // One bank row holds kBSubBlocks consecutive dense B blocks. static constexpr uint32_t kBSubBlocks = @@ -856,7 +861,7 @@ class TcuUnit::Impl { wgmma_desc_[wid][0] = trace->src_data.at(0).at(0).u32; } wgmma_desc_[wid][1] = trace->src_data.at(1).at(0).u32; - this->decode_descs(wid, tpuArgs.fmt_s, tpuArgs.is_a_smem); + this->decode_descs(wid, tpuArgs.fmt_s, tpuArgs.is_a_smem, is_sparse); } auto& req = reqs.at(b); @@ -970,14 +975,41 @@ class TcuUnit::Impl { return ready_mask; } + // The tile buffers cannot fetch a descriptor that breaks the alignment rule, + // so SimX must not read it from its exact address either. + static void check_desc(uint32_t desc, bool is_b, bool is_sparse) { + uint32_t slice_words = vt::wgmma_slice_words(is_b, is_sparse, cfg::tcK, kFedpWords); + auto fault = vt::smem_desc_check(desc, slice_words, kBankRowWords * 4); + if (fault == vt::smem_desc_fault::none) { + return; + } + std::cerr << "*** WGMMA " << (is_b ? "B" : "A") << " descriptor 0x" << std::hex << desc << std::dec + << " (offset " << vt::smem_desc_offset(desc) << " B, stride " << vt::smem_desc_stride(desc) << " B): "; + switch (fault) { + case vt::smem_desc_fault::block_base: + std::cerr << "a block-major base must be aligned to an LMEM bank row (" << (kBankRowWords * 4) << " B)"; + break; + case vt::smem_desc_fault::row_base: + std::cerr << "a row-major base must be a multiple of the " << (slice_words * 4) << "-byte K-slice"; + break; + default: + std::cerr << "a row-major stride must be a multiple of the " << (slice_words * 4) << "-byte K-slice"; + break; + } + std::cerr << std::endl; + std::abort(); + } + // Decode a warp's WGMMA descriptors into LMEM base and row stride. - void decode_descs(uint32_t wid, uint32_t fmt_s, bool is_a_smem) { + void decode_descs(uint32_t wid, uint32_t fmt_s, bool is_a_smem, bool is_sparse) { uint32_t e_bits = elem_bits(fmt_s); uint32_t a_desc = wgmma_desc_[wid][0]; uint32_t b_desc = wgmma_desc_[wid][1]; if (is_a_smem) { + check_desc(a_desc, false, is_sparse); lmem_desc_[wid][0] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(a_desc), vt::smem_desc_stride(a_desc) * 8 / e_bits, false}; } + check_desc(b_desc, true, is_sparse); lmem_desc_[wid][1] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(b_desc), vt::smem_desc_stride(b_desc) * 8 / e_bits, false}; } @@ -1199,16 +1231,12 @@ class TcuUnit::Impl { uint32_t ratio = elem_ratio(fmt_s); uint32_t k_words = is_sparse ? cfg::tcK : kFedpWords; - uint32_t e_bits = elem_bits(fmt_s); // Decode smem descriptors (B always from smem, A optionally). if (is_setup_uop) { wgmma_desc_[wid][0] = a_desc; wgmma_desc_[wid][1] = b_desc; - if (is_a_smem) { - lmem_desc_[wid][0] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(a_desc), vt::smem_desc_stride(a_desc) * 8 / e_bits, false}; - } - lmem_desc_[wid][1] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + vt::smem_desc_offset(b_desc), vt::smem_desc_stride(b_desc) * 8 / e_bits, false}; + this->decode_descs(wid, fmt_s, is_a_smem, is_sparse); uint32_t nrc = (cd_nregs == 0) ? 8 : (cd_nregs == 1) ? 16 : 32; cur_xtile_n_ = (nrc * VX_CFG_NUM_THREADS) / wg_cfg::xtileM; return; diff --git a/sw/common/lmem_slot.h b/sw/common/lmem_slot.h new file mode 100644 index 0000000000..797778f051 --- /dev/null +++ b/sw/common/lmem_slot.h @@ -0,0 +1,41 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The pitch of the equal per-CTA LMEM slots the CTA dispatcher partitions LMEM +// into. + +#pragma once + +#include +#include + +namespace vortex { + +// Alignment of every CTA's LMEM slot, and so of its __local_mem() base. An +// LMEM DMA agent (DXA, the WGMMA tile buffers) moves whole bank rows: a +// block-major WGMMA operand must start one, and DXA multicast steps receivers +// by whole rows. +inline constexpr uint32_t lmem_slot_align() { + constexpr uint32_t bank_row = VX_CFG_LMEM_NUM_BANKS * (VX_CFG_XLEN / 8); + constexpr bool has_dma = VX_CFG_EXT_DXA_ENABLED || VX_CFG_TCU_WGMMA_ENABLED; + return (has_dma && bank_row > VX_CFG_MEM_BLOCK_SIZE) ? bank_row : VX_CFG_MEM_BLOCK_SIZE; +} + +static_assert((lmem_slot_align() & (lmem_slot_align() - 1)) == 0, + "the LMEM slot alignment must be a power of two"); + +inline constexpr uint32_t lmem_slot_pitch(uint32_t lmem_size) { + return (lmem_size + lmem_slot_align() - 1u) & ~(lmem_slot_align() - 1u); +} + +} // namespace vortex diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 79bf70a181..06d8576cc5 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -174,6 +174,41 @@ inline constexpr uint32_t smem_desc_stride(uint32_t desc) { return (desc >> smem_desc_offset_bits) << smem_desc_stride_lg; } +// 32-bit words of its K dimension one row of a row-major operand supplies to +// a WGMMA micro-op: a sparse A row is compressed, a sparse B row carries both +// candidates of each K step. +inline constexpr uint32_t wgmma_slice_words(bool is_b, bool is_sparse, + uint32_t tc_k, uint32_t fedp_k) { + return is_sparse ? (is_b ? 2 * tc_k : tc_k) : fedp_k; +} + +// What the TCU tile buffers can fetch. A row-major operand (stride != 0) is +// read one row per LMEM access, so each row's K-slice must lie in one bank +// row: the base and the stride must be multiples of the slice. A block-major +// operand (stride == 0) is read in whole bank rows, so its base must start one. +enum class smem_desc_fault { + none, + block_base, // block-major base not aligned to an LMEM bank row + row_base, // row-major base not a multiple of the K-slice + row_stride // row-major stride not a multiple of the K-slice +}; + +inline constexpr smem_desc_fault smem_desc_check(uint32_t desc, uint32_t slice_words, + uint32_t bank_row_bytes) { + uint32_t offset = smem_desc_offset(desc); + uint32_t stride_words = smem_desc_stride(desc) >> smem_desc_stride_lg; + if (stride_words == 0) { + return (offset % bank_row_bytes) ? smem_desc_fault::block_base : smem_desc_fault::none; + } + if (offset % (slice_words * 4)) { + return smem_desc_fault::row_base; + } + if (stride_words % slice_words) { + return smem_desc_fault::row_stride; + } + return smem_desc_fault::none; +} + // The TCU accumulates and returns fp32 (int32 for integer inputs); it has no // other output format. inline constexpr bool tcu_output_valid(uint32_t it, uint32_t ot) { diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 52092ef52d..a5fd89466c 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -30,7 +30,15 @@ struct smem_matrix_desc { }; // Build a smem descriptor from a pointer and a row stride in bytes, a multiple of 4. +// The TCU fetches only aligned operands (smem_desc_check), checked by wgmma_sync: +// row-major (leading_bytes != 0): ptr and leading_bytes are multiples of the +// operand's K-slice, fedpK words (sparse: tcK words for A, 2*tcK for B). +// A multiple of one tile row (tileK elements, tileK/2 for sparse A) holds +// in every configuration, so stepping a descriptor along K by tileK does. +// block-major (leading_bytes == 0): ptr starts an LMEM bank row, +// NUM_THREADS * XLEN/8 bytes. static inline __attribute__((always_inline)) smem_matrix_desc vx_make_smem_desc(const void* ptr, uint32_t leading_bytes) { + assert((leading_bytes % 4) == 0 && "smem descriptor stride must be a multiple of 4 bytes"); size_t lmem_base = csr_read_nv(VX_CSR_LOCAL_MEM_BASE); uint32_t offset = static_cast(static_cast(reinterpret_cast(ptr)) - lmem_base); return {smem_desc_pack(offset, leading_bytes)}; @@ -910,6 +918,14 @@ struct wgmma_context { | ((a_is_smem ? 1 : 0) << 3); } + // The TCU requires one LMEM bank per thread. + static constexpr uint32_t smem_bank_row_bytes = NT * (__riscv_xlen / 8); + + static bool smem_desc_valid(smem_matrix_desc desc, bool is_b) { + uint32_t slice_words = wgmma_slice_words(is_b, is_sparse, tcK, fedpK); + return smem_desc_check(desc.value, slice_words, smem_bank_row_bytes) == smem_desc_fault::none; + } + public: // Types using input_t = typename ctx_c::input_t; @@ -1223,6 +1239,11 @@ struct wgmma_context { } static_assert(b_is_smem, "B must be smem_matrix_desc (SR mode is not supported)"); + if constexpr (a_is_smem) { + assert(smem_desc_valid(op_a, false) && "WGMMA A descriptor breaks the TCU alignment rule (vx_make_smem_desc)"); + } + assert(smem_desc_valid(op_b, true) && "WGMMA B descriptor breaks the TCU alignment rule (vx_make_smem_desc)"); + constexpr int flags = wgmma_flags(); // --- SS path: both from smem --- diff --git a/sw/runtime/common/legacy_utils.cpp b/sw/runtime/common/legacy_utils.cpp index 4d771fe65e..3254accb40 100644 --- a/sw/runtime/common/legacy_utils.cpp +++ b/sw/runtime/common/legacy_utils.cpp @@ -12,6 +12,7 @@ // limitations under the License. #include +#include #include #include @@ -176,10 +177,9 @@ int vx_check_occupancy(vx_device_h hdevice, uint32_t block_size, uint32_t max_lo max_localmem, (unsigned long)local_mem_size); return -1; } - // LMEM-limited co-residency uses the same MEM_BLOCK-aligned stride as the - // CTA dispatcher; report when LMEM (not threads) caps occupancy. - uint32_t stride = (max_localmem + VX_CFG_MEM_BLOCK_SIZE - 1u) - & ~uint32_t(VX_CFG_MEM_BLOCK_SIZE - 1u); + // LMEM-limited co-residency uses the same slot pitch as the CTA + // dispatcher; report when LMEM (not threads) caps occupancy. + uint32_t stride = vortex::lmem_slot_pitch(max_localmem); int blocks_by_lmem = (int)(local_mem_size / stride); if (blocks_by_lmem < blocks_per_core) blocks_per_core = blocks_by_lmem; diff --git a/sw/runtime/include/dxa.h b/sw/runtime/include/dxa.h index ee77ba8a0d..07edf1c7db 100644 --- a/sw/runtime/include/dxa.h +++ b/sw/runtime/include/dxa.h @@ -255,7 +255,9 @@ inline int program_5d( } // Program the multicast SMEM stride for an existing DXA descriptor. -// smem_stride_bytes: byte offset between consecutive CTAs' SMEM bases +// smem_stride_bytes: the per-CTA LMEM size the kernel is launched with +// (lmem_size). The device rounds it up to the CTA slot pitch, the distance +// between consecutive CTAs' SMEM bases, as it rounds each CTA's LMEM slot. // Bar stride is always 1 (hardcoded in hardware). // Multicast is active when cta_mask has more than one bit set. inline int set_multicast( diff --git a/tests/regression/cta_cluster/main.cpp b/tests/regression/cta_cluster/main.cpp index e2c364a645..e001733664 100644 --- a/tests/regression/cta_cluster/main.cpp +++ b/tests/regression/cta_cluster/main.cpp @@ -241,10 +241,9 @@ int main(int argc, char* argv[]) { RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); // Default: one launch where one cluster fills one core (= num_warps - // single-warp CTAs). The cta_dispatcher rounds each CTA's LMEM up to - // VX_CFG_MEM_BLOCK_SIZE so consecutive CTAs land at block-aligned - // offsets; the DXA multicast descriptor's smem_stride must match that, - // else peer CTAs receive the scale at the wrong SMEM offset. + // single-warp CTAs). The device rounds each CTA's LMEM slot, and the DXA + // multicast descriptor's smem_stride, up to the same slot alignment, so a + // stride of the per-CTA LMEM size lands peer CTAs at their slot offsets. if (launch_specs.empty()) { const uint32_t K = (uint32_t)num_warps; launch_specs.push_back({K, num_clusters, VX_CFG_MEM_BLOCK_SIZE}); diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 73e6c88a44..e7bf4971c7 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -28,12 +28,22 @@ #ifndef A_OFF_K #define A_OFF_K 0 #endif +// Extra A offset in elements: one that is not a multiple of the K-slice builds +// a descriptor the TCU cannot fetch, which SimX must reject. +#ifndef A_OFF_ELEMS +#define A_OFF_ELEMS 0 +#endif #ifndef B_PAD_K #define B_PAD_K 0 #endif #ifndef B_OFF_K #define B_OFF_K 0 #endif +// Extra per-CTA LMEM in bytes: a footprint that is not a multiple of an LMEM +// bank row still gives every co-resident CTA a bank-row aligned base. +#ifndef SMEM_PAD_BYTES +#define SMEM_PAD_BYTES 0 +#endif typedef struct { uint32_t M, N, K; diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index 9f8aa6998d..b138c8376d 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -11,10 +11,10 @@ using ctx = vt::wgmma_context(arg->A_addr); diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 77d094ee23..94d1ba084c 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -715,7 +715,7 @@ int main(int argc, char *argv[]) { #else uint32_t b_smem = wg_cfg::tileK * per_warp_N; #endif - uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t); + uint32_t smem_size = (a_smem + b_smem) * sizeof(itype_t) + SMEM_PAD_BYTES; vx_event_h launch_ev = nullptr; { vx_launch_info_t li = {}; diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index c72114c769..49702e8046 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -217,6 +217,9 @@ const char *kernel_file = "kernel.vxbin"; uint32_t xm = 64; uint32_t xn = 64; uint32_t xk = 64; +// Extra per-CTA LMEM bytes: a footprint that is not a multiple of the CTA slot +// pitch, passed as the multicast stride, must still reach each receiver. +uint32_t smem_pad = 0; // `warps` (= warps per CTA = WGMMA group size) is derived at runtime from // VX_CAPS_ISSUE_WIDTH after the device opens. WGMMA requires CTA-warp count // to match the hardware's TCU BLOCK_SIZE (= ISSUE_WIDTH), so it's not a @@ -236,16 +239,17 @@ constexpr uint32_t kDescB = 1; static void show_usage() { std::cout << "Vortex SGEMM TCU WGMMA+DXA Test." << std::endl; - std::cout << "Usage: [-m M] [-n N] [-k K] [-h help]" << std::endl; + std::cout << "Usage: [-m M] [-n N] [-k K] [-p smem_pad_bytes] [-h help]" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "m:n:k:h")) != -1) { + while ((c = getopt(argc, argv, "m:n:k:p:h")) != -1) { switch (c) { case 'm': xm = atoi(optarg); break; case 'n': xn = atoi(optarg); break; case 'k': xk = atoi(optarg); break; + case 'p': smem_pad = atoi(optarg); break; case 'h': show_usage(); exit(0); break; default: show_usage(); exit(-1); } @@ -346,7 +350,7 @@ int main(int argc, char *argv[]) { uint32_t block_dim[2] = {warps * (uint32_t)NT, 1}; // SMEM: A tile [cta_M x tileK] + B tile [tileK x tileN] - uint32_t smem_size = (cta_M * cfg::tileK + cfg::tileK * cfg::xtileN) * sizeof(itype_t); + uint32_t smem_size = (cta_M * cfg::tileK + cfg::tileK * cfg::xtileN) * sizeof(itype_t) + smem_pad; std::cout << "input type: " << vt::ITYPE::name << ", output type: " << vt::OTYPE::name << std::endl; std::cout << "WGMMA tile: M=" << cfg::xtileM << ", N=" << cfg::xtileN << ", K=" << cfg::tileK << std::endl; @@ -433,12 +437,8 @@ int main(int argc, char *argv[]) { // Multicast attribute on B descriptor: smem_stride per receiver. Each // co-resident CTA has its own LMEM region; the dispatcher allocates them - // sequentially with stride = per-CTA SMEM size. - const uint32_t b_tile_bytes = cfg::xtileN * cfg::tileK * sizeof(itype_t); - const uint32_t smem_stride = (cta_M * cfg::tileK + cfg::xtileN * cfg::tileK) - * sizeof(itype_t); - RT_CHECK(vortex::dxa::set_multicast(device, kDescB, smem_stride)); - (void)b_tile_bytes; + // sequentially, one per-CTA SMEM size (rounded to the slot pitch) apart. + RT_CHECK(vortex::dxa::set_multicast(device, kDescB, smem_size)); std::cout << "load kernel module" << std::endl; RT_CHECK(vx_module_load_file(device, kernel_file, &module_));