From 3315ff8cd3d8d4238771351da1095c4278d2ebc2 Mon Sep 17 00:00:00 2001 From: jel221 Date: Tue, 17 Nov 2026 19:00:38 -0800 Subject: [PATCH] tcu: decode the 2:4 sparse selects once per FEDP row (tcu -3,021 LUT; pipe_fedp 8 -> 9 levels accepted, re-recorded with ci/logic_gate.py -b tcu --update-baseline --against-rev 7300bc849) --- ci/baselines/synthesis/beacon/tensor.json | 12 +- hw/rtl/tcu/VX_tcu_core.sv | 23 +- hw/rtl/tcu/VX_tcu_sp_mux.sv | 249 +++++++--------------- hw/rtl/tcu/VX_tcu_sp_sel.sv | 147 +++++++++++++ 4 files changed, 247 insertions(+), 184 deletions(-) create mode 100644 hw/rtl/tcu/VX_tcu_sp_sel.sv diff --git a/ci/baselines/synthesis/beacon/tensor.json b/ci/baselines/synthesis/beacon/tensor.json index 78b4e48cc7..7fe0d2e34c 100644 --- a/ci/baselines/synthesis/beacon/tensor.json +++ b/ci/baselines/synthesis/beacon/tensor.json @@ -6,7 +6,7 @@ "dut": "tcu", "result": { "bram": 0.0, - "build_time_s": 35, + "build_time_s": 40, "deep_endpoints": 2294, "deepest_paths": [ "25 tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_acc/g_pipe.g_no_reset.pipe[39] -> tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_norm_round/g_pipe.g_no_reset.pipe.d[6]", @@ -21,17 +21,17 @@ "25 tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_acc/g_pipe.g_no_reset.pipe[39] -> tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_norm_round/g_pipe.g_no_reset.pipe.d[17]" ], "dsp": 513, - "endpoints": 41750, + "endpoints": 41758, "env": { "circt": "CIRCT 892a65a01", "device": "xcu55c-fsvh2892-2L-e", "foldsim": "foldsim d8350bc0f", "xlen": 32 }, - "ff": 44471, + "ff": 44475, "fix_levels": 12, "interfaces": {}, - "lut": 120216, + "lut": 117205, "lutram": 2083, "max_levels": 25, "pred_fmax_mhz": 277.7, @@ -49,7 +49,7 @@ "tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/pipe_norm_round": 25, "tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/shared_mul/g_mul_f8_sf_*_mul_f8/g_lane_*_pipe_ctrl": 8, "tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/shared_mul/pipe_seam": 13, - "tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_pipe_fedp": 8, + "tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_pipe_fedp": 9, "tcu_unit/g_blocks_*_tcu_block/tcu_core/landq_credits": 12, "tcu_unit/g_blocks_*_tcu_block/tcu_core/mdata_queue": 11, "tcu_unit/g_blocks_*_tcu_block/tcu_core/mdata_queue/g_depth_n_dp_ram": 9, @@ -57,7 +57,7 @@ "tcu_unit/lane_dispatch/g_blocks_*_buf_out/g_eb2_stream_buffer": 10 }, "uram": 0, - "watch_endpoints": 13618, + "watch_endpoints": 13626, "watch_levels": 8 } }, diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index 1436cdd42b..7256ccdf4f 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -457,6 +457,20 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [PE_M-1:0][PE_N-1:0][31:0] d_pe; for (genvar i = 0; i < PE_M; ++i) begin : g_i + `ifdef VX_CFG_TCU_SPARSE_ENABLE + // The row's candidate selects, shared by its FEDPs (all columns). + wire [TCU_TC_K-1:0][7:0][5:0] sp_sel; + VX_tcu_sp_sel #( + .ROW_IDX (i), + .ROW_STRIDE (PE_M), + .ROW_SELS (PE_RATIO_M) + ) tcu_sp_sel ( + .row_sel (slice_m), + .fmt_s (fmt_s), + .vld_mask (vld_meta_block), + .sel (sp_sel) + ); + `endif for (genvar j = 0; j < PE_N; ++j) begin : g_j // the output element (row_i, col_j) this PE computes this cycle wire [ROW_W-1:0] row_i = ROW_W'(i) + ROW_W'(PE_M) * ROW_W'(slice_m); @@ -563,16 +577,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `ifdef VX_CFG_TCU_SPARSE_ENABLE VX_tcu_sp_mux #( - .INSTANCE_ID (INSTANCE_ID), - .ROW_IDX (i), - .ROW_STRIDE (PE_M), - .ROW_SELS (PE_RATIO_M) + .INSTANCE_ID (INSTANCE_ID) ) tcu_sp_mux ( - .row_sel (slice_m), - .fmt_s (fmt_s), + .sel (sp_sel), .b_col_in1 (b_col_1), .b_col_in2 (b_col_2), - .vld_mask (vld_meta_block), .b_col_out (b_col_sparse) ); for (genvar k_idx = 0; k_idx < FEDP_K; ++k_idx) begin : g_sparse_b_select diff --git a/hw/rtl/tcu/VX_tcu_sp_mux.sv b/hw/rtl/tcu/VX_tcu_sp_mux.sv index f4cea4ec22..1b08783bba 100644 --- a/hw/rtl/tcu/VX_tcu_sp_mux.sv +++ b/hw/rtl/tcu/VX_tcu_sp_mux.sv @@ -15,192 +15,99 @@ `ifdef VX_CFG_TCU_SPARSE_ENABLE -// -// 2:4 structured sparsity gather mux. -// -// For a given (i, j) FEDP cell at row ROW_IDX, this module selects -// the sparse B-column elements based on the validity metadata mask. -// A cell that computes ROW_SELS rows in turn (VX_CFG_TCU_PE_RATIO) is at -// row ROW_IDX + row_sel * ROW_STRIDE. -// -// Three parallel gather datapaths (I_RATIO = 2, 4, 8) are always -// computed; the output mux selects based on fmt_s at runtime. -// -// The vld_mask input is a unified metadata slice: it comes from -// VX_tcu_sp_meta (for WMMA_SP) or from VX_tcu_tbuf (for WGMMA_SP), -// pre-muxed by VX_tcu_core. This module is format-agnostic w.r.t. -// the metadata source. -// +// 2:4 structured sparsity gather mux of one (i, j) FEDP cell. +// Output nibble p of B word k takes one of six fixed nibbles of the two +// candidate words b_col_in1[k] and b_col_in2[k] (the union over the element +// widths); VX_tcu_sp_sel decodes which, one-hot, once for the FEDP's row, +// from the metadata and fmt_s. module VX_tcu_sp_mux import VX_gpu_pkg::*, VX_tcu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ROW_IDX = 0, - parameter ROW_STRIDE = 1, - parameter ROW_SELS = 1 + parameter `STRING INSTANCE_ID = "" ) ( - input wire [`LOG2UP(ROW_SELS)-1:0] row_sel, - input wire [4:0] fmt_s, + input wire [TCU_TC_K-1:0][7:0][5:0] sel, input wire [TCU_TC_K-1:0][31:0] b_col_in1, input wire [TCU_TC_K-1:0][31:0] b_col_in2, - input wire [TCU_MAX_META_BLOCK_WIDTH-1:0] vld_mask, output logic [TCU_TC_K-1:0][31:0] b_col_out ); `UNUSED_SPARAM (INSTANCE_ID); - // Per-I_RATIO metadata row widths - localparam MRW_R1 = TCU_TC_K * 2 * 1; - localparam MRW_R2 = TCU_TC_K * 2 * 2; - localparam MRW_R4 = TCU_TC_K * 2 * 4; - localparam MRW_R8 = TCU_TC_K * 2 * 8; - - // Extract metadata row slices for each I_RATIO variant - wire [MRW_R1-1:0] meta_row_r1; - wire [MRW_R2-1:0] meta_row_r2; - wire [MRW_R4-1:0] meta_row_r4; - wire [MRW_R8-1:0] meta_row_r8; - - if (ROW_SELS == 1) begin : g_row_fixed - localparam USED_LO = MRW_R8 * ROW_IDX; - localparam USED_HI = MRW_R8 * (ROW_IDX + 1); - if (USED_LO > 0) begin : g_vld_mask_lo_unused - `UNUSED_VAR (vld_mask[USED_LO-1:0]) - end - if (USED_HI < TCU_MAX_META_BLOCK_WIDTH) begin : g_vld_mask_hi_unused - `UNUSED_VAR (vld_mask[TCU_MAX_META_BLOCK_WIDTH-1:USED_HI]) - end - `UNUSED_VAR (row_sel) - assign meta_row_r1 = vld_mask[MRW_R1 * ROW_IDX +: MRW_R1]; - assign meta_row_r2 = vld_mask[MRW_R2 * ROW_IDX +: MRW_R2]; - assign meta_row_r4 = vld_mask[MRW_R4 * ROW_IDX +: MRW_R4]; - assign meta_row_r8 = vld_mask[MRW_R8 * ROW_IDX +: MRW_R8]; - end else begin : g_row_sel - wire [31:0] row = ROW_IDX + 32'(row_sel) * ROW_STRIDE; - `UNUSED_VAR (vld_mask) - assign meta_row_r1 = vld_mask[MRW_R1 * row +: MRW_R1]; - assign meta_row_r2 = vld_mask[MRW_R2 * row +: MRW_R2]; - assign meta_row_r4 = vld_mask[MRW_R4 * row +: MRW_R4]; - assign meta_row_r8 = vld_mask[MRW_R8 * row +: MRW_R8]; - end - // For I_RATIO=1 (tf32), only the lo bits [TCU_TC_K-1:0] are used per k-step; - // the hi bits are the complement and are not needed by the mux logic. - `UNUSED_VAR (meta_row_r1[MRW_R1-1:TCU_TC_K]) - - // Four parallel gather outputs - wire [TCU_TC_K-1:0][31:0] b_col_r1, b_col_r2, b_col_r4, b_col_r8; - - for (genvar k = 0; k < TCU_TC_K; ++k) begin : g_bmux - - // ---- I_RATIO=1 path (ELT_W=32) ---- - if (1) begin : g_r1 - wire mask_lo = meta_row_r1[k]; - assign b_col_r1[k] = mask_lo ? b_col_in1[k] : b_col_in2[k]; - end - - // ---- I_RATIO=2 path (ELT_W=16) ---- - if (1) begin : g_r2 - localparam I_R = 2; - localparam EW = 16; - - wire [EW-1:0] b1_elems[I_R]; - wire [EW-1:0] b2_elems[I_R]; - for (genvar i = 0; i < I_R; ++i) begin : g_elems_r2 - assign b1_elems[i] = b_col_in1[k][i*EW +: EW]; - assign b2_elems[i] = b_col_in2[k][i*EW +: EW]; + for (genvar k = 0; k < TCU_TC_K; ++k) begin : g_k + for (genvar p = 0; p < 8; ++p) begin : g_p + // Candidate c of output nibble p: a fixed nibble of in1[k] or in2[k]. + wire [5:0][3:0] cand; + if (p == 0) begin : g_p0 + assign cand[0] = b_col_in1[k][3:0]; + assign cand[1] = b_col_in2[k][3:0]; + assign cand[2] = b_col_in1[k][19:16]; + assign cand[3] = b_col_in2[k][19:16]; + assign cand[4] = b_col_in1[k][11:8]; + assign cand[5] = b_col_in1[k][7:4]; end - - wire [I_R-1:0] mask_lo = meta_row_r2[I_R * k +: I_R]; - wire [I_R-1:0] mask_hi = meta_row_r2[I_R * (TCU_TC_K + k) +: I_R]; - wire [2*I_R-1:0] grp_mask = {mask_hi, mask_lo}; - - wire [EW-1:0] sel_0 = grp_mask[0] ? b1_elems[0] : - grp_mask[1] ? b1_elems[1] : - grp_mask[2] ? b2_elems[0] : b2_elems[1]; - - wire [EW-1:0] sel_1 = grp_mask[3] ? b2_elems[1] : - grp_mask[2] ? b2_elems[0] : - grp_mask[1] ? b1_elems[1] : b1_elems[0]; - - assign b_col_r2[k] = {sel_1, sel_0}; - end - - // ---- I_RATIO=4 path (ELT_W=8) ---- - if (1) begin : g_r4 - localparam I_R = 4; - localparam EW = 8; - - wire [EW-1:0] b1_elems[I_R]; - wire [EW-1:0] b2_elems[I_R]; - for (genvar i = 0; i < I_R; ++i) begin : g_elems_r4 - assign b1_elems[i] = b_col_in1[k][i*EW +: EW]; - assign b2_elems[i] = b_col_in2[k][i*EW +: EW]; + if (p == 1) begin : g_p1 + assign cand[0] = b_col_in1[k][7:4]; + assign cand[1] = b_col_in2[k][7:4]; + assign cand[2] = b_col_in1[k][23:20]; + assign cand[3] = b_col_in2[k][23:20]; + assign cand[4] = b_col_in1[k][15:12]; + assign cand[5] = b_col_in1[k][11:8]; end - - wire [I_R-1:0] grp_mask_lo = meta_row_r4[I_R * k +: I_R]; - wire [I_R-1:0] grp_mask_hi = meta_row_r4[I_R * (TCU_TC_K + k) +: I_R]; - - wire [EW-1:0] lo_0 = grp_mask_lo[0] ? b1_elems[0] : - grp_mask_lo[1] ? b1_elems[1] : - b1_elems[2]; - wire [EW-1:0] lo_1 = grp_mask_lo[3] ? b1_elems[3] : - grp_mask_lo[2] ? b1_elems[2] : - b1_elems[1]; - - wire [EW-1:0] hi_0 = grp_mask_hi[0] ? b2_elems[0] : - grp_mask_hi[1] ? b2_elems[1] : - b2_elems[2]; - wire [EW-1:0] hi_1 = grp_mask_hi[3] ? b2_elems[3] : - grp_mask_hi[2] ? b2_elems[2] : - b2_elems[1]; - - assign b_col_r4[k] = {hi_1, hi_0, lo_1, lo_0}; - end - - // ---- I_RATIO=8 path (ELT_W=4) ---- - if (1) begin : g_r8 - localparam I_R = 8; - localparam EW = 4; - - wire [EW-1:0] b1_elems[I_R]; - wire [EW-1:0] b2_elems[I_R]; - for (genvar i = 0; i < I_R; ++i) begin : g_elems_r8 - assign b1_elems[i] = b_col_in1[k][i*EW +: EW]; - assign b2_elems[i] = b_col_in2[k][i*EW +: EW]; + if (p == 2) begin : g_p2 + assign cand[0] = b_col_in1[k][11:8]; + assign cand[1] = b_col_in2[k][11:8]; + assign cand[2] = b_col_in1[k][27:24]; + assign cand[3] = b_col_in2[k][27:24]; + assign cand[4] = b_col_in1[k][19:16]; + assign cand[5] = b_col_in1[k][23:20]; + end + if (p == 3) begin : g_p3 + assign cand[0] = b_col_in1[k][15:12]; + assign cand[1] = b_col_in2[k][15:12]; + assign cand[2] = b_col_in1[k][31:28]; + assign cand[3] = b_col_in2[k][31:28]; + assign cand[4] = b_col_in1[k][23:20]; + assign cand[5] = b_col_in1[k][27:24]; + end + if (p == 4) begin : g_p4 + assign cand[0] = b_col_in1[k][19:16]; + assign cand[1] = b_col_in2[k][19:16]; + assign cand[2] = b_col_in2[k][3:0]; + assign cand[3] = b_col_in1[k][3:0]; + assign cand[4] = b_col_in2[k][11:8]; + assign cand[5] = b_col_in2[k][7:4]; + end + if (p == 5) begin : g_p5 + assign cand[0] = b_col_in1[k][23:20]; + assign cand[1] = b_col_in2[k][23:20]; + assign cand[2] = b_col_in2[k][7:4]; + assign cand[3] = b_col_in1[k][7:4]; + assign cand[4] = b_col_in2[k][15:12]; + assign cand[5] = b_col_in2[k][11:8]; + end + if (p == 6) begin : g_p6 + assign cand[0] = b_col_in1[k][27:24]; + assign cand[1] = b_col_in2[k][27:24]; + assign cand[2] = b_col_in2[k][11:8]; + assign cand[3] = b_col_in1[k][11:8]; + assign cand[4] = b_col_in2[k][19:16]; + assign cand[5] = b_col_in2[k][23:20]; + end + if (p == 7) begin : g_p7 + assign cand[0] = b_col_in1[k][31:28]; + assign cand[1] = b_col_in2[k][31:28]; + assign cand[2] = b_col_in2[k][15:12]; + assign cand[3] = b_col_in1[k][15:12]; + assign cand[4] = b_col_in2[k][23:20]; + assign cand[5] = b_col_in2[k][27:24]; + end + always_comb begin + b_col_out[k][p * 4 +: 4] = '0; + for (int c = 0; c < 6; ++c) begin + b_col_out[k][p * 4 +: 4] |= cand[c] & {4{sel[k][p][c]}}; + end end - - wire [I_R-1:0] grp_mask_lo = meta_row_r8[I_R * k +: I_R]; - wire [I_R-1:0] grp_mask_hi = meta_row_r8[I_R * (TCU_TC_K + k) +: I_R]; - - wire [3:0] sg0_mask = grp_mask_lo[3:0]; - wire [3:0] sg1_mask = grp_mask_lo[7:4]; - wire [3:0] sg2_mask = grp_mask_hi[3:0]; - wire [3:0] sg3_mask = grp_mask_hi[7:4]; - - wire [EW-1:0] sg0_0 = sg0_mask[0] ? b1_elems[0] : sg0_mask[1] ? b1_elems[1] : b1_elems[2]; - wire [EW-1:0] sg0_1 = sg0_mask[3] ? b1_elems[3] : sg0_mask[2] ? b1_elems[2] : b1_elems[1]; - wire [EW-1:0] sg1_0 = sg1_mask[0] ? b1_elems[4] : sg1_mask[1] ? b1_elems[5] : b1_elems[6]; - wire [EW-1:0] sg1_1 = sg1_mask[3] ? b1_elems[7] : sg1_mask[2] ? b1_elems[6] : b1_elems[5]; - - wire [EW-1:0] sg2_0 = sg2_mask[0] ? b2_elems[0] : sg2_mask[1] ? b2_elems[1] : b2_elems[2]; - wire [EW-1:0] sg2_1 = sg2_mask[3] ? b2_elems[3] : sg2_mask[2] ? b2_elems[2] : b2_elems[1]; - wire [EW-1:0] sg3_0 = sg3_mask[0] ? b2_elems[4] : sg3_mask[1] ? b2_elems[5] : b2_elems[6]; - wire [EW-1:0] sg3_1 = sg3_mask[3] ? b2_elems[7] : sg3_mask[2] ? b2_elems[6] : b2_elems[5]; - - assign b_col_r8[k] = {sg3_1, sg3_0, sg2_1, sg2_0, sg1_1, sg1_0, sg0_1, sg0_0}; end end - // Output mux: select path based on format - always_comb begin - automatic int unsigned elt_width = tcu_fmt_width(fmt_s); - case (elt_width) - 32: b_col_out = b_col_r1; - 16: b_col_out = b_col_r2; - 8: b_col_out = b_col_r4; - 4: b_col_out = b_col_r8; - default: b_col_out = 'x; - endcase - end - endmodule `endif // VX_CFG_TCU_SPARSE_ENABLE diff --git a/hw/rtl/tcu/VX_tcu_sp_sel.sv b/hw/rtl/tcu/VX_tcu_sp_sel.sv new file mode 100644 index 0000000000..a5b28dd64f --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_sp_sel.sv @@ -0,0 +1,147 @@ +// Copyright 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_TCU_SPARSE_ENABLE + +// 2:4 structured sparsity: candidate selects of one FEDP row. +// Across the element widths (32/16/8/4 bits), output nibble p of B word k is +// one of six fixed nibbles of the two candidate words b_col_in1[k] and +// b_col_in2[k] (VX_tcu_sp_mux's candidate tables). This module decodes, once +// per row, which candidate each output nibble takes, one-hot; every FEDP of +// the row (all columns) shares the result, so the per-FEDP mux is an AND-OR. +// A cell that computes ROW_SELS rows in turn (VX_CFG_TCU_PE_RATIO) is at +// row ROW_IDX + row_sel * ROW_STRIDE. + +module VX_tcu_sp_sel import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter ROW_IDX = 0, + parameter ROW_STRIDE = 1, + parameter ROW_SELS = 1 +) ( + input wire [`LOG2UP(ROW_SELS)-1:0] row_sel, + input wire [4:0] fmt_s, + input wire [TCU_MAX_META_BLOCK_WIDTH-1:0] vld_mask, + output logic [TCU_TC_K-1:0][7:0][5:0] sel +); + // Per-I_RATIO metadata row widths + localparam MRW_R1 = TCU_TC_K * 2 * 1; + localparam MRW_R2 = TCU_TC_K * 2 * 2; + localparam MRW_R4 = TCU_TC_K * 2 * 4; + localparam MRW_R8 = TCU_TC_K * 2 * 8; + + wire [MRW_R1-1:0] meta_row_r1; + wire [MRW_R2-1:0] meta_row_r2; + wire [MRW_R4-1:0] meta_row_r4; + wire [MRW_R8-1:0] meta_row_r8; + + if (ROW_SELS == 1) begin : g_row_fixed + localparam USED_LO = MRW_R8 * ROW_IDX; + localparam USED_HI = MRW_R8 * (ROW_IDX + 1); + if (USED_LO > 0) begin : g_vld_mask_lo_unused + `UNUSED_VAR (vld_mask[USED_LO-1:0]) + end + if (USED_HI < TCU_MAX_META_BLOCK_WIDTH) begin : g_vld_mask_hi_unused + `UNUSED_VAR (vld_mask[TCU_MAX_META_BLOCK_WIDTH-1:USED_HI]) + end + `UNUSED_VAR (row_sel) + assign meta_row_r1 = vld_mask[MRW_R1 * ROW_IDX +: MRW_R1]; + assign meta_row_r2 = vld_mask[MRW_R2 * ROW_IDX +: MRW_R2]; + assign meta_row_r4 = vld_mask[MRW_R4 * ROW_IDX +: MRW_R4]; + assign meta_row_r8 = vld_mask[MRW_R8 * ROW_IDX +: MRW_R8]; + end else begin : g_row_sel + wire [31:0] row = ROW_IDX + 32'(row_sel) * ROW_STRIDE; + `UNUSED_VAR (vld_mask) + assign meta_row_r1 = vld_mask[MRW_R1 * row +: MRW_R1]; + assign meta_row_r2 = vld_mask[MRW_R2 * row +: MRW_R2]; + assign meta_row_r4 = vld_mask[MRW_R4 * row +: MRW_R4]; + assign meta_row_r8 = vld_mask[MRW_R8 * row +: MRW_R8]; + end + // For I_RATIO=1 (tf32), only the lo bits [TCU_TC_K-1:0] are used per k-step; + // the hi bits are the complement and are not needed. + `UNUSED_VAR (meta_row_r1[MRW_R1-1:TCU_TC_K]) + + // Each K word's group masks, per element width. + wire [TCU_TC_K-1:0] m1; + wire [TCU_TC_K-1:0][1:0] m2lo, m2hi; + wire [TCU_TC_K-1:0][3:0] m4lo, m4hi; + wire [TCU_TC_K-1:0][7:0] m8lo, m8hi; + for (genvar k = 0; k < TCU_TC_K; ++k) begin : g_masks + assign m1[k] = meta_row_r1[k]; + assign m2lo[k] = meta_row_r2[2 * k +: 2]; + assign m2hi[k] = meta_row_r2[2 * (TCU_TC_K + k) +: 2]; + assign m4lo[k] = meta_row_r4[4 * k +: 4]; + assign m4hi[k] = meta_row_r4[4 * (TCU_TC_K + k) +: 4]; + assign m8lo[k] = meta_row_r8[8 * k +: 8]; + assign m8hi[k] = meta_row_r8[8 * (TCU_TC_K + k) +: 8]; + end + + // Per width, the first kept element of a group takes the lowest set mask + // bit and the second the highest, with fixed fallbacks for a mask with + // fewer bits. The width comes from fmt_s, ahead of the metadata: it is + // decoded apart and AND-ORs the per-width selects, so the metadata path + // sees no width case. + wire is_w32 = (tcu_fmt_width(fmt_s) == 32); + wire is_w16 = (tcu_fmt_width(fmt_s) == 16); + wire is_w8 = (tcu_fmt_width(fmt_s) == 8); + wire is_w4 = (tcu_fmt_width(fmt_s) == 4); + logic [TCU_TC_K-1:0][7:0][5:0] s32, s16, s8, s4; + always_comb begin + for (int k = 0; k < TCU_TC_K; ++k) begin + s32[k][0] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][1] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][2] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][3] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][4] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][5] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][6] = m1[k] ? 6'b000001 : 6'b000010; + s32[k][7] = m1[k] ? 6'b000001 : 6'b000010; + s16[k][0] = m2lo[k][0] ? 6'b000001 : m2lo[k][1] ? 6'b000100 : m2hi[k][0] ? 6'b000010 : 6'b001000; + s16[k][1] = m2lo[k][0] ? 6'b000001 : m2lo[k][1] ? 6'b000100 : m2hi[k][0] ? 6'b000010 : 6'b001000; + s16[k][2] = m2lo[k][0] ? 6'b000001 : m2lo[k][1] ? 6'b000100 : m2hi[k][0] ? 6'b000010 : 6'b001000; + s16[k][3] = m2lo[k][0] ? 6'b000001 : m2lo[k][1] ? 6'b000100 : m2hi[k][0] ? 6'b000010 : 6'b001000; + s16[k][4] = m2hi[k][1] ? 6'b000010 : m2hi[k][0] ? 6'b000100 : m2lo[k][1] ? 6'b000001 : 6'b001000; + s16[k][5] = m2hi[k][1] ? 6'b000010 : m2hi[k][0] ? 6'b000100 : m2lo[k][1] ? 6'b000001 : 6'b001000; + s16[k][6] = m2hi[k][1] ? 6'b000010 : m2hi[k][0] ? 6'b000100 : m2lo[k][1] ? 6'b000001 : 6'b001000; + s16[k][7] = m2hi[k][1] ? 6'b000010 : m2hi[k][0] ? 6'b000100 : m2lo[k][1] ? 6'b000001 : 6'b001000; + s8[k][0] = m4lo[k][0] ? 6'b000001 : m4lo[k][1] ? 6'b010000 : 6'b000100; + s8[k][1] = m4lo[k][0] ? 6'b000001 : m4lo[k][1] ? 6'b010000 : 6'b000100; + s8[k][2] = m4lo[k][3] ? 6'b000100 : m4lo[k][2] ? 6'b010000 : 6'b000001; + s8[k][3] = m4lo[k][3] ? 6'b000100 : m4lo[k][2] ? 6'b010000 : 6'b000001; + s8[k][4] = m4hi[k][0] ? 6'b000100 : m4hi[k][1] ? 6'b010000 : 6'b000010; + s8[k][5] = m4hi[k][0] ? 6'b000100 : m4hi[k][1] ? 6'b010000 : 6'b000010; + s8[k][6] = m4hi[k][3] ? 6'b000010 : m4hi[k][2] ? 6'b010000 : 6'b000100; + s8[k][7] = m4hi[k][3] ? 6'b000010 : m4hi[k][2] ? 6'b010000 : 6'b000100; + s4[k][0] = m8lo[k][0] ? 6'b000001 : m8lo[k][1] ? 6'b100000 : 6'b010000; + s4[k][1] = m8lo[k][3] ? 6'b010000 : m8lo[k][2] ? 6'b100000 : 6'b000001; + s4[k][2] = m8lo[k][4] ? 6'b010000 : m8lo[k][5] ? 6'b100000 : 6'b000100; + s4[k][3] = m8lo[k][7] ? 6'b000100 : m8lo[k][6] ? 6'b100000 : 6'b010000; + s4[k][4] = m8hi[k][0] ? 6'b000100 : m8hi[k][1] ? 6'b100000 : 6'b010000; + s4[k][5] = m8hi[k][3] ? 6'b010000 : m8hi[k][2] ? 6'b100000 : 6'b000100; + s4[k][6] = m8hi[k][4] ? 6'b010000 : m8hi[k][5] ? 6'b100000 : 6'b000010; + s4[k][7] = m8hi[k][7] ? 6'b000010 : m8hi[k][6] ? 6'b100000 : 6'b010000; + end + end + always_comb begin + for (int k = 0; k < TCU_TC_K; ++k) begin + for (int p = 0; p < 8; ++p) begin + sel[k][p] = ({6{is_w32}} & s32[k][p]) | ({6{is_w16}} & s16[k][p]) + | ({6{is_w8}} & s8[k][p]) | ({6{is_w4}} & s4[k][p]) + | {5'b0, ~(is_w32 | is_w16 | is_w8 | is_w4)}; + end + end + end + +endmodule + +`endif // VX_CFG_TCU_SPARSE_ENABLE