Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions ci/baselines/synthesis/beacon/tensor.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"dut": "tcu",
"result": {
"bram": 0.0,
"build_time_s": 35,
"build_time_s": 40,
"deep_endpoints": 2294,
"deepest_paths": [
"25 tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_acc/g_pipe.g_no_reset.pipe[39] -> tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_norm_round/g_pipe.g_no_reset.pipe.d[6]",
Expand All @@ -21,17 +21,17 @@
"25 tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_acc/g_pipe.g_no_reset.pipe[39] -> tcu_unit/g_blocks_0.tcu_block/tcu_core/g_i_0.g_j_0.fedp/pipe_norm_round/g_pipe.g_no_reset.pipe.d[17]"
],
"dsp": 513,
"endpoints": 41750,
"endpoints": 41758,
"env": {
"circt": "CIRCT 892a65a01",
"device": "xcu55c-fsvh2892-2L-e",
"foldsim": "foldsim d8350bc0f",
"xlen": 32
},
"ff": 44471,
"ff": 44475,
"fix_levels": 12,
"interfaces": {},
"lut": 120216,
"lut": 117205,
"lutram": 2083,
"max_levels": 25,
"pred_fmax_mhz": 277.7,
Expand All @@ -49,15 +49,15 @@
"tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/pipe_norm_round": 25,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/shared_mul/g_mul_f8_sf_*_mul_f8/g_lane_*_pipe_ctrl": 8,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_fedp/shared_mul/pipe_seam": 13,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_pipe_fedp": 8,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/g_i_*_g_j_*_pipe_fedp": 9,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/landq_credits": 12,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/mdata_queue": 11,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/mdata_queue/g_depth_n_dp_ram": 9,
"tcu_unit/g_blocks_*_tcu_block/tcu_core/mdata_queue/pending_size": 12,
"tcu_unit/lane_dispatch/g_blocks_*_buf_out/g_eb2_stream_buffer": 10
},
"uram": 0,
"watch_endpoints": 13618,
"watch_endpoints": 13626,
"watch_levels": 8
}
},
Expand Down
23 changes: 16 additions & 7 deletions hw/rtl/tcu/VX_tcu_core.sv
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,20 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
wire [PE_M-1:0][PE_N-1:0][31:0] d_pe;

for (genvar i = 0; i < PE_M; ++i) begin : g_i
`ifdef VX_CFG_TCU_SPARSE_ENABLE
// The row's candidate selects, shared by its FEDPs (all columns).
wire [TCU_TC_K-1:0][7:0][5:0] sp_sel;
VX_tcu_sp_sel #(
.ROW_IDX (i),
.ROW_STRIDE (PE_M),
.ROW_SELS (PE_RATIO_M)
) tcu_sp_sel (
.row_sel (slice_m),
.fmt_s (fmt_s),
.vld_mask (vld_meta_block),
.sel (sp_sel)
);
`endif
for (genvar j = 0; j < PE_N; ++j) begin : g_j
// the output element (row_i, col_j) this PE computes this cycle
wire [ROW_W-1:0] row_i = ROW_W'(i) + ROW_W'(PE_M) * ROW_W'(slice_m);
Expand Down Expand Up @@ -563,16 +577,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #(

`ifdef VX_CFG_TCU_SPARSE_ENABLE
VX_tcu_sp_mux #(
.INSTANCE_ID (INSTANCE_ID),
.ROW_IDX (i),
.ROW_STRIDE (PE_M),
.ROW_SELS (PE_RATIO_M)
.INSTANCE_ID (INSTANCE_ID)
) tcu_sp_mux (
.row_sel (slice_m),
.fmt_s (fmt_s),
.sel (sp_sel),
.b_col_in1 (b_col_1),
.b_col_in2 (b_col_2),
.vld_mask (vld_meta_block),
.b_col_out (b_col_sparse)
);
for (genvar k_idx = 0; k_idx < FEDP_K; ++k_idx) begin : g_sparse_b_select
Expand Down
249 changes: 78 additions & 171 deletions hw/rtl/tcu/VX_tcu_sp_mux.sv
Original file line number Diff line number Diff line change
Expand Up @@ -15,192 +15,99 @@

`ifdef VX_CFG_TCU_SPARSE_ENABLE

//
// 2:4 structured sparsity gather mux.
//
// For a given (i, j) FEDP cell at row ROW_IDX, this module selects
// the sparse B-column elements based on the validity metadata mask.
// A cell that computes ROW_SELS rows in turn (VX_CFG_TCU_PE_RATIO) is at
// row ROW_IDX + row_sel * ROW_STRIDE.
//
// Three parallel gather datapaths (I_RATIO = 2, 4, 8) are always
// computed; the output mux selects based on fmt_s at runtime.
//
// The vld_mask input is a unified metadata slice: it comes from
// VX_tcu_sp_meta (for WMMA_SP) or from VX_tcu_tbuf (for WGMMA_SP),
// pre-muxed by VX_tcu_core. This module is format-agnostic w.r.t.
// the metadata source.
//
// 2:4 structured sparsity gather mux of one (i, j) FEDP cell.
// Output nibble p of B word k takes one of six fixed nibbles of the two
// candidate words b_col_in1[k] and b_col_in2[k] (the union over the element
// widths); VX_tcu_sp_sel decodes which, one-hot, once for the FEDP's row,
// from the metadata and fmt_s.

module VX_tcu_sp_mux import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
parameter `STRING INSTANCE_ID = "",
parameter ROW_IDX = 0,
parameter ROW_STRIDE = 1,
parameter ROW_SELS = 1
parameter `STRING INSTANCE_ID = ""
) (
input wire [`LOG2UP(ROW_SELS)-1:0] row_sel,
input wire [4:0] fmt_s,
input wire [TCU_TC_K-1:0][7:0][5:0] sel,
input wire [TCU_TC_K-1:0][31:0] b_col_in1,
input wire [TCU_TC_K-1:0][31:0] b_col_in2,
input wire [TCU_MAX_META_BLOCK_WIDTH-1:0] vld_mask,
output logic [TCU_TC_K-1:0][31:0] b_col_out
);
`UNUSED_SPARAM (INSTANCE_ID);

// Per-I_RATIO metadata row widths
localparam MRW_R1 = TCU_TC_K * 2 * 1;
localparam MRW_R2 = TCU_TC_K * 2 * 2;
localparam MRW_R4 = TCU_TC_K * 2 * 4;
localparam MRW_R8 = TCU_TC_K * 2 * 8;

// Extract metadata row slices for each I_RATIO variant
wire [MRW_R1-1:0] meta_row_r1;
wire [MRW_R2-1:0] meta_row_r2;
wire [MRW_R4-1:0] meta_row_r4;
wire [MRW_R8-1:0] meta_row_r8;

if (ROW_SELS == 1) begin : g_row_fixed
localparam USED_LO = MRW_R8 * ROW_IDX;
localparam USED_HI = MRW_R8 * (ROW_IDX + 1);
if (USED_LO > 0) begin : g_vld_mask_lo_unused
`UNUSED_VAR (vld_mask[USED_LO-1:0])
end
if (USED_HI < TCU_MAX_META_BLOCK_WIDTH) begin : g_vld_mask_hi_unused
`UNUSED_VAR (vld_mask[TCU_MAX_META_BLOCK_WIDTH-1:USED_HI])
end
`UNUSED_VAR (row_sel)
assign meta_row_r1 = vld_mask[MRW_R1 * ROW_IDX +: MRW_R1];
assign meta_row_r2 = vld_mask[MRW_R2 * ROW_IDX +: MRW_R2];
assign meta_row_r4 = vld_mask[MRW_R4 * ROW_IDX +: MRW_R4];
assign meta_row_r8 = vld_mask[MRW_R8 * ROW_IDX +: MRW_R8];
end else begin : g_row_sel
wire [31:0] row = ROW_IDX + 32'(row_sel) * ROW_STRIDE;
`UNUSED_VAR (vld_mask)
assign meta_row_r1 = vld_mask[MRW_R1 * row +: MRW_R1];
assign meta_row_r2 = vld_mask[MRW_R2 * row +: MRW_R2];
assign meta_row_r4 = vld_mask[MRW_R4 * row +: MRW_R4];
assign meta_row_r8 = vld_mask[MRW_R8 * row +: MRW_R8];
end
// For I_RATIO=1 (tf32), only the lo bits [TCU_TC_K-1:0] are used per k-step;
// the hi bits are the complement and are not needed by the mux logic.
`UNUSED_VAR (meta_row_r1[MRW_R1-1:TCU_TC_K])

// Four parallel gather outputs
wire [TCU_TC_K-1:0][31:0] b_col_r1, b_col_r2, b_col_r4, b_col_r8;

for (genvar k = 0; k < TCU_TC_K; ++k) begin : g_bmux

// ---- I_RATIO=1 path (ELT_W=32) ----
if (1) begin : g_r1
wire mask_lo = meta_row_r1[k];
assign b_col_r1[k] = mask_lo ? b_col_in1[k] : b_col_in2[k];
end

// ---- I_RATIO=2 path (ELT_W=16) ----
if (1) begin : g_r2
localparam I_R = 2;
localparam EW = 16;

wire [EW-1:0] b1_elems[I_R];
wire [EW-1:0] b2_elems[I_R];
for (genvar i = 0; i < I_R; ++i) begin : g_elems_r2
assign b1_elems[i] = b_col_in1[k][i*EW +: EW];
assign b2_elems[i] = b_col_in2[k][i*EW +: EW];
for (genvar k = 0; k < TCU_TC_K; ++k) begin : g_k
for (genvar p = 0; p < 8; ++p) begin : g_p
// Candidate c of output nibble p: a fixed nibble of in1[k] or in2[k].
wire [5:0][3:0] cand;
if (p == 0) begin : g_p0
assign cand[0] = b_col_in1[k][3:0];
assign cand[1] = b_col_in2[k][3:0];
assign cand[2] = b_col_in1[k][19:16];
assign cand[3] = b_col_in2[k][19:16];
assign cand[4] = b_col_in1[k][11:8];
assign cand[5] = b_col_in1[k][7:4];
end

wire [I_R-1:0] mask_lo = meta_row_r2[I_R * k +: I_R];
wire [I_R-1:0] mask_hi = meta_row_r2[I_R * (TCU_TC_K + k) +: I_R];
wire [2*I_R-1:0] grp_mask = {mask_hi, mask_lo};

wire [EW-1:0] sel_0 = grp_mask[0] ? b1_elems[0] :
grp_mask[1] ? b1_elems[1] :
grp_mask[2] ? b2_elems[0] : b2_elems[1];

wire [EW-1:0] sel_1 = grp_mask[3] ? b2_elems[1] :
grp_mask[2] ? b2_elems[0] :
grp_mask[1] ? b1_elems[1] : b1_elems[0];

assign b_col_r2[k] = {sel_1, sel_0};
end

// ---- I_RATIO=4 path (ELT_W=8) ----
if (1) begin : g_r4
localparam I_R = 4;
localparam EW = 8;

wire [EW-1:0] b1_elems[I_R];
wire [EW-1:0] b2_elems[I_R];
for (genvar i = 0; i < I_R; ++i) begin : g_elems_r4
assign b1_elems[i] = b_col_in1[k][i*EW +: EW];
assign b2_elems[i] = b_col_in2[k][i*EW +: EW];
if (p == 1) begin : g_p1
assign cand[0] = b_col_in1[k][7:4];
assign cand[1] = b_col_in2[k][7:4];
assign cand[2] = b_col_in1[k][23:20];
assign cand[3] = b_col_in2[k][23:20];
assign cand[4] = b_col_in1[k][15:12];
assign cand[5] = b_col_in1[k][11:8];
end

wire [I_R-1:0] grp_mask_lo = meta_row_r4[I_R * k +: I_R];
wire [I_R-1:0] grp_mask_hi = meta_row_r4[I_R * (TCU_TC_K + k) +: I_R];

wire [EW-1:0] lo_0 = grp_mask_lo[0] ? b1_elems[0] :
grp_mask_lo[1] ? b1_elems[1] :
b1_elems[2];
wire [EW-1:0] lo_1 = grp_mask_lo[3] ? b1_elems[3] :
grp_mask_lo[2] ? b1_elems[2] :
b1_elems[1];

wire [EW-1:0] hi_0 = grp_mask_hi[0] ? b2_elems[0] :
grp_mask_hi[1] ? b2_elems[1] :
b2_elems[2];
wire [EW-1:0] hi_1 = grp_mask_hi[3] ? b2_elems[3] :
grp_mask_hi[2] ? b2_elems[2] :
b2_elems[1];

assign b_col_r4[k] = {hi_1, hi_0, lo_1, lo_0};
end

// ---- I_RATIO=8 path (ELT_W=4) ----
if (1) begin : g_r8
localparam I_R = 8;
localparam EW = 4;

wire [EW-1:0] b1_elems[I_R];
wire [EW-1:0] b2_elems[I_R];
for (genvar i = 0; i < I_R; ++i) begin : g_elems_r8
assign b1_elems[i] = b_col_in1[k][i*EW +: EW];
assign b2_elems[i] = b_col_in2[k][i*EW +: EW];
if (p == 2) begin : g_p2
assign cand[0] = b_col_in1[k][11:8];
assign cand[1] = b_col_in2[k][11:8];
assign cand[2] = b_col_in1[k][27:24];
assign cand[3] = b_col_in2[k][27:24];
assign cand[4] = b_col_in1[k][19:16];
assign cand[5] = b_col_in1[k][23:20];
end
if (p == 3) begin : g_p3
assign cand[0] = b_col_in1[k][15:12];
assign cand[1] = b_col_in2[k][15:12];
assign cand[2] = b_col_in1[k][31:28];
assign cand[3] = b_col_in2[k][31:28];
assign cand[4] = b_col_in1[k][23:20];
assign cand[5] = b_col_in1[k][27:24];
end
if (p == 4) begin : g_p4
assign cand[0] = b_col_in1[k][19:16];
assign cand[1] = b_col_in2[k][19:16];
assign cand[2] = b_col_in2[k][3:0];
assign cand[3] = b_col_in1[k][3:0];
assign cand[4] = b_col_in2[k][11:8];
assign cand[5] = b_col_in2[k][7:4];
end
if (p == 5) begin : g_p5
assign cand[0] = b_col_in1[k][23:20];
assign cand[1] = b_col_in2[k][23:20];
assign cand[2] = b_col_in2[k][7:4];
assign cand[3] = b_col_in1[k][7:4];
assign cand[4] = b_col_in2[k][15:12];
assign cand[5] = b_col_in2[k][11:8];
end
if (p == 6) begin : g_p6
assign cand[0] = b_col_in1[k][27:24];
assign cand[1] = b_col_in2[k][27:24];
assign cand[2] = b_col_in2[k][11:8];
assign cand[3] = b_col_in1[k][11:8];
assign cand[4] = b_col_in2[k][19:16];
assign cand[5] = b_col_in2[k][23:20];
end
if (p == 7) begin : g_p7
assign cand[0] = b_col_in1[k][31:28];
assign cand[1] = b_col_in2[k][31:28];
assign cand[2] = b_col_in2[k][15:12];
assign cand[3] = b_col_in1[k][15:12];
assign cand[4] = b_col_in2[k][23:20];
assign cand[5] = b_col_in2[k][27:24];
end
always_comb begin
b_col_out[k][p * 4 +: 4] = '0;
for (int c = 0; c < 6; ++c) begin
b_col_out[k][p * 4 +: 4] |= cand[c] & {4{sel[k][p][c]}};
end
end

wire [I_R-1:0] grp_mask_lo = meta_row_r8[I_R * k +: I_R];
wire [I_R-1:0] grp_mask_hi = meta_row_r8[I_R * (TCU_TC_K + k) +: I_R];

wire [3:0] sg0_mask = grp_mask_lo[3:0];
wire [3:0] sg1_mask = grp_mask_lo[7:4];
wire [3:0] sg2_mask = grp_mask_hi[3:0];
wire [3:0] sg3_mask = grp_mask_hi[7:4];

wire [EW-1:0] sg0_0 = sg0_mask[0] ? b1_elems[0] : sg0_mask[1] ? b1_elems[1] : b1_elems[2];
wire [EW-1:0] sg0_1 = sg0_mask[3] ? b1_elems[3] : sg0_mask[2] ? b1_elems[2] : b1_elems[1];
wire [EW-1:0] sg1_0 = sg1_mask[0] ? b1_elems[4] : sg1_mask[1] ? b1_elems[5] : b1_elems[6];
wire [EW-1:0] sg1_1 = sg1_mask[3] ? b1_elems[7] : sg1_mask[2] ? b1_elems[6] : b1_elems[5];

wire [EW-1:0] sg2_0 = sg2_mask[0] ? b2_elems[0] : sg2_mask[1] ? b2_elems[1] : b2_elems[2];
wire [EW-1:0] sg2_1 = sg2_mask[3] ? b2_elems[3] : sg2_mask[2] ? b2_elems[2] : b2_elems[1];
wire [EW-1:0] sg3_0 = sg3_mask[0] ? b2_elems[4] : sg3_mask[1] ? b2_elems[5] : b2_elems[6];
wire [EW-1:0] sg3_1 = sg3_mask[3] ? b2_elems[7] : sg3_mask[2] ? b2_elems[6] : b2_elems[5];

assign b_col_r8[k] = {sg3_1, sg3_0, sg2_1, sg2_0, sg1_1, sg1_0, sg0_1, sg0_0};
end
end

// Output mux: select path based on format
always_comb begin
automatic int unsigned elt_width = tcu_fmt_width(fmt_s);
case (elt_width)
32: b_col_out = b_col_r1;
16: b_col_out = b_col_r2;
8: b_col_out = b_col_r4;
4: b_col_out = b_col_r8;
default: b_col_out = 'x;
endcase
end

endmodule

`endif // VX_CFG_TCU_SPARSE_ENABLE
Loading
Loading