From a99ecc7c9d98d427713d0b6c8bfe45e1ad58a2e4 Mon Sep 17 00:00:00 2001 From: tinebp Date: Thu, 19 Nov 2026 03:41:57 -0800 Subject: [PATCH] fpu/fpnew: complete a lane group only when every lane is done; fix div/sqrt boxing and 0/0 flags VX_fpu_fpnew ran one fpnew_top per lane but took valid, ready, tag and fflags from lane 0 only. Div/sqrt has operand-dependent latency (special operands finish early), so lanes finished out of step: the group was released on lane 0's handshake and the other lanes returned zeros or stale results, and fflags raised by any lane but lane 0 were lost. Aggregating the lane valids alone is not enough, because inside one fpnew_top the op groups have different latencies and can return results in a different order on different lanes. Split each lane into two fpnew instances: - a lockstep instance with ADDMUL/NONCOMP/CONV only. Their latency does not depend on the data and every lane sees the same handshake, so lane 0's handshake is exact (a runtime assertion checks it); - a div/sqrt-only instance. A VX_stream_fork hands each instruction to every lane exactly once, and a VX_stream_join completes the group when all lanes have a result. Each lane's divider is in order, so the k-th result of every lane belongs to the same instruction. VX_stream_arb merges the two response streams (replacing the response buffer), and fflags are ORed across the instruction's active lanes, with the mask carried in the tag. Two divider bugs found on the way, fixed in the wrapper (the PULP submodule is untouched): - at XLEN 64, fpnew's divider ignores NaN-boxing, so an improperly boxed single operand was used as a number. It now reads as the canonical NaN. - 0/0 raised NV and DZ together; the spec raises NV only. DZ is cleared when NV is set. Two flag and boxing defects outside the FPnew wrapper, found in review: - with fewer FPU lanes than threads, an instruction runs as lane packets and VX_fpu_block ORed their flags into one shared register until the last packet. Packets of different warps' instructions complete interleaved (the cores have different latencies, and FPnew's div/sqrt path is merged by priority), so one warp's flags went to another and were lost from their own. fcsr accrues flags by OR, so each packet now writes its own flags as it returns; the scoreboard still releases fflags readers on the last packet, which writes no later than before. - the STD and DSP backends fed F32 FMA and div/sqrt operands to the units as raw low words, so an improperly NaN-boxed operand was used as a number at FLEN 64. fp_unbox_s/fp_unbox_fmt in VX_fpu_pkg substitute the canonical NaN at those inputs; FLEN 32 builds are unchanged. New dogfood tests fdiv-special (special and normal divides mixed across lanes), fp-nanbox (64-bit only: an unboxed operand of sqrt, div and each FMA operand) and fflags-lanes (flags raised by a single non-zero lane, masked lanes must not contribute, warps mixing divides and adds). On master with FPnew (XLEN 64, 8 lanes) the first and last fail with 264 and 388 errors and the div/sqrt rows of fp-nanbox with 342; with the fix they pass, as do the full dogfood suite, rv64uf/rv64ud and rv32uf on FPnew, and the new tests on SimX and the DPI, STD and DSP backends at XLEN 32 and 64. fflags-lanes with two FPU lanes for four threads fails 7 lanes on FPnew with the shared accumulator and passes with per-packet writes (the STD backend's fixed latencies did not interleave packets in this test). CI: dogfood-fpnew runs the suite on rtlsim with FPnew at XLEN 32 and 64; dogfood-fflags-pbw-{std,fpnew} run fflags-lanes with two FPU lanes; dogfood-nanbox-dsp runs fp-nanbox on DSP at XLEN 64 (STD and FPnew are covered by the default and FPnew suites). No model_parity case builds FPnew, and the cycle change is limited to that backend. The dogfood Makefile now declares its headers, so editing testcases.h rebuilds the host binary. Beacon (core, XLEN 32, which does not build FPnew): depth, LUT and FF unchanged. Fixes #437. Co-Authored-By: Claude Opus 5.5 --- ci/testcases/regression.yaml | 42 +++++ docs/assets/img/fpu_architecture.svg | 5 +- docs/designs/floating_point_unit.md | 12 +- hw/rtl/fpu/VX_fpu_block.sv | 27 +--- hw/rtl/fpu/VX_fpu_dsp.sv | 16 +- hw/rtl/fpu/VX_fpu_fpnew.sv | 228 +++++++++++++++++++++------ hw/rtl/fpu/VX_fpu_pkg.sv | 20 +++ hw/rtl/fpu/VX_fpu_std.sv | 12 +- tests/regression/dogfood/Makefile | 3 + tests/regression/dogfood/common.h | 7 + tests/regression/dogfood/kernel.cpp | 87 ++++++++++ tests/regression/dogfood/testcases.h | 183 +++++++++++++++++++++ 12 files changed, 554 insertions(+), 88 deletions(-) diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 686087444e..27ffaa978a 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -307,6 +307,48 @@ tests: - xrt app: dogfood args: -n1 -tbar +# The FPnew FPU backend. Its div/sqrt latency depends on the operands, so +# fdiv-special makes the lanes of a warp finish out of step; fflags-lanes checks +# the warp's flags across lanes, and fp-nanbox the 64-bit NaN-boxing. +- id: dogfood-fpnew + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_FPU_TYPE_FPNEW +# Fewer FPU lanes than threads: an instruction runs as several lane packets, +# and packets of different warps' instructions complete interleaved, so each +# must carry its flags to its own warp. +- id: dogfood-fflags-pbw-std + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -tfflags-lanes + configs: -DVX_CFG_FPU_TYPE_STD -DVX_CFG_NUM_FPU_LANES=2 + xlen: + - 32 +- id: dogfood-fflags-pbw-fpnew + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -tfflags-lanes + configs: -DVX_CFG_FPU_TYPE_FPNEW -DVX_CFG_NUM_FPU_LANES=2 + xlen: + - 32 +# NaN-boxing of single-precision operands on the DSP backend's FMA and +# div/sqrt inputs, with 64-bit FP registers. +- id: dogfood-nanbox-dsp + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -tfp-nanbox + configs: -DVX_CFG_FPU_TYPE_DSP + xlen: + - 64 - id: gbar-phase-1 via: blackbox drivers: diff --git a/docs/assets/img/fpu_architecture.svg b/docs/assets/img/fpu_architecture.svg index c6981b7417..e062a29fc8 100644 --- a/docs/assets/img/fpu_architecture.svg +++ b/docs/assets/img/fpu_architecture.svg @@ -58,9 +58,8 @@ response tag → header; VX_lane_gather → commit - fflags OR-ed across the packets of one - instruction, written to the warp's fcsr - on the last packet + each packet's fflags OR-ed into the + warp's fcsr as the packet returns VX_fpu_csr_if diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index f2df789672..251da8b4be 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -56,7 +56,7 @@ wide (the SIMD width). | `VX_lane_dispatch` | maps issue slots onto FPU blocks and SIMD lanes onto FPU lanes; an instruction wider than the lane count is split into packets | | tag store (`VX_index_buffer`, `VX_CFG_FPU_QUEUE_SIZE`) | holds the instruction header while the operands are in the backend; bounds the operations in flight | | rounding-mode resolve | a request with `frm = DYN` takes the issuing warp's rounding mode from `fcsr`; `MISC` operations reuse the `frm` field as a sub-operation and are left alone | -| flag accumulation | flags are OR-ed across the packets of one instruction and written to the warp's `fcsr` on the last packet, through a registered `VX_fpu_csr_if` | +| flag write | each packet's flags are OR-ed into the warp's `fcsr` as the packet returns, through a registered `VX_fpu_csr_if`; packets of different instructions can complete interleaved, so a per-block accumulator would hand one warp's flags to another | | `VX_lane_gather` | reassembles the blocks onto the commit interface | The header carries the decoded writeback bit through the tag store rather @@ -132,10 +132,12 @@ build elaborates the F32-only RTL and carries **no** 64-bit datapath. The active format is selected **per instruction from `dst_fmt`**, never from the operand width. A single-precision value in a 64-bit lane is NaN-boxed — -upper 32 bits all ones. Every unit boxes its F32 results on the way out. The -arithmetic cores read the low 32 bits of an F32 operand; the -non-computational unit additionally checks the box and treats an unboxed F32 -operand as the canonical quiet NaN, as the specification requires. +upper 32 bits all ones. Every unit boxes its F32 results on the way out. Every +unit also checks the box of an F32 operand and treats an unboxed one as the +canonical quiet NaN, as the specification requires: the conversion and +non-computational units in place, the FMA and div/sqrt inputs of the STD and +DSP backends through `fp_unbox_s`/`fp_unbox_fmt` in `VX_fpu_pkg`, and the +FPnew wrapper in front of the PULP divider, which ignores the box. **Double precision requires `XLEN = 64`.** Operand lanes are `XLEN` wide, so there is no place to carry a 64-bit value on a 32-bit build; diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv index 2f46ddf6f2..66131f6382 100644 --- a/hw/rtl/fpu/VX_fpu_block.sv +++ b/hw/rtl/fpu/VX_fpu_block.sv @@ -13,7 +13,7 @@ `include "VX_fpu_define.vh" -// One FPU block: the FP datapath with its tag store and fflags accumulation. +// One FPU block: the FP datapath with its tag store and fflags writes. module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_LANES = 1, @@ -29,7 +29,6 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( VX_fpu_csr_if.master fpu_csr_if ); `UNUSED_SPARAM (INSTANCE_ID) - localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); wire fpu_req_valid, fpu_req_ready; @@ -219,27 +218,13 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `endif - // handle CSR update - fflags_t fpu_rsp_fflags_q; - - if (PID_BITS != 0) begin : g_fflags_pid - fflags_t fpu_rsp_fflags_r; - always @(posedge clk) begin - if (reset) begin - fpu_rsp_fflags_r <= '0; - end else if (fpu_rsp_fire) begin - fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); - end - end - assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; - end else begin : g_fflags_no_pid - assign fpu_rsp_fflags_q = fpu_rsp_fflags; - end - + // handle CSR update: fflags accrue in the CSR, so each lane packet writes + // its own flags as it returns, in whatever order the packets of different + // instructions complete. VX_fpu_csr_if fpu_csr_tmp_if(); - assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; + assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_rsp_has_fflags; `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; + assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags; VX_pipe_register #( .DATAW (1 + NW_WIDTH + $bits(fflags_t)), diff --git a/hw/rtl/fpu/VX_fpu_dsp.sv b/hw/rtl/fpu/VX_fpu_dsp.sv index 6099ee9c20..aa87aca5d2 100644 --- a/hw/rtl/fpu/VX_fpu_dsp.sv +++ b/hw/rtl/fpu/VX_fpu_dsp.sv @@ -250,9 +250,9 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .op_type (pe_op), .op_mod (pe_op_mod), .frm (pe_frm), - .dataa (pe_data_in[i][0+:32]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:32]), - .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]), + .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))), + .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))), + .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))), .result (res32), .fflags (ff32) ); @@ -510,6 +510,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_PES_DIV-1:0] div_pe_ce; assign div_pe_ce = div_pe_enable; + wire div_pe_is_s = (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_div_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign div_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -526,8 +528,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (div_pe_mask_out[i]), .fmt (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (div_pe_shared[0+:INST_FRM_BITS]), - .dataa (div_pe_data_in[i][0+:`VX_CFG_FLEN]), - .datab (div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(div_pe_data_in[i][0+:`VX_CFG_FLEN], div_pe_is_s)), + .datab (fp_unbox_fmt(div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], div_pe_is_s)), .result (div_pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (div_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) ); @@ -598,6 +600,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_PES_SQRT-1:0] sqrt_pe_ce; assign sqrt_pe_ce = sqrt_pe_enable; + wire sqrt_pe_is_s = (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_SQRT; ++i) begin : g_sqrt_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign sqrt_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -614,7 +618,7 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (sqrt_pe_mask_out[i]), .fmt (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (sqrt_pe_shared[0+:INST_FRM_BITS]), - .dataa (sqrt_pe_data_in[i][0+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(sqrt_pe_data_in[i][0+:`VX_CFG_FLEN], sqrt_pe_is_s)), .result (sqrt_pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (sqrt_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) ); diff --git a/hw/rtl/fpu/VX_fpu_fpnew.sv b/hw/rtl/fpu/VX_fpu_fpnew.sv index e6abd025c9..6d24cc9c4c 100644 --- a/hw/rtl/fpu/VX_fpu_fpnew.sv +++ b/hw/rtl/fpu/VX_fpu_fpnew.sv @@ -88,32 +88,42 @@ module VX_fpu_fpnew `endif }; - localparam fpnew_pkg::fpu_implementation_t FPU_IMPLEMENTATION = '{ + // Div/sqrt has operand-dependent latency, so it runs on its own per-lane + // instance; the remaining groups are fixed-latency and stay in lockstep. + localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_LOCKSTEP = '{ PipeRegs:'{'{`VX_CFG_FMA_LATENCY, 0, 0, 0, 0}, // ADDMUL - '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT + '{default: 0}, // DIVSQRT '{default: `VX_CFG_FNCP_LATENCY}, // NONCOMP '{default: `VX_CFG_FCVT_LATENCY}}, // CONV UnitTypes:'{'{default: fpnew_pkg::PARALLEL}, // ADDMUL - '{default: fpnew_pkg::MERGED}, // DIVSQRT + '{default: fpnew_pkg::DISABLED}, // DIVSQRT '{default: fpnew_pkg::PARALLEL}, // NONCOMP '{default: fpnew_pkg::MERGED}}, // CONV PipeConfig: fpnew_pkg::DISTRIBUTED }; - wire fpu_ready_in, fpu_valid_in; - wire fpu_ready_out, fpu_valid_out; + localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_DIVSQRT = '{ + PipeRegs:'{'{default: 0}, // ADDMUL + '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT + '{default: 0}, // NONCOMP + '{default: 0}}, // CONV + UnitTypes:'{'{default: fpnew_pkg::DISABLED}, // ADDMUL + '{default: fpnew_pkg::MERGED}, // DIVSQRT + '{default: fpnew_pkg::DISABLED}, // NONCOMP + '{default: fpnew_pkg::DISABLED}}, // CONV + PipeConfig: fpnew_pkg::DISTRIBUTED + }; - reg [TAG_WIDTH-1:0] fpu_tag_in, fpu_tag_out; + localparam LS_TAGW = TAG_WIDTH + 1 + NUM_LANES; // {tag, has_fflags, mask} + localparam DS_TAGW = TAG_WIDTH + NUM_LANES; // {tag, mask} + localparam DS_DATAW = DS_TAGW + $bits(fflags_t) + `VX_CFG_XLEN; logic [2:0][NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_operands; - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_result; - fpnew_pkg::status_t fpu_status; - fpnew_pkg::operation_e fpu_op; reg [INST_FRM_BITS-1:0] fpu_rnd; reg fpu_op_mod; - reg fpu_has_fflags, fpu_has_fflags_out; + reg fpu_has_fflags; fpnew_pkg::fp_format_e fpu_src_fmt, fpu_dst_fmt; fpnew_pkg::int_format_e fpu_int_fmt; @@ -192,22 +202,64 @@ module VX_fpu_fpnew endcase end - `UNUSED_VAR (mask_in) - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_fpnew_coreses - wire [(TAG_WIDTH+1)-1:0] fpu_tag; - wire fpu_valid_out_uq; - wire fpu_ready_in_uq; - fpnew_pkg::status_t fpu_status_uq; - `UNUSED_VAR (fpu_tag) - `UNUSED_VAR (fpu_valid_out_uq) - `UNUSED_VAR (fpu_ready_in_uq) - `UNUSED_VAR (fpu_status_uq) + wire is_divsqrt = (op_type == INST_FPU_DIV) || (op_type == INST_FPU_SQRT); + + wire ls_valid_in = valid_in && ~is_divsqrt; + wire [NUM_LANES-1:0] ls_ready_in; + wire [NUM_LANES-1:0] ls_valid_out; + wire ls_ready_out; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ls_result; + fflags_t [NUM_LANES-1:0] ls_status; + wire [NUM_LANES-1:0][LS_TAGW-1:0] ls_tag; + + wire ds_valid_in, ds_ready_in; + wire [NUM_LANES-1:0] ds_lane_valid_in, ds_lane_ready_in; + wire [NUM_LANES-1:0] ds_lane_valid_out, ds_lane_ready_out; + wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_lane_data_out; + wire ds_valid_out, ds_ready_out; + wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_data_out; + + // Each lane's div/sqrt unit is in order and sees every instruction once, so + // the k-th result of every lane belongs to the same instruction. + VX_stream_fork #( + .NUM_OUTPUTS (NUM_LANES), + .DATAW (1), + .EAGER (1) + ) ds_fork ( + .clk (clk), + .reset (reset), + .valid_in (ds_valid_in), + .data_in (1'b0), + .ready_in (ds_ready_in), + .valid_out (ds_lane_valid_in), + `UNUSED_PIN (data_out), + .ready_out (ds_lane_ready_in) + ); + + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes + wire [`VX_CFG_XLEN-1:0] lane_ds_result; + fflags_t lane_ds_status; + wire [DS_TAGW-1:0] lane_ds_tag; + wire [1:0][`VX_CFG_XLEN-1:0] ds_operands; + + for (genvar j = 0; j < 2; ++j) begin : g_ds_operands + `ifdef VX_CFG_XLEN_64 + // fpnew's div/sqrt ignores NaN-boxing: an improperly boxed single + // operand must read as the canonical NaN. Sqrt has no operand B, + // but the unit still tests it for NaN. + wire is_used = (j == 0) || (fpu_op == fpnew_pkg::DIV); + wire unboxed = is_used && (fpu_dst_fmt == fpnew_pkg::FP32) && ~(& fpu_operands[j][i][63:32]); + assign ds_operands[j] = unboxed ? 64'hFFFFFFFF7FC00000 : fpu_operands[j][i]; + `else + assign ds_operands[j] = fpu_operands[j][i]; + `endif + end fpnew_top #( .Features (FPU_FEATURES), - .Implementation (FPU_IMPLEMENTATION), - .TagType (logic[(TAG_WIDTH+1)-1:0]) - ) fpnew_core ( + .Implementation (FPU_IMPL_LOCKSTEP), + .TagType (logic[LS_TAGW-1:0]) + ) fpnew_lockstep ( .clk_i (clk), .rst_ni (~reset), .operands_i ({fpu_operands[2][i], fpu_operands[1][i], fpu_operands[0][i]}), @@ -219,43 +271,123 @@ module VX_fpu_fpnew .int_fmt_i (fpu_int_fmt), .vectorial_op_i (1'b0), .simd_mask_i (1'b1), - .tag_i ({fpu_tag_in, fpu_has_fflags}), - .in_valid_i (fpu_valid_in), - .in_ready_o (fpu_ready_in_uq), + .tag_i ({tag_in, fpu_has_fflags, mask_in}), + .in_valid_i (ls_valid_in), + .in_ready_o (ls_ready_in[i]), + .flush_i (1'b0), + .result_o (ls_result[i]), + .status_o (ls_status[i]), + .tag_o (ls_tag[i]), + .out_valid_o (ls_valid_out[i]), + .out_ready_i (ls_ready_out), + `UNUSED_PIN (busy_o) + ); + + fpnew_top #( + .Features (FPU_FEATURES), + .Implementation (FPU_IMPL_DIVSQRT), + .TagType (logic[DS_TAGW-1:0]) + ) fpnew_divsqrt ( + .clk_i (clk), + .rst_ni (~reset), + .operands_i ({fpu_operands[2][i], ds_operands}), + .rnd_mode_i (fpnew_pkg::roundmode_e'(fpu_rnd)), + .op_i (fpu_op), + .op_mod_i (fpu_op_mod), + .src_fmt_i (fpu_src_fmt), + .dst_fmt_i (fpu_dst_fmt), + .int_fmt_i (fpu_int_fmt), + .vectorial_op_i (1'b0), + .simd_mask_i (1'b1), + .tag_i ({tag_in, mask_in}), + .in_valid_i (ds_lane_valid_in[i]), + .in_ready_o (ds_lane_ready_in[i]), .flush_i (1'b0), - .result_o (fpu_result[i]), - .status_o (fpu_status_uq), - .tag_o (fpu_tag), - .out_valid_o (fpu_valid_out_uq), - .out_ready_i (fpu_ready_out), + .result_o (lane_ds_result), + .status_o (lane_ds_status), + .tag_o (lane_ds_tag), + .out_valid_o (ds_lane_valid_out[i]), + .out_ready_i (ds_lane_ready_out[i]), `UNUSED_PIN (busy_o) ); - if (i == 0) begin : g_output_0 - assign {fpu_tag_out, fpu_has_fflags_out} = fpu_tag; - assign fpu_valid_out = fpu_valid_out_uq; - assign fpu_ready_in = fpu_ready_in_uq; - assign fpu_status = fpu_status_uq; + // The div/sqrt unit also raises DZ on 0/0, which is only invalid. + fflags_t lane_ds_fflags; + always @(*) begin + lane_ds_fflags = lane_ds_status; + lane_ds_fflags.DZ = lane_ds_status.DZ && ~lane_ds_status.NV; + end + + assign ds_lane_data_out[i] = {lane_ds_tag, lane_ds_fflags, lane_ds_result}; + + if (i != 0) begin : g_unused_tags + `UNUSED_VAR (ls_tag[i]) end end - assign fpu_valid_in = valid_in; - assign ready_in = fpu_ready_in; - assign fpu_tag_in = tag_in; + `RUNTIME_ASSERT(((ls_valid_out == '0) || (ls_valid_out == '1)) && ((ls_ready_in == '0) || (ls_ready_in == '1)), + ("%t: fpnew lanes out of lockstep: valid_out=%b, ready_in=%b", $time, ls_valid_out, ls_ready_in)) + + VX_stream_join #( + .NUM_INPUTS (NUM_LANES), + .DATAW (DS_DATAW) + ) ds_join ( + .clk (clk), + .reset (reset), + .valid_in (ds_lane_valid_out), + .data_in (ds_lane_data_out), + .ready_in (ds_lane_ready_out), + .valid_out (ds_valid_out), + .data_out (ds_data_out), + .ready_out (ds_ready_out) + ); + + assign ds_valid_in = valid_in && is_divsqrt; + assign ready_in = is_divsqrt ? ds_ready_in : ls_ready_in[0]; + + // Lockstep response + wire [TAG_WIDTH-1:0] ls_tag_out; + wire ls_has_fflags; + wire [NUM_LANES-1:0] ls_mask; + fflags_t ls_fflags; + assign {ls_tag_out, ls_has_fflags, ls_mask} = ls_tag[0]; + + `FPU_MERGE_FFLAGS(ls_fflags, ls_status, ls_mask, NUM_LANES); + + // Div/sqrt response + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ds_result; + fflags_t [NUM_LANES-1:0] ds_status; + wire [NUM_LANES-1:0][DS_TAGW-1:0] ds_tag; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_ds_unpack + assign {ds_tag[i], ds_status[i], ds_result[i]} = ds_data_out[i]; + if (i != 0) begin : g_unused_tags + `UNUSED_VAR (ds_tag[i]) + end + end + + wire [TAG_WIDTH-1:0] ds_tag_out; + wire [NUM_LANES-1:0] ds_mask; + fflags_t ds_fflags; + assign {ds_tag_out, ds_mask} = ds_tag[0]; + + `FPU_MERGE_FFLAGS(ds_fflags, ds_status, ds_mask, NUM_LANES); - VX_elastic_buffer #( - .DATAW (RSP_DATAW), - .SIZE (`TO_OUT_BUF_SIZE(OUT_BUF)), - .OUT_REG (`TO_OUT_BUF_REG(OUT_BUF)) - ) rsp_buf ( + VX_stream_arb #( + .NUM_INPUTS (2), + .DATAW (RSP_DATAW), + .ARBITER ("P"), + .OUT_BUF (OUT_BUF) + ) rsp_arb ( .clk (clk), .reset (reset), - .valid_in (fpu_valid_out), - .ready_in (fpu_ready_out), - .data_in ({fpu_result, fpu_has_fflags_out, fpu_status, fpu_tag_out}), + .valid_in ({ls_valid_out[0], ds_valid_out}), + .ready_in ({ls_ready_out, ds_ready_out}), + .data_in ({{ls_result, ls_has_fflags, ls_fflags, ls_tag_out}, + {ds_result, 1'b1, ds_fflags, ds_tag_out}}), .data_out ({result, has_fflags, fflags, tag_out}), .valid_out (valid_out), - .ready_out (ready_out) + .ready_out (ready_out), + `UNUSED_PIN (sel_out) ); endmodule diff --git a/hw/rtl/fpu/VX_fpu_pkg.sv b/hw/rtl/fpu/VX_fpu_pkg.sv index f5dca05e39..f1bd50024d 100644 --- a/hw/rtl/fpu/VX_fpu_pkg.sv +++ b/hw/rtl/fpu/VX_fpu_pkg.sv @@ -40,6 +40,26 @@ package VX_fpu_pkg; `DECL_EXECUTE_T (fpu, `VX_CFG_NUM_FPU_LANES); + // A single-precision operand that is not NaN-boxed in a 64-bit FP register + // reads as the canonical NaN. + function automatic logic [31:0] fp_unbox_s(input logic [63:0] x); + if ((`VX_CFG_FLEN > 32) && ~(&x[63:32])) begin + return 32'h7FC00000; + end + return x[31:0]; + endfunction + + // The same for the operand of a unit that takes either format: only a + // single-precision operand is unboxed. + function automatic logic [`VX_CFG_FLEN-1:0] fp_unbox_fmt(input logic [`VX_CFG_FLEN-1:0] x, input logic is_s); + logic [63:0] x64; + x64 = 64'(x); + if (is_s) begin + x64[31:0] = fp_unbox_s(x64); + end + return `VX_CFG_FLEN'(x64); + endfunction + endpackage `endif // VX_FPU_PKG_VH diff --git a/hw/rtl/fpu/VX_fpu_std.sv b/hw/rtl/fpu/VX_fpu_std.sv index e003584354..dfa8dfef88 100644 --- a/hw/rtl/fpu/VX_fpu_std.sv +++ b/hw/rtl/fpu/VX_fpu_std.sv @@ -238,9 +238,9 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .op_type (pe_op), .op_mod (pe_op_mod), .frm (pe_frm), - .dataa (pe_data_in[i][0+:32]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:32]), - .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]), + .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))), + .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))), + .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))), .result (res32), .fflags (ff32) ); @@ -434,6 +434,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( assign fflags_lanes[i] = data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]; end + wire pe_is_s = (pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -448,8 +450,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (pe_mask_out[i]), .fmt (pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (pe_shared[0+:INST_FRM_BITS]), - .dataa (pe_data_in[i][0+:`VX_CFG_FLEN]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(pe_data_in[i][0+:`VX_CFG_FLEN], pe_is_s)), + .datab (fp_unbox_fmt(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], pe_is_s)), .is_sqrt (pe_shared[INST_FRM_BITS+INST_FMT_BITS]), .result (pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/tests/regression/dogfood/Makefile b/tests/regression/dogfood/Makefile index 4a16b81af2..eb87ac7e15 100644 --- a/tests/regression/dogfood/Makefile +++ b/tests/regression/dogfood/Makefile @@ -9,6 +9,9 @@ SRCS := $(SRC_DIR)/main.cpp VX_SRCS := $(SRC_DIR)/kernel.cpp +HDRS := $(SRC_DIR)/testcases.h $(SRC_DIR)/common.h +VX_HDRS := $(SRC_DIR)/common.h + OPTS ?= -n64 -xbar -xgbar KERNEL_LIB := vortex2 diff --git a/tests/regression/dogfood/common.h b/tests/regression/dogfood/common.h index 6824eca1d7..67b417c249 100644 --- a/tests/regression/dogfood/common.h +++ b/tests/regression/dogfood/common.h @@ -10,4 +10,11 @@ typedef struct { uint64_t dst_addr; } kernel_arg_t; +// The fflags-lanes test masks these lanes off while their operands would raise +// a flag. +inline bool fflags_lane_masked(uint32_t lane, uint32_t i, uint32_t num_threads) { + return (num_threads >= 4) && (((i % 4) == 0 && lane == 1) || ((i % 4) == 2 && lane == 0) + || ((i % 4) == 3 && lane == 1)); +} + #endif \ No newline at end of file diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp index a5ffa38a22..bc28317fb5 100644 --- a/tests/regression/dogfood/kernel.cpp +++ b/tests/regression/dogfood/kernel.cpp @@ -255,6 +255,92 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { } } +void kernel_fdiv_special(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (float*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + auto c = __ieee754_sqrtf(a) / b; + dst_ptr[offset+i] = c; + } +} + +#ifdef VX_CFG_FLEN_64 +// Zero the upper word of a single-precision register, leaving it improperly NaN-boxed. +inline float unbox_f32(float x) { + float r; + uint64_t t; + asm volatile ("fmv.x.w %1, %2\n slli %1, %1, 32\n srli %1, %1, 32\n fmv.d.x %0, %1" : "=f"(r), "=&r"(t) : "f"(x)); + return r; +} +#endif + +// Six cases per element: sqrt, div and each FMA operand (a, b, c) take an +// improperly NaN-boxed input once, plus one case with every input boxed. +void kernel_fp_nanbox(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (float*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + auto ua = a, ub = b; + #ifdef VX_CFG_FLEN_64 + ua = unbox_f32(a); + ub = unbox_f32(b); + #endif + float c; + switch ((offset + i) % 6) { + case 0: c = __ieee754_sqrtf(ua) / b; break; + case 1: c = __ieee754_sqrtf(a) / ub; break; + case 2: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(ua), "f"(b), "f"(b)); break; + case 3: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(ub), "f"(a)); break; + case 4: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(b), "f"(ub)); break; + default: c = __ieee754_sqrtf(a) / b; break; + } + dst_ptr[offset+i] = c; + } +} + +void kernel_fflags_lanes(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (uint32_t*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t num_threads = vx_num_threads(); + uint32_t lane = gid % num_threads; + uint32_t wid = gid / num_threads; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + // keep the operands live in the lanes the branch below masks off + asm volatile ("" : "+f"(a), "+f"(b)); + if (!fflags_lane_masked(lane, i, num_threads)) { + float c; + uint32_t flags; + if ((i + wid) & 1) { + asm volatile ("csrwi fflags, 0\n fadd.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b)); + } else { + asm volatile ("csrwi fflags, 0\n fdiv.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b)); + } + dst_ptr[offset+i] = flags; + } + } +} + void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; @@ -446,6 +532,7 @@ static const PFN_Kernel sc_tests[] = { kernel_fdiv, kernel_fdiv2, kernel_fsqrt, kernel_ftoi, kernel_ftou, kernel_itof, kernel_utof, kernel_fclamp, kernel_iclamp, kernel_trigo, kernel_bar, kernel_gbar, + kernel_fdiv_special, kernel_fp_nanbox, kernel_fflags_lanes, }; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h index 29e8e819c1..bf274d1d16 100644 --- a/tests/regression/dogfood/testcases.h +++ b/tests/regression/dogfood/testcases.h @@ -4,6 +4,7 @@ #include #include #include +#include "common.h" void cleanup(); @@ -576,6 +577,185 @@ class Test_FSQRT : public ITestCase { } }; +class Test_FDIV_SPECIAL : public ITestCase { +public: + Test_FDIV_SPECIAL(TestSuite* suite) : ITestCase(suite, "fdiv-special") {} + + // Zero, infinity, NaN and negative operands take the divide and square-root + // early-out paths; the hash scatters them across the threads of a warp. + int setup(uint32_t n, void* src1, void* src2) override { + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t i = 0; i < n; ++i) { + float x = 1.0f + fround((n - i) * (1.0f/n)); + float y = fround((n + i) * (1.0f/n)); + switch ((i * 0x9E3779B1u) >> 28) { + case 0: y = 0.0f; break; + case 1: x = 0.0f; y = 0.0f; break; + case 2: x = INFINITY; break; + case 3: x = NAN; break; + case 4: x = 0.0f; break; + case 5: x = -x; break; + default: break; + } + a[i] = x; + b[i] = y; + } + return 0; + } + + int verify(uint32_t n, void* dst, const void* src1, const void* src2) override { + int errors = 0; + auto a = (float*)src1; + auto b = (float*)src2; + auto c = (float*)dst; + for (uint32_t i = 0; i < n; ++i) { + auto ref = sqrtf(a[i]) / b[i]; + bool match = std::isnan(ref) ? std::isnan(c[i]) : almost_equal(c[i], ref); + if (!match) { + std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + } + return errors; + } +}; + +class Test_FP_NANBOX : public ITestCase { +public: + Test_FP_NANBOX(TestSuite* suite) : ITestCase(suite, "fp-nanbox") {} + + int setup(uint32_t n, void* src1, void* src2) override { + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t i = 0; i < n; ++i) { + a[i] = 1.0f + fround((n - i) * (1.0f/n)); + b[i] = fround((n + i) * (1.0f/n)); + } + return 0; + } + + // With 64-bit FP registers, the kernel leaves one input of five elements in + // six improperly NaN-boxed, which must read as the canonical NaN. + int verify(uint32_t n, void* dst, const void* src1, const void* src2) override { + int errors = 0; + auto a = (float*)src1; + auto b = (float*)src2; + auto c = (float*)dst; + for (uint32_t i = 0; i < n; ++i) { + uint32_t k = i % 6; + #ifdef VX_CFG_FLEN_64 + if (k != 5) { + Float_t r{c[i]}; + if (r.i != 0x7fc00000) { + std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << std::hex << ": expected=0x7fc00000, actual=0x" << r.i << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + continue; + } + #endif + float ref; + switch (k) { + case 2: ref = fmaf(a[i], b[i], b[i]); break; + case 3: ref = fmaf(a[i], b[i], a[i]); break; + case 4: ref = fmaf(a[i], b[i], b[i]); break; + default: ref = sqrtf(a[i]) / b[i]; break; + } + if (!almost_equal(c[i], ref)) { + std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + } + return errors; + } +}; + +class Test_FFLAGS_LANES : public ITestCase { +public: + Test_FFLAGS_LANES(TestSuite* suite) : ITestCase(suite, "fflags-lanes") {} + + enum { FL_NX = 0x01, FL_OF = 0x04, FL_DZ = 0x08, FL_NV = 0x10 }; + enum { K_NONE, K_NX, K_X1, K_X2 }; + + // A warp divides or adds by the parity of its iteration plus its index, so + // warps running side by side mix the two; X1 is DZ or OF, X2 is NV. + static uint32_t recipe(bool is_add, int kind, float* a, float* b) { + static const float div_a[] = {1.0f, 1.0f, 1.0f, 0.0f}; + static const float div_b[] = {2.0f, 3.0f, 0.0f, 0.0f}; + static const uint32_t div_f[] = {0, FL_NX, FL_DZ, FL_NV}; + static const float add_a[] = {1.0f, 1.0f, std::numeric_limits::max(), INFINITY}; + static const float add_b[] = {2.0f, 0x1p-30f, std::numeric_limits::max(), -INFINITY}; + static const uint32_t add_f[] = {0, FL_NX, FL_OF | FL_NX, FL_NV}; + *a = is_add ? add_a[kind] : div_a[kind]; + *b = is_add ? add_b[kind] : div_b[kind]; + return is_add ? add_f[kind] : div_f[kind]; + } + + // Flags raised by one lane; only non-zero lanes raise any, and a masked lane + // gets operands raising a flag no active lane raises. + uint32_t lane_flags(uint32_t k, uint32_t n, float* a, float* b) const { + uint32_t L = num_threads_; + uint32_t count = n / (num_cores_ * num_warps_ * L); + uint32_t t = k / count, i = k % count; + uint32_t lane = t % L, wid = t / L; + uint32_t sc = ((i >> 2) + wid) % 4; + bool is_add = ((i + wid) & 1) != 0; + int kind = K_NONE; + if ((sc == 0 && lane == 2) || (sc == 2 && lane == L - 1)) { + kind = K_X1; + } else if (sc == 1 && lane == 3) { + kind = K_X2; + } else if ((sc == 2 && lane == 2) || (sc == 3 && lane == 0)) { + kind = K_NX; + } + if (fflags_lane_masked(lane, i, L)) { + kind = (sc == 1) ? K_X1 : K_X2; + } + return recipe(is_add, kind, a, b); + } + + int setup(uint32_t n, void* src1, void* src2) override { + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_CORES, &num_cores_)); + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_WARPS, &num_warps_)); + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_THREADS, &num_threads_)); + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t k = 0; k < n; ++k) { + lane_flags(k, n, &a[k], &b[k]); + } + return 0; + } + + // fflags is per warp: every active lane reads the OR over the active lanes. + int verify(uint32_t n, void* dst, const void* /*src1*/, const void* /*src2*/) override { + int errors = 0; + auto c = (uint32_t*)dst; + uint32_t L = num_threads_; + uint32_t count = n / (num_cores_ * num_warps_ * L); + for (uint32_t k = 0; k < n; ++k) { + uint32_t t = k / count, i = k % count; + if (fflags_lane_masked(t % L, i, L)) + continue; + uint32_t ref = 0; + for (uint32_t l = 0; l < L; ++l) { + if (fflags_lane_masked(l, i, L)) + continue; + float a, b; + ref |= lane_flags(((t / L) * L + l) * count + i, n, &a, &b); + } + if (c[k] != ref) { + std::cout << "error at result #" << std::dec << k << " (lane " << (t % L) << ", iteration " << i << ")" << std::hex << ": expected fflags=0x" << ref << ", actual=0x" << c[k] << std::endl; + ++errors; + } + } + return errors; + } + + uint64_t num_cores_; + uint64_t num_warps_; + uint64_t num_threads_; +}; + class Test_FTOI : public ITestCase { public: Test_FTOI(TestSuite* suite) : ITestCase(suite, "ftoi") {} @@ -897,6 +1077,9 @@ TestSuite::TestSuite(vx_device_h device) this->add_test(new Test_TRIGO(this)); this->add_test(new Test_BAR(this)); this->add_test(new Test_GBAR(this)); + this->add_test(new Test_FDIV_SPECIAL(this)); + this->add_test(new Test_FP_NANBOX(this)); + this->add_test(new Test_FFLAGS_LANES(this)); } TestSuite::~TestSuite() {