diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 686087444e..27ffaa978a 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -307,6 +307,48 @@ tests: - xrt app: dogfood args: -n1 -tbar +# The FPnew FPU backend. Its div/sqrt latency depends on the operands, so +# fdiv-special makes the lanes of a warp finish out of step; fflags-lanes checks +# the warp's flags across lanes, and fp-nanbox the 64-bit NaN-boxing. +- id: dogfood-fpnew + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_FPU_TYPE_FPNEW +# Fewer FPU lanes than threads: an instruction runs as several lane packets, +# and packets of different warps' instructions complete interleaved, so each +# must carry its flags to its own warp. +- id: dogfood-fflags-pbw-std + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -tfflags-lanes + configs: -DVX_CFG_FPU_TYPE_STD -DVX_CFG_NUM_FPU_LANES=2 + xlen: + - 32 +- id: dogfood-fflags-pbw-fpnew + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -n16 -tfflags-lanes + configs: -DVX_CFG_FPU_TYPE_FPNEW -DVX_CFG_NUM_FPU_LANES=2 + xlen: + - 32 +# NaN-boxing of single-precision operands on the DSP backend's FMA and +# div/sqrt inputs, with 64-bit FP registers. +- id: dogfood-nanbox-dsp + via: blackbox + drivers: + - rtlsim + app: dogfood + args: -tfp-nanbox + configs: -DVX_CFG_FPU_TYPE_DSP + xlen: + - 64 - id: gbar-phase-1 via: blackbox drivers: diff --git a/docs/assets/img/fpu_architecture.svg b/docs/assets/img/fpu_architecture.svg index c6981b7417..e062a29fc8 100644 --- a/docs/assets/img/fpu_architecture.svg +++ b/docs/assets/img/fpu_architecture.svg @@ -58,9 +58,8 @@ response tag → header; VX_lane_gather → commit - fflags OR-ed across the packets of one - instruction, written to the warp's fcsr - on the last packet + each packet's fflags OR-ed into the + warp's fcsr as the packet returns VX_fpu_csr_if diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index f2df789672..251da8b4be 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -56,7 +56,7 @@ wide (the SIMD width). | `VX_lane_dispatch` | maps issue slots onto FPU blocks and SIMD lanes onto FPU lanes; an instruction wider than the lane count is split into packets | | tag store (`VX_index_buffer`, `VX_CFG_FPU_QUEUE_SIZE`) | holds the instruction header while the operands are in the backend; bounds the operations in flight | | rounding-mode resolve | a request with `frm = DYN` takes the issuing warp's rounding mode from `fcsr`; `MISC` operations reuse the `frm` field as a sub-operation and are left alone | -| flag accumulation | flags are OR-ed across the packets of one instruction and written to the warp's `fcsr` on the last packet, through a registered `VX_fpu_csr_if` | +| flag write | each packet's flags are OR-ed into the warp's `fcsr` as the packet returns, through a registered `VX_fpu_csr_if`; packets of different instructions can complete interleaved, so a per-block accumulator would hand one warp's flags to another | | `VX_lane_gather` | reassembles the blocks onto the commit interface | The header carries the decoded writeback bit through the tag store rather @@ -132,10 +132,12 @@ build elaborates the F32-only RTL and carries **no** 64-bit datapath. The active format is selected **per instruction from `dst_fmt`**, never from the operand width. A single-precision value in a 64-bit lane is NaN-boxed — -upper 32 bits all ones. Every unit boxes its F32 results on the way out. The -arithmetic cores read the low 32 bits of an F32 operand; the -non-computational unit additionally checks the box and treats an unboxed F32 -operand as the canonical quiet NaN, as the specification requires. +upper 32 bits all ones. Every unit boxes its F32 results on the way out. Every +unit also checks the box of an F32 operand and treats an unboxed one as the +canonical quiet NaN, as the specification requires: the conversion and +non-computational units in place, the FMA and div/sqrt inputs of the STD and +DSP backends through `fp_unbox_s`/`fp_unbox_fmt` in `VX_fpu_pkg`, and the +FPnew wrapper in front of the PULP divider, which ignores the box. **Double precision requires `XLEN = 64`.** Operand lanes are `XLEN` wide, so there is no place to carry a 64-bit value on a 32-bit build; diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv index 2f46ddf6f2..66131f6382 100644 --- a/hw/rtl/fpu/VX_fpu_block.sv +++ b/hw/rtl/fpu/VX_fpu_block.sv @@ -13,7 +13,7 @@ `include "VX_fpu_define.vh" -// One FPU block: the FP datapath with its tag store and fflags accumulation. +// One FPU block: the FP datapath with its tag store and fflags writes. module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_LANES = 1, @@ -29,7 +29,6 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( VX_fpu_csr_if.master fpu_csr_if ); `UNUSED_SPARAM (INSTANCE_ID) - localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); wire fpu_req_valid, fpu_req_ready; @@ -219,27 +218,13 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `endif - // handle CSR update - fflags_t fpu_rsp_fflags_q; - - if (PID_BITS != 0) begin : g_fflags_pid - fflags_t fpu_rsp_fflags_r; - always @(posedge clk) begin - if (reset) begin - fpu_rsp_fflags_r <= '0; - end else if (fpu_rsp_fire) begin - fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); - end - end - assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; - end else begin : g_fflags_no_pid - assign fpu_rsp_fflags_q = fpu_rsp_fflags; - end - + // handle CSR update: fflags accrue in the CSR, so each lane packet writes + // its own flags as it returns, in whatever order the packets of different + // instructions complete. VX_fpu_csr_if fpu_csr_tmp_if(); - assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; + assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_rsp_has_fflags; `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; + assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags; VX_pipe_register #( .DATAW (1 + NW_WIDTH + $bits(fflags_t)), diff --git a/hw/rtl/fpu/VX_fpu_dsp.sv b/hw/rtl/fpu/VX_fpu_dsp.sv index 6099ee9c20..aa87aca5d2 100644 --- a/hw/rtl/fpu/VX_fpu_dsp.sv +++ b/hw/rtl/fpu/VX_fpu_dsp.sv @@ -250,9 +250,9 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .op_type (pe_op), .op_mod (pe_op_mod), .frm (pe_frm), - .dataa (pe_data_in[i][0+:32]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:32]), - .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]), + .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))), + .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))), + .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))), .result (res32), .fflags (ff32) ); @@ -510,6 +510,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_PES_DIV-1:0] div_pe_ce; assign div_pe_ce = div_pe_enable; + wire div_pe_is_s = (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_div_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign div_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -526,8 +528,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (div_pe_mask_out[i]), .fmt (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (div_pe_shared[0+:INST_FRM_BITS]), - .dataa (div_pe_data_in[i][0+:`VX_CFG_FLEN]), - .datab (div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(div_pe_data_in[i][0+:`VX_CFG_FLEN], div_pe_is_s)), + .datab (fp_unbox_fmt(div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], div_pe_is_s)), .result (div_pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (div_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) ); @@ -598,6 +600,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_PES_SQRT-1:0] sqrt_pe_ce; assign sqrt_pe_ce = sqrt_pe_enable; + wire sqrt_pe_is_s = (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_SQRT; ++i) begin : g_sqrt_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign sqrt_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -614,7 +618,7 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (sqrt_pe_mask_out[i]), .fmt (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (sqrt_pe_shared[0+:INST_FRM_BITS]), - .dataa (sqrt_pe_data_in[i][0+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(sqrt_pe_data_in[i][0+:`VX_CFG_FLEN], sqrt_pe_is_s)), .result (sqrt_pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (sqrt_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) ); diff --git a/hw/rtl/fpu/VX_fpu_fpnew.sv b/hw/rtl/fpu/VX_fpu_fpnew.sv index e6abd025c9..6d24cc9c4c 100644 --- a/hw/rtl/fpu/VX_fpu_fpnew.sv +++ b/hw/rtl/fpu/VX_fpu_fpnew.sv @@ -88,32 +88,42 @@ module VX_fpu_fpnew `endif }; - localparam fpnew_pkg::fpu_implementation_t FPU_IMPLEMENTATION = '{ + // Div/sqrt has operand-dependent latency, so it runs on its own per-lane + // instance; the remaining groups are fixed-latency and stay in lockstep. + localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_LOCKSTEP = '{ PipeRegs:'{'{`VX_CFG_FMA_LATENCY, 0, 0, 0, 0}, // ADDMUL - '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT + '{default: 0}, // DIVSQRT '{default: `VX_CFG_FNCP_LATENCY}, // NONCOMP '{default: `VX_CFG_FCVT_LATENCY}}, // CONV UnitTypes:'{'{default: fpnew_pkg::PARALLEL}, // ADDMUL - '{default: fpnew_pkg::MERGED}, // DIVSQRT + '{default: fpnew_pkg::DISABLED}, // DIVSQRT '{default: fpnew_pkg::PARALLEL}, // NONCOMP '{default: fpnew_pkg::MERGED}}, // CONV PipeConfig: fpnew_pkg::DISTRIBUTED }; - wire fpu_ready_in, fpu_valid_in; - wire fpu_ready_out, fpu_valid_out; + localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_DIVSQRT = '{ + PipeRegs:'{'{default: 0}, // ADDMUL + '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT + '{default: 0}, // NONCOMP + '{default: 0}}, // CONV + UnitTypes:'{'{default: fpnew_pkg::DISABLED}, // ADDMUL + '{default: fpnew_pkg::MERGED}, // DIVSQRT + '{default: fpnew_pkg::DISABLED}, // NONCOMP + '{default: fpnew_pkg::DISABLED}}, // CONV + PipeConfig: fpnew_pkg::DISTRIBUTED + }; - reg [TAG_WIDTH-1:0] fpu_tag_in, fpu_tag_out; + localparam LS_TAGW = TAG_WIDTH + 1 + NUM_LANES; // {tag, has_fflags, mask} + localparam DS_TAGW = TAG_WIDTH + NUM_LANES; // {tag, mask} + localparam DS_DATAW = DS_TAGW + $bits(fflags_t) + `VX_CFG_XLEN; logic [2:0][NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_operands; - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_result; - fpnew_pkg::status_t fpu_status; - fpnew_pkg::operation_e fpu_op; reg [INST_FRM_BITS-1:0] fpu_rnd; reg fpu_op_mod; - reg fpu_has_fflags, fpu_has_fflags_out; + reg fpu_has_fflags; fpnew_pkg::fp_format_e fpu_src_fmt, fpu_dst_fmt; fpnew_pkg::int_format_e fpu_int_fmt; @@ -192,22 +202,64 @@ module VX_fpu_fpnew endcase end - `UNUSED_VAR (mask_in) - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_fpnew_coreses - wire [(TAG_WIDTH+1)-1:0] fpu_tag; - wire fpu_valid_out_uq; - wire fpu_ready_in_uq; - fpnew_pkg::status_t fpu_status_uq; - `UNUSED_VAR (fpu_tag) - `UNUSED_VAR (fpu_valid_out_uq) - `UNUSED_VAR (fpu_ready_in_uq) - `UNUSED_VAR (fpu_status_uq) + wire is_divsqrt = (op_type == INST_FPU_DIV) || (op_type == INST_FPU_SQRT); + + wire ls_valid_in = valid_in && ~is_divsqrt; + wire [NUM_LANES-1:0] ls_ready_in; + wire [NUM_LANES-1:0] ls_valid_out; + wire ls_ready_out; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ls_result; + fflags_t [NUM_LANES-1:0] ls_status; + wire [NUM_LANES-1:0][LS_TAGW-1:0] ls_tag; + + wire ds_valid_in, ds_ready_in; + wire [NUM_LANES-1:0] ds_lane_valid_in, ds_lane_ready_in; + wire [NUM_LANES-1:0] ds_lane_valid_out, ds_lane_ready_out; + wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_lane_data_out; + wire ds_valid_out, ds_ready_out; + wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_data_out; + + // Each lane's div/sqrt unit is in order and sees every instruction once, so + // the k-th result of every lane belongs to the same instruction. + VX_stream_fork #( + .NUM_OUTPUTS (NUM_LANES), + .DATAW (1), + .EAGER (1) + ) ds_fork ( + .clk (clk), + .reset (reset), + .valid_in (ds_valid_in), + .data_in (1'b0), + .ready_in (ds_ready_in), + .valid_out (ds_lane_valid_in), + `UNUSED_PIN (data_out), + .ready_out (ds_lane_ready_in) + ); + + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes + wire [`VX_CFG_XLEN-1:0] lane_ds_result; + fflags_t lane_ds_status; + wire [DS_TAGW-1:0] lane_ds_tag; + wire [1:0][`VX_CFG_XLEN-1:0] ds_operands; + + for (genvar j = 0; j < 2; ++j) begin : g_ds_operands + `ifdef VX_CFG_XLEN_64 + // fpnew's div/sqrt ignores NaN-boxing: an improperly boxed single + // operand must read as the canonical NaN. Sqrt has no operand B, + // but the unit still tests it for NaN. + wire is_used = (j == 0) || (fpu_op == fpnew_pkg::DIV); + wire unboxed = is_used && (fpu_dst_fmt == fpnew_pkg::FP32) && ~(& fpu_operands[j][i][63:32]); + assign ds_operands[j] = unboxed ? 64'hFFFFFFFF7FC00000 : fpu_operands[j][i]; + `else + assign ds_operands[j] = fpu_operands[j][i]; + `endif + end fpnew_top #( .Features (FPU_FEATURES), - .Implementation (FPU_IMPLEMENTATION), - .TagType (logic[(TAG_WIDTH+1)-1:0]) - ) fpnew_core ( + .Implementation (FPU_IMPL_LOCKSTEP), + .TagType (logic[LS_TAGW-1:0]) + ) fpnew_lockstep ( .clk_i (clk), .rst_ni (~reset), .operands_i ({fpu_operands[2][i], fpu_operands[1][i], fpu_operands[0][i]}), @@ -219,43 +271,123 @@ module VX_fpu_fpnew .int_fmt_i (fpu_int_fmt), .vectorial_op_i (1'b0), .simd_mask_i (1'b1), - .tag_i ({fpu_tag_in, fpu_has_fflags}), - .in_valid_i (fpu_valid_in), - .in_ready_o (fpu_ready_in_uq), + .tag_i ({tag_in, fpu_has_fflags, mask_in}), + .in_valid_i (ls_valid_in), + .in_ready_o (ls_ready_in[i]), + .flush_i (1'b0), + .result_o (ls_result[i]), + .status_o (ls_status[i]), + .tag_o (ls_tag[i]), + .out_valid_o (ls_valid_out[i]), + .out_ready_i (ls_ready_out), + `UNUSED_PIN (busy_o) + ); + + fpnew_top #( + .Features (FPU_FEATURES), + .Implementation (FPU_IMPL_DIVSQRT), + .TagType (logic[DS_TAGW-1:0]) + ) fpnew_divsqrt ( + .clk_i (clk), + .rst_ni (~reset), + .operands_i ({fpu_operands[2][i], ds_operands}), + .rnd_mode_i (fpnew_pkg::roundmode_e'(fpu_rnd)), + .op_i (fpu_op), + .op_mod_i (fpu_op_mod), + .src_fmt_i (fpu_src_fmt), + .dst_fmt_i (fpu_dst_fmt), + .int_fmt_i (fpu_int_fmt), + .vectorial_op_i (1'b0), + .simd_mask_i (1'b1), + .tag_i ({tag_in, mask_in}), + .in_valid_i (ds_lane_valid_in[i]), + .in_ready_o (ds_lane_ready_in[i]), .flush_i (1'b0), - .result_o (fpu_result[i]), - .status_o (fpu_status_uq), - .tag_o (fpu_tag), - .out_valid_o (fpu_valid_out_uq), - .out_ready_i (fpu_ready_out), + .result_o (lane_ds_result), + .status_o (lane_ds_status), + .tag_o (lane_ds_tag), + .out_valid_o (ds_lane_valid_out[i]), + .out_ready_i (ds_lane_ready_out[i]), `UNUSED_PIN (busy_o) ); - if (i == 0) begin : g_output_0 - assign {fpu_tag_out, fpu_has_fflags_out} = fpu_tag; - assign fpu_valid_out = fpu_valid_out_uq; - assign fpu_ready_in = fpu_ready_in_uq; - assign fpu_status = fpu_status_uq; + // The div/sqrt unit also raises DZ on 0/0, which is only invalid. + fflags_t lane_ds_fflags; + always @(*) begin + lane_ds_fflags = lane_ds_status; + lane_ds_fflags.DZ = lane_ds_status.DZ && ~lane_ds_status.NV; + end + + assign ds_lane_data_out[i] = {lane_ds_tag, lane_ds_fflags, lane_ds_result}; + + if (i != 0) begin : g_unused_tags + `UNUSED_VAR (ls_tag[i]) end end - assign fpu_valid_in = valid_in; - assign ready_in = fpu_ready_in; - assign fpu_tag_in = tag_in; + `RUNTIME_ASSERT(((ls_valid_out == '0) || (ls_valid_out == '1)) && ((ls_ready_in == '0) || (ls_ready_in == '1)), + ("%t: fpnew lanes out of lockstep: valid_out=%b, ready_in=%b", $time, ls_valid_out, ls_ready_in)) + + VX_stream_join #( + .NUM_INPUTS (NUM_LANES), + .DATAW (DS_DATAW) + ) ds_join ( + .clk (clk), + .reset (reset), + .valid_in (ds_lane_valid_out), + .data_in (ds_lane_data_out), + .ready_in (ds_lane_ready_out), + .valid_out (ds_valid_out), + .data_out (ds_data_out), + .ready_out (ds_ready_out) + ); + + assign ds_valid_in = valid_in && is_divsqrt; + assign ready_in = is_divsqrt ? ds_ready_in : ls_ready_in[0]; + + // Lockstep response + wire [TAG_WIDTH-1:0] ls_tag_out; + wire ls_has_fflags; + wire [NUM_LANES-1:0] ls_mask; + fflags_t ls_fflags; + assign {ls_tag_out, ls_has_fflags, ls_mask} = ls_tag[0]; + + `FPU_MERGE_FFLAGS(ls_fflags, ls_status, ls_mask, NUM_LANES); + + // Div/sqrt response + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ds_result; + fflags_t [NUM_LANES-1:0] ds_status; + wire [NUM_LANES-1:0][DS_TAGW-1:0] ds_tag; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_ds_unpack + assign {ds_tag[i], ds_status[i], ds_result[i]} = ds_data_out[i]; + if (i != 0) begin : g_unused_tags + `UNUSED_VAR (ds_tag[i]) + end + end + + wire [TAG_WIDTH-1:0] ds_tag_out; + wire [NUM_LANES-1:0] ds_mask; + fflags_t ds_fflags; + assign {ds_tag_out, ds_mask} = ds_tag[0]; + + `FPU_MERGE_FFLAGS(ds_fflags, ds_status, ds_mask, NUM_LANES); - VX_elastic_buffer #( - .DATAW (RSP_DATAW), - .SIZE (`TO_OUT_BUF_SIZE(OUT_BUF)), - .OUT_REG (`TO_OUT_BUF_REG(OUT_BUF)) - ) rsp_buf ( + VX_stream_arb #( + .NUM_INPUTS (2), + .DATAW (RSP_DATAW), + .ARBITER ("P"), + .OUT_BUF (OUT_BUF) + ) rsp_arb ( .clk (clk), .reset (reset), - .valid_in (fpu_valid_out), - .ready_in (fpu_ready_out), - .data_in ({fpu_result, fpu_has_fflags_out, fpu_status, fpu_tag_out}), + .valid_in ({ls_valid_out[0], ds_valid_out}), + .ready_in ({ls_ready_out, ds_ready_out}), + .data_in ({{ls_result, ls_has_fflags, ls_fflags, ls_tag_out}, + {ds_result, 1'b1, ds_fflags, ds_tag_out}}), .data_out ({result, has_fflags, fflags, tag_out}), .valid_out (valid_out), - .ready_out (ready_out) + .ready_out (ready_out), + `UNUSED_PIN (sel_out) ); endmodule diff --git a/hw/rtl/fpu/VX_fpu_pkg.sv b/hw/rtl/fpu/VX_fpu_pkg.sv index f5dca05e39..f1bd50024d 100644 --- a/hw/rtl/fpu/VX_fpu_pkg.sv +++ b/hw/rtl/fpu/VX_fpu_pkg.sv @@ -40,6 +40,26 @@ package VX_fpu_pkg; `DECL_EXECUTE_T (fpu, `VX_CFG_NUM_FPU_LANES); + // A single-precision operand that is not NaN-boxed in a 64-bit FP register + // reads as the canonical NaN. + function automatic logic [31:0] fp_unbox_s(input logic [63:0] x); + if ((`VX_CFG_FLEN > 32) && ~(&x[63:32])) begin + return 32'h7FC00000; + end + return x[31:0]; + endfunction + + // The same for the operand of a unit that takes either format: only a + // single-precision operand is unboxed. + function automatic logic [`VX_CFG_FLEN-1:0] fp_unbox_fmt(input logic [`VX_CFG_FLEN-1:0] x, input logic is_s); + logic [63:0] x64; + x64 = 64'(x); + if (is_s) begin + x64[31:0] = fp_unbox_s(x64); + end + return `VX_CFG_FLEN'(x64); + endfunction + endpackage `endif // VX_FPU_PKG_VH diff --git a/hw/rtl/fpu/VX_fpu_std.sv b/hw/rtl/fpu/VX_fpu_std.sv index e003584354..dfa8dfef88 100644 --- a/hw/rtl/fpu/VX_fpu_std.sv +++ b/hw/rtl/fpu/VX_fpu_std.sv @@ -238,9 +238,9 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .op_type (pe_op), .op_mod (pe_op_mod), .frm (pe_frm), - .dataa (pe_data_in[i][0+:32]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:32]), - .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]), + .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))), + .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))), + .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))), .result (res32), .fflags (ff32) ); @@ -434,6 +434,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( assign fflags_lanes[i] = data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]; end + wire pe_is_s = (pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S); + for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_units if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad assign pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0; @@ -448,8 +450,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .mask (pe_mask_out[i]), .fmt (pe_shared[INST_FRM_BITS+:INST_FMT_BITS]), .frm (pe_shared[0+:INST_FRM_BITS]), - .dataa (pe_data_in[i][0+:`VX_CFG_FLEN]), - .datab (pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]), + .dataa (fp_unbox_fmt(pe_data_in[i][0+:`VX_CFG_FLEN], pe_is_s)), + .datab (fp_unbox_fmt(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], pe_is_s)), .is_sqrt (pe_shared[INST_FRM_BITS+INST_FMT_BITS]), .result (pe_data_out[i][0+:`VX_CFG_FLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/tests/regression/dogfood/Makefile b/tests/regression/dogfood/Makefile index 4a16b81af2..eb87ac7e15 100644 --- a/tests/regression/dogfood/Makefile +++ b/tests/regression/dogfood/Makefile @@ -9,6 +9,9 @@ SRCS := $(SRC_DIR)/main.cpp VX_SRCS := $(SRC_DIR)/kernel.cpp +HDRS := $(SRC_DIR)/testcases.h $(SRC_DIR)/common.h +VX_HDRS := $(SRC_DIR)/common.h + OPTS ?= -n64 -xbar -xgbar KERNEL_LIB := vortex2 diff --git a/tests/regression/dogfood/common.h b/tests/regression/dogfood/common.h index 6824eca1d7..67b417c249 100644 --- a/tests/regression/dogfood/common.h +++ b/tests/regression/dogfood/common.h @@ -10,4 +10,11 @@ typedef struct { uint64_t dst_addr; } kernel_arg_t; +// The fflags-lanes test masks these lanes off while their operands would raise +// a flag. +inline bool fflags_lane_masked(uint32_t lane, uint32_t i, uint32_t num_threads) { + return (num_threads >= 4) && (((i % 4) == 0 && lane == 1) || ((i % 4) == 2 && lane == 0) + || ((i % 4) == 3 && lane == 1)); +} + #endif \ No newline at end of file diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp index a5ffa38a22..bc28317fb5 100644 --- a/tests/regression/dogfood/kernel.cpp +++ b/tests/regression/dogfood/kernel.cpp @@ -255,6 +255,92 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { } } +void kernel_fdiv_special(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (float*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + auto c = __ieee754_sqrtf(a) / b; + dst_ptr[offset+i] = c; + } +} + +#ifdef VX_CFG_FLEN_64 +// Zero the upper word of a single-precision register, leaving it improperly NaN-boxed. +inline float unbox_f32(float x) { + float r; + uint64_t t; + asm volatile ("fmv.x.w %1, %2\n slli %1, %1, 32\n srli %1, %1, 32\n fmv.d.x %0, %1" : "=f"(r), "=&r"(t) : "f"(x)); + return r; +} +#endif + +// Six cases per element: sqrt, div and each FMA operand (a, b, c) take an +// improperly NaN-boxed input once, plus one case with every input boxed. +void kernel_fp_nanbox(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (float*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + auto ua = a, ub = b; + #ifdef VX_CFG_FLEN_64 + ua = unbox_f32(a); + ub = unbox_f32(b); + #endif + float c; + switch ((offset + i) % 6) { + case 0: c = __ieee754_sqrtf(ua) / b; break; + case 1: c = __ieee754_sqrtf(a) / ub; break; + case 2: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(ua), "f"(b), "f"(b)); break; + case 3: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(ub), "f"(a)); break; + case 4: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(b), "f"(ub)); break; + default: c = __ieee754_sqrtf(a) / b; break; + } + dst_ptr[offset+i] = c; + } +} + +void kernel_fflags_lanes(kernel_arg_t* __UNIFORM__ arg) { + size_t count = arg->task_size; + auto src0_ptr = (float*)arg->src0_addr; + auto src1_ptr = (float*)arg->src1_addr; + auto dst_ptr = (uint32_t*)arg->dst_addr; + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t num_threads = vx_num_threads(); + uint32_t lane = gid % num_threads; + uint32_t wid = gid / num_threads; + auto offset = gid * count; + + for (size_t i = 0; i < count; ++i) { + auto a = src0_ptr[offset+i]; + auto b = src1_ptr[offset+i]; + // keep the operands live in the lanes the branch below masks off + asm volatile ("" : "+f"(a), "+f"(b)); + if (!fflags_lane_masked(lane, i, num_threads)) { + float c; + uint32_t flags; + if ((i + wid) & 1) { + asm volatile ("csrwi fflags, 0\n fadd.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b)); + } else { + asm volatile ("csrwi fflags, 0\n fdiv.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b)); + } + dst_ptr[offset+i] = flags; + } + } +} + void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; @@ -446,6 +532,7 @@ static const PFN_Kernel sc_tests[] = { kernel_fdiv, kernel_fdiv2, kernel_fsqrt, kernel_ftoi, kernel_ftou, kernel_itof, kernel_utof, kernel_fclamp, kernel_iclamp, kernel_trigo, kernel_bar, kernel_gbar, + kernel_fdiv_special, kernel_fp_nanbox, kernel_fflags_lanes, }; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h index 29e8e819c1..bf274d1d16 100644 --- a/tests/regression/dogfood/testcases.h +++ b/tests/regression/dogfood/testcases.h @@ -4,6 +4,7 @@ #include #include #include +#include "common.h" void cleanup(); @@ -576,6 +577,185 @@ class Test_FSQRT : public ITestCase { } }; +class Test_FDIV_SPECIAL : public ITestCase { +public: + Test_FDIV_SPECIAL(TestSuite* suite) : ITestCase(suite, "fdiv-special") {} + + // Zero, infinity, NaN and negative operands take the divide and square-root + // early-out paths; the hash scatters them across the threads of a warp. + int setup(uint32_t n, void* src1, void* src2) override { + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t i = 0; i < n; ++i) { + float x = 1.0f + fround((n - i) * (1.0f/n)); + float y = fround((n + i) * (1.0f/n)); + switch ((i * 0x9E3779B1u) >> 28) { + case 0: y = 0.0f; break; + case 1: x = 0.0f; y = 0.0f; break; + case 2: x = INFINITY; break; + case 3: x = NAN; break; + case 4: x = 0.0f; break; + case 5: x = -x; break; + default: break; + } + a[i] = x; + b[i] = y; + } + return 0; + } + + int verify(uint32_t n, void* dst, const void* src1, const void* src2) override { + int errors = 0; + auto a = (float*)src1; + auto b = (float*)src2; + auto c = (float*)dst; + for (uint32_t i = 0; i < n; ++i) { + auto ref = sqrtf(a[i]) / b[i]; + bool match = std::isnan(ref) ? std::isnan(c[i]) : almost_equal(c[i], ref); + if (!match) { + std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + } + return errors; + } +}; + +class Test_FP_NANBOX : public ITestCase { +public: + Test_FP_NANBOX(TestSuite* suite) : ITestCase(suite, "fp-nanbox") {} + + int setup(uint32_t n, void* src1, void* src2) override { + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t i = 0; i < n; ++i) { + a[i] = 1.0f + fround((n - i) * (1.0f/n)); + b[i] = fround((n + i) * (1.0f/n)); + } + return 0; + } + + // With 64-bit FP registers, the kernel leaves one input of five elements in + // six improperly NaN-boxed, which must read as the canonical NaN. + int verify(uint32_t n, void* dst, const void* src1, const void* src2) override { + int errors = 0; + auto a = (float*)src1; + auto b = (float*)src2; + auto c = (float*)dst; + for (uint32_t i = 0; i < n; ++i) { + uint32_t k = i % 6; + #ifdef VX_CFG_FLEN_64 + if (k != 5) { + Float_t r{c[i]}; + if (r.i != 0x7fc00000) { + std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << std::hex << ": expected=0x7fc00000, actual=0x" << r.i << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + continue; + } + #endif + float ref; + switch (k) { + case 2: ref = fmaf(a[i], b[i], b[i]); break; + case 3: ref = fmaf(a[i], b[i], a[i]); break; + case 4: ref = fmaf(a[i], b[i], b[i]); break; + default: ref = sqrtf(a[i]) / b[i]; break; + } + if (!almost_equal(c[i], ref)) { + std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; + ++errors; + } + } + return errors; + } +}; + +class Test_FFLAGS_LANES : public ITestCase { +public: + Test_FFLAGS_LANES(TestSuite* suite) : ITestCase(suite, "fflags-lanes") {} + + enum { FL_NX = 0x01, FL_OF = 0x04, FL_DZ = 0x08, FL_NV = 0x10 }; + enum { K_NONE, K_NX, K_X1, K_X2 }; + + // A warp divides or adds by the parity of its iteration plus its index, so + // warps running side by side mix the two; X1 is DZ or OF, X2 is NV. + static uint32_t recipe(bool is_add, int kind, float* a, float* b) { + static const float div_a[] = {1.0f, 1.0f, 1.0f, 0.0f}; + static const float div_b[] = {2.0f, 3.0f, 0.0f, 0.0f}; + static const uint32_t div_f[] = {0, FL_NX, FL_DZ, FL_NV}; + static const float add_a[] = {1.0f, 1.0f, std::numeric_limits::max(), INFINITY}; + static const float add_b[] = {2.0f, 0x1p-30f, std::numeric_limits::max(), -INFINITY}; + static const uint32_t add_f[] = {0, FL_NX, FL_OF | FL_NX, FL_NV}; + *a = is_add ? add_a[kind] : div_a[kind]; + *b = is_add ? add_b[kind] : div_b[kind]; + return is_add ? add_f[kind] : div_f[kind]; + } + + // Flags raised by one lane; only non-zero lanes raise any, and a masked lane + // gets operands raising a flag no active lane raises. + uint32_t lane_flags(uint32_t k, uint32_t n, float* a, float* b) const { + uint32_t L = num_threads_; + uint32_t count = n / (num_cores_ * num_warps_ * L); + uint32_t t = k / count, i = k % count; + uint32_t lane = t % L, wid = t / L; + uint32_t sc = ((i >> 2) + wid) % 4; + bool is_add = ((i + wid) & 1) != 0; + int kind = K_NONE; + if ((sc == 0 && lane == 2) || (sc == 2 && lane == L - 1)) { + kind = K_X1; + } else if (sc == 1 && lane == 3) { + kind = K_X2; + } else if ((sc == 2 && lane == 2) || (sc == 3 && lane == 0)) { + kind = K_NX; + } + if (fflags_lane_masked(lane, i, L)) { + kind = (sc == 1) ? K_X1 : K_X2; + } + return recipe(is_add, kind, a, b); + } + + int setup(uint32_t n, void* src1, void* src2) override { + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_CORES, &num_cores_)); + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_WARPS, &num_warps_)); + RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_THREADS, &num_threads_)); + auto a = (float*)src1; + auto b = (float*)src2; + for (uint32_t k = 0; k < n; ++k) { + lane_flags(k, n, &a[k], &b[k]); + } + return 0; + } + + // fflags is per warp: every active lane reads the OR over the active lanes. + int verify(uint32_t n, void* dst, const void* /*src1*/, const void* /*src2*/) override { + int errors = 0; + auto c = (uint32_t*)dst; + uint32_t L = num_threads_; + uint32_t count = n / (num_cores_ * num_warps_ * L); + for (uint32_t k = 0; k < n; ++k) { + uint32_t t = k / count, i = k % count; + if (fflags_lane_masked(t % L, i, L)) + continue; + uint32_t ref = 0; + for (uint32_t l = 0; l < L; ++l) { + if (fflags_lane_masked(l, i, L)) + continue; + float a, b; + ref |= lane_flags(((t / L) * L + l) * count + i, n, &a, &b); + } + if (c[k] != ref) { + std::cout << "error at result #" << std::dec << k << " (lane " << (t % L) << ", iteration " << i << ")" << std::hex << ": expected fflags=0x" << ref << ", actual=0x" << c[k] << std::endl; + ++errors; + } + } + return errors; + } + + uint64_t num_cores_; + uint64_t num_warps_; + uint64_t num_threads_; +}; + class Test_FTOI : public ITestCase { public: Test_FTOI(TestSuite* suite) : ITestCase(suite, "ftoi") {} @@ -897,6 +1077,9 @@ TestSuite::TestSuite(vx_device_h device) this->add_test(new Test_TRIGO(this)); this->add_test(new Test_BAR(this)); this->add_test(new Test_GBAR(this)); + this->add_test(new Test_FDIV_SPECIAL(this)); + this->add_test(new Test_FP_NANBOX(this)); + this->add_test(new Test_FFLAGS_LANES(this)); } TestSuite::~TestSuite() {