diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml
index 686087444e..27ffaa978a 100644
--- a/ci/testcases/regression.yaml
+++ b/ci/testcases/regression.yaml
@@ -307,6 +307,48 @@ tests:
- xrt
app: dogfood
args: -n1 -tbar
+# The FPnew FPU backend. Its div/sqrt latency depends on the operands, so
+# fdiv-special makes the lanes of a warp finish out of step; fflags-lanes checks
+# the warp's flags across lanes, and fp-nanbox the 64-bit NaN-boxing.
+- id: dogfood-fpnew
+ via: blackbox
+ drivers:
+ - rtlsim
+ app: dogfood
+ args: -n16 -xbar -xgbar
+ configs: -DVX_CFG_FPU_TYPE_FPNEW
+# Fewer FPU lanes than threads: an instruction runs as several lane packets,
+# and packets of different warps' instructions complete interleaved, so each
+# must carry its flags to its own warp.
+- id: dogfood-fflags-pbw-std
+ via: blackbox
+ drivers:
+ - rtlsim
+ app: dogfood
+ args: -n16 -tfflags-lanes
+ configs: -DVX_CFG_FPU_TYPE_STD -DVX_CFG_NUM_FPU_LANES=2
+ xlen:
+ - 32
+- id: dogfood-fflags-pbw-fpnew
+ via: blackbox
+ drivers:
+ - rtlsim
+ app: dogfood
+ args: -n16 -tfflags-lanes
+ configs: -DVX_CFG_FPU_TYPE_FPNEW -DVX_CFG_NUM_FPU_LANES=2
+ xlen:
+ - 32
+# NaN-boxing of single-precision operands on the DSP backend's FMA and
+# div/sqrt inputs, with 64-bit FP registers.
+- id: dogfood-nanbox-dsp
+ via: blackbox
+ drivers:
+ - rtlsim
+ app: dogfood
+ args: -tfp-nanbox
+ configs: -DVX_CFG_FPU_TYPE_DSP
+ xlen:
+ - 64
- id: gbar-phase-1
via: blackbox
drivers:
diff --git a/docs/assets/img/fpu_architecture.svg b/docs/assets/img/fpu_architecture.svg
index c6981b7417..e062a29fc8 100644
--- a/docs/assets/img/fpu_architecture.svg
+++ b/docs/assets/img/fpu_architecture.svg
@@ -58,9 +58,8 @@
response
tag → header; VX_lane_gather → commit
- fflags OR-ed across the packets of one
- instruction, written to the warp's fcsr
- on the last packet
+ each packet's fflags OR-ed into the
+ warp's fcsr as the packet returns
VX_fpu_csr_if
diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md
index f2df789672..251da8b4be 100644
--- a/docs/designs/floating_point_unit.md
+++ b/docs/designs/floating_point_unit.md
@@ -56,7 +56,7 @@ wide (the SIMD width).
| `VX_lane_dispatch` | maps issue slots onto FPU blocks and SIMD lanes onto FPU lanes; an instruction wider than the lane count is split into packets |
| tag store (`VX_index_buffer`, `VX_CFG_FPU_QUEUE_SIZE`) | holds the instruction header while the operands are in the backend; bounds the operations in flight |
| rounding-mode resolve | a request with `frm = DYN` takes the issuing warp's rounding mode from `fcsr`; `MISC` operations reuse the `frm` field as a sub-operation and are left alone |
-| flag accumulation | flags are OR-ed across the packets of one instruction and written to the warp's `fcsr` on the last packet, through a registered `VX_fpu_csr_if` |
+| flag write | each packet's flags are OR-ed into the warp's `fcsr` as the packet returns, through a registered `VX_fpu_csr_if`; packets of different instructions can complete interleaved, so a per-block accumulator would hand one warp's flags to another |
| `VX_lane_gather` | reassembles the blocks onto the commit interface |
The header carries the decoded writeback bit through the tag store rather
@@ -132,10 +132,12 @@ build elaborates the F32-only RTL and carries **no** 64-bit datapath.
The active format is selected **per instruction from `dst_fmt`**, never from
the operand width. A single-precision value in a 64-bit lane is NaN-boxed —
-upper 32 bits all ones. Every unit boxes its F32 results on the way out. The
-arithmetic cores read the low 32 bits of an F32 operand; the
-non-computational unit additionally checks the box and treats an unboxed F32
-operand as the canonical quiet NaN, as the specification requires.
+upper 32 bits all ones. Every unit boxes its F32 results on the way out. Every
+unit also checks the box of an F32 operand and treats an unboxed one as the
+canonical quiet NaN, as the specification requires: the conversion and
+non-computational units in place, the FMA and div/sqrt inputs of the STD and
+DSP backends through `fp_unbox_s`/`fp_unbox_fmt` in `VX_fpu_pkg`, and the
+FPnew wrapper in front of the PULP divider, which ignores the box.
**Double precision requires `XLEN = 64`.** Operand lanes are `XLEN` wide, so
there is no place to carry a 64-bit value on a 32-bit build;
diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv
index 2f46ddf6f2..66131f6382 100644
--- a/hw/rtl/fpu/VX_fpu_block.sv
+++ b/hw/rtl/fpu/VX_fpu_block.sv
@@ -13,7 +13,7 @@
`include "VX_fpu_define.vh"
-// One FPU block: the FP datapath with its tag store and fflags accumulation.
+// One FPU block: the FP datapath with its tag store and fflags writes.
module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
parameter `STRING INSTANCE_ID = "",
parameter NUM_LANES = 1,
@@ -29,7 +29,6 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
VX_fpu_csr_if.master fpu_csr_if
);
`UNUSED_SPARAM (INSTANCE_ID)
- localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES);
localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE);
wire fpu_req_valid, fpu_req_ready;
@@ -219,27 +218,13 @@ module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
`endif
- // handle CSR update
- fflags_t fpu_rsp_fflags_q;
-
- if (PID_BITS != 0) begin : g_fflags_pid
- fflags_t fpu_rsp_fflags_r;
- always @(posedge clk) begin
- if (reset) begin
- fpu_rsp_fflags_r <= '0;
- end else if (fpu_rsp_fire) begin
- fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags);
- end
- end
- assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags;
- end else begin : g_fflags_no_pid
- assign fpu_rsp_fflags_q = fpu_rsp_fflags;
- end
-
+ // handle CSR update: fflags accrue in the CSR, so each lane packet writes
+ // its own flags as it returns, in whatever order the packets of different
+ // instructions complete.
VX_fpu_csr_if fpu_csr_tmp_if();
- assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags;
+ assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_rsp_has_fflags;
`ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS)
- assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q;
+ assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags;
VX_pipe_register #(
.DATAW (1 + NW_WIDTH + $bits(fflags_t)),
diff --git a/hw/rtl/fpu/VX_fpu_dsp.sv b/hw/rtl/fpu/VX_fpu_dsp.sv
index 6099ee9c20..aa87aca5d2 100644
--- a/hw/rtl/fpu/VX_fpu_dsp.sv
+++ b/hw/rtl/fpu/VX_fpu_dsp.sv
@@ -250,9 +250,9 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
.op_type (pe_op),
.op_mod (pe_op_mod),
.frm (pe_frm),
- .dataa (pe_data_in[i][0+:32]),
- .datab (pe_data_in[i][`VX_CFG_XLEN+:32]),
- .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]),
+ .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))),
+ .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))),
+ .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))),
.result (res32),
.fflags (ff32)
);
@@ -510,6 +510,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
wire [NUM_PES_DIV-1:0] div_pe_ce;
assign div_pe_ce = div_pe_enable;
+ wire div_pe_is_s = (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S);
+
for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_div_units
if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad
assign div_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0;
@@ -526,8 +528,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
.mask (div_pe_mask_out[i]),
.fmt (div_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]),
.frm (div_pe_shared[0+:INST_FRM_BITS]),
- .dataa (div_pe_data_in[i][0+:`VX_CFG_FLEN]),
- .datab (div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]),
+ .dataa (fp_unbox_fmt(div_pe_data_in[i][0+:`VX_CFG_FLEN], div_pe_is_s)),
+ .datab (fp_unbox_fmt(div_pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], div_pe_is_s)),
.result (div_pe_data_out[i][0+:`VX_CFG_FLEN]),
.fflags (div_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS])
);
@@ -598,6 +600,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
wire [NUM_PES_SQRT-1:0] sqrt_pe_ce;
assign sqrt_pe_ce = sqrt_pe_enable;
+ wire sqrt_pe_is_s = (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S);
+
for (genvar i = 0; i < NUM_PES_SQRT; ++i) begin : g_sqrt_units
if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad
assign sqrt_pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0;
@@ -614,7 +618,7 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
.mask (sqrt_pe_mask_out[i]),
.fmt (sqrt_pe_shared[INST_FRM_BITS+:INST_FMT_BITS]),
.frm (sqrt_pe_shared[0+:INST_FRM_BITS]),
- .dataa (sqrt_pe_data_in[i][0+:`VX_CFG_FLEN]),
+ .dataa (fp_unbox_fmt(sqrt_pe_data_in[i][0+:`VX_CFG_FLEN], sqrt_pe_is_s)),
.result (sqrt_pe_data_out[i][0+:`VX_CFG_FLEN]),
.fflags (sqrt_pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS])
);
diff --git a/hw/rtl/fpu/VX_fpu_fpnew.sv b/hw/rtl/fpu/VX_fpu_fpnew.sv
index e6abd025c9..6d24cc9c4c 100644
--- a/hw/rtl/fpu/VX_fpu_fpnew.sv
+++ b/hw/rtl/fpu/VX_fpu_fpnew.sv
@@ -88,32 +88,42 @@ module VX_fpu_fpnew
`endif
};
- localparam fpnew_pkg::fpu_implementation_t FPU_IMPLEMENTATION = '{
+ // Div/sqrt has operand-dependent latency, so it runs on its own per-lane
+ // instance; the remaining groups are fixed-latency and stay in lockstep.
+ localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_LOCKSTEP = '{
PipeRegs:'{'{`VX_CFG_FMA_LATENCY, 0, 0, 0, 0}, // ADDMUL
- '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT
+ '{default: 0}, // DIVSQRT
'{default: `VX_CFG_FNCP_LATENCY}, // NONCOMP
'{default: `VX_CFG_FCVT_LATENCY}}, // CONV
UnitTypes:'{'{default: fpnew_pkg::PARALLEL}, // ADDMUL
- '{default: fpnew_pkg::MERGED}, // DIVSQRT
+ '{default: fpnew_pkg::DISABLED}, // DIVSQRT
'{default: fpnew_pkg::PARALLEL}, // NONCOMP
'{default: fpnew_pkg::MERGED}}, // CONV
PipeConfig: fpnew_pkg::DISTRIBUTED
};
- wire fpu_ready_in, fpu_valid_in;
- wire fpu_ready_out, fpu_valid_out;
+ localparam fpnew_pkg::fpu_implementation_t FPU_IMPL_DIVSQRT = '{
+ PipeRegs:'{'{default: 0}, // ADDMUL
+ '{default: unsigned'(LATENCY_FDIVSQRT)}, // DIVSQRT
+ '{default: 0}, // NONCOMP
+ '{default: 0}}, // CONV
+ UnitTypes:'{'{default: fpnew_pkg::DISABLED}, // ADDMUL
+ '{default: fpnew_pkg::MERGED}, // DIVSQRT
+ '{default: fpnew_pkg::DISABLED}, // NONCOMP
+ '{default: fpnew_pkg::DISABLED}}, // CONV
+ PipeConfig: fpnew_pkg::DISTRIBUTED
+ };
- reg [TAG_WIDTH-1:0] fpu_tag_in, fpu_tag_out;
+ localparam LS_TAGW = TAG_WIDTH + 1 + NUM_LANES; // {tag, has_fflags, mask}
+ localparam DS_TAGW = TAG_WIDTH + NUM_LANES; // {tag, mask}
+ localparam DS_DATAW = DS_TAGW + $bits(fflags_t) + `VX_CFG_XLEN;
logic [2:0][NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_operands;
- wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_result;
- fpnew_pkg::status_t fpu_status;
-
fpnew_pkg::operation_e fpu_op;
reg [INST_FRM_BITS-1:0] fpu_rnd;
reg fpu_op_mod;
- reg fpu_has_fflags, fpu_has_fflags_out;
+ reg fpu_has_fflags;
fpnew_pkg::fp_format_e fpu_src_fmt, fpu_dst_fmt;
fpnew_pkg::int_format_e fpu_int_fmt;
@@ -192,22 +202,64 @@ module VX_fpu_fpnew
endcase
end
- `UNUSED_VAR (mask_in)
- for (genvar i = 0; i < NUM_LANES; ++i) begin : g_fpnew_coreses
- wire [(TAG_WIDTH+1)-1:0] fpu_tag;
- wire fpu_valid_out_uq;
- wire fpu_ready_in_uq;
- fpnew_pkg::status_t fpu_status_uq;
- `UNUSED_VAR (fpu_tag)
- `UNUSED_VAR (fpu_valid_out_uq)
- `UNUSED_VAR (fpu_ready_in_uq)
- `UNUSED_VAR (fpu_status_uq)
+ wire is_divsqrt = (op_type == INST_FPU_DIV) || (op_type == INST_FPU_SQRT);
+
+ wire ls_valid_in = valid_in && ~is_divsqrt;
+ wire [NUM_LANES-1:0] ls_ready_in;
+ wire [NUM_LANES-1:0] ls_valid_out;
+ wire ls_ready_out;
+ wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ls_result;
+ fflags_t [NUM_LANES-1:0] ls_status;
+ wire [NUM_LANES-1:0][LS_TAGW-1:0] ls_tag;
+
+ wire ds_valid_in, ds_ready_in;
+ wire [NUM_LANES-1:0] ds_lane_valid_in, ds_lane_ready_in;
+ wire [NUM_LANES-1:0] ds_lane_valid_out, ds_lane_ready_out;
+ wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_lane_data_out;
+ wire ds_valid_out, ds_ready_out;
+ wire [NUM_LANES-1:0][DS_DATAW-1:0] ds_data_out;
+
+ // Each lane's div/sqrt unit is in order and sees every instruction once, so
+ // the k-th result of every lane belongs to the same instruction.
+ VX_stream_fork #(
+ .NUM_OUTPUTS (NUM_LANES),
+ .DATAW (1),
+ .EAGER (1)
+ ) ds_fork (
+ .clk (clk),
+ .reset (reset),
+ .valid_in (ds_valid_in),
+ .data_in (1'b0),
+ .ready_in (ds_ready_in),
+ .valid_out (ds_lane_valid_in),
+ `UNUSED_PIN (data_out),
+ .ready_out (ds_lane_ready_in)
+ );
+
+ for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes
+ wire [`VX_CFG_XLEN-1:0] lane_ds_result;
+ fflags_t lane_ds_status;
+ wire [DS_TAGW-1:0] lane_ds_tag;
+ wire [1:0][`VX_CFG_XLEN-1:0] ds_operands;
+
+ for (genvar j = 0; j < 2; ++j) begin : g_ds_operands
+ `ifdef VX_CFG_XLEN_64
+ // fpnew's div/sqrt ignores NaN-boxing: an improperly boxed single
+ // operand must read as the canonical NaN. Sqrt has no operand B,
+ // but the unit still tests it for NaN.
+ wire is_used = (j == 0) || (fpu_op == fpnew_pkg::DIV);
+ wire unboxed = is_used && (fpu_dst_fmt == fpnew_pkg::FP32) && ~(& fpu_operands[j][i][63:32]);
+ assign ds_operands[j] = unboxed ? 64'hFFFFFFFF7FC00000 : fpu_operands[j][i];
+ `else
+ assign ds_operands[j] = fpu_operands[j][i];
+ `endif
+ end
fpnew_top #(
.Features (FPU_FEATURES),
- .Implementation (FPU_IMPLEMENTATION),
- .TagType (logic[(TAG_WIDTH+1)-1:0])
- ) fpnew_core (
+ .Implementation (FPU_IMPL_LOCKSTEP),
+ .TagType (logic[LS_TAGW-1:0])
+ ) fpnew_lockstep (
.clk_i (clk),
.rst_ni (~reset),
.operands_i ({fpu_operands[2][i], fpu_operands[1][i], fpu_operands[0][i]}),
@@ -219,43 +271,123 @@ module VX_fpu_fpnew
.int_fmt_i (fpu_int_fmt),
.vectorial_op_i (1'b0),
.simd_mask_i (1'b1),
- .tag_i ({fpu_tag_in, fpu_has_fflags}),
- .in_valid_i (fpu_valid_in),
- .in_ready_o (fpu_ready_in_uq),
+ .tag_i ({tag_in, fpu_has_fflags, mask_in}),
+ .in_valid_i (ls_valid_in),
+ .in_ready_o (ls_ready_in[i]),
+ .flush_i (1'b0),
+ .result_o (ls_result[i]),
+ .status_o (ls_status[i]),
+ .tag_o (ls_tag[i]),
+ .out_valid_o (ls_valid_out[i]),
+ .out_ready_i (ls_ready_out),
+ `UNUSED_PIN (busy_o)
+ );
+
+ fpnew_top #(
+ .Features (FPU_FEATURES),
+ .Implementation (FPU_IMPL_DIVSQRT),
+ .TagType (logic[DS_TAGW-1:0])
+ ) fpnew_divsqrt (
+ .clk_i (clk),
+ .rst_ni (~reset),
+ .operands_i ({fpu_operands[2][i], ds_operands}),
+ .rnd_mode_i (fpnew_pkg::roundmode_e'(fpu_rnd)),
+ .op_i (fpu_op),
+ .op_mod_i (fpu_op_mod),
+ .src_fmt_i (fpu_src_fmt),
+ .dst_fmt_i (fpu_dst_fmt),
+ .int_fmt_i (fpu_int_fmt),
+ .vectorial_op_i (1'b0),
+ .simd_mask_i (1'b1),
+ .tag_i ({tag_in, mask_in}),
+ .in_valid_i (ds_lane_valid_in[i]),
+ .in_ready_o (ds_lane_ready_in[i]),
.flush_i (1'b0),
- .result_o (fpu_result[i]),
- .status_o (fpu_status_uq),
- .tag_o (fpu_tag),
- .out_valid_o (fpu_valid_out_uq),
- .out_ready_i (fpu_ready_out),
+ .result_o (lane_ds_result),
+ .status_o (lane_ds_status),
+ .tag_o (lane_ds_tag),
+ .out_valid_o (ds_lane_valid_out[i]),
+ .out_ready_i (ds_lane_ready_out[i]),
`UNUSED_PIN (busy_o)
);
- if (i == 0) begin : g_output_0
- assign {fpu_tag_out, fpu_has_fflags_out} = fpu_tag;
- assign fpu_valid_out = fpu_valid_out_uq;
- assign fpu_ready_in = fpu_ready_in_uq;
- assign fpu_status = fpu_status_uq;
+ // The div/sqrt unit also raises DZ on 0/0, which is only invalid.
+ fflags_t lane_ds_fflags;
+ always @(*) begin
+ lane_ds_fflags = lane_ds_status;
+ lane_ds_fflags.DZ = lane_ds_status.DZ && ~lane_ds_status.NV;
+ end
+
+ assign ds_lane_data_out[i] = {lane_ds_tag, lane_ds_fflags, lane_ds_result};
+
+ if (i != 0) begin : g_unused_tags
+ `UNUSED_VAR (ls_tag[i])
end
end
- assign fpu_valid_in = valid_in;
- assign ready_in = fpu_ready_in;
- assign fpu_tag_in = tag_in;
+ `RUNTIME_ASSERT(((ls_valid_out == '0) || (ls_valid_out == '1)) && ((ls_ready_in == '0) || (ls_ready_in == '1)),
+ ("%t: fpnew lanes out of lockstep: valid_out=%b, ready_in=%b", $time, ls_valid_out, ls_ready_in))
+
+ VX_stream_join #(
+ .NUM_INPUTS (NUM_LANES),
+ .DATAW (DS_DATAW)
+ ) ds_join (
+ .clk (clk),
+ .reset (reset),
+ .valid_in (ds_lane_valid_out),
+ .data_in (ds_lane_data_out),
+ .ready_in (ds_lane_ready_out),
+ .valid_out (ds_valid_out),
+ .data_out (ds_data_out),
+ .ready_out (ds_ready_out)
+ );
+
+ assign ds_valid_in = valid_in && is_divsqrt;
+ assign ready_in = is_divsqrt ? ds_ready_in : ls_ready_in[0];
+
+ // Lockstep response
+ wire [TAG_WIDTH-1:0] ls_tag_out;
+ wire ls_has_fflags;
+ wire [NUM_LANES-1:0] ls_mask;
+ fflags_t ls_fflags;
+ assign {ls_tag_out, ls_has_fflags, ls_mask} = ls_tag[0];
+
+ `FPU_MERGE_FFLAGS(ls_fflags, ls_status, ls_mask, NUM_LANES);
+
+ // Div/sqrt response
+ wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] ds_result;
+ fflags_t [NUM_LANES-1:0] ds_status;
+ wire [NUM_LANES-1:0][DS_TAGW-1:0] ds_tag;
+ for (genvar i = 0; i < NUM_LANES; ++i) begin : g_ds_unpack
+ assign {ds_tag[i], ds_status[i], ds_result[i]} = ds_data_out[i];
+ if (i != 0) begin : g_unused_tags
+ `UNUSED_VAR (ds_tag[i])
+ end
+ end
+
+ wire [TAG_WIDTH-1:0] ds_tag_out;
+ wire [NUM_LANES-1:0] ds_mask;
+ fflags_t ds_fflags;
+ assign {ds_tag_out, ds_mask} = ds_tag[0];
+
+ `FPU_MERGE_FFLAGS(ds_fflags, ds_status, ds_mask, NUM_LANES);
- VX_elastic_buffer #(
- .DATAW (RSP_DATAW),
- .SIZE (`TO_OUT_BUF_SIZE(OUT_BUF)),
- .OUT_REG (`TO_OUT_BUF_REG(OUT_BUF))
- ) rsp_buf (
+ VX_stream_arb #(
+ .NUM_INPUTS (2),
+ .DATAW (RSP_DATAW),
+ .ARBITER ("P"),
+ .OUT_BUF (OUT_BUF)
+ ) rsp_arb (
.clk (clk),
.reset (reset),
- .valid_in (fpu_valid_out),
- .ready_in (fpu_ready_out),
- .data_in ({fpu_result, fpu_has_fflags_out, fpu_status, fpu_tag_out}),
+ .valid_in ({ls_valid_out[0], ds_valid_out}),
+ .ready_in ({ls_ready_out, ds_ready_out}),
+ .data_in ({{ls_result, ls_has_fflags, ls_fflags, ls_tag_out},
+ {ds_result, 1'b1, ds_fflags, ds_tag_out}}),
.data_out ({result, has_fflags, fflags, tag_out}),
.valid_out (valid_out),
- .ready_out (ready_out)
+ .ready_out (ready_out),
+ `UNUSED_PIN (sel_out)
);
endmodule
diff --git a/hw/rtl/fpu/VX_fpu_pkg.sv b/hw/rtl/fpu/VX_fpu_pkg.sv
index f5dca05e39..f1bd50024d 100644
--- a/hw/rtl/fpu/VX_fpu_pkg.sv
+++ b/hw/rtl/fpu/VX_fpu_pkg.sv
@@ -40,6 +40,26 @@ package VX_fpu_pkg;
`DECL_EXECUTE_T (fpu, `VX_CFG_NUM_FPU_LANES);
+ // A single-precision operand that is not NaN-boxed in a 64-bit FP register
+ // reads as the canonical NaN.
+ function automatic logic [31:0] fp_unbox_s(input logic [63:0] x);
+ if ((`VX_CFG_FLEN > 32) && ~(&x[63:32])) begin
+ return 32'h7FC00000;
+ end
+ return x[31:0];
+ endfunction
+
+ // The same for the operand of a unit that takes either format: only a
+ // single-precision operand is unboxed.
+ function automatic logic [`VX_CFG_FLEN-1:0] fp_unbox_fmt(input logic [`VX_CFG_FLEN-1:0] x, input logic is_s);
+ logic [63:0] x64;
+ x64 = 64'(x);
+ if (is_s) begin
+ x64[31:0] = fp_unbox_s(x64);
+ end
+ return `VX_CFG_FLEN'(x64);
+ endfunction
+
endpackage
`endif // VX_FPU_PKG_VH
diff --git a/hw/rtl/fpu/VX_fpu_std.sv b/hw/rtl/fpu/VX_fpu_std.sv
index e003584354..dfa8dfef88 100644
--- a/hw/rtl/fpu/VX_fpu_std.sv
+++ b/hw/rtl/fpu/VX_fpu_std.sv
@@ -238,9 +238,9 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
.op_type (pe_op),
.op_mod (pe_op_mod),
.frm (pe_frm),
- .dataa (pe_data_in[i][0+:32]),
- .datab (pe_data_in[i][`VX_CFG_XLEN+:32]),
- .datac (pe_data_in[i][2*`VX_CFG_XLEN+:32]),
+ .dataa (fp_unbox_s(64'(pe_data_in[i][0+:`VX_CFG_XLEN]))),
+ .datab (fp_unbox_s(64'(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_XLEN]))),
+ .datac (fp_unbox_s(64'(pe_data_in[i][2*`VX_CFG_XLEN+:`VX_CFG_XLEN]))),
.result (res32),
.fflags (ff32)
);
@@ -434,6 +434,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
assign fflags_lanes[i] = data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS];
end
+ wire pe_is_s = (pe_shared[INST_FRM_BITS+:INST_FMT_BITS] == INST_FMT_S);
+
for (genvar i = 0; i < NUM_PES_DIV; ++i) begin : g_units
if (`VX_CFG_XLEN > `VX_CFG_FLEN) begin : g_pad
assign pe_data_out[i][`VX_CFG_FLEN +: (`VX_CFG_XLEN-`VX_CFG_FLEN)] = '0;
@@ -448,8 +450,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #(
.mask (pe_mask_out[i]),
.fmt (pe_shared[INST_FRM_BITS+:INST_FMT_BITS]),
.frm (pe_shared[0+:INST_FRM_BITS]),
- .dataa (pe_data_in[i][0+:`VX_CFG_FLEN]),
- .datab (pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN]),
+ .dataa (fp_unbox_fmt(pe_data_in[i][0+:`VX_CFG_FLEN], pe_is_s)),
+ .datab (fp_unbox_fmt(pe_data_in[i][`VX_CFG_XLEN+:`VX_CFG_FLEN], pe_is_s)),
.is_sqrt (pe_shared[INST_FRM_BITS+INST_FMT_BITS]),
.result (pe_data_out[i][0+:`VX_CFG_FLEN]),
.fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS])
diff --git a/tests/regression/dogfood/Makefile b/tests/regression/dogfood/Makefile
index 4a16b81af2..eb87ac7e15 100644
--- a/tests/regression/dogfood/Makefile
+++ b/tests/regression/dogfood/Makefile
@@ -9,6 +9,9 @@ SRCS := $(SRC_DIR)/main.cpp
VX_SRCS := $(SRC_DIR)/kernel.cpp
+HDRS := $(SRC_DIR)/testcases.h $(SRC_DIR)/common.h
+VX_HDRS := $(SRC_DIR)/common.h
+
OPTS ?= -n64 -xbar -xgbar
KERNEL_LIB := vortex2
diff --git a/tests/regression/dogfood/common.h b/tests/regression/dogfood/common.h
index 6824eca1d7..67b417c249 100644
--- a/tests/regression/dogfood/common.h
+++ b/tests/regression/dogfood/common.h
@@ -10,4 +10,11 @@ typedef struct {
uint64_t dst_addr;
} kernel_arg_t;
+// The fflags-lanes test masks these lanes off while their operands would raise
+// a flag.
+inline bool fflags_lane_masked(uint32_t lane, uint32_t i, uint32_t num_threads) {
+ return (num_threads >= 4) && (((i % 4) == 0 && lane == 1) || ((i % 4) == 2 && lane == 0)
+ || ((i % 4) == 3 && lane == 1));
+}
+
#endif
\ No newline at end of file
diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp
index a5ffa38a22..bc28317fb5 100644
--- a/tests/regression/dogfood/kernel.cpp
+++ b/tests/regression/dogfood/kernel.cpp
@@ -255,6 +255,92 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) {
}
}
+void kernel_fdiv_special(kernel_arg_t* __UNIFORM__ arg) {
+ size_t count = arg->task_size;
+ auto src0_ptr = (float*)arg->src0_addr;
+ auto src1_ptr = (float*)arg->src1_addr;
+ auto dst_ptr = (float*)arg->dst_addr;
+ uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+ auto offset = gid * count;
+
+ for (size_t i = 0; i < count; ++i) {
+ auto a = src0_ptr[offset+i];
+ auto b = src1_ptr[offset+i];
+ auto c = __ieee754_sqrtf(a) / b;
+ dst_ptr[offset+i] = c;
+ }
+}
+
+#ifdef VX_CFG_FLEN_64
+// Zero the upper word of a single-precision register, leaving it improperly NaN-boxed.
+inline float unbox_f32(float x) {
+ float r;
+ uint64_t t;
+ asm volatile ("fmv.x.w %1, %2\n slli %1, %1, 32\n srli %1, %1, 32\n fmv.d.x %0, %1" : "=f"(r), "=&r"(t) : "f"(x));
+ return r;
+}
+#endif
+
+// Six cases per element: sqrt, div and each FMA operand (a, b, c) take an
+// improperly NaN-boxed input once, plus one case with every input boxed.
+void kernel_fp_nanbox(kernel_arg_t* __UNIFORM__ arg) {
+ size_t count = arg->task_size;
+ auto src0_ptr = (float*)arg->src0_addr;
+ auto src1_ptr = (float*)arg->src1_addr;
+ auto dst_ptr = (float*)arg->dst_addr;
+ uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+ auto offset = gid * count;
+
+ for (size_t i = 0; i < count; ++i) {
+ auto a = src0_ptr[offset+i];
+ auto b = src1_ptr[offset+i];
+ auto ua = a, ub = b;
+ #ifdef VX_CFG_FLEN_64
+ ua = unbox_f32(a);
+ ub = unbox_f32(b);
+ #endif
+ float c;
+ switch ((offset + i) % 6) {
+ case 0: c = __ieee754_sqrtf(ua) / b; break;
+ case 1: c = __ieee754_sqrtf(a) / ub; break;
+ case 2: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(ua), "f"(b), "f"(b)); break;
+ case 3: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(ub), "f"(a)); break;
+ case 4: asm volatile ("fmadd.s %0, %1, %2, %3" : "=f"(c) : "f"(a), "f"(b), "f"(ub)); break;
+ default: c = __ieee754_sqrtf(a) / b; break;
+ }
+ dst_ptr[offset+i] = c;
+ }
+}
+
+void kernel_fflags_lanes(kernel_arg_t* __UNIFORM__ arg) {
+ size_t count = arg->task_size;
+ auto src0_ptr = (float*)arg->src0_addr;
+ auto src1_ptr = (float*)arg->src1_addr;
+ auto dst_ptr = (uint32_t*)arg->dst_addr;
+ uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+ uint32_t num_threads = vx_num_threads();
+ uint32_t lane = gid % num_threads;
+ uint32_t wid = gid / num_threads;
+ auto offset = gid * count;
+
+ for (size_t i = 0; i < count; ++i) {
+ auto a = src0_ptr[offset+i];
+ auto b = src1_ptr[offset+i];
+ // keep the operands live in the lanes the branch below masks off
+ asm volatile ("" : "+f"(a), "+f"(b));
+ if (!fflags_lane_masked(lane, i, num_threads)) {
+ float c;
+ uint32_t flags;
+ if ((i + wid) & 1) {
+ asm volatile ("csrwi fflags, 0\n fadd.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b));
+ } else {
+ asm volatile ("csrwi fflags, 0\n fdiv.s %0, %2, %3\n frflags %1" : "=&f"(c), "=&r"(flags) : "f"(a), "f"(b));
+ }
+ dst_ptr[offset+i] = flags;
+ }
+ }
+}
+
void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) {
size_t count = arg->task_size;
auto src0_ptr = (float*)arg->src0_addr;
@@ -446,6 +532,7 @@ static const PFN_Kernel sc_tests[] = {
kernel_fdiv, kernel_fdiv2, kernel_fsqrt, kernel_ftoi,
kernel_ftou, kernel_itof, kernel_utof, kernel_fclamp,
kernel_iclamp, kernel_trigo, kernel_bar, kernel_gbar,
+ kernel_fdiv_special, kernel_fp_nanbox, kernel_fflags_lanes,
};
__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) {
diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h
index 29e8e819c1..bf274d1d16 100644
--- a/tests/regression/dogfood/testcases.h
+++ b/tests/regression/dogfood/testcases.h
@@ -4,6 +4,7 @@
#include
#include
#include
+#include "common.h"
void cleanup();
@@ -576,6 +577,185 @@ class Test_FSQRT : public ITestCase {
}
};
+class Test_FDIV_SPECIAL : public ITestCase {
+public:
+ Test_FDIV_SPECIAL(TestSuite* suite) : ITestCase(suite, "fdiv-special") {}
+
+ // Zero, infinity, NaN and negative operands take the divide and square-root
+ // early-out paths; the hash scatters them across the threads of a warp.
+ int setup(uint32_t n, void* src1, void* src2) override {
+ auto a = (float*)src1;
+ auto b = (float*)src2;
+ for (uint32_t i = 0; i < n; ++i) {
+ float x = 1.0f + fround((n - i) * (1.0f/n));
+ float y = fround((n + i) * (1.0f/n));
+ switch ((i * 0x9E3779B1u) >> 28) {
+ case 0: y = 0.0f; break;
+ case 1: x = 0.0f; y = 0.0f; break;
+ case 2: x = INFINITY; break;
+ case 3: x = NAN; break;
+ case 4: x = 0.0f; break;
+ case 5: x = -x; break;
+ default: break;
+ }
+ a[i] = x;
+ b[i] = y;
+ }
+ return 0;
+ }
+
+ int verify(uint32_t n, void* dst, const void* src1, const void* src2) override {
+ int errors = 0;
+ auto a = (float*)src1;
+ auto b = (float*)src2;
+ auto c = (float*)dst;
+ for (uint32_t i = 0; i < n; ++i) {
+ auto ref = sqrtf(a[i]) / b[i];
+ bool match = std::isnan(ref) ? std::isnan(c[i]) : almost_equal(c[i], ref);
+ if (!match) {
+ std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl;
+ ++errors;
+ }
+ }
+ return errors;
+ }
+};
+
+class Test_FP_NANBOX : public ITestCase {
+public:
+ Test_FP_NANBOX(TestSuite* suite) : ITestCase(suite, "fp-nanbox") {}
+
+ int setup(uint32_t n, void* src1, void* src2) override {
+ auto a = (float*)src1;
+ auto b = (float*)src2;
+ for (uint32_t i = 0; i < n; ++i) {
+ a[i] = 1.0f + fround((n - i) * (1.0f/n));
+ b[i] = fround((n + i) * (1.0f/n));
+ }
+ return 0;
+ }
+
+ // With 64-bit FP registers, the kernel leaves one input of five elements in
+ // six improperly NaN-boxed, which must read as the canonical NaN.
+ int verify(uint32_t n, void* dst, const void* src1, const void* src2) override {
+ int errors = 0;
+ auto a = (float*)src1;
+ auto b = (float*)src2;
+ auto c = (float*)dst;
+ for (uint32_t i = 0; i < n; ++i) {
+ uint32_t k = i % 6;
+ #ifdef VX_CFG_FLEN_64
+ if (k != 5) {
+ Float_t r{c[i]};
+ if (r.i != 0x7fc00000) {
+ std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << std::hex << ": expected=0x7fc00000, actual=0x" << r.i << ", a=" << a[i] << ", b=" << b[i] << std::endl;
+ ++errors;
+ }
+ continue;
+ }
+ #endif
+ float ref;
+ switch (k) {
+ case 2: ref = fmaf(a[i], b[i], b[i]); break;
+ case 3: ref = fmaf(a[i], b[i], a[i]); break;
+ case 4: ref = fmaf(a[i], b[i], b[i]); break;
+ default: ref = sqrtf(a[i]) / b[i]; break;
+ }
+ if (!almost_equal(c[i], ref)) {
+ std::cout << "error at result #" << std::dec << i << " (case " << k << ")" << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl;
+ ++errors;
+ }
+ }
+ return errors;
+ }
+};
+
+class Test_FFLAGS_LANES : public ITestCase {
+public:
+ Test_FFLAGS_LANES(TestSuite* suite) : ITestCase(suite, "fflags-lanes") {}
+
+ enum { FL_NX = 0x01, FL_OF = 0x04, FL_DZ = 0x08, FL_NV = 0x10 };
+ enum { K_NONE, K_NX, K_X1, K_X2 };
+
+ // A warp divides or adds by the parity of its iteration plus its index, so
+ // warps running side by side mix the two; X1 is DZ or OF, X2 is NV.
+ static uint32_t recipe(bool is_add, int kind, float* a, float* b) {
+ static const float div_a[] = {1.0f, 1.0f, 1.0f, 0.0f};
+ static const float div_b[] = {2.0f, 3.0f, 0.0f, 0.0f};
+ static const uint32_t div_f[] = {0, FL_NX, FL_DZ, FL_NV};
+ static const float add_a[] = {1.0f, 1.0f, std::numeric_limits::max(), INFINITY};
+ static const float add_b[] = {2.0f, 0x1p-30f, std::numeric_limits::max(), -INFINITY};
+ static const uint32_t add_f[] = {0, FL_NX, FL_OF | FL_NX, FL_NV};
+ *a = is_add ? add_a[kind] : div_a[kind];
+ *b = is_add ? add_b[kind] : div_b[kind];
+ return is_add ? add_f[kind] : div_f[kind];
+ }
+
+ // Flags raised by one lane; only non-zero lanes raise any, and a masked lane
+ // gets operands raising a flag no active lane raises.
+ uint32_t lane_flags(uint32_t k, uint32_t n, float* a, float* b) const {
+ uint32_t L = num_threads_;
+ uint32_t count = n / (num_cores_ * num_warps_ * L);
+ uint32_t t = k / count, i = k % count;
+ uint32_t lane = t % L, wid = t / L;
+ uint32_t sc = ((i >> 2) + wid) % 4;
+ bool is_add = ((i + wid) & 1) != 0;
+ int kind = K_NONE;
+ if ((sc == 0 && lane == 2) || (sc == 2 && lane == L - 1)) {
+ kind = K_X1;
+ } else if (sc == 1 && lane == 3) {
+ kind = K_X2;
+ } else if ((sc == 2 && lane == 2) || (sc == 3 && lane == 0)) {
+ kind = K_NX;
+ }
+ if (fflags_lane_masked(lane, i, L)) {
+ kind = (sc == 1) ? K_X1 : K_X2;
+ }
+ return recipe(is_add, kind, a, b);
+ }
+
+ int setup(uint32_t n, void* src1, void* src2) override {
+ RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_CORES, &num_cores_));
+ RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_WARPS, &num_warps_));
+ RT_CHECK(vx_dev_caps(suite_->device(), VX_CAPS_NUM_THREADS, &num_threads_));
+ auto a = (float*)src1;
+ auto b = (float*)src2;
+ for (uint32_t k = 0; k < n; ++k) {
+ lane_flags(k, n, &a[k], &b[k]);
+ }
+ return 0;
+ }
+
+ // fflags is per warp: every active lane reads the OR over the active lanes.
+ int verify(uint32_t n, void* dst, const void* /*src1*/, const void* /*src2*/) override {
+ int errors = 0;
+ auto c = (uint32_t*)dst;
+ uint32_t L = num_threads_;
+ uint32_t count = n / (num_cores_ * num_warps_ * L);
+ for (uint32_t k = 0; k < n; ++k) {
+ uint32_t t = k / count, i = k % count;
+ if (fflags_lane_masked(t % L, i, L))
+ continue;
+ uint32_t ref = 0;
+ for (uint32_t l = 0; l < L; ++l) {
+ if (fflags_lane_masked(l, i, L))
+ continue;
+ float a, b;
+ ref |= lane_flags(((t / L) * L + l) * count + i, n, &a, &b);
+ }
+ if (c[k] != ref) {
+ std::cout << "error at result #" << std::dec << k << " (lane " << (t % L) << ", iteration " << i << ")" << std::hex << ": expected fflags=0x" << ref << ", actual=0x" << c[k] << std::endl;
+ ++errors;
+ }
+ }
+ return errors;
+ }
+
+ uint64_t num_cores_;
+ uint64_t num_warps_;
+ uint64_t num_threads_;
+};
+
class Test_FTOI : public ITestCase {
public:
Test_FTOI(TestSuite* suite) : ITestCase(suite, "ftoi") {}
@@ -897,6 +1077,9 @@ TestSuite::TestSuite(vx_device_h device)
this->add_test(new Test_TRIGO(this));
this->add_test(new Test_BAR(this));
this->add_test(new Test_GBAR(this));
+ this->add_test(new Test_FDIV_SPECIAL(this));
+ this->add_test(new Test_FP_NANBOX(this));
+ this->add_test(new Test_FFLAGS_LANES(this));
}
TestSuite::~TestSuite() {