diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 90366ed1af..7238e12251 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -11,6 +11,10 @@ tests: - id: hw via: script run: "make -C hw/unittest" + - id: hw-mmu-park + via: script + run: "make -C hw/unittest/mmu_park run" + touches: [hw/rtl/vm, hw/rtl/libs/VX_rr_arbiter.sv, hw/unittest/mmu_park] # FPU arithmetic units (VX_fma/fdiv/fsqrt/fcvt/fdivsqrt): the `hw` lane above only # builds the unit tests; this lane executes them so a softfloat-divergent # regression actually fails CI (each driver checks bit-exact result+fflags over diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 4af3363c54..78a875fe8c 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -291,7 +291,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // A hit joins its page's live entry only to stay behind the lane's own // parked requests (per-lane program order). Joining otherwise would funnel // every access to a hot page through the one-per-cycle miss station - // drain, lowest lane first. A fetch needs no order (one request in flight + // drain. A fetch needs no order (one request in flight // per warp). A registered park joins always: its stage is a cycle stale. assign mshr_order[l] = (EXEC_SIDE == 0) && ((PARK_REG != 0) || lane_parked[l]); assign cam_acc[l] = req_acc[l]; @@ -301,43 +301,52 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign cat_pfault[l] = xlat && !mshr_join[l] && cam_hit[l] && !cam_perm[l]; end - // Park arbitration: at most one lane parks a miss per cycle (lowest lane). - // The parked fields are taken with a one-hot select straight off the lane - // categories; the encoded lane only rides in the payload. The one-hot comes - // from a prefix OR (MODEL 2), so a lane's park grant, and with it the lane's - // accept, depends only on the lanes below it. + // Rotate park priority so a stream of misses cannot starve another lane. localparam PARK_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + FIELDS_W; wire [NUM_REQS-1:0] park_sel; wire sel_valid; wire sel_ready; wire [LANE_W-1:0] sel_lane; - VX_priority_encoder #( - .N (NUM_REQS), - .MODEL (2) - ) park_enc ( - .data_in (cat_park), - .onehot_out (park_sel), - .index_out (sel_lane), - .valid_out (sel_valid) + wire [NUM_REQS-1:0] park_requests; + wire grant_valid; + wire grant_ready; + if (NUM_REQS > 1 && PARK_REG == 0) begin : g_reserve + // Select from the preceding cycle's misses while their current probes run. + // A fill may turn the reserved miss into a hit before it reaches the station. + reg [NUM_REQS-1:0] requests_r; + always @(posedge clk) begin + if (reset) begin + requests_r <= '0; + end else if (grant_ready) begin + requests_r <= cat_park; + end + end + assign park_requests = requests_r; + assign sel_valid = |(cat_park & park_sel); + assign grant_ready = !sel_valid || sel_ready; + `UNUSED_VAR (grant_valid) + end else begin : g_select + assign park_requests = cat_park; + assign sel_valid = grant_valid; + assign grant_ready = sel_ready; + end + VX_rr_arbiter #( + .NUM_REQS (NUM_REQS) + ) park_arb ( + .clk (clk), + .reset (reset), + .requests (park_requests), + .grant_onehot (park_sel), + .grant_index (sel_lane), + .grant_valid (grant_valid), + .grant_ready (grant_ready) ); wire [NUM_REQS-1:0][PARK_W-1:0] park_src; for (genvar l = 0; l < NUM_REQS; ++l) begin : g_park_src assign park_src[l] = {req_vpn[l], req_acc[l], req_amo[l], req_fields[l]}; end - // park_sel[l] is cat_park[l] && park_first[l]. A lane's accept reads - // park_first under its own cat_park, so its translation result enters the - // accept only once, at the category select. - wire [NUM_REQS-1:0] park_first; - for (genvar l = 0; l < NUM_REQS; ++l) begin : g_park_first - if (l == 0) begin : g_first - assign park_first[l] = 1'b1; - end else begin : g_rest - assign park_first[l] = ~(| cat_park[l-1:0]); - end - end - wire [PARK_W-1:0] park_out; VX_onehot_mux #( .DATAW (PARK_W), @@ -553,9 +562,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign req_if[l].req_ready = cat_bypass[l] ? (!replay_to_lane[l] && pipe_ready[l]) : cat_hit[l] ? (!replay_to_lane[l] && pipe_ready[l]) - : cat_park[l] ? (park_first[l] && sel_ready) - : cat_pfault[l] ? pf_accept[l] - : 1'b0; + : (cat_park[l] && park_sel[l] && sel_ready) || pf_accept[l]; // Response path wires straight back, except a kill teardown or a // permission fault injects a zeroed response ahead of the cache reply. diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 43b55d645c..cf6eeb5fed 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -1,6 +1,7 @@ all: $(MAKE) -C generic_queue $(MAKE) -C mem_scheduler + $(MAKE) -C mmu_park $(MAKE) -C cache $(MAKE) -C core $(MAKE) -C issue @@ -36,6 +37,7 @@ all: run: $(MAKE) -C generic_queue run $(MAKE) -C mem_scheduler run + $(MAKE) -C mmu_park run $(MAKE) -C cache run $(MAKE) -C core run $(MAKE) -C issue run @@ -197,6 +199,7 @@ run-tcu-dsp: clean: $(MAKE) -C generic_queue clean $(MAKE) -C mem_scheduler clean + $(MAKE) -C mmu_park clean $(MAKE) -C cache clean $(MAKE) -C core clean $(MAKE) -C issue clean diff --git a/hw/unittest/mmu_park/Makefile b/hw/unittest/mmu_park/Makefile new file mode 100644 index 0000000000..78c2e67a90 --- /dev/null +++ b/hw/unittest/mmu_park/Makefile @@ -0,0 +1,14 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mmu_park +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(VORTEX_HOME)/hw/unittest/common +SRCS := $(SRC_DIR)/main.cpp +RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/vm/VX_tlb_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(SRC_DIR) + +TOP := VX_mmu_park_top +include ../common.mk diff --git a/hw/unittest/mmu_park/VX_mmu_park_top.sv b/hw/unittest/mmu_park/VX_mmu_park_top.sv new file mode 100644 index 0000000000..d6ffc6ba4e --- /dev/null +++ b/hw/unittest/mmu_park/VX_mmu_park_top.sv @@ -0,0 +1,80 @@ +`include "VX_define.vh" + +module VX_mmu_park_top import VX_gpu_pkg::*, VX_tlb_pkg::*; ( + input wire clk, + input wire reset, + input wire walk_enable, + output reg [7:0] accepted [4] +); + localparam NUM_PORTS = 4; + localparam ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - 2; + VX_mem_bus_if #(.DATA_SIZE(4), .ADDR_WIDTH(ADDR_WIDTH), .TAG_WIDTH(UUID_WIDTH+1)) core_bus[NUM_PORTS](); + VX_mem_bus_if #(.DATA_SIZE(4), .ADDR_WIDTH(ADDR_WIDTH), .TAG_WIDTH(UUID_WIDTH+1)) mem_bus[NUM_PORTS](); + VX_tlb_bus_if #(.ID_WIDTH(1)) walk(); + VX_tlb_flush_if flush(); + VX_mmu_fault_if fault(); + + // Disjoint streams of cold pages keep every port requesting a walk. + for (genvar p = 0; p < NUM_PORTS; ++p) begin : g_port + wire [ADDR_WIDTH-1:0] vpn = ADDR_WIDTH'(accepted[p]) * NUM_PORTS + ADDR_WIDTH'(p+1); + assign core_bus[p].req_valid = !reset; + assign core_bus[p].req_data = '{addr: vpn << (`VX_VM_PAGE_LOG2_SIZE - 2), default: '0}; + assign core_bus[p].rsp_ready = 1'b1; + assign mem_bus[p].req_ready = 1'b1; + assign mem_bus[p].rsp_valid = 1'b0; + assign mem_bus[p].rsp_data = '0; + always @(posedge clk) begin + if (reset) begin + accepted[p] <= '0; + end else if (core_bus[p].req_valid && core_bus[p].req_ready) begin + accepted[p] <= accepted[p] + 1'b1; + end + end + end + + wire walk_ready; + assign walk.req_ready = walk_enable && walk_ready; + VX_pipe_buffer #(.DATAW(1)) walk_reply ( + .clk (clk), + .reset (reset), + .valid_in (walk.req_valid && walk_enable), + .ready_in (walk_ready), + .data_in (walk.req_data.id), + .valid_out (walk.rsp_valid), + .ready_out (walk.rsp_ready), + .data_out (walk.rsp_data.id) + ); + assign walk.rsp_data.ppn = '0; + assign walk.rsp_data.flags = 8'h13; // user-readable leaf + assign walk.rsp_data.level = '0; + assign walk.rsp_data.fault = 1'b0; + assign flush.req = 1'b0; + `UNUSED_VAR ({flush.done, fault.va, fault.amo, fault.access}) + `UNUSED_VAR ({walk.req_data.vpn, walk.req_data.access, walk.req_data.amo}) + always @(posedge clk) begin + if (!reset) begin + assert (!fault.valid); + end + end + + VX_mmu #( + .NUM_REQS (NUM_PORTS), + .MSHR_SIZE (2), + .DATA_SIZE (4), + .TAG_WIDTH (UUID_WIDTH+1) + ) dut ( + .clk (clk), + .reset (reset), + .vm_active (1'b1), +`ifdef PERF_ENABLE + `UNUSED_PIN (mmu_perf), +`endif + .core_bus_if (core_bus), + .mem_bus_if (mem_bus), + .tlb_bus_if (walk), + .flush_if (flush), + .fault_if (fault), + `UNUSED_PIN (lane_idle), + `UNUSED_PIN (empty) + ); +endmodule diff --git a/hw/unittest/mmu_park/main.cpp b/hw/unittest/mmu_park/main.cpp new file mode 100644 index 0000000000..81b3f7294f --- /dev/null +++ b/hw/unittest/mmu_park/main.cpp @@ -0,0 +1,29 @@ +#include "vl_simulator.h" +#include "VVX_mmu_park_top.h" +#include +#include + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + vl_simulator sim; + sim->walk_enable = 0; + auto tick = sim.reset(0); + bool passed = true; + // Saturate the two miss slots, then let walks complete one at a time. + for (unsigned cycle = 0; cycle < 256; ++cycle) { + sim->walk_enable = cycle >= 16 && cycle % 8 == 0; + tick = sim.step(tick, 2); + const auto& counts = sim->accepted; + auto bounds = std::minmax_element(counts.data(), counts.data() + 4); + if (passed && *bounds.second - *bounds.first > 1) { + std::printf("unbalanced grants at cycle %u: %u %u %u %u\n", + cycle, counts[0], counts[1], counts[2], counts[3]); + passed = false; + } + } + for (unsigned p = 0; p < 4; ++p) { + std::printf("port %u accepted %u misses\n", p, sim->accepted[p]); + passed &= sim->accepted[p] >= 2; + } + return passed ? 0 : 1; +} diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index 00a85836bb..0851640628 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -37,6 +37,7 @@ Mmu::Mmu(const SimContext& ctx, , rsp_reg_(rsp_reg) , write_enable_(write_enable) , park_reg_(park_reg) + , reserved_port_(num_ports) , tlb_(tlb_size) , mshr_(VX_CFG_L1_TLB_MSHR_SIZE) , replay_(num_ports) @@ -66,6 +67,8 @@ void Mmu::on_reset() { s.reset(); } stage_.reset(); + next_park_port_ = 0; + reserved_port_ = num_ports_; reports_ = {}; // Physical pages move under a virtual address between launches, so // cached translations cannot outlive one — the shared levels clear here @@ -387,7 +390,7 @@ void Mmu::on_tick() { // 4) Forward requests. This is a hit-under-miss stage: a hit issues // while older requests are still parked, so only same-address order is // guaranteed — those share a VPN, hence one entry and its arrival-order - // parked list. Replays drain ahead of new input on the same port. + // parked list. Replays take the outgoing port, but a new miss can still park. // With a registered park, the stage's state at the start of the cycle // decides both whether it takes a park and which page holds. bool stage_free = !stage_.has_value(); @@ -398,12 +401,12 @@ void Mmu::on_tick() { if (park_reg_) { stage_drain(); } + uint32_t selected = num_ports_; + bool reserved_miss = false; for (uint32_t p = 0; p < num_ports_; ++p) { - if (!replay_.at(p).empty()) { - if (ReqOut.at(p).try_send(replay_.at(p).front())) { - replay_.at(p).pop(); - } - continue; + bool replaying = !replay_.at(p).empty(); + if (replaying && ReqOut.at(p).try_send(replay_.at(p).front())) { + replay_.at(p).pop(); } if (ReqIn.at(p).empty()) { @@ -417,7 +420,7 @@ void Mmu::on_tick() { if (req.flags.flush && write_enable_ && !exec_side_ && port_parked(p)) { continue; } - if (ReqOut.at(p).try_send(req)) { + if (!replaying && ReqOut.at(p).try_send(req)) { ReqIn.at(p).pop(); } continue; @@ -455,30 +458,49 @@ void Mmu::on_tick() { MemReq translated = req; translated.addr = (res.ppn << VX_VM_PAGE_LOG2_SIZE) | (req.addr & ((1ULL << VX_VM_PAGE_LOG2_SIZE) - 1)); - if (ReqOut.at(p).try_send(translated)) { + if (!replaying && ReqOut.at(p).try_send(translated)) { ReqIn.at(p).pop(); } } else { - // Park the miss and keep the port flowing (hit-under-miss). - // Same-VPN misses chain on one entry, preserving arrival order; - // a full entry or a full station stalls this port only. - if (park_reg_) { - if (stage_free) { - stage_ = Staged{p, req, vpn, SimPlatform::instance().cycles() + 1}; - stage_free = false; - ReqIn.at(p).pop(); - } - continue; + if (selected == num_ports_ || (selected < next_park_port_ && p >= next_park_port_)) { + selected = p; } + if (reserved_port_ == p) { + reserved_miss = true; + } + } + } + + const bool reserve = num_ports_ > 1 && !park_reg_; + uint32_t port = selected; + if (reserve) { + port = reserved_miss ? reserved_port_ : num_ports_; + } + if (port != num_ports_) { + const auto& req = ReqIn.at(port).peek(); + uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + if (park_reg_) { + if (!stage_free) { + return; + } + stage_ = Staged{port, req, vpn, SimPlatform::instance().cycles() + 1}; + } else { int id = mshr_find(vpn); if (id < 0) { id = mshr_alloc(vpn); } - if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) { - mshr_[id].parked.emplace_back(p, req); - ReqIn.at(p).pop(); + if (id < 0 || mshr_[id].parked.size() >= MAX_PARKED_PER_ENTRY) { + return; } + mshr_[id].parked.emplace_back(port, req); } + ReqIn.at(port).pop(); + } + if (selected != num_ports_) { + next_park_port_ = selected + 1 == num_ports_ ? 0 : selected + 1; + } + if (reserve) { + reserved_port_ = selected; } } diff --git a/sim/simx/mem/mmu.h b/sim/simx/mem/mmu.h index 25b331bef8..2ff2e2f8ab 100644 --- a/sim/simx/mem/mmu.h +++ b/sim/simx/mem/mmu.h @@ -125,6 +125,8 @@ class Mmu : public SimObject { // earliest; the stage takes a request only while empty, and a lane whose // page is staged holds meanwhile. bool park_reg_; + uint32_t next_park_port_ = 0; + uint32_t reserved_port_; struct Staged { uint32_t port; MemReq req;