Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ci/testcases/unittest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ tests:
- id: hw
via: script
run: "make -C hw/unittest"
- id: hw-mmu-park
via: script
run: "make -C hw/unittest/mmu_park run"
touches: [hw/rtl/vm, hw/rtl/libs/VX_rr_arbiter.sv, hw/unittest/mmu_park]
# FPU arithmetic units (VX_fma/fdiv/fsqrt/fcvt/fdivsqrt): the `hw` lane above only
# builds the unit tests; this lane executes them so a softfloat-divergent
# regression actually fails CI (each driver checks bit-exact result+fflags over
Expand Down
65 changes: 36 additions & 29 deletions hw/rtl/vm/VX_mmu.sv
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #(
// A hit joins its page's live entry only to stay behind the lane's own
// parked requests (per-lane program order). Joining otherwise would funnel
// every access to a hot page through the one-per-cycle miss station
// drain, lowest lane first. A fetch needs no order (one request in flight
// drain. A fetch needs no order (one request in flight
// per warp). A registered park joins always: its stage is a cycle stale.
assign mshr_order[l] = (EXEC_SIDE == 0) && ((PARK_REG != 0) || lane_parked[l]);
assign cam_acc[l] = req_acc[l];
Expand All @@ -301,43 +301,52 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #(
assign cat_pfault[l] = xlat && !mshr_join[l] && cam_hit[l] && !cam_perm[l];
end

// Park arbitration: at most one lane parks a miss per cycle (lowest lane).
// The parked fields are taken with a one-hot select straight off the lane
// categories; the encoded lane only rides in the payload. The one-hot comes
// from a prefix OR (MODEL 2), so a lane's park grant, and with it the lane's
// accept, depends only on the lanes below it.
// Rotate park priority so a stream of misses cannot starve another lane.
localparam PARK_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + FIELDS_W;

wire [NUM_REQS-1:0] park_sel;
wire sel_valid;
wire sel_ready;
wire [LANE_W-1:0] sel_lane;
VX_priority_encoder #(
.N (NUM_REQS),
.MODEL (2)
) park_enc (
.data_in (cat_park),
.onehot_out (park_sel),
.index_out (sel_lane),
.valid_out (sel_valid)
wire [NUM_REQS-1:0] park_requests;
wire grant_valid;
wire grant_ready;
if (NUM_REQS > 1 && PARK_REG == 0) begin : g_reserve
// Select from the preceding cycle's misses while their current probes run.
// A fill may turn the reserved miss into a hit before it reaches the station.
reg [NUM_REQS-1:0] requests_r;
always @(posedge clk) begin
if (reset) begin
requests_r <= '0;
end else if (grant_ready) begin
requests_r <= cat_park;
end
end
assign park_requests = requests_r;
assign sel_valid = |(cat_park & park_sel);
assign grant_ready = !sel_valid || sel_ready;
`UNUSED_VAR (grant_valid)
end else begin : g_select
assign park_requests = cat_park;
assign sel_valid = grant_valid;
assign grant_ready = sel_ready;
end
VX_rr_arbiter #(
.NUM_REQS (NUM_REQS)
) park_arb (
.clk (clk),
.reset (reset),
.requests (park_requests),
.grant_onehot (park_sel),
.grant_index (sel_lane),
.grant_valid (grant_valid),
.grant_ready (grant_ready)
);

wire [NUM_REQS-1:0][PARK_W-1:0] park_src;
for (genvar l = 0; l < NUM_REQS; ++l) begin : g_park_src
assign park_src[l] = {req_vpn[l], req_acc[l], req_amo[l], req_fields[l]};
end
// park_sel[l] is cat_park[l] && park_first[l]. A lane's accept reads
// park_first under its own cat_park, so its translation result enters the
// accept only once, at the category select.
wire [NUM_REQS-1:0] park_first;
for (genvar l = 0; l < NUM_REQS; ++l) begin : g_park_first
if (l == 0) begin : g_first
assign park_first[l] = 1'b1;
end else begin : g_rest
assign park_first[l] = ~(| cat_park[l-1:0]);
end
end

wire [PARK_W-1:0] park_out;
VX_onehot_mux #(
.DATAW (PARK_W),
Expand Down Expand Up @@ -553,9 +562,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #(
assign req_if[l].req_ready =
cat_bypass[l] ? (!replay_to_lane[l] && pipe_ready[l])
: cat_hit[l] ? (!replay_to_lane[l] && pipe_ready[l])
: cat_park[l] ? (park_first[l] && sel_ready)
: cat_pfault[l] ? pf_accept[l]
: 1'b0;
: (cat_park[l] && park_sel[l] && sel_ready) || pf_accept[l];

// Response path wires straight back, except a kill teardown or a
// permission fault injects a zeroed response ahead of the cache reply.
Expand Down
3 changes: 3 additions & 0 deletions hw/unittest/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
all:
$(MAKE) -C generic_queue
$(MAKE) -C mem_scheduler
$(MAKE) -C mmu_park
$(MAKE) -C cache
$(MAKE) -C core
$(MAKE) -C issue
Expand Down Expand Up @@ -36,6 +37,7 @@ all:
run:
$(MAKE) -C generic_queue run
$(MAKE) -C mem_scheduler run
$(MAKE) -C mmu_park run
$(MAKE) -C cache run
$(MAKE) -C core run
$(MAKE) -C issue run
Expand Down Expand Up @@ -197,6 +199,7 @@ run-tcu-dsp:
clean:
$(MAKE) -C generic_queue clean
$(MAKE) -C mem_scheduler clean
$(MAKE) -C mmu_park clean
$(MAKE) -C cache clean
$(MAKE) -C core clean
$(MAKE) -C issue clean
Expand Down
14 changes: 14 additions & 0 deletions hw/unittest/mmu_park/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
ROOT_DIR := $(realpath ../../..)
include $(ROOT_DIR)/config.mk

PROJECT := mmu_park
RTL_DIR := $(VORTEX_HOME)/hw/rtl
SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT)

CXXFLAGS := -I$(VORTEX_HOME)/hw/unittest/common
SRCS := $(SRC_DIR)/main.cpp
RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/vm/VX_tlb_pkg.sv
RTL_INCLUDE := -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(SRC_DIR)

TOP := VX_mmu_park_top
include ../common.mk
80 changes: 80 additions & 0 deletions hw/unittest/mmu_park/VX_mmu_park_top.sv
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
`include "VX_define.vh"

module VX_mmu_park_top import VX_gpu_pkg::*, VX_tlb_pkg::*; (
input wire clk,
input wire reset,
input wire walk_enable,
output reg [7:0] accepted [4]
);
localparam NUM_PORTS = 4;
localparam ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - 2;
VX_mem_bus_if #(.DATA_SIZE(4), .ADDR_WIDTH(ADDR_WIDTH), .TAG_WIDTH(UUID_WIDTH+1)) core_bus[NUM_PORTS]();
VX_mem_bus_if #(.DATA_SIZE(4), .ADDR_WIDTH(ADDR_WIDTH), .TAG_WIDTH(UUID_WIDTH+1)) mem_bus[NUM_PORTS]();
VX_tlb_bus_if #(.ID_WIDTH(1)) walk();
VX_tlb_flush_if flush();
VX_mmu_fault_if fault();

// Disjoint streams of cold pages keep every port requesting a walk.
for (genvar p = 0; p < NUM_PORTS; ++p) begin : g_port
wire [ADDR_WIDTH-1:0] vpn = ADDR_WIDTH'(accepted[p]) * NUM_PORTS + ADDR_WIDTH'(p+1);
assign core_bus[p].req_valid = !reset;
assign core_bus[p].req_data = '{addr: vpn << (`VX_VM_PAGE_LOG2_SIZE - 2), default: '0};
assign core_bus[p].rsp_ready = 1'b1;
assign mem_bus[p].req_ready = 1'b1;
assign mem_bus[p].rsp_valid = 1'b0;
assign mem_bus[p].rsp_data = '0;
always @(posedge clk) begin
if (reset) begin
accepted[p] <= '0;
end else if (core_bus[p].req_valid && core_bus[p].req_ready) begin
accepted[p] <= accepted[p] + 1'b1;
end
end
end

wire walk_ready;
assign walk.req_ready = walk_enable && walk_ready;
VX_pipe_buffer #(.DATAW(1)) walk_reply (
.clk (clk),
.reset (reset),
.valid_in (walk.req_valid && walk_enable),
.ready_in (walk_ready),
.data_in (walk.req_data.id),
.valid_out (walk.rsp_valid),
.ready_out (walk.rsp_ready),
.data_out (walk.rsp_data.id)
);
assign walk.rsp_data.ppn = '0;
assign walk.rsp_data.flags = 8'h13; // user-readable leaf
assign walk.rsp_data.level = '0;
assign walk.rsp_data.fault = 1'b0;
assign flush.req = 1'b0;
`UNUSED_VAR ({flush.done, fault.va, fault.amo, fault.access})
`UNUSED_VAR ({walk.req_data.vpn, walk.req_data.access, walk.req_data.amo})
always @(posedge clk) begin
if (!reset) begin
assert (!fault.valid);
end
end

VX_mmu #(
.NUM_REQS (NUM_PORTS),
.MSHR_SIZE (2),
.DATA_SIZE (4),
.TAG_WIDTH (UUID_WIDTH+1)
) dut (
.clk (clk),
.reset (reset),
.vm_active (1'b1),
`ifdef PERF_ENABLE
`UNUSED_PIN (mmu_perf),
`endif
.core_bus_if (core_bus),
.mem_bus_if (mem_bus),
.tlb_bus_if (walk),
.flush_if (flush),
.fault_if (fault),
`UNUSED_PIN (lane_idle),
`UNUSED_PIN (empty)
);
endmodule
29 changes: 29 additions & 0 deletions hw/unittest/mmu_park/main.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#include "vl_simulator.h"
#include "VVX_mmu_park_top.h"
#include <algorithm>
#include <cstdio>

int main(int argc, char** argv) {
Verilated::commandArgs(argc, argv);
vl_simulator<VVX_mmu_park_top> sim;
sim->walk_enable = 0;
auto tick = sim.reset(0);
bool passed = true;
// Saturate the two miss slots, then let walks complete one at a time.
for (unsigned cycle = 0; cycle < 256; ++cycle) {
sim->walk_enable = cycle >= 16 && cycle % 8 == 0;
tick = sim.step(tick, 2);
const auto& counts = sim->accepted;
auto bounds = std::minmax_element(counts.data(), counts.data() + 4);
if (passed && *bounds.second - *bounds.first > 1) {
std::printf("unbalanced grants at cycle %u: %u %u %u %u\n",
cycle, counts[0], counts[1], counts[2], counts[3]);
passed = false;
}
}
for (unsigned p = 0; p < 4; ++p) {
std::printf("port %u accepted %u misses\n", p, sim->accepted[p]);
passed &= sim->accepted[p] >= 2;
}
return passed ? 0 : 1;
}
64 changes: 43 additions & 21 deletions sim/simx/mem/mmu.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Mmu::Mmu(const SimContext& ctx,
, rsp_reg_(rsp_reg)
, write_enable_(write_enable)
, park_reg_(park_reg)
, reserved_port_(num_ports)
, tlb_(tlb_size)
, mshr_(VX_CFG_L1_TLB_MSHR_SIZE)
, replay_(num_ports)
Expand Down Expand Up @@ -66,6 +67,8 @@ void Mmu::on_reset() {
s.reset();
}
stage_.reset();
next_park_port_ = 0;
reserved_port_ = num_ports_;
reports_ = {};
// Physical pages move under a virtual address between launches, so
// cached translations cannot outlive one — the shared levels clear here
Expand Down Expand Up @@ -387,7 +390,7 @@ void Mmu::on_tick() {
// 4) Forward requests. This is a hit-under-miss stage: a hit issues
// while older requests are still parked, so only same-address order is
// guaranteed — those share a VPN, hence one entry and its arrival-order
// parked list. Replays drain ahead of new input on the same port.
// parked list. Replays take the outgoing port, but a new miss can still park.
// With a registered park, the stage's state at the start of the cycle
// decides both whether it takes a park and which page holds.
bool stage_free = !stage_.has_value();
Expand All @@ -398,12 +401,12 @@ void Mmu::on_tick() {
if (park_reg_) {
stage_drain();
}
uint32_t selected = num_ports_;
bool reserved_miss = false;
for (uint32_t p = 0; p < num_ports_; ++p) {
if (!replay_.at(p).empty()) {
if (ReqOut.at(p).try_send(replay_.at(p).front())) {
replay_.at(p).pop();
}
continue;
bool replaying = !replay_.at(p).empty();
if (replaying && ReqOut.at(p).try_send(replay_.at(p).front())) {
replay_.at(p).pop();
}

if (ReqIn.at(p).empty()) {
Expand All @@ -417,7 +420,7 @@ void Mmu::on_tick() {
if (req.flags.flush && write_enable_ && !exec_side_ && port_parked(p)) {
continue;
}
if (ReqOut.at(p).try_send(req)) {
if (!replaying && ReqOut.at(p).try_send(req)) {
ReqIn.at(p).pop();
}
continue;
Expand Down Expand Up @@ -455,30 +458,49 @@ void Mmu::on_tick() {
MemReq translated = req;
translated.addr = (res.ppn << VX_VM_PAGE_LOG2_SIZE) |
(req.addr & ((1ULL << VX_VM_PAGE_LOG2_SIZE) - 1));
if (ReqOut.at(p).try_send(translated)) {
if (!replaying && ReqOut.at(p).try_send(translated)) {
ReqIn.at(p).pop();
}
} else {
// Park the miss and keep the port flowing (hit-under-miss).
// Same-VPN misses chain on one entry, preserving arrival order;
// a full entry or a full station stalls this port only.
if (park_reg_) {
if (stage_free) {
stage_ = Staged{p, req, vpn, SimPlatform::instance().cycles() + 1};
stage_free = false;
ReqIn.at(p).pop();
}
continue;
if (selected == num_ports_ || (selected < next_park_port_ && p >= next_park_port_)) {
selected = p;
}
if (reserved_port_ == p) {
reserved_miss = true;
}
}
}

const bool reserve = num_ports_ > 1 && !park_reg_;
uint32_t port = selected;
if (reserve) {
port = reserved_miss ? reserved_port_ : num_ports_;
}
if (port != num_ports_) {
const auto& req = ReqIn.at(port).peek();
uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE;
if (park_reg_) {
if (!stage_free) {
return;
}
stage_ = Staged{port, req, vpn, SimPlatform::instance().cycles() + 1};
} else {
int id = mshr_find(vpn);
if (id < 0) {
id = mshr_alloc(vpn);
}
if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) {
mshr_[id].parked.emplace_back(p, req);
ReqIn.at(p).pop();
if (id < 0 || mshr_[id].parked.size() >= MAX_PARKED_PER_ENTRY) {
return;
}
mshr_[id].parked.emplace_back(port, req);
}
ReqIn.at(port).pop();
}
if (selected != num_ports_) {
next_park_port_ = selected + 1 == num_ports_ ? 0 : selected + 1;
}
if (reserve) {
reserved_port_ = selected;
}
}

Expand Down
2 changes: 2 additions & 0 deletions sim/simx/mem/mmu.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ class Mmu : public SimObject<Mmu> {
// earliest; the stage takes a request only while empty, and a lane whose
// page is staged holds meanwhile.
bool park_reg_;
uint32_t next_park_port_ = 0;
uint32_t reserved_port_;
struct Staged {
uint32_t port;
MemReq req;
Expand Down