diff --git a/ci/testcases/debug.yaml b/ci/testcases/debug.yaml index 21b4ccd8fc..14f1ab41dd 100644 --- a/ci/testcases/debug.yaml +++ b/ci/testcases/debug.yaml @@ -107,3 +107,13 @@ tests: cores: 4 l2cache: true args: -n128 +- id: simx_functional-3 + via: blackbox + drivers: + - simx + app: sgemm + configs: -DSIMX_FUNCTIONAL -DSIMX_MT=4 -DSIMX_QUANTUM=100 + shape: + cores: 4 + l2cache: true + args: -n128 diff --git a/docs/debugging.md b/docs/debugging.md index 63e0f983d7..1a7e60817c 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -51,6 +51,15 @@ before switching to a timed build for cycle-level analysis. Cycle counts in a functional run are non-physical: never use them for performance comparisons, `perf_gate`, or `model_parity`. +For the fastest multi-threaded functional runs, add `-DSIMX_QUANTUM=`: +domains run `Q` cycles between barriers and a cross-domain call waits for +the next quantum boundary. Results stay bit-identical across thread +counts for a given `Q`, but changing `Q` changes the cycle count. `Q=100` is +a good start; the timed kernel rejects `Q > 1`. + + // Functional, 16 host threads, barrier every 100 cycles + $ CONFIGS="-DSIMX_FUNCTIONAL -DSIMX_MT=16 -DSIMX_QUANTUM=100" ./ci/blackbox.sh --driver=simx --app=raycast --cores=16 --l2cache + ## RTL Debugging To debug the processor RTL, you need to use VLSIM or RTLSIM driver. VLSIM simulates the full processor including the AFU command processor (using `/rtl/afu/opae/vortex_afu.sv` as top module). RTLSIM simulates the Vortex processor only (using `/rtl/Vortex.v` as top module). diff --git a/docs/designs/simobject.md b/docs/designs/simobject.md index 6943ca70d3..dee538fb5a 100644 --- a/docs/designs/simobject.md +++ b/docs/designs/simobject.md @@ -327,10 +327,10 @@ must complete in-cycle). Default to `delay=1` for normal flow. ### Inflight counter and `idle()` -`SimChannelBase::inflight_count()` is a process-global counter -incremented on `reserve()` and decremented on queue pop. Useful for -deadlock detection (a tick that drops to zero traffic and stays there -when work is expected) and for end-of-simulation drain assertions. +Every endpoint counts the packets sent toward it but not yet consumed: +incremented on `reserve()`, decremented on queue pop. The count lives in the +endpoint's execution domain (one cache line per domain, no atomics), because +only that domain's context reserves toward or pops from its endpoints. Host-side quiescence is a framework service: `SimPlatform::idle()` is true when no packet is in flight **and** no deferred cross-domain delivery is @@ -413,8 +413,8 @@ packet delivery. 2. Calls `do_reset()` on every active-reset object. 3. Resets `cycles_ = 0`. -Inflight-count is *not* reset by the platform — clear it externally if -your test depends on it. Module `on_reset()` should clear all internal +The in-flight counts are *not* reset by the platform, and `idle()` is their +only reader. Module `on_reset()` should clear all internal queues and counters; channels are reset as a side effect (storage is reconstructed at construction; reset doesn't recreate them). @@ -470,6 +470,15 @@ owner, cross-domain deliveries are merged in a canonical `(due, src, seq)` order, and each cycle is fenced by barriers. A contributor who follows the Cardinal Rule gets deterministic parallelism for free. +Cross-domain calls travel through per-(target, source) mailbox lanes, each +written by one source and drained by one target, so no lock or shared counter +sits on the path. A lane is triple-buffered by epoch (a cycle, or a quantum in +the functional kernel): a source writes the current epoch, and the target +drains the previous one after the barrier that closed it. A call therefore runs +at the first epoch boundary at or after its due cycle, in `(due, src, seq)` +order; with one-cycle epochs this is exactly the order of a single shared +inbox. + What makes a module MT-safe is not new code — it is the *absence* of illegal code, enforced by lint and asserts: @@ -505,6 +514,15 @@ the same executor with timing removed, for full-speed architectural runs - The MT executor, domains, canonical ordering, and topology validation are retained — `-DSIMX_MT` composes and cycles remain bit-identical across thread counts. +- `-DSIMX_QUANTUM=` (resolved in `sim/simx/types.h`, handed to + `SimPlatform::set_quantum()`) lets domains run `Q` cycles between + barriers; a cross-domain call runs at the first quantum boundary at or + after its due cycle. + Results are bit-identical for every thread count at a given `Q`, but `Q` + itself changes the (non-physical) cycle count. A larger `Q` amortizes + synchronization but delays every crossing — an L1 miss into the uncore + domain waits for the boundary — so `Q` around 100 is a good start. The + timed kernel keeps per-cycle lockstep and rejects `Q > 1`. - Cycle counts are monotonic but **non-physical**: a functional build must never feed `perf_gate`, `model_parity`, or baseline regeneration. diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index f69478c063..277592dc25 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -48,7 +48,8 @@ it serves as the RTL oracle for cycle-parity debugging. With `SIMX_MT > 1` (a build define set via `CONFIGS`) the engine executes execution domains (one per socket, uncore = domain 0) on parallel workers in cycle lockstep, with results bit-identical to - serial execution. + serial execution. The functional kernel can instead synchronize every + `SIMX_QUANTUM` cycles (see [SimObject](simobject.md)). - **`SimEventLink`** — a one-way, typed control-plane link (barrier arrive/resume, FWD arm/done, doorbells): both ends are declared members, the receiver binds a member-function handler, elaboration wires diff --git a/docs/install_vortex.md b/docs/install_vortex.md index 0bb69943e4..a5b4d00d33 100644 --- a/docs/install_vortex.md +++ b/docs/install_vortex.md @@ -43,20 +43,25 @@ Useful configure options: `--xlen={32,64}` selects the ISA width, `--prefix=` the install location, and `--profile=` builds the machine described by `profiles/.json` by default (see - [Hardware Profiles](simulation.md#hardware-profiles)). Two SimX build knobs are plain + [Hardware Profiles](simulation.md#hardware-profiles)). Three SimX build knobs are plain config defines, passed like any `VX_CFG_*` option through `CONFIGS`: `-DSIMX_MT=` runs the simulator on `T` host threads (default serial; - cycle results are bit-identical for every thread count), and + cycle results are bit-identical for every thread count), `-DSIMX_FUNCTIONAL` builds the **functional** simulation kernel (timing disabled, for full-speed architectural runs such as conformance suites — functional cycle counts are non-physical and must never be used for - performance comparisons): + performance comparisons), and `-DSIMX_QUANTUM=` (functional kernel + only) synchronizes the threads every `Q` cycles instead of every cycle: ``` $ CONFIGS="-DSIMX_MT=4" ./ci/blackbox.sh --driver=simx --app=sgemm --cores=4 --l2cache $ CONFIGS="-DSIMX_FUNCTIONAL" ./ci/blackbox.sh --driver=simx --app=sgemm + $ CONFIGS="-DSIMX_FUNCTIONAL -DSIMX_MT=16 -DSIMX_QUANTUM=100" ./ci/blackbox.sh --driver=simx --app=sgemm --cores=16 --l2cache ``` + Multi-threading pays off only with several sockets (one execution domain + each); a 1-core build has nothing to run in parallel. + `make install` lays out the Vortex SDK under `$VORTEX_PATH` (default `/install`, override with `../configure --prefix=...`): diff --git a/sim/common/fixed_vector.h b/sim/common/fixed_vector.h new file mode 100644 index 0000000000..76799e1a45 --- /dev/null +++ b/sim/common/fixed_vector.h @@ -0,0 +1,114 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace vortex { + +// A vector with inline storage whose capacity is a compile-time bound (a +// VX_CFG_* thread or operand count): the std::vector subset the simulator +// uses, without a heap allocation per object or per copy. The size is still +// dynamic up to the capacity; exceeding it is an assertion failure. +template +class FixedVector { +public: + using value_type = T; + using size_type = uint32_t; + using iterator = T*; + using const_iterator = const T*; + using reference = T&; + using const_reference = const T&; + + FixedVector() : size_(0) {} + + explicit FixedVector(size_type n) : size_(0) { + this->assign(n, T{}); + } + + FixedVector(size_type n, const T& value) : size_(0) { + this->assign(n, value); + } + + FixedVector(std::initializer_list init) : size_(0) { + __assert(init.size() <= N, "FixedVector: capacity " << N << " exceeded"); + for (auto& v : init) { + data_[size_++] = v; + } + } + + static constexpr size_type capacity() { return N; } + + size_type size() const { return size_; } + bool empty() const { return size_ == 0; } + + T* data() { return data_.data(); } + const T* data() const { return data_.data(); } + + iterator begin() { return data_.data(); } + iterator end() { return data_.data() + size_; } + const_iterator begin() const { return data_.data(); } + const_iterator end() const { return data_.data() + size_; } + + reference operator[](size_type i) { return data_[i]; } + const_reference operator[](size_type i) const { return data_[i]; } + + reference at(size_type i) { + __assert(i < size_, "FixedVector: index " << i << " out of range " << size_); + return data_[i]; + } + const_reference at(size_type i) const { + __assert(i < size_, "FixedVector: index " << i << " out of range " << size_); + return data_[i]; + } + + reference front() { return data_[0]; } + const_reference front() const { return data_[0]; } + reference back() { return data_[size_ - 1]; } + const_reference back() const { return data_[size_ - 1]; } + + void assign(size_type n, const T& value) { + __assert(n <= N, "FixedVector: capacity " << N << " exceeded"); + for (size_type i = 0; i < n; ++i) { + data_[i] = value; + } + size_ = n; + } + + void resize(size_type n, const T& value = T{}) { + __assert(n <= N, "FixedVector: capacity " << N << " exceeded"); + for (size_type i = size_; i < n; ++i) { + data_[i] = value; + } + size_ = n; + } + + void push_back(const T& value) { + __assert(size_ < N, "FixedVector: capacity " << N << " exceeded"); + data_[size_++] = value; + } + + void pop_back() { --size_; } + void clear() { size_ = 0; } + +private: + std::array data_; + size_type size_; +}; + +} // namespace vortex diff --git a/sim/common/simobject.h b/sim/common/simobject.h index 16cff3443e..a953436c8d 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -38,6 +38,16 @@ #include "mempool.h" #include "util.h" #include "smallfunc.h" + +// The platform's per-thread execution context is read on every send and +// cycles() query. initial-exec turns each access into one %fs-relative load +// instead of a __tls_get_addr call. It also puts libsimx's whole TLS segment +// in glibc's static block, whose spare room for dlopen'ed libraries is a few +// hundred bytes shared by all of them: keep thread_local state in libsimx to +// a few pointers. +#ifndef SIMX_TLS_IE +#define SIMX_TLS_IE __attribute__((tls_model("initial-exec"))) +#endif #include "ringqueue.h" namespace vortex { @@ -121,13 +131,6 @@ class SimChannelBase { virtual uint32_t size() const = 0; virtual uint32_t capacity() const = 0; - // Process-global count of packets that have been sent but not yet consumed. - // Atomic: incremented/decremented from domain worker threads; only read for - // the termination check at a barrier point. - static std::atomic& inflight_count() { - static std::atomic count{0}; - return count; - } protected: // Defined after SimPlatform: registers with the platform channel registry @@ -145,12 +148,29 @@ class SimChannelBase { SimChannelBase* sink_; SimChannelBase* source_; SimChannelBase* endpoint_cache_ = nullptr; + // Packets sent toward this endpoint and not yet consumed are counted in the + // endpoint's domain (resolved on first use): only that domain's context + // touches the counter, so it needs no atomic and shares no cache line. + int64_t* inflight_ = nullptr; + int64_t& inflight() { + if (__builtin_expect(inflight_ == nullptr, 0)) { + inflight_ = &this->resolve_inflight(); + } + return *inflight_; + } + int64_t& resolve_inflight(); bool boundary_stage_ = false; SmallFunction pop_cb_; // Terminal endpoint of the bind chain (cached: bindings are immutable // after construction). - SimChannelBase* endpoint() { + // Bindings are immutable after elaboration, so the endpoint is resolved + // once and occupancy queries go straight to it. + SimChannelBase* endpoint() const { + return const_cast(this)->resolve_endpoint(); + } + + SimChannelBase* resolve_endpoint() { if (endpoint_cache_ == nullptr) { auto* e = this; while (e->sink_) { @@ -230,7 +250,7 @@ class SimCallEvent final : public SimEventBase { class SimPlatform { public: - static SimPlatform& instance() { + __attribute__((always_inline)) static SimPlatform& instance() { static SimPlatform s_inst; return s_inst; } @@ -258,7 +278,13 @@ class SimPlatform { // the only valid quiescence test — host loops must not reassemble it from // framework internals. bool idle() const { - return SimChannelBase::inflight_count() == 0 && this->cross_pending() == 0; + int64_t inflight = 0; + for (auto& dom : domains_) { + if (dom) { + inflight += dom->inflight; + } + } + return inflight == 0 && this->cross_pending() == 0; } // Domain topology API — used only by topology containers (Socket/Cluster); @@ -286,6 +312,12 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // The timed kernel keeps per-cycle lockstep: its cross-domain latencies are + // physical, so only the functional kernel accepts a larger quantum. + void set_quantum(uint32_t cycles) { + __assert(cycles <= 1, "a synchronization quantum requires the functional kernel"); + } + // Objects built under a LaunchKeepScope (the last-level cache and the // memory below it, when the command processor writes into that cache) // carry their state and in-flight work across reset_launch(). @@ -372,12 +404,23 @@ class SimPlatform { std::vector scan; // creation order within the domain std::vector> wheel; LinkedList imm; + // Packets in flight toward this domain's endpoints (see SimChannelBase). + alignas(64) int64_t inflight = 0; uint32_t delta = 0; uint32_t id = 0; Domain() : wheel(WHEEL_SIZE) {} }; Domain& domain_at(uint32_t d) { + if (__builtin_expect(d < domains_.size(), 1)) { + if (auto* dom = domains_[d].get()) { + return *dom; + } + } + return this->domain_grow(d); + } + + __attribute__((noinline)) Domain& domain_grow(uint32_t d) { if (domains_.size() <= d) { domains_.resize(d + 1); for (uint32_t i = 0; i < domains_.size(); ++i) { @@ -423,10 +466,9 @@ class SimPlatform { uint32_t build_domain_ = 0; bool build_launch_kept_ = false; bool launch_resetting_ = false; - static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; - static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; - static inline thread_local uint64_t tl_cross_seq_ = 0; - static inline thread_local uint64_t tl_cycles_ = 0; + static inline thread_local uint32_t tl_exec_domain_ SIMX_TLS_IE = HOST_DOMAIN; + static inline thread_local SimObjectBase* tl_exec_object_ SIMX_TLS_IE = nullptr; + static inline thread_local uint64_t tl_cycles_ SIMX_TLS_IE = 0; bool audit_enabled_ = false; bool audit_requested_ = false; std::map audit_hist_; @@ -437,37 +479,102 @@ class SimPlatform { // event-link primitives; host loops test quiescence through idle(). using CrossFn = SmallFunction; + // Cross-domain mailboxes. Each (target, source) pair owns a lane, so a + // source appends without locks and the target drains alone. A lane is + // triple-buffered by epoch (a cycle): during epoch e a source writes + // epoch e (its tick) and e+1 (its delivery phase), while the target drains + // epoch e-1 after the barrier that closed it. An entry written at cycle t + // is due at t+1, which is always in the epoch after the one it was written + // in, so draining epoch e-1 at the start of epoch e reproduces the + // (due, source, order) delivery of a single shared inbox exactly. + static constexpr uint32_t kCrossEpochs = 3; + static constexpr uint32_t kHostLane = 0; // lane 1 + d belongs to domain d + + struct CrossEntry { + uint64_t due; + CrossFn fn; + }; + struct alignas(64) CrossLane { + std::vector buf[kCrossEpochs]; + }; + // mail_[target][lane] + std::vector>> mail_; + + static uint64_t cross_epoch(uint64_t cycle) { + return cycle; + } + + // Undelivered cross-domain calls (host context only: workers parked). uint64_t cross_pending() const { - return cross_pending_.load(std::memory_order_acquire); + uint64_t n = 0; + for (auto& lanes : mail_) { + for (auto& lane : lanes) { + if (lane) { + for (auto& b : lane->buf) { + n += b.size(); + } + } + } + } + return n; } // Host-context calls (tl_exec_domain_ == HOST_DOMAIN) must come from one - // thread at a time — the per-thread sequence numbers that order same-cycle - // entries are only comparable within a single source thread. + // thread at a time; they share the host lane. void cross_call(uint32_t target_domain, CrossFn&& fn) { assert(target_domain < num_domains_); - auto& inbox = this->cross_inbox_at(target_domain); uint32_t src = tl_exec_domain_; - uint64_t seq = ++tl_cross_seq_; - { - std::lock_guard g(inbox.lock); - inbox.entries.push_back({this->cycles() + 1, src, seq, std::move(fn)}); + uint64_t now = this->cycles(); + auto& lane = this->cross_lane(target_domain, (src == HOST_DOMAIN) ? kHostLane : (src + 1)); + lane.buf[cross_epoch(now) % kCrossEpochs].push_back({now + 1, std::move(fn)}); + } + + CrossLane& cross_lane(uint32_t target, uint32_t lane) { + if (__builtin_expect(target < mail_.size(), 1)) { + auto& lanes = mail_[target]; + if (__builtin_expect(lane < lanes.size() && lanes[lane] != nullptr, 1)) { + return *lanes[lane]; + } } - cross_pending_.fetch_add(1, std::memory_order_release); + return this->cross_lane_grow(target, lane); } - struct CrossEntry { - uint64_t due; - uint32_t src; - uint64_t seq; - CrossFn fn; - }; - struct CrossInbox { - std::mutex lock; - std::vector entries; - }; - std::vector> cross_inbox_; - std::atomic cross_pending_{0}; + // Elaboration or serial execution only: parallel workers find every lane + // materialized by init_mt. + __attribute__((noinline)) CrossLane& cross_lane_grow(uint32_t target, uint32_t lane) { + __assert(!workers_started_, "cross-domain lane created during parallel execution"); + if (mail_.size() <= target) { + mail_.resize(target + 1); + } + auto& lanes = mail_[target]; + if (lanes.size() <= lane) { + lanes.resize(lane + 1); + } + if (!lanes[lane]) { + lanes[lane] = std::make_unique(); + } + return *lanes[lane]; + } + + void materialize_cross_lanes() { + for (uint32_t t = 0; t < num_domains_; ++t) { + for (uint32_t l = 0; l <= num_domains_; ++l) { + (void)this->cross_lane_grow(t, l); + } + } + } + + void clear_cross_lanes() { + for (auto& lanes : mail_) { + for (auto& lane : lanes) { + if (lane) { + for (auto& b : lane->buf) { + b.clear(); + } + } + } + } + } // Channel registry (elaboration-time only) for topology validation. std::vector channels_; @@ -487,59 +594,54 @@ class SimPlatform { void validate_topology(); - CrossInbox& cross_inbox_at(uint32_t d) { - if (cross_inbox_.size() <= d) { - cross_inbox_.resize(d + 1); - } - if (!cross_inbox_[d]) { - cross_inbox_[d] = std::make_unique(); - } - return *cross_inbox_[d]; - } - - // Run the calls due this cycle for one domain, in (due, src, seq) order — - // the canonical order that makes serial and parallel execution agree. + // Run the calls due this epoch for one domain: the previous epoch's buffer + // of each lane, in canonical order — the order that makes serial and + // parallel execution agree. void drain_cross_calls_for(uint32_t d) { - if (cross_inbox_.size() <= d || !cross_inbox_[d]) { + if (d >= mail_.size()) { return; } - auto& inbox = *cross_inbox_[d]; - std::vector due_list; - { - std::lock_guard g(inbox.lock); - auto& v = inbox.entries; - for (size_t i = 0; i < v.size();) { - if (v[i].due <= this->cycles()) { - due_list.push_back(std::move(v[i])); - v[i] = std::move(v.back()); - v.pop_back(); - } else { - ++i; - } + uint32_t nlanes = uint32_t(mail_[d].size()); + uint32_t idx = uint32_t((cross_epoch(this->cycles()) + kCrossEpochs - 1) % kCrossEpochs); + bool any = false; + for (auto& lane : mail_[d]) { + if (lane && !lane->buf[idx].empty()) { + any = true; + break; } } - if (due_list.empty()) { + if (!any) { + return; + } + // One epoch is one cycle: every entry has the same due cycle, so the + // canonical order is source order, host last. + for (uint32_t l = 1; l <= nlanes; ++l) { + this->run_lane(d, l % nlanes, idx); + } + } + + // A call may create a lane in serial execution, which can reallocate + // mail_: index it afresh per lane. The lanes themselves never move. + void run_lane(uint32_t d, uint32_t l, uint32_t idx) { + auto* lane = mail_[d][l].get(); + if (lane == nullptr) { return; } - std::sort(due_list.begin(), due_list.end(), - [](const CrossEntry& a, const CrossEntry& b) { - return (a.due != b.due) ? (a.due < b.due) - : (a.src != b.src) ? (a.src < b.src) - : (a.seq < b.seq); - }); - for (auto& e : due_list) { + auto& buf = lane->buf[idx]; + for (auto& e : buf) { e.fn(); - cross_pending_.fetch_sub(1, std::memory_order_release); } + buf.clear(); } // ── Multi-threaded executor ──────────────────────────────────────────── // One persistent worker per extra thread; domains are assigned round-robin // (coordinator keeps stripe 0). Two phases per cycle, separated by spinning - // epoch-counting barriers: A) drain-inbox + tick each domain, B) deliver - // each domain's due wheel events. All per-domain state is touched only by - // its owner; the only shared structures are the cross inboxes (locked), the - // inflight/cross counters (atomic), and the barrier itself. + // epoch-counting barriers: A) drain the mailboxes + tick each domain, + // B) deliver each domain's due wheel events. All per-domain state, the + // inflight count included, is touched only by its owner; the only shared + // structures are the mailbox lanes (one writer and one reader each, kept + // apart by epoch) and the barrier itself. uint32_t mt_threads_ = 1; uint32_t req_workers_ = 1; uint32_t mt_max_threads_ = std::numeric_limits::max(); @@ -555,6 +657,8 @@ class SimPlatform { // OS thread on every call without desynchronizing the workers. The last // thread to arrive runs on_release before publishing the new epoch, so its // writes are visible to every thread on the other side of the barrier. + static constexpr uint32_t kBarrierSpins = 4096; + template void barrier_wait(Fn&& on_release) { int nthreads = int(mt_threads_); @@ -564,10 +668,18 @@ class SimPlatform { on_release(); barrier_epoch_.store(epoch + 1, std::memory_order_release); } else { + // Spin briefly (the common case: peers arrive within microseconds), + // then yield so a waiting thread does not starve a descheduled peer + // on an oversubscribed host. + uint32_t spins = 0; while (barrier_epoch_.load(std::memory_order_acquire) == epoch) { - #if defined(__x86_64__) - __builtin_ia32_pause(); - #endif + if (++spins < kBarrierSpins) { + #if defined(__x86_64__) + __builtin_ia32_pause(); + #endif + } else { + std::this_thread::yield(); + } } } } @@ -604,12 +716,10 @@ class SimPlatform { } audit_enabled_ = false; // the audit histogram is not thread-safe mt_threads_ = count; - // Materialize every domain and inbox up front: workers must never resize + // Materialize every domain and mailbox lane up front: workers must never resize // these containers or construct a slot concurrently. (void)this->domain_at(num_domains_ - 1); - for (uint32_t d = 0; d < num_domains_; ++d) { - (void)this->cross_inbox_at(d); - } + this->materialize_cross_lanes(); stripe_domains_.assign(count, {}); for (uint32_t d = 0; d < num_domains_; ++d) { stripe_domains_[d % count].push_back(domains_[d].get()); @@ -622,7 +732,7 @@ class SimPlatform { break; } // Tick and deliver back-to-back: a domain's wheel is written only - // by its own context (cross traffic goes through the inboxes), so + // by its own context (cross traffic goes through the mailboxes), so // no synchronization is needed between the two phases — each // worker advances its thread-local cycle view instead. uint64_t cycle = cycles_; @@ -697,6 +807,7 @@ class SimChannel : public SimChannelBase { sink_ = nullptr; source_ = nullptr; endpoint_cache_ = nullptr; + inflight_ = nullptr; pop_cb_ = {}; storage_ = RingQueue(other.storage_.capacity()); pending_count_ = 0; @@ -738,7 +849,7 @@ class SimChannel : public SimChannelBase { bool full() const override { if (sink_) { - return sink_->full(); + return this->endpoint()->full(); } assert(!SimPlatform::instance().mt_cross_access(module_)); return this->occupancy() + this->freed_now() >= storage_.capacity(); @@ -759,12 +870,7 @@ class SimChannel : public SimChannelBase { // boundary, so a deferred send always finds space. if (platform.is_cross_exec(this->endpoint()->module_)) { __assert(delay >= 1, "cross-domain send requires a registered delay"); - // Boxed: keeps the capture pointer-sized for any packet type; boundary - // traffic is a handful of packets per cycle. - platform.cross_call(this->endpoint()->module_->domain(), - [this, p = std::make_shared(pkt), delay]() { - this->send(*p, delay - 1); - }); + this->cross_send(Pkt(pkt), delay); return; } __assert(!this->full(), "channel is full"); @@ -779,10 +885,7 @@ class SimChannel : public SimChannelBase { auto& platform = SimPlatform::instance(); if (platform.is_cross_exec(this->endpoint()->module_)) { __assert(delay >= 1, "cross-domain send requires a registered delay"); - platform.cross_call(this->endpoint()->module_->domain(), - [this, p = std::make_shared(std::move(pkt)), delay]() { - this->send(*p, delay - 1); - }); + this->cross_send(std::move(pkt), delay); return; } __assert(!this->full(), "channel is full"); @@ -805,6 +908,25 @@ class SimChannel : public SimChannelBase { return true; } + // Hand a send to the endpoint's domain: it re-issues the send locally with + // one less delay, landing on the same absolute cycle. A packet that fits + // the call's inline buffer rides by value; a larger one is boxed. + void cross_send(Pkt&& pkt, uint64_t delay) { + auto& platform = SimPlatform::instance(); + uint32_t target = this->endpoint()->module_->domain(); + if constexpr (sizeof(Pkt) <= kInlineCrossPkt) { + platform.cross_call(target, [this, p = std::move(pkt), delay]() { + this->send(p, delay - 1); + }); + } else { + platform.cross_call(target, [this, p = std::make_shared(std::move(pkt)), delay]() { + this->send(*p, delay - 1); + }); + } + } + + static constexpr size_t kInlineCrossPkt = 128; + bool empty() const override { this->assert_endpoint(); return this->queue_empty(); @@ -832,24 +954,26 @@ class SimChannel : public SimChannelBase { uint32_t size() const override { if (sink_) { - return sink_->size(); + return this->endpoint()->size(); } assert(!SimPlatform::instance().mt_cross_access(module_)); return this->occupancy(); } uint32_t capacity() const override { - if (sink_) return sink_->capacity(); + if (sink_) { + return this->endpoint()->capacity(); + } return storage_.capacity(); } protected: void reserve() override { if (sink_) { - sink_->reserve(); + this->endpoint()->reserve(); } else { ++pending_count_; - ++SimChannelBase::inflight_count(); + ++this->inflight(); wake_module(module_); } } @@ -907,7 +1031,7 @@ class SimChannel : public SimChannelBase { ++freed_count_; } storage_.pop(); - --SimChannelBase::inflight_count(); + --this->inflight(); if (pop_cb_) { pop_cb_(); } @@ -1263,16 +1387,10 @@ inline void SimPlatform::reset() { object->tick_active_ = true; } - // Drop undelivered cross-domain calls from the previous run. The inbox - // slots themselves are kept: workers index them concurrently once started, + // Drop undelivered cross-domain calls from the previous run. The lanes + // themselves are kept: workers index them concurrently once started, // so the vector must never shrink after init_mt materialized it. - for (auto& inbox : cross_inbox_) { - if (inbox) { - std::lock_guard g(inbox->lock); - inbox->entries.clear(); - } - } - cross_pending_ = 0; + this->clear_cross_lanes(); // Reset timing cycles_ = 0; @@ -1329,18 +1447,23 @@ inline void SimPlatform::tick_domain(Domain& dom) { tl_exec_domain_ = dom.id; tl_exec_object_ = nullptr; // Start-of-cycle for this domain: integrate deferred cross-domain work. - if (this->cross_pending() != 0) { + { this->drain_cross_calls_for(dom.id); } if (dom.delta != 0) { fire_immediate_events(dom); } + // Every object on a domain's scan list belongs to that domain, so the + // execution domain set above holds for the whole scan. The current object + // is tracked only for the boundary audit, its sole reader. + const bool track_object = audit_enabled_; for (auto* object : dom.scan) { if (!object->tick_active_) { continue; } - tl_exec_domain_ = object->domain_; - tl_exec_object_ = object; + if (track_object) { + tl_exec_object_ = object; + } object->do_tick(); if (dom.delta != 0) { fire_immediate_events(dom); @@ -1374,8 +1497,7 @@ inline void SimPlatform::cleanup() { } } domains_.clear(); - cross_inbox_.clear(); - cross_pending_ = 0; + mail_.clear(); num_domains_ = 1; cycles_ = 0; // Allow a subsequent initialize() to elaborate and parallelize afresh. @@ -1417,6 +1539,10 @@ inline SimChannelBase::SimChannelBase(SimObjectBase* module) SimPlatform::instance().register_channel(this); } +inline int64_t& SimChannelBase::resolve_inflight() { + return SimPlatform::instance().domain_at(module_->domain()).inflight; +} + inline SimChannelBase::~SimChannelBase() { SimPlatform::instance().unregister_channel(this); } diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 89615a5ec9..e26efdb80a 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -38,6 +38,17 @@ #include "mempool.h" #include "util.h" #include "smallfunc.h" +#include "ringqueue.h" + +// The platform's per-thread execution context is read on every send and +// cycles() query. initial-exec turns each access into one %fs-relative load +// instead of a __tls_get_addr call. It also puts libsimx's whole TLS segment +// in glibc's static block, whose spare room for dlopen'ed libraries is a few +// hundred bytes shared by all of them: keep thread_local state in libsimx to +// a few pointers. +#ifndef SIMX_TLS_IE +#define SIMX_TLS_IE __attribute__((tls_model("initial-exec"))) +#endif // Functional simulation kernel: same executor as the timed kernel in // simobject.h (domains, lockstep multi-threading, canonical cross-domain @@ -136,13 +147,6 @@ class SimChannelBase { virtual uint32_t size() const = 0; virtual uint32_t capacity() const = 0; - // Process-global count of packets that have been sent but not yet consumed. - // Atomic: incremented/decremented from domain worker threads; only read for - // the termination check at a barrier point. - static std::atomic& inflight_count() { - static std::atomic count{0}; - return count; - } protected: // Defined after SimPlatform: registers with the platform channel registry @@ -160,12 +164,29 @@ class SimChannelBase { SimChannelBase* sink_; SimChannelBase* source_; SimChannelBase* endpoint_cache_ = nullptr; + // Packets sent toward this endpoint and not yet consumed are counted in the + // endpoint's domain (resolved on first use): only that domain's context + // touches the counter, so it needs no atomic and shares no cache line. + int64_t* inflight_ = nullptr; + int64_t& inflight() { + if (__builtin_expect(inflight_ == nullptr, 0)) { + inflight_ = &this->resolve_inflight(); + } + return *inflight_; + } + int64_t& resolve_inflight(); bool boundary_stage_ = false; SmallFunction pop_cb_; // Terminal endpoint of the bind chain (cached: bindings are immutable // after construction). - SimChannelBase* endpoint() { + // Bindings are immutable after elaboration, so the endpoint is resolved + // once and occupancy queries go straight to it. + SimChannelBase* endpoint() const { + return const_cast(this)->resolve_endpoint(); + } + + SimChannelBase* resolve_endpoint() { if (endpoint_cache_ == nullptr) { auto* e = this; while (e->sink_) { @@ -245,7 +266,7 @@ class SimCallEvent final : public SimEventBase { class SimPlatform { public: - static SimPlatform& instance() { + __attribute__((always_inline)) static SimPlatform& instance() { static SimPlatform s_inst; return s_inst; } @@ -269,7 +290,13 @@ class SimPlatform { // the only valid quiescence test — host loops must not reassemble it from // framework internals. bool idle() const { - return SimChannelBase::inflight_count() == 0 && this->cross_pending() == 0; + int64_t inflight = 0; + for (auto& dom : domains_) { + if (dom) { + inflight += dom->inflight; + } + } + return inflight == 0 && this->cross_pending() == 0; } // Domain topology API — used only by topology containers (Socket/Cluster); @@ -297,6 +324,16 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Synchronization quantum (must be set before the first tick): execution + // domains run this many cycles between barriers, and a cross-domain call + // runs at the first quantum boundary at or after its due cycle. Serial and parallel + // runs stay bit-identical for a given quantum; 1 keeps per-cycle lockstep. + // Legal only here — the functional kernel's latencies are non-physical. + void set_quantum(uint32_t cycles) { + __assert(!mt_init_done_, "the quantum is fixed once execution starts"); + quantum_ = std::max(1u, cycles); + } + // Objects built under a LaunchKeepScope (the last-level cache and the // memory below it, when the command processor writes into that cache) // carry their state and in-flight work across reset_launch(). @@ -382,16 +419,29 @@ class SimPlatform { // list are private to one execution domain, so a domain can be ticked by // its own worker without touching another domain's structures. Events are // routed to the ENDPOINT's domain at schedule time. + struct CrossEntry; struct Domain { std::vector scan; // creation order within the domain std::vector> wheel; LinkedList imm; + // Packets in flight toward this domain's endpoints (see SimChannelBase). + alignas(64) int64_t inflight = 0; + std::vector cross_order; // drain scratch (quantum mode) uint32_t delta = 0; uint32_t id = 0; Domain() : wheel(WHEEL_SIZE) {} }; Domain& domain_at(uint32_t d) { + if (__builtin_expect(d < domains_.size(), 1)) { + if (auto* dom = domains_[d].get()) { + return *dom; + } + } + return this->domain_grow(d); + } + + __attribute__((noinline)) Domain& domain_grow(uint32_t d) { if (domains_.size() <= d) { domains_.resize(d + 1); for (uint32_t i = 0; i < domains_.size(); ++i) { @@ -413,8 +463,10 @@ class SimPlatform { } ~SimPlatform() { cleanup(); } + void cleanup(); - void tick_domain(Domain& dom); + void tick_domain(Domain& dom, bool drain = true); + void run_quantum(Domain& dom, uint64_t base); void fire_immediate_events(Domain& dom); template @@ -436,10 +488,9 @@ class SimPlatform { uint32_t build_domain_ = 0; bool build_launch_kept_ = false; bool launch_resetting_ = false; - static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; - static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; - static inline thread_local uint64_t tl_cross_seq_ = 0; - static inline thread_local uint64_t tl_cycles_ = 0; + static inline thread_local uint32_t tl_exec_domain_ SIMX_TLS_IE = HOST_DOMAIN; + static inline thread_local SimObjectBase* tl_exec_object_ SIMX_TLS_IE = nullptr; + static inline thread_local uint64_t tl_cycles_ SIMX_TLS_IE = 0; bool audit_enabled_ = false; bool audit_requested_ = false; std::map audit_hist_; @@ -450,37 +501,102 @@ class SimPlatform { // event-link primitives; host loops test quiescence through idle(). using CrossFn = SmallFunction; + // Cross-domain mailboxes. Each (target, source) pair owns a lane, so a + // source appends without locks and the target drains alone. A lane is + // triple-buffered by epoch (a quantum): during epoch e a source writes + // epoch e, and e+1 from the delivery phase that closes the quantum, while + // the target drains epoch e-1 after the barrier that closed it. So a call + // runs at the first quantum boundary at or after its due cycle, sorted by + // (due, source, order); with a quantum of 1 that is exactly the delivery of + // a single shared inbox. + static constexpr uint32_t kCrossEpochs = 3; + static constexpr uint32_t kHostLane = 0; // lane 1 + d belongs to domain d + + struct CrossEntry { + uint64_t due; + CrossFn fn; + }; + struct alignas(64) CrossLane { + std::vector buf[kCrossEpochs]; + }; + // mail_[target][lane] + std::vector>> mail_; + + uint64_t cross_epoch(uint64_t cycle) const { + return cycle / quantum_; + } + + // Undelivered cross-domain calls (host context only: workers parked). uint64_t cross_pending() const { - return cross_pending_.load(std::memory_order_acquire); + uint64_t n = 0; + for (auto& lanes : mail_) { + for (auto& lane : lanes) { + if (lane) { + for (auto& b : lane->buf) { + n += b.size(); + } + } + } + } + return n; } // Host-context calls (tl_exec_domain_ == HOST_DOMAIN) must come from one - // thread at a time — the per-thread sequence numbers that order same-cycle - // entries are only comparable within a single source thread. + // thread at a time; they share the host lane. void cross_call(uint32_t target_domain, CrossFn&& fn) { assert(target_domain < num_domains_); - auto& inbox = this->cross_inbox_at(target_domain); uint32_t src = tl_exec_domain_; - uint64_t seq = ++tl_cross_seq_; - { - std::lock_guard g(inbox.lock); - inbox.entries.push_back({this->cycles() + 1, src, seq, std::move(fn)}); + uint64_t now = this->cycles(); + auto& lane = this->cross_lane(target_domain, (src == HOST_DOMAIN) ? kHostLane : (src + 1)); + lane.buf[cross_epoch(now) % kCrossEpochs].push_back({now + 1, std::move(fn)}); + } + + CrossLane& cross_lane(uint32_t target, uint32_t lane) { + if (__builtin_expect(target < mail_.size(), 1)) { + auto& lanes = mail_[target]; + if (__builtin_expect(lane < lanes.size() && lanes[lane] != nullptr, 1)) { + return *lanes[lane]; + } } - cross_pending_.fetch_add(1, std::memory_order_release); + return this->cross_lane_grow(target, lane); } - struct CrossEntry { - uint64_t due; - uint32_t src; - uint64_t seq; - CrossFn fn; - }; - struct CrossInbox { - std::mutex lock; - std::vector entries; - }; - std::vector> cross_inbox_; - std::atomic cross_pending_{0}; + // Elaboration or serial execution only: parallel workers find every lane + // materialized by init_mt. + __attribute__((noinline)) CrossLane& cross_lane_grow(uint32_t target, uint32_t lane) { + __assert(!workers_started_, "cross-domain lane created during parallel execution"); + if (mail_.size() <= target) { + mail_.resize(target + 1); + } + auto& lanes = mail_[target]; + if (lanes.size() <= lane) { + lanes.resize(lane + 1); + } + if (!lanes[lane]) { + lanes[lane] = std::make_unique(); + } + return *lanes[lane]; + } + + void materialize_cross_lanes() { + for (uint32_t t = 0; t < num_domains_; ++t) { + for (uint32_t l = 0; l <= num_domains_; ++l) { + (void)this->cross_lane_grow(t, l); + } + } + } + + void clear_cross_lanes() { + for (auto& lanes : mail_) { + for (auto& lane : lanes) { + if (lane) { + for (auto& b : lane->buf) { + b.clear(); + } + } + } + } + } // Channel registry (elaboration-time only) for topology validation. std::vector channels_; @@ -500,61 +616,83 @@ class SimPlatform { void validate_topology(); - CrossInbox& cross_inbox_at(uint32_t d) { - if (cross_inbox_.size() <= d) { - cross_inbox_.resize(d + 1); + // Run the calls due this epoch for one domain: the previous epoch's buffer + // of each lane, in canonical order — the order that makes serial and + // parallel execution agree. + void drain_cross_calls_for(uint32_t d) { + if (d >= mail_.size()) { + return; } - if (!cross_inbox_[d]) { - cross_inbox_[d] = std::make_unique(); + uint32_t nlanes = uint32_t(mail_[d].size()); + uint32_t idx = uint32_t((cross_epoch(this->cycles()) + kCrossEpochs - 1) % kCrossEpochs); + bool any = false; + for (auto& lane : mail_[d]) { + if (lane && !lane->buf[idx].empty()) { + any = true; + break; + } } - return *cross_inbox_[d]; - } - - // Run the calls due this cycle for one domain, in (due, src, seq) order — - // the canonical order that makes serial and parallel execution agree. - void drain_cross_calls_for(uint32_t d) { - if (cross_inbox_.size() <= d || !cross_inbox_[d]) { + if (!any) { return; } - auto& inbox = *cross_inbox_[d]; - std::vector due_list; - { - std::lock_guard g(inbox.lock); - auto& v = inbox.entries; - for (size_t i = 0; i < v.size();) { - if (v[i].due <= this->cycles()) { - due_list.push_back(std::move(v[i])); - v[i] = std::move(v.back()); - v.pop_back(); - } else { - ++i; + if (quantum_ == 1) { + // One epoch is one cycle: every entry has the same due cycle, so the + // canonical order is source order, host last. + for (uint32_t l = 1; l <= nlanes; ++l) { + this->run_lane(d, l % nlanes, idx); + } + return; + } + // A quantum spans several due cycles: order by due, then source (host + // last), then append order. The calls write only the current epoch's + // buffers, so the entries being run stay in place. + auto& order = domain_at(d).cross_order; + order.clear(); + for (uint32_t l = 1; l <= nlanes; ++l) { + auto* lane = mail_[d][l % nlanes].get(); + if (lane) { + for (auto& e : lane->buf[idx]) { + order.push_back(&e); } } } - if (due_list.empty()) { + std::stable_sort(order.begin(), order.end(), + [](const CrossEntry* a, const CrossEntry* b) { return a->due < b->due; }); + for (auto* e : order) { + e->fn(); + } + for (auto& lane : mail_[d]) { + if (lane) { + lane->buf[idx].clear(); + } + } + } + + // A call may create a lane in serial execution, which can reallocate + // mail_: index it afresh per lane. The lanes themselves never move. + void run_lane(uint32_t d, uint32_t l, uint32_t idx) { + auto* lane = mail_[d][l].get(); + if (lane == nullptr) { return; } - std::sort(due_list.begin(), due_list.end(), - [](const CrossEntry& a, const CrossEntry& b) { - return (a.due != b.due) ? (a.due < b.due) - : (a.src != b.src) ? (a.src < b.src) - : (a.seq < b.seq); - }); - for (auto& e : due_list) { + auto& buf = lane->buf[idx]; + for (auto& e : buf) { e.fn(); - cross_pending_.fetch_sub(1, std::memory_order_release); } + buf.clear(); } // ── Multi-threaded executor ──────────────────────────────────────────── // One persistent worker per extra thread; domains are assigned round-robin // (coordinator keeps stripe 0). Two phases per cycle, separated by spinning - // epoch-counting barriers: A) drain-inbox + tick each domain, B) deliver - // each domain's due wheel events. All per-domain state is touched only by - // its owner; the only shared structures are the cross inboxes (locked), the - // inflight/cross counters (atomic), and the barrier itself. + // epoch-counting barriers: A) drain the mailboxes + tick each domain, + // B) deliver each domain's due wheel events. All per-domain state, the + // inflight count included, is touched only by its owner; the only shared + // structures are the mailbox lanes (one writer and one reader each, kept + // apart by epoch) and the barrier itself. uint32_t mt_threads_ = 1; uint32_t req_workers_ = 1; + uint64_t quantum_ = 1; uint32_t mt_max_threads_ = std::numeric_limits::max(); bool mt_init_done_ = false; bool workers_started_ = false; @@ -568,6 +706,8 @@ class SimPlatform { // OS thread on every call without desynchronizing the workers. The last // thread to arrive runs on_release before publishing the new epoch, so its // writes are visible to every thread on the other side of the barrier. + static constexpr uint32_t kBarrierSpins = 4096; + template void barrier_wait(Fn&& on_release) { int nthreads = int(mt_threads_); @@ -577,10 +717,18 @@ class SimPlatform { on_release(); barrier_epoch_.store(epoch + 1, std::memory_order_release); } else { + // Spin briefly (the common case: peers arrive within microseconds), + // then yield so a waiting thread does not starve a descheduled peer + // on an oversubscribed host. + uint32_t spins = 0; while (barrier_epoch_.load(std::memory_order_acquire) == epoch) { - #if defined(__x86_64__) - __builtin_ia32_pause(); - #endif + if (++spins < kBarrierSpins) { + #if defined(__x86_64__) + __builtin_ia32_pause(); + #endif + } else { + std::this_thread::yield(); + } } } } @@ -617,12 +765,10 @@ class SimPlatform { } audit_enabled_ = false; // the audit histogram is not thread-safe mt_threads_ = count; - // Materialize every domain and inbox up front: workers must never resize + // Materialize every domain and mailbox lane up front: workers must never resize // these containers or construct a slot concurrently. (void)this->domain_at(num_domains_ - 1); - for (uint32_t d = 0; d < num_domains_; ++d) { - (void)this->cross_inbox_at(d); - } + this->materialize_cross_lanes(); stripe_domains_.assign(count, {}); for (uint32_t d = 0; d < num_domains_; ++d) { stripe_domains_[d % count].push_back(domains_[d].get()); @@ -635,10 +781,19 @@ class SimPlatform { break; } // Tick and deliver back-to-back: a domain's wheel is written only - // by its own context (cross traffic goes through the inboxes), so + // by its own context (cross traffic goes through the mailboxes), so // no synchronization is needed between the two phases — each // worker advances its thread-local cycle view instead. uint64_t cycle = cycles_; + if (quantum_ > 1) { + for (auto* dom : stripe_domains_[w]) { + this->run_quantum(*dom, cycle); + } + tl_exec_domain_ = HOST_DOMAIN; + tl_exec_object_ = nullptr; + this->barrier_wait([this]() { cycles_ += quantum_; }); + continue; + } tl_cycles_ = cycle; for (auto* dom : stripe_domains_[w]) { this->tick_domain(*dom); @@ -692,10 +847,12 @@ class SimChannel : public SimChannelBase { // compatibility and ignored. SimChannel(SimObjectBase* module, uint32_t /*capacity*/ = 2) : SimChannelBase(module) + , storage_(kInitCapacity) , pending_count_(0) {} SimChannel(const SimChannel& other) : SimChannelBase(other.module_) + , storage_(kInitCapacity) , pending_count_(0) , convert_fn_(other.convert_fn_) , tx_cb_(other.tx_cb_) { @@ -709,6 +866,7 @@ class SimChannel : public SimChannelBase { sink_ = nullptr; source_ = nullptr; endpoint_cache_ = nullptr; + inflight_ = nullptr; pop_cb_ = {}; storage_.clear(); pending_count_ = 0; @@ -765,12 +923,7 @@ class SimChannel : public SimChannelBase { // channel state is thereby touched only by its owning thread. if (platform.is_cross_exec(this->endpoint()->module_)) { __assert(delay >= 1, "cross-domain send requires a registered delay"); - // Boxed: keeps the capture pointer-sized for any packet type; boundary - // traffic is a handful of packets per cycle. - platform.cross_call(this->endpoint()->module_->domain(), - [this, p = std::make_shared(pkt), delay]() { - this->send(*p, delay - 1); - }); + this->cross_send(Pkt(pkt), delay); return; } if (platform.audit_enabled()) { @@ -787,10 +940,7 @@ class SimChannel : public SimChannelBase { auto& platform = SimPlatform::instance(); if (platform.is_cross_exec(this->endpoint()->module_)) { __assert(delay >= 1, "cross-domain send requires a registered delay"); - platform.cross_call(this->endpoint()->module_->domain(), - [this, p = std::make_shared(std::move(pkt)), delay]() { - this->send(*p, delay - 1); - }); + this->cross_send(std::move(pkt), delay); return; } if (platform.audit_enabled()) { @@ -810,6 +960,25 @@ class SimChannel : public SimChannelBase { return true; } + // Hand a send to the endpoint's domain: it re-issues the send locally with + // one less delay, landing on the same absolute cycle. A packet that fits + // the call's inline buffer rides by value; a larger one is boxed. + void cross_send(Pkt&& pkt, uint64_t delay) { + auto& platform = SimPlatform::instance(); + uint32_t target = this->endpoint()->module_->domain(); + if constexpr (sizeof(Pkt) <= kInlineCrossPkt) { + platform.cross_call(target, [this, p = std::move(pkt), delay]() { + this->send(p, delay - 1); + }); + } else { + platform.cross_call(target, [this, p = std::make_shared(std::move(pkt)), delay]() { + this->send(*p, delay - 1); + }); + } + } + + static constexpr size_t kInlineCrossPkt = 128; + bool empty() const override { this->assert_endpoint(); return this->queue_empty(); @@ -837,7 +1006,7 @@ class SimChannel : public SimChannelBase { uint32_t size() const override { if (sink_) { - return sink_->size(); + return this->endpoint()->size(); } assert(!SimPlatform::instance().mt_cross_access(module_)); return this->occupancy(); @@ -850,10 +1019,10 @@ class SimChannel : public SimChannelBase { protected: void reserve() override { if (sink_) { - sink_->reserve(); + this->endpoint()->reserve(); } else { ++pending_count_; - ++SimChannelBase::inflight_count(); + ++this->inflight(); wake_module(module_); } } @@ -893,11 +1062,11 @@ class SimChannel : public SimChannelBase { uint32_t occupancy() const { return this->queue_size() + pending_count_; } bool queue_empty() const { return storage_.empty(); } - uint32_t queue_size() const { return uint32_t(storage_.size()); } + uint32_t queue_size() const { return storage_.size(); } const Pkt& queue_front() const { return storage_.front(); } void queue_pop() { - storage_.pop_front(); - --SimChannelBase::inflight_count(); + storage_.pop(); + --this->inflight(); if (pop_cb_) { pop_cb_(); } @@ -906,10 +1075,16 @@ class SimChannel : public SimChannelBase { // High-water check: with back-pressure disabled a feedback loop that // outruns its consumer must fail loudly instead of growing silently. __assert(storage_.size() < (1u << 20), "functional channel high-water exceeded"); - storage_.push_back(pkt); + if (__builtin_expect(storage_.full(), 0)) { + storage_.grow(); + } + storage_.push(pkt); } - std::deque storage_; + // Unbounded ring: starts small and doubles on demand, so steady-state + // traffic reuses the same slots with no per-packet allocation. + static constexpr uint32_t kInitCapacity = 4; + RingQueue storage_; uint32_t pending_count_; SmallFunction convert_fn_; TxCallback tx_cb_; @@ -1251,16 +1426,10 @@ inline void SimPlatform::reset() { object->tick_active_ = true; } - // Drop undelivered cross-domain calls from the previous run. The inbox - // slots themselves are kept: workers index them concurrently once started, + // Drop undelivered cross-domain calls from the previous run. The lanes + // themselves are kept: workers index them concurrently once started, // so the vector must never shrink after init_mt materialized it. - for (auto& inbox : cross_inbox_) { - if (inbox) { - std::lock_guard g(inbox->lock); - inbox->entries.clear(); - } - } - cross_pending_ = 0; + this->clear_cross_lanes(); // Reset timing cycles_ = 0; @@ -1278,6 +1447,15 @@ inline void SimPlatform::tick() { // per-phase cycle view is thread-local (see cycles()). this->barrier_wait(); // release workers uint64_t cycle = cycles_; + if (quantum_ > 1) { + for (auto* dom : stripe_domains_[0]) { + this->run_quantum(*dom, cycle); + } + tl_exec_domain_ = HOST_DOMAIN; + tl_exec_object_ = nullptr; + this->barrier_wait([this]() { cycles_ += quantum_; }); + return; + } tl_cycles_ = cycle; for (auto* dom : stripe_domains_[0]) { this->tick_domain(*dom); @@ -1292,6 +1470,20 @@ inline void SimPlatform::tick() { return; } + if (quantum_ > 1) { + // Domain-major: domains do not interact inside a quantum, so running + // each one to the quantum's end in turn matches the parallel schedule. + uint64_t base = cycles_; + for (auto& dom : domains_) { + this->run_quantum(*dom, base); + } + cycles_ = base + quantum_; + tl_cycles_ = cycles_; + tl_exec_domain_ = HOST_DOMAIN; + tl_exec_object_ = nullptr; + return; + } + // Serial reference path: tick each domain — its due cross calls, immediate // events, then its scan list in creation order (the order fixes the // delta-event interleaving within the domain). Domains only interact @@ -1312,22 +1504,40 @@ inline void SimPlatform::tick() { tl_exec_object_ = nullptr; } -inline void SimPlatform::tick_domain(Domain& dom) { +// One quantum of a domain: cross-domain work produced before the quantum is +// integrated at its first cycle only (anything sent during the quantum is due +// after it and waits for the next one), so the domain never observes another +// domain's progress inside the quantum. +inline void SimPlatform::run_quantum(Domain& dom, uint64_t base) { + for (uint64_t q = 0; q < quantum_; ++q) { + tl_cycles_ = base + q; + this->tick_domain(dom, q == 0); + tl_cycles_ = base + q + 1; + this->fire_bucket(dom, base + q + 1); + } +} + +inline void SimPlatform::tick_domain(Domain& dom, bool drain) { tl_exec_domain_ = dom.id; tl_exec_object_ = nullptr; // Start-of-cycle for this domain: integrate deferred cross-domain work. - if (this->cross_pending() != 0) { + if (drain) { this->drain_cross_calls_for(dom.id); } if (dom.delta != 0) { fire_immediate_events(dom); } + // Every object on a domain's scan list belongs to that domain, so the + // execution domain set above holds for the whole scan. The current object + // is tracked only for the boundary audit, its sole reader. + const bool track_object = audit_enabled_; for (auto* object : dom.scan) { if (!object->tick_active_) { continue; } - tl_exec_domain_ = object->domain_; - tl_exec_object_ = object; + if (track_object) { + tl_exec_object_ = object; + } object->do_tick(); if (dom.delta != 0) { fire_immediate_events(dom); @@ -1361,8 +1571,7 @@ inline void SimPlatform::cleanup() { } } domains_.clear(); - cross_inbox_.clear(); - cross_pending_ = 0; + mail_.clear(); num_domains_ = 1; cycles_ = 0; // Allow a subsequent initialize() to elaborate and parallelize afresh. @@ -1404,6 +1613,10 @@ inline SimChannelBase::SimChannelBase(SimObjectBase* module) SimPlatform::instance().register_channel(this); } +inline int64_t& SimChannelBase::resolve_inflight() { + return SimPlatform::instance().domain_at(module_->domain()).inflight; +} + inline SimChannelBase::~SimChannelBase() { SimPlatform::instance().unregister_channel(this); } diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 6c43d6b69e..b4eb08f122 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -126,11 +126,16 @@ endif # default simx binary does not carry it. VORTEX_GEM5_SRCS := $(SRC_DIR)/gem5/vortex_gpgpu.cpp $(SRC_DIR)/gem5/dev_mem.cpp +# Optimization of a release build. libsimx is a shared library: without +# -fno-semantic-interposition the compiler may not inline a call to any of its +# own exported functions, and -fno-plt calls them without a PLT stub. +SIMX_OPT ?= -O3 -fno-semantic-interposition -fno-plt + # Debugging ifdef DEBUG CXXFLAGS += -g -O0 -DVX_DBG_DEBUG_LEVEL=$(DEBUG) else - CXXFLAGS += -O2 -DNDEBUG + CXXFLAGS += $(SIMX_OPT) -DNDEBUG endif # Enable perf counters diff --git a/sim/simx/instr_trace.h b/sim/simx/instr_trace.h index 37fd1a330b..b8c46472d3 100644 --- a/sim/simx/instr_trace.h +++ b/sim/simx/instr_trace.h @@ -37,17 +37,17 @@ struct instr_trace_t { RegOpd dst_reg; - std::vector src_regs; + FixedVector src_regs; FUType fu_type; OpType op_type; // Operands data - std::vector> src_data; + FixedVector src_data; // Destination data - std::vector dst_data; + RegDataVec dst_data; // Byte write enable uint8_t dst_bytesel; @@ -92,7 +92,7 @@ struct instr_trace_t { , src_regs(NUM_SRC_REGS, {RegType::None, 0}) , fu_type(FUType::ALU) , op_type({}) - , src_data(NUM_SRC_REGS, std::vector(VX_CFG_NUM_THREADS)) + , src_data(NUM_SRC_REGS, RegDataVec(VX_CFG_NUM_THREADS)) , dst_data(VX_CFG_NUM_THREADS) , dst_bytesel(0xFF) , pid(-1) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index bd52da131a..4e64a407bc 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -2497,6 +2497,7 @@ class CachePortArb : public SimObject { offer.held = false; } held_ = false; + idle_ticks_ = 0; std::fill(realign_.begin(), realign_.end(), realign_t()); } @@ -2520,6 +2521,17 @@ class CachePortArb : public SimObject { --init_cycles_; return; } + // Sleep once the ports, the tree and the offers have been empty for two + // ticks: the second idle tick settles the realign registers, so a skipped + // tick cannot change them. A request reserved toward any port re-arms it. + if (this->quiescent()) { + if (++idle_ticks_ >= 2) { + idle_ticks_ = 0; + this->tick_sleep(); + } + } else { + idle_ticks_ = 0; + } uint32_t nb = ReqOut.size(); uint32_t nl = levels_.size(); held_ = false; @@ -2571,8 +2583,10 @@ class CachePortArb : public SimObject { const bool hold = this->any_token(); this->update_realign(); // merged ports riding each leader's bank access, by offset in its group - std::vector followers(n, false); - std::vector merged(n, 0); + auto& followers = followers_; + auto& merged = merged_; + followers.assign(n, 0); + merged.assign(n, 0); if (merge_group_ > 1 && !hold) { for (uint32_t i = 0; i < n; ++i) { if (!this->is_plain_read(i) || this->realign_hold(i)) { @@ -2601,8 +2615,10 @@ class CachePortArb : public SimObject { // leave on their own port uint32_t slices = this->arb_slices(0); uint32_t slice_ports = (nl != 0) ? kMaxFanout : n; - std::vector want(nb * slices, 0); - std::vector wants(nb, 0); + auto& want = want_; + auto& wants = wants_; + want.assign(nb * slices, 0); + wants.assign(nb, 0); for (uint32_t i = 0; i < n; ++i) { auto& in = ReqIn.at(i); if (in.empty() || followers.at(i)) { @@ -2778,7 +2794,10 @@ class CachePortArb : public SimObject { return; } uint32_t n = ReqIn.size(); - std::vector start(n), owes_head(n); + auto& start = start_; + auto& owes_head = owes_head_; + start.assign(n, 0); + owes_head.assign(n, 0); for (uint32_t i = 0; i < n; ++i) { auto& r = realign_.at(i); auto& in = ReqIn.at(i); @@ -2821,6 +2840,21 @@ class CachePortArb : public SimObject { return (merge_group_ == 2) && !merge_line_; } + // Nothing at the ports, in the tree or offered to a bank. + bool quiescent() const { + for (auto& in : ReqIn) { + if (in.size() != 0) { + return false; + } + } + for (auto& offer : offers_) { + if (offer.valid || offer.held) { + return false; + } + } + return this->drained(); + } + // A plain read: no write, flush, IO, local or atomic attribute. bool is_plain_read(uint32_t i) const { if (ReqIn.at(i).empty()) { @@ -2857,6 +2891,14 @@ class CachePortArb : public SimObject { bool hold = false; }; std::vector realign_; + // per-tick scratch, kept to avoid an allocation per cycle + std::vector followers_; + std::vector merged_; + std::vector want_; + std::vector wants_; + std::vector start_; + std::vector owes_head_; + uint32_t idle_ticks_ = 0; struct offer_t { SimChannel* bank_in = nullptr; bool valid = false; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index f173bb77bb..f6506cfc15 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -71,7 +71,7 @@ void MemCoalescer::on_tick() { auto& entry = pending_rd_reqs_.at(rsp0.tag); BitVector<> lane_mask(output_size_); - std::vector> lane_data(output_size_); + FixedVector, LSU_MAX_LANES> lane_data(output_size_); for (uint32_t j = o; j < output_size_; ++j) { if (RspIn.at(j).empty()) { continue; @@ -172,13 +172,13 @@ void MemCoalescer::on_tick() { const bool in_is_amo = in_req.is_amo(); BitVector<> out_mask(output_size_); - std::vector out_addrs(output_size_); - std::vector> out_data(output_size_); - std::vector out_byteen(output_size_, 0); + FixedVector out_addrs(output_size_); + FixedVector, LSU_MAX_LANES> out_data(output_size_); + FixedVector out_byteen(output_size_, 0); // Comparand rides with the lane that owns the output. AMOs never coalesce // across lanes, so there is exactly one owner and no merge to do. - std::vector out_amo_cmp(output_size_, 0); - std::vector out_tids(output_size_, 0); + FixedVector out_amo_cmp(output_size_, 0); + FixedVector out_tids(output_size_, 0); BitVector<> cur_mask(input_size_); diff --git a/sim/simx/opc_unit.cpp b/sim/simx/opc_unit.cpp index 9585ba21fb..4519dba145 100644 --- a/sim/simx/opc_unit.cpp +++ b/sim/simx/opc_unit.cpp @@ -29,15 +29,12 @@ OpcUnit::OpcUnit(const SimContext &ctx, const char* name, uint32_t num_warp_slots, uint32_t num_threads) : SimObject(ctx, name) , Input(this) - , Output(this) - , num_threads_(num_threads) { + , Output(this) { + __assert(num_threads == VX_CFG_NUM_THREADS, "register file sized for VX_CFG_NUM_THREADS"); // The scoreboard's output skid buffer: two entries, the one being collected // among them, with a registered ready. Input.set_registered_ready(true); - regs_.reserve(num_warp_slots); - for (uint32_t i = 0; i < num_warp_slots; ++i) { - regs_.emplace_back(num_threads); - } + regs_.resize(num_warp_slots); this->on_reset(); } @@ -100,7 +97,7 @@ void OpcUnit::on_tick() { } } -void OpcUnit::read_src(std::vector& out, +void OpcUnit::read_src(RegDataVec& out, uint32_t wid, uint32_t src_index, const RegOpd& reg) const { @@ -109,13 +106,13 @@ void OpcUnit::read_src(std::vector& out, switch (reg.type) { case RegType::Integer: { const auto& src = slot.ireg_file[reg.idx]; - for (uint32_t t = 0; t < num_threads_; ++t) { + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { out[t].u = src[t]; } } break; case RegType::Float: { const auto& src = slot.freg_file[reg.idx]; - for (uint32_t t = 0; t < num_threads_; ++t) { + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { out[t].u64 = src[t]; } } break; @@ -141,7 +138,7 @@ void OpcUnit::writeback(instr_trace_t* trace, uint32_t wid) { return; uint32_t warp_slot = wid_to_opc_slot(wid); auto& rdest = trace->dst_reg; - auto num_threads = num_threads_; + constexpr uint32_t num_threads = VX_CFG_NUM_THREADS; // Bytesel: 0xFF (default) collapses to full overwrite. Partial-width writers // (packLD uops) supply a sub-mask so we OR-merge into the regfile slot. uint64_t mask = expand_bytesel64(trace->dst_bytesel); diff --git a/sim/simx/opc_unit.h b/sim/simx/opc_unit.h index ed15f26819..4d5daa57ad 100644 --- a/sim/simx/opc_unit.h +++ b/sim/simx/opc_unit.h @@ -39,7 +39,7 @@ class OpcUnit : public SimObject { void writeback(instr_trace_t* trace, uint32_t wid); // Read one source operand for `wid` into `out[t]` - void read_src(std::vector& out, + void read_src(RegDataVec& out, uint32_t wid, uint32_t src_index, const RegOpd& reg) const; @@ -61,17 +61,14 @@ class OpcUnit : public SimObject { virtual void on_tick(); private: + // Register files sized by the build's thread count: lane loops get a + // constant trip count and no per-register indirection. struct warp_regs_t { - std::vector> ireg_file; // [reg][thread] - std::vector> freg_file; // [reg][thread] - warp_regs_t(uint32_t num_threads) - : ireg_file(MAX_NUM_REGS, std::vector(num_threads, 0)) - , freg_file(MAX_NUM_REGS, std::vector(num_threads, 0)) - {} + std::array, MAX_NUM_REGS> ireg_file{}; // [reg][thread] + std::array, MAX_NUM_REGS> freg_file{}; // [reg][thread] }; std::vector regs_; // indexed by warp_slot - uint32_t num_threads_; uint32_t total_stalls_ = 0; instr_trace_t* cur_trace_ = nullptr; diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index 56a9d971dd..fc4f3ea7a8 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -36,7 +36,7 @@ namespace { // Emit a "DEBUG SrcN Reg: ={values...} (#uuid)" line for trace_csv.py. inline void log_src_operand(uint32_t src_index, const RegOpd& reg, - const std::vector& values, + const RegDataVec& values, const ThreadMask& tmask, uint64_t uuid) { // All five params are referenced only inside DPH/DPN, which expand to diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index fae316663a..b87020221e 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -43,6 +43,7 @@ ProcessorImpl::ProcessorImpl() { SimPlatform::instance().initialize(); SimPlatform::instance().set_num_workers(SIMX_NUM_WORKERS); + SimPlatform::instance().set_quantum(SIMX_QUANTUM_CYCLES); assert(VX_CFG_PLATFORM_MEMORY_DATA_SIZE == VX_CFG_MEM_BLOCK_SIZE); diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 9fb06bbcab..be70301739 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -1103,10 +1103,10 @@ class TcuUnit::Impl { uint32_t step_m, uint32_t step_n, uint32_t step_k, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse) { if (is_sparse) { if (!vt::sparse_format_supported(fmt_s)) { @@ -1183,10 +1183,10 @@ class TcuUnit::Impl { uint32_t step_k, uint32_t a_desc, uint32_t b_desc, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse, uint32_t cd_nregs, uint32_t is_a_smem, @@ -1588,8 +1588,8 @@ class TcuUnit::Impl { uint32_t fmt_s, uint32_t fmt_d, const reg_data_t* a_tile, const reg_data_t* b_tile, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse, uint32_t k_words = cfg::tcK, uint32_t fedp_words = cfg::tcK, @@ -1909,10 +1909,10 @@ void TcuUnit::wmma(uint32_t wid, uint32_t step_m, uint32_t step_n, uint32_t step_k, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse) { impl_->wmma(wid, fmt_s, fmt_d, step_m, step_n, step_k, rs1_data, rs2_data, rs3_data, @@ -1928,10 +1928,10 @@ void TcuUnit::wgmma(uint32_t wid, uint32_t step_k, uint32_t a_desc, uint32_t b_desc, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse, uint32_t cd_nregs, uint32_t is_a_smem, diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 91db5de22f..45070564d6 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -101,10 +101,10 @@ class TcuUnit : public FuncUnit { uint32_t step_m, uint32_t step_n, uint32_t step_k, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse); void wgmma(uint32_t wid, @@ -115,10 +115,10 @@ class TcuUnit : public FuncUnit { uint32_t step_k, uint32_t a_desc, uint32_t b_desc, - const std::vector& rs1_data, - const std::vector& rs2_data, - const std::vector& rs3_data, - std::vector& rd_data, + const RegDataVec& rs1_data, + const RegDataVec& rs2_data, + const RegDataVec& rs3_data, + RegDataVec& rd_data, bool is_sparse, uint32_t cd_nregs, uint32_t is_a_smem, diff --git a/sim/simx/types.h b/sim/simx/types.h index 51edd62a21..7b71a8a4d4 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -43,6 +43,7 @@ #include #include +#include #include #include "debug.h" @@ -53,11 +54,18 @@ #ifndef SIMX_MT #define SIMX_MT 0 #endif +// Synchronization quantum of the functional kernel, in cycles (e.g. +// CONFIGS="-DSIMX_FUNCTIONAL -DSIMX_MT=8 -DSIMX_QUANTUM=100"); 1 keeps +// per-cycle lockstep. The timed kernel accepts only 1. +#ifndef SIMX_QUANTUM +#define SIMX_QUANTUM 1 +#endif #include "constants.h" namespace vortex { constexpr uint32_t SIMX_NUM_WORKERS = (SIMX_MT > 1) ? SIMX_MT : 1; +constexpr uint32_t SIMX_QUANTUM_CYCLES = (SIMX_QUANTUM > 1) ? SIMX_QUANTUM : 1; // One memory block (a cache line / DRAM transfer unit). Carried by // MemReq/MemRsp in TLM data-path mode. Use `shared_ptr` so @@ -110,6 +118,10 @@ union reg_data_t { int64_t i64; }; +// One register value per thread: inline storage sized by the build's thread +// count, so traces and operand reads never touch the heap. +using RegDataVec = FixedVector; + /////////////////////////////////////////////////////////////////////////////// struct warp_barrier_t { @@ -1163,17 +1175,21 @@ class Arbiter { /////////////////////////////////////////////////////////////////////////////// +// Lanes of an LSU request or response: the core's LSU lanes or the D-cache +// channels behind them, whichever is wider. +inline constexpr uint32_t LSU_MAX_LANES = (LSU_CHANNELS > DCACHE_CHANNELS) ? LSU_CHANNELS : DCACHE_CHANNELS; + struct LsuReq { MemOp op; MemFlags flags; - std::vector addrs; - std::vector> data; - std::vector byteen; + FixedVector addrs; + FixedVector, LSU_MAX_LANES> data; + FixedVector byteen; // Comparand per lane. Only compare-and-swap reads it; it is a separate // field because the data operand is already carrying the swap value. - std::vector amo_cmp; + FixedVector amo_cmp; BitVector<> mask; - std::vector tids; + FixedVector tids; uint32_t tag; uint32_t cid; uint32_t wid; @@ -1224,7 +1240,7 @@ struct LsuRsp { uint64_t uuid; // TLM data: per-lane block populated for loads/AMO returns. - std::vector> data; + FixedVector, LSU_MAX_LANES> data; LsuRsp(uint32_t size) : mask(size) diff --git a/sw/common/mempool.h b/sw/common/mempool.h index f8761d857f..49d4c05cb4 100644 --- a/sw/common/mempool.h +++ b/sw/common/mempool.h @@ -23,53 +23,45 @@ namespace vortex { -// Memory pool for fixed-size objects with fallback to new/delete. +// Memory pool for fixed-size objects. The pool grows by slabs of PoolSize +// blocks and recycles them; it never falls back to the heap per object. // Every block carries an owner tag ahead of the payload, so a block may be -// freed on any thread: same-pool frees take the unsynchronized fast path, -// frees of another pool's block go through that pool's lock-free return -// stack, and heap-fallback blocks are deleted directly. +// freed on any thread: same-pool frees take the unsynchronized fast path and +// frees of another pool's block go through that pool's lock-free return stack. template class MemoryPool { public: MemoryPool() { - chunk_ = static_cast(aligned_alloc(kAlign, kStride * PoolSize)); - for (size_t i = 0; i < PoolSize; ++i) { - char* block = chunk_ + i * kStride; - owner_of(block) = this; - next_of(block) = (i < PoolSize - 1) ? (chunk_ + (i + 1) * kStride) : nullptr; - } - free_list_ = chunk_; + this->grow(); } ~MemoryPool() noexcept { - free(chunk_); + for (auto* chunk : chunks_) { + free(chunk); + } } T* allocate() { - if (!free_list_) { - // Reclaim blocks returned by other threads. + if (__builtin_expect(free_list_ == nullptr, 0)) { + // Reclaim blocks returned by other threads, else add a slab. free_list_ = static_cast(remote_free_.exchange(nullptr, std::memory_order_acquire)); + if (free_list_ == nullptr) { + this->grow(); + } } - if (free_list_) { - char* block = free_list_; - free_list_ = next_of(block); - return payload_of(block); - } - char* block = static_cast(::operator new(kStride, std::align_val_t(kAlign))); - owner_of(block) = nullptr; + char* block = free_list_; + free_list_ = next_of(block); return payload_of(block); } void deallocate(T* ptr) noexcept { char* block = reinterpret_cast(ptr) - kHeader; auto* owner = owner_of(block); - if (owner == this) { + if (__builtin_expect(owner == this, 1)) { next_of(block) = free_list_; free_list_ = block; - } else if (owner) { - owner->remote_free_push(block); } else { - ::operator delete(block, std::align_val_t(kAlign)); + owner->remote_free_push(block); } } @@ -91,6 +83,20 @@ class MemoryPool { return reinterpret_cast(block + kHeader); } + __attribute__((noinline)) void grow() { + auto chunk = static_cast(aligned_alloc(kAlign, kStride * PoolSize)); + if (chunk == nullptr) { + throw std::bad_alloc(); + } + chunks_.push_back(chunk); + for (size_t i = 0; i < PoolSize; ++i) { + char* block = chunk + i * kStride; + owner_of(block) = this; + next_of(block) = (i < PoolSize - 1) ? (chunk + (i + 1) * kStride) : free_list_; + } + free_list_ = chunk; + } + // Push-only Treiber stack; the owner drains it with a single exchange. void remote_free_push(char* block) noexcept { void* head = remote_free_.load(std::memory_order_relaxed); @@ -100,12 +106,38 @@ class MemoryPool { std::memory_order_release, std::memory_order_relaxed)); } - char* chunk_ = nullptr; + std::vector chunks_; char* free_list_ = nullptr; std::atomic remote_free_{nullptr}; }; -// Custom allocator using the memory pool with fallback +// Hands every pool a thread bound back to its registry when the thread exits. +// One list serves all pool types: a thread_local holder per type would cost +// static TLS space for each type. +class PoolParking { +public: + using release_fn = void (*)(void*); + + static void add(release_fn release, void* pool) { + static thread_local PoolParking parking; + parking.entries_.push_back({release, pool}); + } + + ~PoolParking() { + for (auto& e : entries_) { + e.release(e.pool); + } + } + +private: + struct entry_t { + release_fn release; + void* pool; + }; + std::vector entries_; +}; + +// STL allocator backed by a per-thread MemoryPool. template class PoolAllocator { public: @@ -165,14 +197,24 @@ class PoolAllocator { return *registry; } + // Hot path: a trivially-initialized per-thread pointer (no TLS wrapper + // call); the pool is bound, and its release at thread exit registered, once. static MemoryPool& get_pool() { - struct holder_t { - MemoryPool* pool; - holder_t() : pool(registry().acquire()) {} - ~holder_t() { registry().release(pool); } - }; - static thread_local holder_t holder; - return *holder.pool; + static thread_local MemoryPool* cached + __attribute__((tls_model("initial-exec"))) = nullptr; + if (__builtin_expect(cached != nullptr, 1)) { + return *cached; + } + cached = bind_pool(); + return *cached; + } + + __attribute__((noinline)) static MemoryPool* bind_pool() { + auto* pool = registry().acquire(); + PoolParking::add([](void* p) { + registry().release(static_cast*>(p)); + }, pool); + return pool; } }; diff --git a/sw/common/ringqueue.h b/sw/common/ringqueue.h index 6cb8db76e8..399fa12197 100644 --- a/sw/common/ringqueue.h +++ b/sw/common/ringqueue.h @@ -107,6 +107,21 @@ class RingQueue { } } + // Double the capacity, keeping the queued elements in order (for queues + // that are unbounded by design). + void grow() { + uint32_t new_cap = (capacity_ != 0) ? capacity_ * 2 : 4; + std::unique_ptr grown(new Storage[new_cap]); + for (uint32_t i = 0, idx = head_; i < size_; ++i, idx = inc(idx)) { + new (&grown[i]) T(std::move(*ptr(idx))); + ptr(idx)->~T(); + } + storage_ = std::move(grown); + capacity_ = new_cap; + head_ = 0; + tail_ = size_; + } + private: using Storage = std::aligned_storage_t; diff --git a/sw/common/util.h b/sw/common/util.h index 05be94e421..735abae836 100644 --- a/sw/common/util.h +++ b/sw/common/util.h @@ -31,13 +31,17 @@ void unused(Args &&...) {} #define __unused(...) unused(__VA_ARGS__) -#define __assert(cond, msg) \ - if (!(cond)) { \ - std::cerr << "Assertion failed: " << msg << "\n"; \ - std::cerr << "File: " << __FILE__ << "\n"; \ - std::cerr << "Line: " << __LINE__ << "\n"; \ - std::cerr << "Function: " << __func__ << "\n"; \ - std::abort(); \ +// The failure report runs in a cold, out-of-line lambda: a call site keeps +// only the test and a call, so small functions that assert stay inlinable. +#define __assert(cond, msg) \ + if (__builtin_expect(!(cond), 0)) { \ + [&, vx_func_ = __func__]() __attribute__((noinline, cold)) { \ + std::cerr << "Assertion failed: " << msg << "\n"; \ + std::cerr << "File: " << __FILE__ << "\n"; \ + std::cerr << "Line: " << __LINE__ << "\n"; \ + std::cerr << "Function: " << vx_func_ << "\n"; \ + std::abort(); \ + }(); \ } const char *fileExtension(const char *filepath);