From 7943a0c4e0861ae846fae36e8ec07c7063e3877d Mon Sep 17 00:00:00 2001 From: Jupiter Belic Date: Tue, 27 Oct 2026 09:34:32 +1000 Subject: [PATCH] cpu: recognise IRIX 6.5's idle loop and park it IRIX 6.5 never parked under idle-pause. The detector hashed PC+GPRs and required an exact repeat, but 6.5's idle path is a ~470-byte scheduler loop (~89 instructions) with calls to spl and the ll/sc lock and stores to kernel globals; its registers evolve every iteration, so the hash never repeated. It also matched short device-poll loops (lbu/andi/bne -3 on a SCSI bit) that must keep running. Replace it with a PC-cluster sample. The batch loop calls sample after each ten-instruction group; sample records the PC every 32 calls (~320 instructions per record, ~10,000 per ring of 32). update sorts the samples into PC-space clusters and parks when one kernel cluster holds at least 12 samples with 12 distinct PCs. User-mode samples are tolerated up to a quarter of the ring. Short DELAY/poll loops, the ll/sc lock spin and user code are never parked. All previous rules remain: IE/IM, pending-line wake, masked-wake back-off, Compare armed. Parking starts 60 s after the CPU thread starts (IRIS_IDLE_WARMUP_SECS overrides) so boot is not sliced. idle on|off|status switches parking at runtime and reports counters; idleprof stats prints the park counters. The ci_clock ip7 unit tests now advance Count by retired cycles, so the required ci_clock,idle-pause suite passes. Measured on IRIX 6.5 IP22 (R4400, fresh disk, lightning,idle-pause build): settled getty 1-2% host CPU; cold boot 35 s. ci_clock,idle-pause boot 52 s; its getty 16-19% (guest time is retired instructions). IRIX 5.3 not tested (no image). --- CHANGELOG.md | 19 + FEATURES.md | 3 +- HACKING.md | 10 + clanker/perf/idle-pause-work.md | 41 +- debug.md | 3 +- src/cpu/idle_park.rs | 822 +++++++++++++++++++++++++++++--- src/cpu/mips_core.rs | 39 +- src/cpu/mips_exec.rs | 97 +++- src/cpu/mips_exec_test.rs | 23 +- 9 files changed, 958 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c151ac..32be8955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,25 @@ hashes are given where a change is easiest to understand by reading the commit. ## October 2026 +### Idle parking + +- **IRIX 6.5's idle loop is recognised and parked.** With `--features + idle-pause`, a settled 6.5 IP22 getty now uses ~1-2 % host CPU instead of + ~100 %. The previous detector hashed PC and registers and required an exact + repeat; 6.5's idle path is a ~470-byte scheduler loop whose registers + evolve every iteration, so it never fired, while the same hash matched + short device-poll loops that must keep running. The replacement samples the + PC every 32 sample calls (about 320 instructions in the batch loop), groups + the samples into PC-space clusters, and parks when one long kernel cluster + dominates the ring; short countdown/device-poll loops and user-mode code + are never parked. Parking waits 60 s after CPU start + (`IRIS_IDLE_WARMUP_SECS` overrides) so boot is not slowed. +- **New monitor commands.** `idle on|off|status` switches parking at runtime + and reports park/sleep/wake counters; `idleprof stats` prints the park + counters. +- The `ci_clock` unit tests that materialise Count now advance it by retired + cycles, so `--features ci_clock,idle-pause` passes. + ### JIT v2 - **FPU integer conversions are compiled inline.** `CVT.W/L.S/D`, diff --git a/FEATURES.md b/FEATURES.md index d4518a65..8f5dafe9 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -483,7 +483,8 @@ and offline-tool controls are included so they are not mistaken for GUI settings | `IRIS_REX3_TRACE` | Trace REX3 GFIFO and CPU accesses from board start to this path (`gputrace` builds). | [src/dev/ng1/rex3_trace.rs](src/dev/ng1/rex3_trace.rs) | | `IRIS_NO_EXIT_ON_POWEROFF` | Any presence prevents process exit on guest power-off/CI quit; set by GUI. | [src/ci.rs](src/ci.rs) | | `IRIS_NO_FETCH_VERIFY` | 1 suppresses fetchverify checks in a fetchverify build. | [src/cpu/mips_exec.rs](src/cpu/mips_exec.rs) | -| `IRIS_NO_IDLE` | Any presence disables idle parking. | [src/cpu/idle_park.rs](src/cpu/idle_park.rs) | +| `IRIS_NO_IDLE` | Any presence disables idle parking at startup; `idle on` re-enables it. | [src/cpu/idle_park.rs](src/cpu/idle_park.rs) | +| `IRIS_IDLE_WARMUP_SECS` | Seconds after CPU start before idle parking engages (default 60; 0 disables the warm-up). | [src/cpu/idle_park.rs](src/cpu/idle_park.rs) | | `IRIS_NO_INLINE_MEM` | Any presence disables JIT inline memory emission. | [src/cpu/jitv2/codegen.rs](src/cpu/jitv2/codegen.rs) | | `IRIS_NO_JIT` | Any presence disables the REX3 shader JIT and the IMPACT raster JIT; does not select the MIPS engine. | [src/dev/ng1/rex3.rs](src/dev/ng1/rex3.rs) | | `IRIS_OPT_SPEED` | Presence selects speed optimization in offline PCP/corpus measurements. | [src/bin/jitv2_pcp_dump.rs](src/bin/jitv2_pcp_dump.rs) | diff --git a/HACKING.md b/HACKING.md index 6298e4ac..f6252a76 100644 --- a/HACKING.md +++ b/HACKING.md @@ -218,6 +218,16 @@ when the guest is in its idle loop. A producer that adds work, or a device that raises an interrupt, unparks the thread it is waiting on. See `clanker/perf/` for what went wrong each time a parking rule was missed. +The idle detector samples the guest PC every 32 sample calls (about 320 +instructions in the batch loop) and parks only when one long kernel cluster of +code dominates the sample ring (at least 12 of 32 samples and 12 distinct PCs, +no user-mode majority), with interrupts enabled, a mask bit set, nothing +pending and Compare armed. That accepts IRIX 5.3's 14-instruction loop and +6.5's ~89-instruction scheduler path while rejecting short DELAY and +device-poll loops. Parking starts 60 s after the CPU thread starts +(`IRIS_IDLE_WARMUP_SECS` overrides), so boot runs at full speed. The monitor's +`idle on|off|status` controls and reports it. + ## 5. Locks The rules, in order of importance: diff --git a/clanker/perf/idle-pause-work.md b/clanker/perf/idle-pause-work.md index f17f58f7..4776a687 100644 --- a/clanker/perf/idle-pause-work.md +++ b/clanker/perf/idle-pause-work.md @@ -165,7 +165,12 @@ break out as soon as an unmasked interrupt is pending. `IRIS_NO_IDLE` disables i dropped from **~100 % to ~2 %** (headless), the system stayed responsive (serial input echoes → wake-on-IRQ works) and the clock stayed correct (`date` in shell). -### Detector — WORKING (architectural-state repeat, k0/k1 excluded) +### Detector — architectural-state repeat (SUPERSEDED, see below) + +> **Superseded 2026-10-27.** The hash below never fires on IRIX 6.5, whose idle +> path is a long scheduler loop with evolving registers, and it *does* match +> short device-poll loops that must keep running. The PC-cluster detector now +> in `src/cpu/idle_park.rs` replaces it; see "Detector — PC-cluster sample". The crux was distinguishing the kernel idle loop from a busy **delay loop**: - IRIX boot calls a calibrated `DELAY()` at `0x88003d70`: `bgezl v1,-1; subu v1,v1,v0` @@ -207,3 +212,37 @@ targets. **Other TODO:** the v1 JIT path that bypassed this run loop entirely has been removed. Whether/how `jitv2` (`--features jitv2`, a from-scratch region compiler) needs its own idle-park hook has not yet been investigated. + +### Detector — PC-cluster sample (current, 2026-10-27) + +`idleprof` at a settled IRIX 6.5 IP22 getty shows the idle path is not a tight +branch: it is a ~470-byte scheduler loop (0x880daedc..0x880db0b4, ~89 +instructions) with calls to `spl` and the ll/sc lock and stores to kernel +globals. Its registers evolve every iteration (the run-queue scan rotates +state), so PC+GPRs never repeats exactly — the old detector could not fire. +It also matched a 3-instruction device poll seen during boot +(`lbu/andi/bne -3` on a SCSI bit), which must keep executing. + +The replacement samples the guest PC every 32 sample calls: the batch loop +calls `sample` after each ten-instruction group, so that is about 320 guest +instructions per record and ~10,000 per ring. `update` sorts the samples into +PC-space clusters (consecutive PCs <=256 bytes apart) and parks when one +**kernel** cluster holds at least 12 of the 32 samples with 12 distinct PCs. +That rejects the 2-instruction DELAY loop, 3-instruction device polls, the +8-instruction ll/sc lock spin and mixed work, while accepting the 5.3 +14-instruction loop and 6.5's 89-instruction scheduler path. User-mode samples +are allowed up to a quarter of the ring (a syscall wake runs a few user +instructions between kernel stretches), but a user-dominated ring is refused. + +The earlier safety rules remain: no parking with IE=0 or every mask clear, +wake on any pending line and on `running=false`, the masked-wake back-off +(no re-park for ~20 ms after waking on a line the guest cannot take), and no +park before Compare is armed. Parking waits 60 s after the CPU thread starts +(`IRIS_IDLE_WARMUP_SECS` overrides) so boot runs at full speed. `idle +on|off|status` and `idleprof stats` expose the switch and counters. + +Measured on the fork's rig (6-core i5-8500T, IRIX 6.5 IP22, R4400, fresh +disk): a settled getty with the `lightning,idle-pause` build uses 1-2 % host +CPU; the wall-clock cold boot is 35 s; `ci_clock,idle-pause` boots in 52 s and +its getty runs at ~16-19 % (guest time is retired instructions, so the timer +handler is a larger share of host time). IRIX 5.3 was not tested (no image). diff --git a/debug.md b/debug.md index 59081f96..26fe95c7 100644 --- a/debug.md +++ b/debug.md @@ -222,7 +222,8 @@ exception of that class is taken. `ex tlb on` stops on every TLB exception. | `undo clear` / `undo resize ` | Clear / resize the undo history **[DEV]** | | `debug >` | Print every executed instruction, or write them to a file **[DEV]** | | `trace start ` / `trace stop` / `trace status` | Record full per-instruction architectural state to a file **[DEV]** | -| `idleprof ` | Sample the PC to find idle and spin loops (`idle-pause` builds) | +| `idleprof ` | Sample the PC to find idle and spin loops (`idle-pause` builds); `stats` prints park counters | +| `idle ` | Turn idle parking on or off at runtime, or show park counters and the detector's last window (`idle-pause` builds) | | `instrstats [report\|clear\|dump]` | Per-instruction decode/execute counters (`--features instr_stats`) **[DEV]** | The undo buffer tracks register changes and memory writes. It does not diff --git a/src/cpu/idle_park.rs b/src/cpu/idle_park.rs index 7e05fec7..7077ff1e 100644 --- a/src/cpu/idle_park.rs +++ b/src/cpu/idle_park.rs @@ -1,7 +1,7 @@ //! Kernel idle-loop detection and in-place CPU thread parking. //! -//! Shared by the interpreter run loop (`mips_exec.rs`) and the JIT dispatch -//! loop (`jit/dispatch.rs`). See `clanker/perf/idle-pause-work.md`. +//! Used by the CPU run loop in `mips_exec.rs`. See +//! `clanker/perf/idle-pause-work.md`. use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; @@ -9,8 +9,55 @@ use std::time::{Duration, Instant}; use crate::cpu::mips_core::{CAUSE_IP_MASK, STATUS_IM_MASK}; use crate::cpu::mips_core::MipsCore; -const IDLE_RING: usize = 32; +/// PCs kept for the idle-window test. The batch run loop calls [`sample`] +/// after each ten-instruction group, so with [`SAMPLE_EVERY`] = 32 the ring +/// spans about 10,000 guest instructions — several iterations of even a long +/// idle loop. The single-step path spans 32. +const PC_RING: usize = 32; +/// Number of [`sample`] calls between recorded PCs. The batch loop calls it +/// after every ten instructions (~320 guest instructions per record); the +/// single-step path calls it after each instruction. +const SAMPLE_EVERY: u32 = 32; +/// A "cluster" is a run of sorted sample PCs no more than this far apart. +/// IRIX's idle path is one hot loop plus a few helper functions (spl, the +/// ll/sc lock) reached by calls, so it is a cluster, not one contiguous +/// window. +const CLUSTER_GAP_BYTES: u64 = 256; +/// The dominant cluster must hold at least this many of the PC_RING samples. +/// A short poll or DELAY loop also fills the ring, but with too few distinct +/// PCs (checked separately). Mixed work spreads its samples over several +/// clusters and cannot reach this. +const CLUSTER_MIN_SAMPLES: usize = 12; +/// ... and at least this many distinct PCs, which rejects 2-3 instruction +/// countdown/device-poll loops and the 8-instruction ll/sc lock spin outright. +/// The IRIX 5.3 idle loop has 14 and 6.5's scheduler path has ~89. +const CLUSTER_MIN_DISTINCT: usize = 12; +/// The dominant-cluster test must hold for this many consecutive batches +/// (1 000 instructions per batch) before parking. One batch is enough to +/// reject a short-lived hot function without delaying idle parking, which +/// would otherwise re-run thousands of idle-loop iterations after every +/// timer interrupt before sleeping again. +const IDLE_STREAK: u32 = 1; const SLICE_NS: u64 = 1_000_000; +/// Host-pacing rate for `hot.cycles` while the CPU is parked: one guest cycle +/// per 10 ns of real time, the rate `NS_PER_GUEST_CYCLE` names under +/// `ci_clock`. In every build this keeps cross-thread cycle readers +/// (WD33C93A's deferred-interrupt spin, CP0 Random) moving during a park. +const STALL_NS_PER_CYCLE: u64 = 10; +/// After a park wakes on a pending line the guest cannot take (`(ip & im) == +/// 0`), do not park again for this many `hot.cycles`. A genuine idle loop +/// wakes for an unmasked interrupt; waking for a masked one means the +/// sampled cluster was a device-poll or DELAY loop that happened to look +/// idle. Sleeping that loop caps it at one batch per slice, and with the +/// wall-clock model the park no longer moves Count to compensate, so boot +/// crawls. About 20 ms of host pacing at the 10 ns/cycle rate. +const PARK_BACKOFF_CYCLES: u64 = 2_000_000; +/// Do not park during the first minute of CPU time. Parking is a steady-state +/// optimisation: an IRIX boot runs long stretches inside single kernel +/// functions and instruction-counted wait loops, and slicing those at 1 ms +/// stretches a 45-second boot to minutes. The guest is doing real work during +/// boot; keep it at full speed, then start parking once it has settled. +const PARK_WARMUP: Duration = Duration::from_secs(60); /// The CPU thread while it is parked in [`IdleParkState::park`], so an /// interrupt source can wake it at once instead of leaving it to notice on its @@ -20,6 +67,26 @@ static PARKER: parking_lot::Mutex> = parking_lot::co /// its sleep. Paired with the writers' order (set the bit, then read this) so /// a wakeup cannot be lost. static PARKED: AtomicBool = AtomicBool::new(false); +/// When this process first built an `IdleParkState`. One `Machine` per +/// process, and the warm-up must not restart every time the monitor stops and +/// starts the CPU thread (which builds a fresh state), so it is process-wide. +static PROCESS_START: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Park statistics, shown by the monitor's `idle status`. They answer "is +/// this guest idle or just being throttled?" without a debugger. +pub static PARK_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static PARK_SLEEP_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static PARK_WAKE_IRQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static PARK_WAKE_OTHER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static UPDATES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static RESET_IRQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static RESET_IP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static RESET_IM0: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static RESET_IE0: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static RESET_USER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static WINDOW_FULL: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static WINDOW_SAMPLES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub static WINDOW_DISTINCT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); /// Wake the parked CPU thread, if it is parked. Call after setting a bit in /// `hot.interrupts` from any thread; one relaxed-cost atomic load when the CPU @@ -33,83 +100,274 @@ pub fn wake() { } } -/// Tracks recent architectural-state hashes to detect polling idle loops. -#[derive(Default)] +/// Detects kernel idle loops from a per-instruction PC sample ring. +/// +/// The previous detector hashed PC + GPRs at each 1000-instruction batch and +/// required an exact repeat. It fails IRIX 6.5's idle path (a ~470-byte +/// scheduler loop with calls and stores whose registers evolve every +/// iteration) and, worse, it *does* match short device-poll loops (e.g. +/// `lbu/andi/bne -3` waiting on a SCSI bit), which must keep executing. +/// +/// This detector samples the PC every [`SAMPLE_EVERY`] calls (about 320 +/// instructions in the batch loop), groups +/// the samples into PC-space clusters, and parks only when one long kernel +/// cluster dominates the ring. User-mode samples are tolerated in a minority +/// (a syscall wake runs a few user instructions between kernel stretches) but +/// a user-dominated ring is refused, so a user process spinning is never +/// parked. pub struct IdleParkState { - ring: [u64; IDLE_RING], - ring_len: usize, - ring_pos: usize, + pcs: [u64; PC_RING], + pc_len: usize, + pc_pos: usize, + /// Instructions until the next PC sample. + sample_countdown: u32, + /// Per-slot user-mode flag, parallel to `pcs`. + user: [bool; PC_RING], + /// Consecutive batches whose dominant cluster was the same loop. + streak: u32, + /// Do not park again before `hot.cycles` reaches this value; set after a + /// park that woke on a masked line (see [`PARK_BACKOFF_CYCLES`]). + park_skip_until_cycle: u64, + /// CPU-thread start, for [`PARK_WARMUP`]. + started: Instant, + /// Warm-up length; `IRIS_IDLE_WARMUP_SECS` overrides it (0 disables). + warmup: Duration, +} + +/// What to do with the current sample window. +pub enum IdleDecision { + /// Keep executing: no idle cluster detected, or an interrupt is ready to + /// deliver so the guest should run, not sleep. + None, + /// One long kernel cluster dominates the window with interrupts enabled + /// and a mask bit set: park the CPU thread; an unmasked interrupt ends the + /// park. + Park, + /// A masked interrupt is already pending (`ip != 0`), so parking would + /// wait for a line the guest only takes after it drops the spl — which it + /// can only do running. Pace instead: sleep in bounded slices, advancing + /// `hot.cycles` at the 10 ns/cycle host-pacing rate so cross-thread + /// readers keep moving. Each slice re-checks `running` and the interrupt + /// word, so a monitor command, a soft reset, or a newly-unmasked + /// interrupt ends it at once. + Pace, +} + +impl Default for IdleParkState { + fn default() -> Self { + Self::new() + } } impl IdleParkState { - /// Hash PC + GPRs (excluding k0/k1 scratch registers). - fn hash_state(core: &MipsCore) -> u64 { - let mut h = core.pc; - for (i, &g) in core.gpr.iter().enumerate() { - if i == 26 || i == 27 { + pub fn new() -> Self { + let warmup = match std::env::var("IRIS_IDLE_WARMUP_SECS") { + Ok(s) => Duration::from_secs(s.parse().unwrap_or(PARK_WARMUP.as_secs())), + Err(_) => PARK_WARMUP, + }; + Self { + pcs: [0; PC_RING], + pc_len: 0, + pc_pos: 0, + sample_countdown: SAMPLE_EVERY, + user: [false; PC_RING], + streak: 0, + park_skip_until_cycle: 0, + started: *PROCESS_START.get_or_init(Instant::now), + warmup, + } + } + + /// Test constructor with the boot warm-up already elapsed. + #[cfg(test)] + pub(crate) fn ready() -> Self { + let mut s = Self::new(); + s.warmup = Duration::ZERO; + s.started = Instant::now(); + s + } + + fn clear_window(&mut self) { + self.pc_len = 0; + self.pc_pos = 0; + self.sample_countdown = SAMPLE_EVERY; + self.streak = 0; + } + + /// Sample the PC every [`SAMPLE_EVERY`] calls. The batch run loop calls + /// this after each ten-instruction group, the single-step path after each + /// instruction; one predictable decrement and a rarely-taken branch on the + /// hot path. + #[inline] + pub fn sample(&mut self, core: &MipsCore) { + self.sample_countdown -= 1; + if self.sample_countdown != 0 { + return; + } + self.sample_countdown = SAMPLE_EVERY; + let user = !matches!(core.get_privilege_mode(), crate::cpu::mips_core::PrivilegeMode::Kernel); + self.pcs[self.pc_pos] = core.pc; + self.user[self.pc_pos] = user; + self.pc_pos = (self.pc_pos + 1) % PC_RING; + if self.pc_len < PC_RING { + self.pc_len += 1; + } + } + + /// Largest kernel cluster in the sample ring: `(samples, distinct_pcs)`, + /// plus the number of user-mode samples. User samples are ignored for the + /// cluster (a syscall wake runs a few user instructions between long + /// kernel stretches) but a ring dominated by user code is refused. + fn largest_cluster(&self) -> (usize, usize, usize) { + let mut kernel: [u64; PC_RING] = [0; PC_RING]; + let mut n = 0usize; + let mut users = 0usize; + for i in 0..PC_RING { + if self.user[i] { + users += 1; + } else { + kernel[n] = self.pcs[i]; + n += 1; + } + } + let sorted = &mut kernel[..n]; + sorted.sort_unstable(); + let mut best = (0usize, 0usize, users); + let mut start = 0usize; + for end in 1..=n { + let boundary = end == n + || sorted[end].saturating_sub(sorted[end - 1]) > CLUSTER_GAP_BYTES; + if !boundary { continue; } - h = h.rotate_left(7) ^ g; + let slice = &sorted[start..end]; + let mut distinct = 0usize; + for i in 0..slice.len() { + if !slice[..i].contains(&slice[i]) { + distinct += 1; + } + } + if slice.len() > best.0 { + best = (slice.len(), distinct, users); + } + start = end; } - h + best } - /// Update idle ring. Returns true when the current state repeated (safe to park). - pub fn update(&mut self, core: &MipsCore) -> bool { - let ie = core.interrupts_enabled(); + /// True when one long, stationary kernel cluster dominates the ring and + /// user code is only a minority of it. + fn window_is_idle(&self) -> bool { + if self.pc_len < PC_RING { + return false; + } + let (samples, distinct, users) = self.largest_cluster(); + users <= PC_RING / 4 + && samples >= CLUSTER_MIN_SAMPLES + && distinct >= CLUSTER_MIN_DISTINCT + } + + /// Update the PC window and decide what to do with the current state. + /// + /// Returns `None` unless the CPU has stayed inside one long, narrow + /// kernel code window for [`PC_RING`] samples with no interrupt ready to + /// deliver — that is a genuine idle loop. With interrupts enabled and a + /// mask bit set the spin is safe to `park`; a pending-but-masked interrupt + /// is paced instead (see [`IdleParkState::pace`]). + pub fn update(&mut self, core: &MipsCore) -> IdleDecision { + // A recent park woke on a line the guest could not take: keep + // executing for the back-off window instead of re-parking a loop + // that is waiting on device progress. + if self.started.elapsed() < self.warmup { + self.clear_window(); + return IdleDecision::None; + } + if core.hot.cycles < self.park_skip_until_cycle { + return IdleDecision::None; + } + UPDATES.fetch_add(1, Ordering::Relaxed); let pending = core.hot.interrupts.load(Ordering::Relaxed) as u32; let ip = (core.cp0_cause | pending) & CAUSE_IP_MASK; let im = core.cp0_status & STATUS_IM_MASK; let interrupt_ready = (ip & im) != 0; - // With every mask bit clear nothing can satisfy `park`'s - // `(ip & im) != 0`, so parking here never wakes. - if im == 0 { - self.ring_len = 0; - self.ring_pos = 0; - return false; + // An interrupt is ready to deliver: run the guest, don't sleep. + if interrupt_ready { + RESET_IRQ.fetch_add(1, Ordering::Relaxed); + self.clear_window(); + return IdleDecision::None; } - if !(ie && !interrupt_ready) { - self.ring_len = 0; - self.ring_pos = 0; - return false; + // With every mask bit clear nothing can satisfy `park`'s + // `(ip & im) != 0`, so parking here never wakes. Do not pace either: + // an IE=0/IM=0 repeated loop is not an idle guest but a busy wait + // (the PROM's device-poll and DELAY loops look exactly like this), + // and sleeping it throttles boot to about one batch per slice. + if im == 0 { + RESET_IM0.fetch_add(1, Ordering::Relaxed); + self.clear_window(); + return IdleDecision::None; } // A pending interrupt that is masked now is still about to be // taken: a loop that raises and drops the spl (IRIX's splhi leaves // only IP6 unmasked on IP22) is mostly seen in its masked stretch, // and takes the line at splx a few instructions on. Parking there - // would wait for something that only running can bring. + // would wait for something that only running can bring, so pace + // instead: bounded sleeps keep the host core free without waiting on + // a wake condition the guest is about to create. if ip != 0 { - self.ring_len = 0; - self.ring_pos = 0; - return false; + RESET_IP.fetch_add(1, Ordering::Relaxed); + self.clear_window(); + return IdleDecision::Pace; } - let h = Self::hash_state(core); - if self.ring[..self.ring_len].contains(&h) { - return true; + // IE=0: no interrupt can be delivered at all, so a park could never + // end. Same busy-wait trap as `im == 0` above; keep executing. + if !core.interrupts_enabled() { + RESET_IE0.fetch_add(1, Ordering::Relaxed); + self.clear_window(); + return IdleDecision::None; } - self.ring[self.ring_pos] = h; - self.ring_pos = (self.ring_pos + 1) % IDLE_RING; - if self.ring_len < IDLE_RING { - self.ring_len += 1; + + // Only kernel code can be parked. A user process spinning in a loop + // (and there are such loops during IRIX boot) is busy by definition; + // the kernel idle path is always kernel mode. This is a privilege + // filter, not an address match, so it is image-independent. + if self.pc_len == PC_RING { + WINDOW_FULL.fetch_add(1, Ordering::Relaxed); + let (samples, distinct, users) = self.largest_cluster(); + WINDOW_SAMPLES.store(samples as u64, Ordering::Relaxed); + WINDOW_DISTINCT.store(distinct as u64, Ordering::Relaxed); + if users > PC_RING / 4 { + RESET_USER.fetch_add(1, Ordering::Relaxed); + } + } + if self.window_is_idle() { + self.streak += 1; + if self.streak >= IDLE_STREAK { + self.streak = IDLE_STREAK; + return IdleDecision::Park; + } + } else { + self.streak = 0; } - false + IdleDecision::None } /// Park in ≤1 ms slices until an interrupt is pending (masked or not) /// or the CPU stops. /// - /// The Count==Compare interrupt needs no special handling here anymore: - /// the compare timer fires on the hptimer thread and ORs IP7 into + /// The Count==Compare interrupt needs no special handling here: the + /// compare timer fires on the hptimer thread and ORs IP7 into /// `hot.interrupts` exactly like a device line, and CP0 Count itself is /// virtual (materialized from the wall clock on read), so nothing has to - /// advance it during the sleep. Only `hot.cycles` is advanced — at a - /// nominal ~100 MIPS — so cross-thread cycle readers (Wd33c93a's - /// deferred-interrupt spin-wait, CP0 Random) keep seeing progress. - pub fn park(&self, core: &mut MipsCore, running: &AtomicBool) { + /// advance it during the sleep. Only `hot.cycles` is advanced — at the + /// 10 ns/guest-cycle host-pacing rate — so cross-thread cycle readers + /// (Wd33c93a's deferred-interrupt spin-wait, CP0 Random) keep seeing + /// progress. Under `ci_clock` the compare deadline is a cycles threshold + /// instead, so the park also stops once it is crossed. + pub fn park(&mut self, core: &mut MipsCore, running: &AtomicBool) { // Only park once the guest has actually armed a Compare deadline. // Before that (PROM), Compare use is ad-hoc and there may be nothing // armed to wake us. cp0_compare is zero out of reset and the guest @@ -119,9 +377,11 @@ impl IdleParkState { return; } + PARK_COUNT.fetch_add(1, Ordering::Relaxed); *PARKER.lock() = Some(std::thread::current()); loop { if !running.load(Ordering::Relaxed) { + PARK_WAKE_OTHER.fetch_add(1, Ordering::Relaxed); break; } // Announce the park before the last look at the pending word: a @@ -136,6 +396,7 @@ impl IdleParkState { // can only do running. (Waiting for `ip & im` slept through the // timer for good in such a loop: an IRIX 6.5 hang.) if ip != 0 { + PARK_WAKE_IRQ.fetch_add(1, Ordering::Relaxed); break; } // ci_clock has no hptimer — the fire point is a cycles threshold @@ -150,12 +411,116 @@ impl IdleParkState { // only polled — but an interrupt now ends it at once. std::thread::park_timeout(Duration::from_nanos(SLICE_NS)); let elapsed_ns = t0.elapsed().as_nanos() as u64; - core.hot.cycles = core.hot.cycles.wrapping_add(elapsed_ns / 10); + PARK_SLEEP_NS.fetch_add(elapsed_ns, Ordering::Relaxed); + core.hot.cycles = core.hot.cycles.wrapping_add(elapsed_ns / STALL_NS_PER_CYCLE); + } + // If the wake was for a line the guest cannot take (masked), this + // was not an idle loop; back off so a device-poll or DELAY loop runs + // at full speed instead of one batch per slice. + let pending = core.hot.interrupts.load(Ordering::SeqCst) as u32; + let ip = (core.cp0_cause | pending) & CAUSE_IP_MASK; + let im = core.cp0_status & STATUS_IM_MASK; + if ip != 0 && (ip & im) == 0 { + self.park_skip_until_cycle = core.hot.cycles.saturating_add(PARK_BACKOFF_CYCLES); } // Every exit, including the ci_clock one, leaves the flag clear: a // stale `true` would put `wake` on the mutex for a running CPU. PARKED.store(false, Ordering::SeqCst); } + + /// Pace a repeated state whose interrupts are masked. + /// + /// Unlike [`IdleParkState::park`], this never waits indefinitely: every + /// slice is bounded, so there is no deadlock risk when nothing can wake + /// the CPU. `hot.cycles` advances during the sleep at the host-pacing + /// rate (`STALL_NS_PER_CYCLE`), so cross-thread cycle readers keep seeing + /// progress and, under `ci_clock`, the Compare threshold still arrives. + /// The loop ends at once when `running` is cleared (monitor command / + /// soft reset), when an interrupt becomes deliverable, when the guest + /// re-enables interrupts (the normal park path takes over next), or — + /// under `ci_clock` — when the compare deadline is reached. + pub fn pace(&self, core: &mut MipsCore, running: &AtomicBool) { + *PARKER.lock() = Some(std::thread::current()); + loop { + if !running.load(Ordering::Relaxed) { + break; + } + // Announce the park before the last look at the pending word + // (Dekker): a writer sets its bit then reads `PARKED`, so either + // we see the bit here or the writer sees `PARKED` and unparks us + // via [`wake`]. + PARKED.store(true, Ordering::SeqCst); + let ie = core.interrupts_enabled(); + let pending = core.hot.interrupts.load(Ordering::SeqCst) as u32; + let ip = (core.cp0_cause | pending) & CAUSE_IP_MASK; + let im = core.cp0_status & STATUS_IM_MASK; + if (ip & im) != 0 { + break; + } + // The guest re-enabled interrupts: hand back to the run loop so + // the next `update` can park normally. + if ie && im != 0 { + break; + } + // ci_clock: stop once the compare deadline is reached, so the run + // loop can deliver IP7. + #[cfg(feature = "ci_clock")] + if core.hot.cycles >= core.count_fire_cycle { + break; + } + let t0 = Instant::now(); + std::thread::park_timeout(Duration::from_nanos(SLICE_NS)); + let elapsed_ns = t0.elapsed().as_nanos() as u64; + core.hot.cycles = core.hot.cycles.wrapping_add(elapsed_ns / STALL_NS_PER_CYCLE); + } + // Every exit leaves the flag clear: a stale `true` would put `wake` + // on the mutex for a running CPU. + PARKED.store(false, Ordering::SeqCst); + } +} + +/// Park the CPU thread while the guest is inside a WAIT instruction, until an +/// enabled unmasked interrupt is pending or a soft reset is requested. +/// +/// The same Dekker wake as [`IdleParkState::park`]: a writer sets its bit and +/// then reads `PARKED`, so either this loop sees the bit or the writer sees +/// `PARKED` and unparks us via [`wake`]. Unlike [`IdleParkState::park`], this +/// runs *inside* a single instruction's stall (`exec_wait`), not the run loop, +/// so there is no `cp0_compare`-armed guard and no `running` flag to poll: it +/// stays parked until the architectural wake condition (or the soft-reset bit, +/// which is set by `MipsCpu::signal` and also goes through [`wake`]). Only +/// `hot.cycles` advances — at the host-pacing rate — so cross-thread cycle +/// readers (Wd33c93a's deferred-interrupt spin) and, under `ci_clock`, CP0 +/// Count keep seeing progress exactly as the old `spin_loop` stall did. +pub fn park_wait(core: &mut MipsCore) { + const SOFT_RESET_BIT: u64 = 1u64 << 63; + *PARKER.lock() = Some(std::thread::current()); + loop { + // Announce the park before the last look at the pending word (Dekker). + PARKED.store(true, Ordering::SeqCst); + let pending = core.hot.interrupts.load(Ordering::SeqCst); + if pending & SOFT_RESET_BIT != 0 { + break; + } + let ip = (core.cp0_cause | (pending as u32)) & CAUSE_IP_MASK; + let im = core.cp0_status & STATUS_IM_MASK; + if (ip & im) != 0 { + break; + } + // ci_clock: stop once the compare deadline is reached so the next + // step delivers it (same rule as `IdleParkState::park`). + #[cfg(feature = "ci_clock")] + if core.hot.cycles >= core.count_fire_cycle { + break; + } + let t0 = Instant::now(); + std::thread::park_timeout(Duration::from_nanos(SLICE_NS)); + let elapsed_ns = t0.elapsed().as_nanos() as u64; + core.hot.cycles = core.hot.cycles.wrapping_add(elapsed_ns / STALL_NS_PER_CYCLE); + } + // Every exit leaves the flag clear: a stale `true` would put `wake` on the + // mutex for a running CPU. + PARKED.store(false, Ordering::SeqCst); } /// Is the CPU thread parked now? (Tests wait for it, not a guessed sleep.) @@ -170,33 +535,162 @@ pub(crate) fn parked() -> bool { #[cfg(test)] pub(crate) static PARK_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::const_mutex(()); +/// Runtime parking switch. `IRIS_NO_IDLE` in the environment disables it at +/// startup; the monitor's `idle on|off` overrides that for the run. +static IDLE_ENABLED: AtomicBool = AtomicBool::new(true); +static IDLE_INIT: std::sync::Once = std::sync::Once::new(); + +fn idle_init() { + IDLE_INIT.call_once(|| { + if std::env::var_os("IRIS_NO_IDLE").is_some() { + IDLE_ENABLED.store(false, Ordering::Relaxed); + } + }); +} + pub fn idle_park_enabled() -> bool { - std::env::var_os("IRIS_NO_IDLE").is_none() + idle_init(); + IDLE_ENABLED.load(Ordering::Relaxed) +} + +/// Enable or disable parking at runtime (monitor `idle on|off`). Enabling +/// unparks a thread already sleeping in `park`, so the change takes effect +/// at once. +pub fn set_idle_park_enabled(on: bool) { + idle_init(); + IDLE_ENABLED.store(on, Ordering::Relaxed); + if on { + wake(); + } } #[cfg(test)] mod tests { use super::*; - use crate::cpu::mips_core::{MipsCore, STATUS_IE, STATUS_IM_SHIFT}; + use crate::cpu::mips_core::{MipsCore, PrivilegeMode, STATUS_IE, STATUS_IM_SHIFT}; - /// A state the detector would otherwise park on: interrupts enabled, none - /// ready, and the same architectural state seen twice. - fn repeated_idle_state(status: u32) -> (IdleParkState, MipsCore) { + /// A kernel-mode core with interrupts enabled and the given mask, no + /// pending lines and no back-off. + fn idle_core(status: u32) -> MipsCore { let mut core = MipsCore::default(); core.cp0_status = status; core.cp0_cause = 0; - core.pc = 0x8000_0100; - let mut st = IdleParkState::default(); - st.update(&core); // first sighting fills the ring - (st, core) + core.pc = 0xffff_ffff_880d_aedc; + core + } + + fn kernel_idle() -> MipsCore { + idle_core(STATUS_IE | (1 << (STATUS_IM_SHIFT + 7))) + } + + /// Fill the sample window with `pcs` (one sample each) for enough + /// consecutive batches to satisfy the streak, returning the last decision. + fn drive(st: &mut IdleParkState, core: &mut MipsCore, pcs: &[u64]) -> IdleDecision { + let mut decision = IdleDecision::None; + for _ in 0..IDLE_STREAK { + for &pc in pcs { + core.pc = pc; + for _ in 0..SAMPLE_EVERY { + st.sample(core); + } + } + decision = st.update(core); + } + decision + } + + /// The 6.5 scheduler idle loop as the batch sampler sees it: 24 distinct + /// PCs spread over ~470 bytes at 0x880daedc..0x880db0b4. + fn six_five_idle_pcs() -> Vec { + (0..PC_RING) + .map(|i| 0xffff_ffff_880d_aedc + (i as u64) * 19) + .collect() } #[test] - fn update_parks_on_a_repeated_state_when_an_interrupt_could_arrive() { - // Control for the test below: with a mask bit set, `park`'s - // `(ip & im) != 0` is satisfiable, so parking is safe and expected. - let (mut st, core) = repeated_idle_state(STATUS_IE | (1 << (STATUS_IM_SHIFT + 7))); - assert!(st.update(&core), "a repeated idle state with IM set should park"); + fn update_parks_a_long_stationary_kernel_window() { + let mut core = kernel_idle(); + let mut st = IdleParkState::ready(); + let pcs = six_five_idle_pcs(); + assert!( + matches!(drive(&mut st, &mut core, &pcs), IdleDecision::Park), + "the 6.5 scheduler idle path must be recognised" + ); + } + + /// The real 6.5 idle path calls helpers far from the scheduler loop + /// (`spl` at 0x8801_03xx, the ll/sc lock at 0x8801_ba90). Those samples + /// form their own small clusters and must not disqualify the loop. + #[test] + fn update_parks_a_dominant_cluster_with_distant_helper_calls() { + let mut core = kernel_idle(); + let mut st = IdleParkState::ready(); + let mut pcs = six_five_idle_pcs(); + pcs[3] = 0xffff_ffff_8801_ba90; + pcs[7] = 0xffff_ffff_8801_03_00; + pcs[11] = 0xffff_ffff_8801_ba98; + pcs[15] = 0xffff_ffff_8801_0318; + assert!( + matches!(drive(&mut st, &mut core, &pcs), IdleDecision::Park), + "distant helper calls must not hide the idle loop" + ); + } + + #[test] + fn update_declines_a_short_device_poll_loop() { + // `lbu v1,0(a3); andi; bne -3` waiting on a device bit: 12 bytes, + // three PCs, and it needs executed instructions, not interrupts. + let mut core = kernel_idle(); + let mut st = IdleParkState::ready(); + let poll = [0xffff_ffff_8805_604c, 0xffff_ffff_8805_6050, 0xffff_ffff_8805_6054]; + let pcs: Vec = (0..PC_RING).map(|i| poll[i % poll.len()]).collect(); + assert!( + matches!(drive(&mut st, &mut core, &pcs), IdleDecision::None), + "a short device-poll loop must keep executing" + ); + } + + #[test] + fn update_declines_a_two_instruction_delay_loop() { + // `bgtz a0,-1; subu a0,a0,a1` (IRIX DELAY): 8 bytes, two PCs. + let mut core = kernel_idle(); + let mut st = IdleParkState::ready(); + let pcs: Vec = (0..PC_RING) + .map(|i| if i % 2 == 0 { 0xffff_ffff_8801_9fb0 } else { 0xffff_ffff_8801_9fb4 }) + .collect(); + assert!( + matches!(drive(&mut st, &mut core, &pcs), IdleDecision::None), + "an instruction-counted DELAY loop must not be parked" + ); + } + + #[test] + fn update_declines_a_user_mode_loop() { + // Same shape as the idle window, but user code: a user process + // spinning is busy by definition. + let mut core = kernel_idle(); + core.cp0_status |= 2 << crate::cpu::mips_core::STATUS_KSU_SHIFT; // user + assert_eq!(core.get_privilege_mode(), PrivilegeMode::User); + let mut st = IdleParkState::ready(); + assert!( + matches!(drive(&mut st, &mut core, &six_five_idle_pcs()), IdleDecision::None), + "user-mode loops are never parked" + ); + } + + #[test] + fn update_declines_a_window_that_keeps_moving() { + // Ordinary kernel execution: PCs spread over more than one page, so + // the ring never settles into a stationary window. + let mut core = kernel_idle(); + let mut st = IdleParkState::ready(); + let pcs: Vec = (0..PC_RING) + .map(|i| 0xffff_ffff_880d_aedc + (i as u64) * 0x400) + .collect(); + assert!( + matches!(drive(&mut st, &mut core, &pcs), IdleDecision::None), + "a moving PC window is not an idle loop" + ); } #[test] @@ -205,20 +699,115 @@ mod tests { // a loop that raises and drops the spl is sampled mostly in its // masked stretch. With the timer's IP4 already pending there, the // guest takes it a few instructions later at splx: nothing to wait - // for, so no park. + // for; pace rather than park. use crate::cpu::mips_core::CAUSE_IP4; - let (mut st, core) = repeated_idle_state(STATUS_IE | (1 << (STATUS_IM_SHIFT + 6))); + let mut core = idle_core(STATUS_IE | (1 << (STATUS_IM_SHIFT + 6))); core.hot.interrupts.store(CAUSE_IP4 as u64, Ordering::SeqCst); - assert!(!st.update(&core), "a pending interrupt is about to be taken"); - assert!(!st.update(&core), "... however often the state repeats"); + let mut st = IdleParkState::ready(); + assert!( + matches!(st.update(&core), IdleDecision::Pace), + "a pending interrupt is about to be taken" + ); + assert!( + matches!(st.update(&core), IdleDecision::Pace), + "... however often the state repeats" + ); } #[test] - fn update_declines_to_park_when_every_interrupt_is_masked() { + fn update_declines_to_park_or_pace_when_every_interrupt_is_masked() { // IE set but IM zero: the guest would take an interrupt, but none can - // be delivered, so the wait `park` performs can never end. - let (mut st, core) = repeated_idle_state(STATUS_IE); - assert!(!st.update(&core), "IM == 0 makes park's wake condition unsatisfiable"); + // be delivered, so the wait `park` performs can never end. This is + // also what a PROM device-poll or DELAY loop looks like; pacing it + // throttles boot to one batch per slice, so keep executing. + let mut core = idle_core(STATUS_IE); + let mut st = IdleParkState::ready(); + assert!( + matches!(drive(&mut st, &mut core, &six_five_idle_pcs()), IdleDecision::None), + "IM == 0 makes park's wake condition unsatisfiable and pacing unsafe" + ); + } + + #[test] + fn update_declines_to_pace_a_repeated_state_with_interrupts_disabled() { + // IE clear: no interrupt can be taken at all. A repeated IE=0 state + // is a busy wait, not an idle guest. + let mut core = idle_core(0); + let mut st = IdleParkState::ready(); + assert!( + matches!(drive(&mut st, &mut core, &six_five_idle_pcs()), IdleDecision::None), + "IE == 0 must not sleep" + ); + } + + #[test] + fn update_returns_none_when_an_interrupt_is_ready_to_deliver() { + // Even a stationary window must not sleep while an unmasked interrupt + // is pending — the guest should take it. + let mut core = kernel_idle(); + core.cp0_cause = crate::cpu::mips_core::CAUSE_IP7; + let mut st = IdleParkState::ready(); + assert!( + matches!(drive(&mut st, &mut core, &six_five_idle_pcs()), IdleDecision::None), + "a ready unmasked interrupt means do not sleep" + ); + } + + /// Pacing returns at once when `running` is cleared (monitor command / + /// soft reset) — no sleep, no cycle advance. + #[test] + fn pace_returns_when_not_running() { + // `pace` publishes this thread in the one global parker slot even when + // it returns at once; take the lock so it cannot steal a concurrent + // park test's slot. + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + let running = AtomicBool::new(false); + let before = core.hot.cycles; + let st = IdleParkState::default(); + st.pace(&mut core, &running); + assert_eq!(core.hot.cycles, before, "pace must not sleep when not running"); + } + + /// Pacing returns at once when an interrupt becomes deliverable. + #[test] + fn pace_returns_when_an_interrupt_is_deliverable() { + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 7)); + core.cp0_cause = crate::cpu::mips_core::CAUSE_IP7; // unmasked IP7 pending + let running = AtomicBool::new(true); + let before = core.hot.cycles; + let st = IdleParkState::default(); + st.pace(&mut core, &running); + assert_eq!( + core.hot.cycles, + before, + "pace must not sleep while an interrupt is deliverable" + ); + } + + /// Pacing advances `hot.cycles` while it sleeps, so cross-thread cycle + /// readers keep seeing progress. + #[test] + fn pace_advances_guest_cycles_while_masked() { + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + core.cp0_status = 0; // interrupts masked: the pace path + let running = std::sync::Arc::new(AtomicBool::new(true)); + let stop = running.clone(); + let before = core.hot.cycles; + let stopper = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(5)); + stop.store(false, Ordering::SeqCst); + wake(); + }); + IdleParkState::ready().pace(&mut core, &running); + stopper.join().unwrap(); + assert!( + core.hot.cycles > before, + "pace must advance guest cycles while it sleeps" + ); } } @@ -255,7 +844,7 @@ mod wake_tests { wake(); }); - let st = IdleParkState::default(); + let mut st = IdleParkState::default(); st.park(&mut core, &running); let returned = Instant::now(); raiser.join().unwrap(); @@ -310,6 +899,99 @@ mod wake_tests { "park slept through a pending interrupt masked only by the spl"); } + /// A park that wakes on a masked line was a false positive: back off so + /// a device-poll or DELAY loop keeps running instead of sleeping a slice + /// per batch. A park woken by an unmasked line leaves parking enabled. + #[test] + fn a_park_woken_by_a_masked_line_backs_off() { + use crate::cpu::mips_core::CAUSE_IP4; + let _one = PARK_TEST_LOCK.lock(); + + // Masked: only IP6 unmasked, IP4 pending. + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 6)); + core.cp0_compare = 1; + core.hot.interrupts.store(CAUSE_IP4 as u64, Ordering::SeqCst); + let running = AtomicBool::new(false); // return at the first check + let mut st = IdleParkState::ready(); + st.park(&mut core, &running); + assert!( + matches!(st.update(&core), IdleDecision::None), + "a masked wake must disable parking for the back-off window" + ); + + // Unmasked: IP7 unmasked and pending. + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 7)); + core.cp0_compare = 1; + core.hot.interrupts.store(CAUSE_IP7 as u64, Ordering::SeqCst); + let mut st = IdleParkState::ready(); + st.park(&mut core, &running); + assert!( + matches!(st.update(&core), IdleDecision::None), + "a deliverable pending interrupt means run, not park (ring cleared)" + ); + } + + /// The CP0 Compare deadline is an `hptimer` one-shot in a wall-clock + /// build. A parked CPU must wake when it fires, without any other thread + /// raising a line: the callback sets IP7 in `hot.interrupts`, and `park` + /// exits on any pending line. The Compare deadline is the hptimer + /// one-shot armed by `schedule_compare_timer`. + #[cfg(not(feature = "ci_clock"))] + #[test] + fn an_armed_compare_timer_ends_the_park() { + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 7)); + core.set_timer_manager(Arc::new(crate::hptimer::TimerManager::new())); + // Arm Compare a few Count ticks ahead: about 1 us at 33 MHz. park's + // own guard requires a non-zero Compare. + let now = core.count_now(); + core.write_cp0(11, now.wrapping_add(33) as u64); + let running = AtomicBool::new(true); + + let t0 = Instant::now(); + IdleParkState::default().park(&mut core, &running); + let elapsed = t0.elapsed(); + assert_ne!( + core.hot.interrupts.load(Ordering::SeqCst) as u32 & CAUSE_IP7, + 0, + "the Compare one-shot must have raised IP7" + ); + assert!( + elapsed < Duration::from_millis(100), + "park waited {elapsed:?} for the armed Compare timer" + ); + } + + /// Under `ci_clock` there is no host timer: the Compare deadline is a + /// `hot.cycles` threshold, and the park must stop once the advancing cycle + /// counter crosses it so the run loop can deliver IP7. + #[cfg(feature = "ci_clock")] + #[test] + fn the_cycle_compare_threshold_ends_the_park() { + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 7)); + core.cp0_compare = 1; // park's guard requires a non-zero Compare + core.count_fire_cycle = core.hot.cycles + 300_000; // 3 ms at 10 ns/cycle + let running = AtomicBool::new(true); + + let t0 = Instant::now(); + IdleParkState::default().park(&mut core, &running); + let elapsed = t0.elapsed(); + assert!( + core.hot.cycles >= 300_000, + "park returned before the cycle threshold: cycles={}", + core.hot.cycles + ); + assert!( + elapsed < Duration::from_millis(200), + "park waited {elapsed:?} for the cycle threshold" + ); + } + #[test] fn an_interrupt_ends_the_park_without_waiting_out_the_slice() { // Without `wake` every sweep contains a phase that waits out most of a diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index afa7f611..8cb76d2e 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -2680,15 +2680,45 @@ mod ip7_ticket_tests { fn core_past_its_compare() -> MipsCore { let mut core = MipsCore::default(); core.cp0_compare = 0x40; - std::thread::sleep(std::time::Duration::from_millis(3)); // `materialize_count` memoizes per cycle count: a core that has never // executed has `hot.cycles == count_read_cycle == 0` and short-circuits // before materializing anything, so Count would stay 0 and no crossing - // could ever be detected. Make it look like it has run. - core.hot.cycles = 1; + // could ever be detected. Make it look like it has run: real time + // under the wall clock, retired cycles under `ci_clock` (~0.33 ticks + // per cycle at the default 33 MHz). + #[cfg(not(feature = "ci_clock"))] + std::thread::sleep(std::time::Duration::from_millis(3)); + #[cfg(not(feature = "ci_clock"))] + { + core.hot.cycles = 1; + } + #[cfg(feature = "ci_clock")] + { + core.hot.cycles = 100_000; + } core } + /// Let enough time pass for Count to move the requested distance: + /// wall-clock time under the default build, retired cycles under + /// `ci_clock` (10 ns each). + fn advance_count(core: &mut MipsCore, count_ticks: u64) { + #[cfg(not(feature = "ci_clock"))] + { + let _ = count_ticks; + std::thread::sleep(std::time::Duration::from_millis(3)); + core.hot.cycles += 1; // see core_past_its_compare: let Count move + } + #[cfg(feature = "ci_clock")] + { + let ns = (count_ticks as u128 * 1_000_000_000) / core.count_hz as u128; + core.hot.cycles = core + .hot + .cycles + .saturating_add((ns as u64 / NS_PER_GUEST_CYCLE).max(1)); + } + } + /// The ticket `schedule_compare_timer` arms its one-shot with is /// `self.ip7_seq`. If something consumed the shared sequence in the /// meantime, that one-shot can never claim: on its first fire `claim_ip7` @@ -2785,8 +2815,7 @@ mod ip7_ticket_tests { fn a_missed_deadline_across_a_count_wrap_still_raises_ip7() { let mut core = core_past_its_compare(); core.write_cp0(9, 0xFFFF_FF00); - std::thread::sleep(std::time::Duration::from_millis(3)); - core.hot.cycles += 1; // see core_past_its_compare: let Count move + advance_count(&mut core, 1_000_000); let now = core.count_peek(); assert!(now < 0x8000_0000, "Count should have wrapped by now: {now:#x}"); diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index a8a593a9..215166e8 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -11522,13 +11522,13 @@ impl Device for MipsCpu Device for MipsCpu Device for MipsCpu { for _ in 0..1000 { let status = guard.$step(); + #[cfg(feature = "idle-pause")] + idle_state.sample(&guard.core); match status { EXEC_BREAKPOINT => { running.store(false, Ordering::SeqCst); @@ -11606,13 +11612,26 @@ impl Device for MipsCpu { + drop(guard); + { + let mut guard = executor.lock(); + idle_state.park(&mut guard.core, &running); + } + guard = executor.lock(); + } + crate::cpu::idle_park::IdleDecision::Pace => { + drop(guard); + { + let mut guard = executor.lock(); + idle_state.pace(&mut guard.core, &running); + } + guard = executor.lock(); + } + crate::cpu::idle_park::IdleDecision::None => {} } - guard = executor.lock(); } // --- end idle park --- // --- perf sampling (comment out to disable) --- @@ -11711,7 +11730,9 @@ impl Device for MipsCpu> [DEV]".to_string()), ("dt".to_string(), "Disassemble traceback: dt [count] | dt file [count]".to_string()), - ("idleprof".to_string(), "Locate idle/spin loops via PC sampling: idleprof ".to_string()), + ("idleprof".to_string(), "Locate idle/spin loops: idleprof ".to_string()), + #[cfg(feature = "idle-pause")] + ("idle".to_string(), "Idle parking: idle ".to_string()), #[cfg(feature = "instr_stats")] ("instrstats".to_string(), "Per-instruction decode/execution counters: instrstats [report|clear|dump] [DEV]".to_string()), ("u".to_string(), "Alias for undo [DEV]".to_string()), @@ -14467,6 +14488,43 @@ impl Device for MipsCpu { + let sub = actual_args.first().copied().unwrap_or("status"); + match sub { + "on" => { + crate::cpu::idle_park::set_idle_park_enabled(true); + writeln!(writer, "idle: parking enabled").unwrap(); + } + "off" => { + crate::cpu::idle_park::set_idle_park_enabled(false); + writeln!(writer, "idle: parking disabled").unwrap(); + } + "status" => { + use std::sync::atomic::Ordering::Relaxed as R; + let n = crate::cpu::idle_park::PARK_COUNT.load(R); + let ns = crate::cpu::idle_park::PARK_SLEEP_NS.load(R); + let irq = crate::cpu::idle_park::PARK_WAKE_IRQ.load(R); + let other = crate::cpu::idle_park::PARK_WAKE_OTHER.load(R); + writeln!(writer, "idle: {} parks={} slept={:.3}s wake_irq={} wake_other={}", + if crate::cpu::idle_park::idle_park_enabled() { "on" } else { "off" }, + n, ns as f64 / 1e9, irq, other).unwrap(); + writeln!(writer, " updates={} full={} largest={} distinct={}", + crate::cpu::idle_park::UPDATES.load(R), + crate::cpu::idle_park::WINDOW_FULL.load(R), + crate::cpu::idle_park::WINDOW_SAMPLES.load(R), + crate::cpu::idle_park::WINDOW_DISTINCT.load(R)).unwrap(); + writeln!(writer, " resets: irq={} ip={} im0={} ie0={} user={}", + crate::cpu::idle_park::RESET_IRQ.load(R), + crate::cpu::idle_park::RESET_IP.load(R), + crate::cpu::idle_park::RESET_IM0.load(R), + crate::cpu::idle_park::RESET_IE0.load(R), + crate::cpu::idle_park::RESET_USER.load(R)).unwrap(); + } + _ => return Err("Usage: idle ".to_string()), + } + Ok(()) + } + #[cfg(feature = "idle-pause")] "idleprof" => { let sub = actual_args.first().copied().unwrap_or("report"); match sub { @@ -14483,7 +14541,16 @@ impl Device for MipsCpu return Err("Usage: idleprof ".to_string()), + "stats" => { + use std::sync::atomic::Ordering; + let n = crate::cpu::idle_park::PARK_COUNT.load(Ordering::Relaxed); + let ns = crate::cpu::idle_park::PARK_SLEEP_NS.load(Ordering::Relaxed); + let irq = crate::cpu::idle_park::PARK_WAKE_IRQ.load(Ordering::Relaxed); + let other = crate::cpu::idle_park::PARK_WAKE_OTHER.load(Ordering::Relaxed); + writeln!(writer, "idleprof: parks={} slept={:.3}s wake_irq={} wake_other={}", + n, ns as f64 / 1e9, irq, other).unwrap(); + } + _ => return Err("Usage: idleprof ".to_string()), } Ok(()) } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index a40df28e..a2178ea3 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6113,8 +6113,11 @@ mod tests { const T4: u32 = 12; // 1, for the handler to store const T5: u32 = 13; // the handler's Status: EXL | IE | IM6 const STATUS: u32 = 12; - const NOPS: u32 = 194; // a pass is NOPS + 6 = 200 instructions: - // 1000-step batches end at one place in it + // A pass is NOPS + 6 = 197 instructions. The PC-cluster detector + // samples every 32 instructions, so the loop length must not share a + // factor with the sample interval in a way that aliases to one PC; + // 197 is coprime with 32 and with the 1000-instruction batch. + const NOPS: u32 = 191; let mtc0 = |rt: u32, rd: u32| (OP_COP0 << 26) | (0x04 << 21) | (rt << 16) | (rd << 11); let lw = |rt: u32, base: u32| (OP_LW << 26) | (base << 21) | (rt << 16); let sw = |rt: u32, base: u32| (OP_SW << 26) | (base << 21) | (rt << 16); @@ -6164,19 +6167,27 @@ mod tests { exec.core.cp0_compare = 1; // (park only once a deadline is armed) ptr_tx.send(Ptr(&exec.core.hot.interrupts as *const AtomicU64)).unwrap(); - let mut idle = IdleParkState::default(); + let mut idle = IdleParkState::ready(); let mut parks = 0u32; while run.load(Ordering::Relaxed) { for _ in 0..1000 { exec.step_int(); + idle.sample(&exec.core); } if exec.core.pc == kseg0(done_pa) || exec.core.pc == kseg0(done_pa + 4) { done_tx.send(Ok(parks)).unwrap(); return; } - if idle.update(&exec.core) { - parks += 1; - idle.park(&mut exec.core, &run); + match idle.update(&exec.core) { + crate::cpu::idle_park::IdleDecision::Park => { + parks += 1; + idle.park(&mut exec.core, &run); + } + crate::cpu::idle_park::IdleDecision::Pace => { + parks += 1; + idle.pace(&mut exec.core, &run); + } + crate::cpu::idle_park::IdleDecision::None => {} } } done_tx.send(Err(exec.core.pc)).unwrap();