From 962ceddb989fe2052643438922cb1c9b2a26b774 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Sat, 31 Oct 2026 08:15:43 -0400 Subject: [PATCH 1/5] Add --cpus and --memory resource limits Sandboxes can now be limited to a number of CPUs and an amount of memory using cgroup v2. --cpus accepts fractional values, and negative values mean all but that many CPUs, so -2 on a 32 CPU machine allows 30. --memory accepts sizes like 512M or 4G and disables swap for the sandbox. Both can also be set with SANDBOX_CPUS / SANDBOX_MEMORY or in config files. Sandboxes with limits get a cgroup at /sys/fs/cgroup/sandbox/-. PID 1 is started in it with CLONE_INTO_CGROUP, which also makes it the root of the sandbox's cgroup namespace, and processes that join the sandbox move into it before entering its namespaces. Stopping the sandbox removes the cgroup. Limits are recorded in the sandbox settings, and changing them requires stopping the sandbox, like network settings. --- README.adoc | 24 ++ cspell.json | 2 + src/actions/config.rs | 10 + src/config/cli.rs | 17 ++ src/config/limits.rs | 183 +++++++++++ src/config/mod.rs | 1 + src/config/resolve.rs | 55 ++++ src/config/structs.rs | 9 + src/sandbox/cgroup.rs | 215 +++++++++++++ src/sandbox/exec.rs | 5 + src/sandbox/get_or_create.rs | 48 ++- src/sandbox/mod.rs | 1 + src/sandbox/settings.rs | 24 ++ src/sandbox/stop.rs | 8 + src/util/clone3.rs | 3 + tests/ai_test_resource_limits.rs | 493 ++++++++++++++++++++++++++++++ tests/ai_test_sandbox_settings.rs | 2 + 17 files changed, 1086 insertions(+), 14 deletions(-) create mode 100644 src/config/limits.rs create mode 100644 src/sandbox/cgroup.rs create mode 100644 tests/ai_test_resource_limits.rs diff --git a/README.adoc b/README.adoc index f862cab..f08c554 100644 --- a/README.adoc +++ b/README.adoc @@ -208,6 +208,24 @@ microphone. Audio works with or without `--net`, and is not affected by in a config file, and `SANDBOX_AUDIO=false` will override that for a single run. +`--cpus=`:: +Limit how many CPUs the sandbox can use. Fractional values such as `1.5` are +allowed. Negative values mean all but that many CPUs, so `--cpus=-2` on a 32 +CPU machine lets the sandbox use 30. This limits the total CPU time the +sandbox gets, it does not pin it to specific CPUs, so tools like `nproc` will +still report every CPU. + +`--memory=`:: +Limit how much memory the sandbox can use, for example `512M` or `4G`. +Suffixes are binary (`1K` is 1024 bytes). Swap is not available to a sandbox +with a memory limit, and when the sandbox runs out of memory the kernel kills +its largest process. ++ +Resource limits use cgroup v2 and apply to every process in the sandbox. They +are set when the sandbox starts, so you need to stop the sandbox to change +them. They may not be available when `sandbox` is run inside a container or +another sandbox. + `--json`:: Formats action output as a JSON blob. Does nothing for sandboxed commands @@ -278,6 +296,10 @@ These config files can set most of the CLI options: mask=[ "~/.ssh", ] + + # Resource limits + cpus=-2 + memory="8G" ``` ## Environment Variables @@ -290,6 +312,8 @@ You can also set most of the options with environment variables if you wish: SANDBOX_NET="none" SANDBOX_LOG_LEVEL="info" SANDBOX_BIND="/my/dir" + SANDBOX_CPUS="-2" + SANDBOX_MEMORY="8G" ``` diff --git a/cspell.json b/cspell.json index 0df102e..a36ea8e 100644 --- a/cspell.json +++ b/cspell.json @@ -32,6 +32,7 @@ "cmdline", "codegen", "codespell", + "cpus", "CREAT", "cstring", "czvf", @@ -169,6 +170,7 @@ "PKGBUILD", "powercap", "prctl", + "procs", "profdata", "profraw", "psub", diff --git a/src/actions/config.rs b/src/actions/config.rs index 2441b7b..615c080 100644 --- a/src/actions/config.rs +++ b/src/actions/config.rs @@ -17,6 +17,8 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { "mask", "no_default_binds", "audio", + "cpus", + "memory", "storage_dir", "sandbox_dir", "upper_cwd", @@ -47,6 +49,12 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { let no_default_binds_str = format!("{}", config.no_default_binds); let audio_str = format!("{}", config.audio); + let cpus_str = config + .cpus + .map_or("unlimited".to_string(), |cpus| cpus.to_string()); + let memory_str = config + .memory + .map_or("unlimited".to_string(), |memory| memory.to_string()); let ignored_str = format!("{}", config.ignored); let config_files_str = config .config_files @@ -78,6 +86,8 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { ("no_default_binds", no_default_binds_str.as_str()) } "audio" => ("audio", audio_str.as_str()), + "cpus" => ("cpus", cpus_str.as_str()), + "memory" => ("memory", memory_str.as_str()), "name" => ("name", config.name.as_str()), "log_level" => ("log_level", config.log_level.as_str()), "ignored" => ("ignored", ignored_str.as_str()), diff --git a/src/config/cli.rs b/src/config/cli.rs index 8fd4231..9d4e819 100644 --- a/src/config/cli.rs +++ b/src/config/cli.rs @@ -81,6 +81,23 @@ pub struct Args { #[arg(long, global = true, action = clap::ArgAction::SetTrue)] pub audio: bool, + /// Limit how many CPUs the sandbox can use. Fractional values such as 1.5 + /// are allowed. Negative values mean all but that many CPUs, so -2 on a + /// 32 CPU machine allows the sandbox to use 30. Requires cgroup v2. + #[arg( + long, + global = true, + allow_negative_numbers = true, + value_name = "CPUS" + )] + pub cpus: Option, + + /// Limit how much memory the sandbox can use, for example 512M or 4G. + /// Swap is not available to sandboxes with a memory limit. Requires + /// cgroup v2. + #[arg(long, global = true, value_name = "SIZE")] + pub memory: Option, + /// Formats action output as a JSON blob. Does nothing for sandboxed commands. #[arg(long, global = true, action = clap::ArgAction::SetTrue)] pub json: bool, diff --git a/src/config/limits.rs b/src/config/limits.rs new file mode 100644 index 0000000..3ab2b5c --- /dev/null +++ b/src/config/limits.rs @@ -0,0 +1,183 @@ +use anyhow::{Result, anyhow}; +use serde::Deserialize; + +/// The smallest CPU limit cgroups will accept: a 1ms quota per 100ms period +const MIN_CPUS: f64 = 0.01; + +/// Number of CPUs this process may run on. This matches `nproc` unless we +/// are already running under a CPU quota, in which case the quota is used. +pub fn available_cpus() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) +} + +/// Parses a CPU limit. Positive values are a number of CPUs and may be +/// fractional. Negative values mean all but that many of the `available` +/// CPUs, so -2 on a 32 CPU machine means 30. +pub fn parse_cpus(spec: &str, available: usize) -> Result { + let invalid = || { + anyhow!( + "Invalid cpus value '{}': expected a number of CPUs such as 2 or 1.5, or a negative number such as -2 to use all but that many CPUs", + spec + ) + }; + + let value: f64 = spec.trim().parse().map_err(|_| invalid())?; + if !value.is_finite() || value == 0.0 { + return Err(invalid()); + } + + let cpus = if value < 0.0 { + available as f64 + value + } else { + value + }; + + if cpus < MIN_CPUS { + return Err(anyhow!( + "Invalid cpus value '{}': this leaves {} of the {} available CPUs for the sandbox, the minimum is {}", + spec, + cpus.max(0.0), + available, + MIN_CPUS + )); + } + + Ok(cpus) +} + +/// Parses a memory size such as 512M, 4G, 1.5GiB or a plain number of bytes. +/// Suffixes are binary (K = 1024) and case insensitive. +pub fn parse_memory(spec: &str) -> Result { + let invalid = || { + anyhow!( + "Invalid memory value '{}': expected a size such as 512M or 4G", + spec + ) + }; + + let trimmed = spec.trim(); + let split = trimmed + .find(|c: char| c.is_ascii_alphabetic()) + .unwrap_or(trimmed.len()); + let (number, suffix) = trimmed.split_at(split); + + let multiplier: u64 = match suffix.to_ascii_lowercase().as_str() { + "" | "b" => 1, + "k" | "kb" | "kib" => 1 << 10, + "m" | "mb" | "mib" => 1 << 20, + "g" | "gb" | "gib" => 1 << 30, + "t" | "tb" | "tib" => 1 << 40, + _ => return Err(invalid()), + }; + + let number: f64 = number.trim().parse().map_err(|_| invalid())?; + let bytes = number * multiplier as f64; + if !bytes.is_finite() || bytes < 1.0 || bytes > u64::MAX as f64 { + return Err(invalid()); + } + + Ok(bytes as u64) +} + +/// Lets config files specify limits as either numbers or strings, e.g. +/// `cpus = -2` or `memory = "4G"` +pub(crate) fn deserialize_number_or_string<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumberOrString { + Integer(i64), + Float(f64), + String(String), + } + + Ok( + Option::::deserialize(deserializer)?.map(|value| { + match value { + NumberOrString::Integer(i) => i.to_string(), + NumberOrString::Float(f) => f.to_string(), + NumberOrString::String(s) => s, + } + }), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_cpus() { + assert_eq!(parse_cpus("2", 32).unwrap(), 2.0); + assert_eq!(parse_cpus(" 1.5 ", 32).unwrap(), 1.5); + assert_eq!(parse_cpus("0.01", 32).unwrap(), 0.01); + assert_eq!(parse_cpus("64", 32).unwrap(), 64.0); + assert_eq!(parse_cpus("-2", 32).unwrap(), 30.0); + assert_eq!(parse_cpus("-0.5", 4).unwrap(), 3.5); + assert_eq!(parse_cpus("-31", 32).unwrap(), 1.0); + } + + #[test] + fn test_parse_cpus_invalid() { + for spec in ["", "abc", "0", "-0", "nan", "inf", "-inf", "2 cpus"] { + let err = parse_cpus(spec, 32).unwrap_err().to_string(); + assert!(err.contains("expected a number of CPUs"), "{}", err); + } + + let err = parse_cpus("-32", 32).unwrap_err().to_string(); + assert!(err.contains("leaves 0 of the 32 available CPUs"), "{}", err); + let err = parse_cpus("-40", 32).unwrap_err().to_string(); + assert!(err.contains("leaves 0 of the 32 available CPUs"), "{}", err); + let err = parse_cpus("0.001", 32).unwrap_err().to_string(); + assert!(err.contains("the minimum is 0.01"), "{}", err); + } + + #[test] + fn test_parse_memory() { + assert_eq!(parse_memory("1024").unwrap(), 1024); + assert_eq!(parse_memory("1024b").unwrap(), 1024); + assert_eq!(parse_memory("4k").unwrap(), 4096); + assert_eq!(parse_memory("4KiB").unwrap(), 4096); + assert_eq!(parse_memory("512M").unwrap(), 512 << 20); + assert_eq!(parse_memory("512mb").unwrap(), 512 << 20); + assert_eq!(parse_memory(" 4G ").unwrap(), 4 << 30); + assert_eq!(parse_memory("4 GiB").unwrap(), 4 << 30); + assert_eq!(parse_memory("1.5G").unwrap(), 3 << 29); + assert_eq!(parse_memory("2T").unwrap(), 2 << 40); + } + + #[test] + fn test_parse_memory_invalid() { + for spec in [ + "", "G", "abc", "0", "0.5", "-1G", "4X", "4GBs", "1e30T", "inf", + ] { + let err = parse_memory(spec).unwrap_err().to_string(); + assert!(err.contains("expected a size"), "{}: {}", spec, err); + } + } + + #[test] + fn test_deserialize_number_or_string() { + #[derive(Deserialize)] + struct Limits { + #[serde( + deserialize_with = "deserialize_number_or_string", + default + )] + value: Option, + } + + let parse = |s: &str| toml::from_str::(s).unwrap().value; + assert_eq!(parse("value = -2"), Some("-2".to_string())); + assert_eq!(parse("value = 1.5"), Some("1.5".to_string())); + assert_eq!(parse("value = \"4G\""), Some("4G".to_string())); + assert_eq!(parse(""), None); + assert!(toml::from_str::("value = true").is_err()); + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs index 517fcf4..bdf4dcf 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod impls; +pub mod limits; pub mod resolve; pub mod structs; diff --git a/src/config/resolve.rs b/src/config/resolve.rs index 73182f2..6f7c0c7 100644 --- a/src/config/resolve.rs +++ b/src/config/resolve.rs @@ -1,4 +1,5 @@ use super::PartialConfig; +use super::limits::{available_cpus, parse_cpus, parse_memory}; use crate::config::{BindMount, BindMountOptions, Network}; use crate::util::{ can_access, can_mkdir, expand_tilde_path, find_mount_point, mkdir, @@ -172,6 +173,16 @@ pub fn resolve_config(cli: Args) -> Result { } } + for (var, key, field) in [ + ("SANDBOX_CPUS", "cpus", &mut partial_config.cpus), + ("SANDBOX_MEMORY", "memory", &mut partial_config.memory), + ] { + if let Some(value) = env::var(var).ok().filter(|v| !v.is_empty()) { + *field = Some(value); + sources.insert(key.into(), "environment".into()); + } + } + // Override with CLI args if provided (highest precedence) if let Some(log_level) = cli.log_level { partial_config.log_level = Some(log_level); @@ -228,6 +239,16 @@ pub fn resolve_config(cli: Args) -> Result { sources.insert("audio".into(), "cli".into()); } + if let Some(cpus) = cli.cpus { + partial_config.cpus = Some(cpus); + sources.insert("cpus".into(), "cli".into()); + } + + if let Some(memory) = cli.memory { + partial_config.memory = Some(memory); + sources.insert("memory".into(), "cli".into()); + } + // Bind mounts and masks from the CLI form the highest precedence layer let mut cli_bind_layer = cli.bind.unwrap_or_default(); if let Some(cli_mask_paths) = cli.mask { @@ -265,6 +286,22 @@ pub fn resolve_config(cli: Args) -> Result { sources.insert("audio".into(), "default".into()); } + let cpus = partial_config + .cpus + .map(|cpus| parse_cpus(&cpus, available_cpus())) + .transpose()?; + if !sources.contains_key("cpus") { + sources.insert("cpus".into(), "default".into()); + } + + let memory = partial_config + .memory + .map(|memory| parse_memory(&memory)) + .transpose()?; + if !sources.contains_key("memory") { + sources.insert("memory".into(), "default".into()); + } + let storage_dir = resolve_sandbox_storage_dir( partial_config.storage_dir, uid_gid_home.uid, @@ -357,6 +394,8 @@ pub fn resolve_config(cli: Args) -> Result { bind_mounts, no_default_binds, audio, + cpus, + memory, config_files, }; @@ -723,6 +762,14 @@ fn merge_configs( base.audio = Some(audio); sources.insert("audio".into(), source.into()); } + if let Some(cpus) = override_config.cpus { + base.cpus = Some(cpus); + sources.insert("cpus".into(), source.into()); + } + if let Some(memory) = override_config.memory { + base.memory = Some(memory); + sources.insert("memory".into(), source.into()); + } // Bind mounts and masks from each config file form their own layer let mut bind_layer = vec![]; if let Some(bind_mounts) = override_config.bind { @@ -922,6 +969,8 @@ mod tests { bind_mounts: vec![], no_default_binds: false, audio: false, + cpus: None, + memory: None, config_files: vec![], }; assert!(validate_config(&config).is_ok()); @@ -998,6 +1047,8 @@ mod tests { mask: Some(vec!["/tmp/test3".to_string()]), no_default_binds: Some(true), audio: Some(true), + cpus: Some("-2".to_string()), + memory: Some("4G".to_string()), }; merge_configs( @@ -1022,6 +1073,10 @@ mod tests { assert!(matches!(base.net, Some(Network::Host))); assert_eq!(base.ignored, Some(true)); assert_eq!(base.audio, Some(true)); + assert_eq!(base.cpus, Some("-2".to_string())); + assert_eq!(base.memory, Some("4G".to_string())); + assert_eq!(sources.get("cpus"), Some(&"test-config".to_string())); + assert_eq!(sources.get("memory"), Some(&"test-config".to_string())); // Each merged config produces its own bind layer let expected_layer = vec![ diff --git a/src/config/structs.rs b/src/config/structs.rs index d8ee92a..94b2223 100644 --- a/src/config/structs.rs +++ b/src/config/structs.rs @@ -1,4 +1,5 @@ use super::impls::deserialize_level_filter; +use super::limits::deserialize_number_or_string; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, path::PathBuf}; @@ -60,6 +61,10 @@ pub struct PartialConfig { pub mask: Option>, pub no_default_binds: Option, pub audio: Option, + #[serde(deserialize_with = "deserialize_number_or_string", default)] + pub cpus: Option, + #[serde(deserialize_with = "deserialize_number_or_string", default)] + pub memory: Option, } #[derive(Clone)] @@ -76,5 +81,9 @@ pub struct Config { pub bind_mounts: Vec, pub no_default_binds: bool, pub audio: bool, + /// Number of CPUs the sandbox may use, None for unlimited + pub cpus: Option, + /// Bytes of memory the sandbox may use, None for unlimited + pub memory: Option, pub config_files: Vec, } diff --git a/src/sandbox/cgroup.rs b/src/sandbox/cgroup.rs new file mode 100644 index 0000000..07cb18c --- /dev/null +++ b/src/sandbox/cgroup.rs @@ -0,0 +1,215 @@ +//! cgroup v2 support for limiting the CPU and memory available to a sandbox. +//! +//! Sandboxes with resource limits get a cgroup at +//! /sys/fs/cgroup/sandbox/-. The sandbox's PID 1 is started in that +//! cgroup, which also makes it the root of the sandbox's cgroup namespace, and +//! every process that later joins the sandbox is moved into it. + +use crate::config::Config; +use crate::sandbox::Sandbox; +use anyhow::{Context, Result, anyhow}; +use log::trace; +use std::os::fd::OwnedFd; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const CGROUP_ROOT: &str = "/sys/fs/cgroup"; +const CPU_PERIOD_US: u64 = 100_000; + +/// The cgroup all sandbox cgroups are created in +fn cgroup_parent() -> String { + let parent = String::from("sandbox"); + + // Lets tests start from a parent cgroup that doesn't exist yet + #[cfg(feature = "coverage")] + let parent = std::env::var("TEST_CGROUP_PARENT").unwrap_or(parent); + + parent +} + +impl Sandbox { + fn cgroup_name(&self) -> String { + format!("{}-{}", self.uid, self.name) + } + + /// Our cgroup's path relative to the cgroup root, as shown in /proc//cgroup + fn cgroup_relative_path(&self) -> PathBuf { + Path::new("/") + .join(cgroup_parent()) + .join(self.cgroup_name()) + } + + pub(crate) fn cgroup_path(&self) -> PathBuf { + Path::new(CGROUP_ROOT) + .join(cgroup_parent()) + .join(self.cgroup_name()) + } + + /// Creates a cgroup with the resource limits from our config. Returns a + /// handle to it for use with CLONE_INTO_CGROUP, or None if the config + /// has no resource limits. + pub(crate) fn create_cgroup( + &self, + config: &Config, + ) -> Result> { + let mut controllers = vec![]; + if config.cpus.is_some() { + controllers.push("cpu"); + } + if config.memory.is_some() { + controllers.push("memory"); + } + if controllers.is_empty() { + return Ok(None); + } + + let root = Path::new(CGROUP_ROOT); + + #[cfg(feature = "coverage")] + let root = if std::env::var_os("TEST_NO_CGROUP2").is_some() { + Path::new("/does-not-exist") + } else { + root + }; + + if !root.join("cgroup.controllers").exists() { + return Err(anyhow!( + "Limiting cpus or memory requires cgroup v2, which is not mounted at {}", + root.display() + )); + } + + let parent = root.join(cgroup_parent()); + enable_controllers(root, &controllers)?; + if !parent.exists() { + let context = + format!("Failed to create cgroup {}", parent.display()); + std::fs::create_dir(&parent).context(context)?; + } + enable_controllers(&parent, &controllers)?; + + let path = self.cgroup_path(); + if path.exists() { + // Left behind by a sandbox that wasn't stopped cleanly, which we + // can remove because it's empty. If it's not empty it belongs to + // a running sandbox with the same name in another storage dir. + std::fs::remove_dir(&path).context(format!( + "Failed to remove existing cgroup {}, is a sandbox with the same name running from a different storage directory?", + path.display() + ))?; + } + trace!("Creating cgroup {}", path.display()); + std::fs::create_dir(&path) + .context(format!("Failed to create cgroup {}", path.display()))?; + + let result = self.write_limits(&path, config).and_then(|_| { + std::fs::File::open(&path) + .map(OwnedFd::from) + .context(format!("Failed to open cgroup {}", path.display())) + }); + if result.is_err() { + remove_cgroup(&path); + } + result.map(Some) + } + + fn write_limits(&self, path: &Path, config: &Config) -> Result<()> { + if let Some(cpus) = config.cpus { + let quota = (cpus * CPU_PERIOD_US as f64).round() as u64; + write_cgroup_file( + path, + "cpu.max", + &format!("{} {}", quota, CPU_PERIOD_US), + )?; + } + if let Some(memory) = config.memory { + write_cgroup_file(path, "memory.max", &memory.to_string())?; + // Swap would let the sandbox use more memory than its limit, but + // the file only exists when swap accounting is enabled. + let swap_accounting = path.join("memory.swap.max").exists(); + + #[cfg(feature = "coverage")] + let swap_accounting = swap_accounting + && std::env::var_os("TEST_NO_SWAP_ACCOUNTING").is_none(); + + if swap_accounting { + write_cgroup_file(path, "memory.swap.max", "0")?; + } + } + Ok(()) + } + + /// Returns the path to our cgroup if our sandbox is running in one + pub(crate) fn running_cgroup(&self) -> Option { + let cgroups = + std::fs::read_to_string(format!("/proc/{}/cgroup", self.pid)) + .ok()?; + let current = cgroups.lines().find_map(|line| line.strip_prefix("0::")); + if current.map(Path::new) == Some(&self.cgroup_relative_path()) { + Some(self.cgroup_path()) + } else { + None + } + } + + /// Moves the current process into our sandbox's cgroup, if it has one, + /// so processes we launch in the sandbox are subject to its limits. + pub(crate) fn join_cgroup(&self) -> Result<()> { + if let Some(path) = self.running_cgroup() { + trace!("Joining cgroup {}", path.display()); + let pid = std::process::id().to_string(); + write_cgroup_file(&path, "cgroup.procs", &pid)?; + } + Ok(()) + } +} + +/// Kills anything left in a cgroup and removes it. Removal can't succeed +/// until killed processes finish exiting, so we wait a little while for that. +pub(crate) fn remove_cgroup(path: &Path) { + trace!("Removing cgroup {}", path.display()); + let _ = std::fs::write(path.join("cgroup.kill"), "1"); + let deadline = Instant::now() + Duration::from_secs(5); + while std::fs::remove_dir(path).is_err() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn write_cgroup_file(dir: &Path, file: &str, value: &str) -> Result<()> { + let path = dir.join(file); + + #[cfg(feature = "coverage")] + let path = if std::env::var_os("TEST_CGROUP_WRITE_FAILURE") + .is_some_and(|f| f == file) + { + PathBuf::from("/does-not-exist").join(file) + } else { + path + }; + + std::fs::write(&path, value).context(format!( + "Failed to write '{}' to {}", + value, + path.display() + )) +} + +/// Enables controllers for a cgroup's children if they aren't already +fn enable_controllers(dir: &Path, controllers: &[&str]) -> Result<()> { + let subtree_control = dir.join("cgroup.subtree_control"); + let context = format!("Failed to read {}", subtree_control.display()); + let enabled = std::fs::read_to_string(&subtree_control).context(context)?; + for controller in controllers { + if !enabled.split_whitespace().any(|c| c == *controller) { + let context = format!( + "Failed to enable the {} cgroup controller. Limiting cpus or memory requires permission to manage {}, which may not be available inside containers or other sandboxes", + controller, + dir.display() + ); + let value = format!("+{}", controller); + write_cgroup_file(dir, "cgroup.subtree_control", &value) + .context(context)?; + } + } + Ok(()) +} diff --git a/src/sandbox/exec.rs b/src/sandbox/exec.rs index 80ecd29..688dcde 100644 --- a/src/sandbox/exec.rs +++ b/src/sandbox/exec.rs @@ -82,6 +82,11 @@ impl Sandbox { libc::prctl(libc::PR_SET_DUMPABLE, 1); }; + /* Join the sandbox's cgroup so we're subject to its resource limits. + * This needs to happen before we enter the sandbox's mount namespace, + * where the cgroup filesystem is read-only. */ + self.join_cgroup()?; + /* Join the sandbox namespaces */ let pid_fd = pidfd_open(self.pid, 0)?; if (unsafe { setns(pid_fd, ns_flags) }) != 0 { diff --git a/src/sandbox/get_or_create.rs b/src/sandbox/get_or_create.rs index b21486a..8d21fd8 100644 --- a/src/sandbox/get_or_create.rs +++ b/src/sandbox/get_or_create.rs @@ -1,3 +1,4 @@ +use super::cgroup::remove_cgroup; use super::mount_overlays::MountHash; use crate::config::{BindMount, BindMountOptions, Config, Network}; use crate::sandbox::Sandbox; @@ -7,9 +8,9 @@ use crate::util::Lock; use crate::util::get_running_sandbox_pid; use crate::util::get_sandbox_pid_path; use crate::util::{ - CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS, CLONE_NEWPID, - CLONE_NEWUTS, Clone3Args, check_path_for_mount_option_compatibility, - clone3, mkdir, mount, + CLONE_INTO_CGROUP, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, + CLONE_NEWNS, CLONE_NEWPID, CLONE_NEWUTS, Clone3Args, + check_path_for_mount_option_compatibility, clone3, mkdir, mount, }; use anyhow::Context; use anyhow::{Result, anyhow}; @@ -191,20 +192,35 @@ impl Sandbox { options: BindMountOptions::ReadWrite, }); + /* Create a cgroup to enforce any resource limits. We do this first + * so we fail before mounting anything if limits aren't available. */ + let cgroup = sandbox + .create_cgroup(&config) + .context("Failed to set up resource limits")?; + trace!("Mounting sandbox"); - let overlay_mounts = sandbox + let started = sandbox .mount_overlays(&config) - .context(format!("failed to mount sandbox: {}", sandbox_name))?; - - /* Create PID 1 for our sandbox */ - sandbox.pid = match sandbox.start_sandbox( - &config, - &get_sandbox_pid_path(storage_dir, sandbox_name), - &overlay_mounts, - ) { + .context(format!("failed to mount sandbox: {}", sandbox_name)) + .and_then(|overlay_mounts| { + /* Create PID 1 for our sandbox */ + sandbox + .start_sandbox( + &config, + &get_sandbox_pid_path(storage_dir, sandbox_name), + &overlay_mounts, + cgroup.as_ref(), + ) + .map_err(|e| anyhow!("Failed to start sandbox: {}", e)) + }); + + sandbox.pid = match started { Ok(pid) => pid, Err(e) => { - return Err(anyhow!("Failed to start sandbox: {}", e)); + if cgroup.is_some() { + remove_cgroup(&sandbox.cgroup_path()); + } + return Err(e); } }; @@ -220,6 +236,7 @@ impl Sandbox { config: &Config, pid_file: &PathBuf, overlay_mounts: &Vec, + cgroup: Option<&OwnedFd>, ) -> Result { // create a pipe between ourselves and the main sandbox process so the main sandbox process can // let us know when it's done setting up the namespaces @@ -238,7 +255,10 @@ impl Sandbox { | CLONE_NEWPID | CLONE_NEWIPC | CLONE_NEWUTS - | CLONE_NEWCGROUP, + | CLONE_NEWCGROUP + // Starting in our cgroup also makes it the root of our cgroup namespace + | cgroup.map_or(0, |_| CLONE_INTO_CGROUP), + cgroup: cgroup.map_or(0, |fd| fd.as_raw_fd() as u64), ..Default::default() }; diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 9e29328..3f7a76e 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -1,3 +1,4 @@ +mod cgroup; pub(crate) mod changes; mod delete; mod exec; diff --git a/src/sandbox/settings.rs b/src/sandbox/settings.rs index 3e2069f..148de3f 100644 --- a/src/sandbox/settings.rs +++ b/src/sandbox/settings.rs @@ -10,6 +10,11 @@ pub struct SandboxSettings { pub mounts: Vec, pub network: crate::config::Network, pub bind_mounts: Vec, + // Defaulted so settings from sandboxes started by older versions load + #[serde(default)] + pub cpus: Option, + #[serde(default)] + pub memory: Option, } impl SandboxSettings { @@ -19,6 +24,8 @@ impl SandboxSettings { mounts: mounts.to_vec(), network: config.net.clone(), bind_mounts: config.bind_mounts.clone(), + cpus: config.cpus, + memory: config.memory, } } @@ -76,6 +83,16 @@ impl SandboxSettings { )); } + if self.cpus != current_settings.cpus + || self.memory != current_settings.memory + { + return Err(anyhow!( + "Resource limits have changed. Existing: {}, Current: {}. Please stop the existing sandbox to change resource limits.", + describe_limits(self.cpus, self.memory), + describe_limits(current_settings.cpus, current_settings.memory), + )); + } + if self.mounts != current_settings.mounts { let mut error_msg = String::from("Mount configuration has changed:\n"); @@ -151,3 +168,10 @@ impl SandboxSettings { Ok(()) } } + +fn describe_limits(cpus: Option, memory: Option) -> String { + let cpus = cpus.map_or("unlimited".to_string(), |c| c.to_string()); + let memory = + memory.map_or("unlimited".to_string(), |m| format!("{} bytes", m)); + format!("cpus={} memory={}", cpus, memory) +} diff --git a/src/sandbox/stop.rs b/src/sandbox/stop.rs index ab099f5..9e25a40 100644 --- a/src/sandbox/stop.rs +++ b/src/sandbox/stop.rs @@ -1,4 +1,5 @@ use crate::sandbox::Sandbox; +use crate::sandbox::cgroup::remove_cgroup; use anyhow::{Context, Result}; use log::trace; use nix::{fcntl::readlink, unistd::Pid}; @@ -17,6 +18,9 @@ impl Sandbox { pid ); + // Find our cgroup before our processes go away + let cgroup = self.running_cgroup(); + #[cfg(feature = "coverage")] if std::env::var_os("TEST_STOP_RACE").is_some() { // pre-kill the sandbox process to simulate a race or otherwise @@ -134,6 +138,10 @@ impl Sandbox { } } } + + if let Some(cgroup) = cgroup { + remove_cgroup(&cgroup); + } } let pid_file = PathBuf::from(format!("{}.pid", self.base.display())); diff --git a/src/util/clone3.rs b/src/util/clone3.rs index a93d060..dfc349e 100644 --- a/src/util/clone3.rs +++ b/src/util/clone3.rs @@ -62,6 +62,9 @@ pub const CLONE_NEWUSER: u64 = libc::CLONE_NEWUSER as u64; /* New user namespace pub const CLONE_NEWPID: u64 = libc::CLONE_NEWPID as u64; /* New pid namespace */ #[allow(dead_code)] pub const CLONE_NEWNET: u64 = libc::CLONE_NEWNET as u64; /* New network namespace */ +/// Start the child in the cgroup given by Clone3Args::cgroup (since Linux 5.7). +/// Not yet defined by the libc crate. +pub const CLONE_INTO_CGROUP: u64 = 0x200000000; #[allow(dead_code)] pub const CLONE_IO: u64 = libc::CLONE_IO as u64; /* Clone I/O context */ diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs new file mode 100644 index 0000000..3222d75 --- /dev/null +++ b/tests/ai_test_resource_limits.rs @@ -0,0 +1,493 @@ +mod fixtures; + +use anyhow::Result; +use fixtures::*; +use rstest::*; +use std::fs; +use std::path::PathBuf; + +const UNAVAILABLE: &str = "Failed to set up resource limits"; + +fn available_cpus() -> usize { + std::thread::available_parallelism().unwrap().get() +} + +fn write_config( + sandbox: &SandboxManager, + prefix: &str, + contents: &str, +) -> Result { + let path = format!("{}.toml", sandbox.test_filename(prefix)); + fs::write(&path, contents)?; + Ok(path) +} + +/// Starts the sandbox with a CPU limit to find out if this machine lets us +/// manage cgroups. When it doesn't, e.g. because we're running inside a +/// container or another sandbox, we check that we fail with a helpful error. +fn limits_available(sandbox: &mut SandboxManager) -> bool { + if sandbox.pass(&["--cpus=1", "true"]) { + return true; + } + assert!( + sandbox.last_stderr.contains(UNAVAILABLE), + "{}", + sandbox.last_stderr + ); + eprintln!("cgroup resource limits are not available, skipping"); + false +} + +/// Finds our sandbox's cgroup on the host, if it has one +fn host_cgroup(sandbox: &SandboxManager) -> Option { + let suffix = format!("-{}", sandbox.name); + fs::read_dir("/sys/fs/cgroup/sandbox") + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .find(|path| path.to_string_lossy().ends_with(&suffix)) +} + +/* + * Configuration + */ + +#[rstest] +fn ai_test_limits_unlimited_by_default( + mut sandbox: SandboxManager, +) -> Result<()> { + sandbox.run(&["config", "cpus", "memory"])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + + // Both keys are part of the full config listing + sandbox.run(&["config"])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + Ok(()) +} + +#[rstest] +fn ai_test_limits_cli(mut sandbox: SandboxManager) -> Result<()> { + sandbox.run(&["--cpus=2", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), "2"); + + sandbox.run(&["--cpus=1.5", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), "1.5"); + + // Negative values are relative to the number of available CPUs, and + // work with or without an equals sign + let expected = (available_cpus() - 1).to_string(); + sandbox.run(&["--cpus=-1", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), expected); + sandbox.run(&["--cpus", "-1", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), expected); + + sandbox.run(&["--memory=512M", "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), (512u64 << 20).to_string()); + + sandbox.run(&["--memory", "1.5G", "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), (3u64 << 29).to_string()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_env(mut sandbox: SandboxManager) -> Result<()> { + sandbox.run_with_env(&["config", "cpus"], "SANDBOX_CPUS", "3")?; + assert_eq!(sandbox.last_stdout.trim(), "3"); + + sandbox.run_with_env(&["config", "memory"], "SANDBOX_MEMORY", "2G")?; + assert_eq!(sandbox.last_stdout.trim(), (2u64 << 30).to_string()); + + // Empty values are ignored + sandbox.run_with_env(&["config", "cpus"], "SANDBOX_CPUS", "")?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + // The CLI takes precedence over the environment + sandbox.run_with_env( + &["--cpus=1", "config", "cpus"], + "SANDBOX_CPUS", + "3", + )?; + assert_eq!(sandbox.last_stdout.trim(), "1"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_config_file(mut sandbox: SandboxManager) -> Result<()> { + let config = + write_config(&sandbox, "limits", "cpus = -1\nmemory = \"4G\"\n")?; + sandbox.run(&["--config", &config, "config", "cpus", "memory"])?; + assert!( + sandbox + .last_stdout + .contains(&format!("cpus={}", available_cpus() - 1)) + ); + assert!( + sandbox + .last_stdout + .contains(&format!("memory={}", 4u64 << 30)) + ); + + // Numbers and strings are both accepted + let config = write_config( + &sandbox, + "limits-types", + "cpus = \"0.5\"\nmemory = 1073741824\n", + )?; + sandbox.run(&["--config", &config, "config", "cpus", "memory"])?; + assert!(sandbox.last_stdout.contains("cpus=0.5")); + assert!(sandbox.last_stdout.contains("memory=1073741824")); + + // The environment takes precedence over config files + sandbox.run_with_env( + &["--config", &config, "config", "cpus"], + "SANDBOX_CPUS", + "2", + )?; + assert_eq!(sandbox.last_stdout.trim(), "2"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_invalid(mut sandbox: SandboxManager) -> Result<()> { + for cpus in ["--cpus=0", "--cpus=abc", "--cpus=0.001"] { + assert!(sandbox.xfail(&[cpus, "config", "cpus"])); + assert!( + sandbox.last_stderr.contains("Invalid cpus value"), + "{}", + sandbox.last_stderr + ); + } + + let all_cpus = format!("--cpus=-{}", available_cpus()); + assert!(sandbox.xfail(&[&all_cpus, "config", "cpus"])); + assert!(sandbox.last_stderr.contains("leaves 0 of the")); + + assert!(sandbox.xfail(&["--memory=lots", "config", "memory"])); + assert!(sandbox.last_stderr.contains("Invalid memory value 'lots'")); + + let config = write_config(&sandbox, "bad-type", "cpus = true\n")?; + assert!(sandbox.xfail(&["--config", &config, "config", "cpus"])); + Ok(()) +} + +/* + * Enforcement + */ + +#[rstest] +fn ai_test_limits_applied(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + let args = ["--cpus=2", "--memory=256M"]; + let run = |sandbox: &mut SandboxManager, command: &[&str]| { + let mut full = args.to_vec(); + full.extend_from_slice(command); + sandbox + .run(&full) + .map(|_| sandbox.last_stdout.trim().to_string()) + }; + + // Our limits are visible inside the sandbox, whose cgroup namespace is + // rooted at the sandbox's cgroup + assert_eq!( + run(&mut sandbox, &["cat", "/sys/fs/cgroup/cpu.max"])?, + "200000 100000" + ); + assert_eq!( + run(&mut sandbox, &["cat", "/sys/fs/cgroup/memory.max"])?, + (256u64 << 20).to_string() + ); + assert_eq!(run(&mut sandbox, &["cat", "/proc/self/cgroup"])?, "0::/"); + + // Processes that join the running sandbox are in its cgroup too + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let mut child = + sandbox.run_in_background(&[args[0], args[1], "sleep", "30"])?; + let mut joined = false; + for _ in 0..100 { + let procs = fs::read_to_string(cgroup.join("cgroup.procs"))?; + if procs.lines().count() > 1 { + joined = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(joined, "sleep should have joined the sandbox cgroup"); + + // Stopping the sandbox removes its cgroup + sandbox.run(&["stop"])?; + let _ = child.wait(); + assert!(!cgroup.exists()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_negative_cpus(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + sandbox.run(&["--cpus=-1", "cat", "/sys/fs/cgroup/cpu.max"])?; + assert_eq!( + sandbox.last_stdout.trim(), + format!("{} 100000", (available_cpus() - 1) * 100000) + ); + Ok(()) +} + +#[rstest] +fn ai_test_limits_memory_enforced(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // tail has to buffer the whole newline-free input, which won't fit + assert!(sandbox.xfail(&[ + "--memory=32M", + "sh", + "-c", + "head -c 256M /dev/zero | tail > /dev/null", + ])); + + sandbox.run(&["--memory=32M", "cat", "/sys/fs/cgroup/memory.events"])?; + let oom_kills: u64 = sandbox + .last_stdout + .lines() + .find_map(|line| line.strip_prefix("oom_kill ")) + .expect("memory.events should have an oom_kill entry") + .parse()?; + assert!(oom_kills >= 1); + Ok(()) +} + +#[rstest] +fn ai_test_limits_changed(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // The sandbox is running with --cpus=1, so other limits are rejected + assert!(sandbox.xfail(&["true"])); + assert!(sandbox.last_stderr.contains("Resource limits have changed")); + assert!( + sandbox + .last_stderr + .contains("Existing: cpus=1 memory=unlimited") + ); + assert!( + sandbox + .last_stderr + .contains("Current: cpus=unlimited memory=unlimited") + ); + + assert!(sandbox.xfail(&["--cpus=1", "--memory=1G", "true"])); + assert!( + sandbox + .last_stderr + .contains(&format!("Current: cpus=1 memory={} bytes", 1u64 << 30)) + ); + + // The same limits are fine, and stopping lets us change them + sandbox.run(&["--cpus=1", "true"])?; + sandbox.run(&["stop"])?; + sandbox.run(&["--cpus=2", "true"])?; + Ok(()) +} + +#[rstest] +fn ai_test_limits_stale_cgroup(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // Kill the sandbox without cleaning up, leaving an empty cgroup behind + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + sandbox.run(&["config", "sandbox_dir"])?; + let pid_file = format!("{}.pid", sandbox.last_stdout.trim()); + let pid = fs::read_to_string(&pid_file)?; + std::process::Command::new("sudo") + .args(["kill", "-9", pid.trim()]) + .status()?; + for _ in 0..100 { + if fs::read_to_string(cgroup.join("cgroup.procs"))?.is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(cgroup.exists()); + + // Starting again replaces the stale cgroup + sandbox.run(&["--cpus=2", "cat", "/sys/fs/cgroup/cpu.max"])?; + assert_eq!(sandbox.last_stdout.trim(), "200000 100000"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_cgroup_in_use(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // A sandbox with the same name in another storage directory would + // need the same cgroup, which is in use + let storage_dir = + std::env::current_dir()?.join(sandbox.test_filename("storage")); + fs::create_dir_all(&storage_dir)?; + let storage_arg = format!("--storage-dir={}", storage_dir.display()); + assert!(sandbox.xfail(&[&storage_arg, "--cpus=1", "true"])); + assert!( + sandbox + .last_stderr + .contains("from a different storage directory") + ); + Ok(()) +} + +#[rstest] +fn ai_test_limits_setup_failures(mut sandbox: SandboxManager) -> Result<()> { + assert!(sandbox.exfail(&["--cpus=1", "true"], "TEST_NO_CGROUP2", "1")); + assert!(sandbox.last_stderr.contains(UNAVAILABLE)); + assert!( + sandbox + .last_stderr + .contains("requires cgroup v2, which is not mounted") + ); + + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // Failing to apply limits cleans up the cgroup + for (file, args) in [ + ("cpu.max", vec!["--cpus=1", "true"]), + ("memory.max", vec!["--memory=1G", "true"]), + ] { + assert!(sandbox.exfail(&args, "TEST_CGROUP_WRITE_FAILURE", file)); + assert!(sandbox.last_stderr.contains(UNAVAILABLE)); + assert!(sandbox.last_stderr.contains("Failed to write")); + assert!(host_cgroup(&sandbox).is_none()); + } + + // As does failing to start the sandbox + assert!(sandbox.exfail( + &["--cpus=1", "true"], + "TEST_START_SANDBOX_FAILURE", + "1" + )); + assert!(sandbox.last_stderr.contains("Failed to start sandbox")); + assert!(host_cgroup(&sandbox).is_none()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_new_parent_cgroup(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // The first sandbox with limits creates the parent cgroup and enables + // the controllers we need in it + let parent = format!("{}-parent", sandbox.name); + let parent_path = PathBuf::from("/sys/fs/cgroup").join(&parent); + assert!(!parent_path.exists()); + sandbox.run_with_env( + &["--cpus=1", "--memory=64M", "cat", "/sys/fs/cgroup/cpu.max"], + "TEST_CGROUP_PARENT", + &parent, + )?; + assert_eq!(sandbox.last_stdout.trim(), "100000 100000"); + let enabled = + fs::read_to_string(parent_path.join("cgroup.subtree_control"))?; + assert!(enabled.split_whitespace().any(|c| c == "cpu")); + assert!(enabled.split_whitespace().any(|c| c == "memory")); + + sandbox.run_with_env(&["stop"], "TEST_CGROUP_PARENT", &parent)?; + assert_eq!( + fs::read_dir(&parent_path)? + .filter(|e| { e.as_ref().is_ok_and(|e| e.path().is_dir()) }) + .count(), + 0 + ); + std::process::Command::new("sudo") + .args(["rmdir", &parent_path.to_string_lossy()]) + .status()?; + Ok(()) +} + +#[rstest] +fn ai_test_limits_no_swap_accounting( + mut sandbox: SandboxManager, +) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // Without swap accounting we can only limit memory, not swap + sandbox.run_with_env( + &["--memory=64M", "cat", "/sys/fs/cgroup/memory.max"], + "TEST_NO_SWAP_ACCOUNTING", + "1", + )?; + assert_eq!(sandbox.last_stdout.trim(), (64u64 << 20).to_string()); + sandbox.run(&["--memory=64M", "cat", "/sys/fs/cgroup/memory.swap.max"])?; + assert_eq!(sandbox.last_stdout.trim(), "max"); + sandbox.run(&["stop"])?; + + // Normally swap is disabled + sandbox.run(&["--memory=64M", "cat", "/sys/fs/cgroup/memory.swap.max"])?; + assert_eq!(sandbox.last_stdout.trim(), "0"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_join_failure(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // The sandbox is running with --cpus=1, joining it requires moving + // into its cgroup + assert!(sandbox.exfail( + &["--cpus=1", "true"], + "TEST_CGROUP_WRITE_FAILURE", + "cgroup.procs" + )); + assert!(sandbox.last_stderr.contains("Failed to write")); + assert!(sandbox.last_stderr.contains("cgroup.procs")); + Ok(()) +} + +#[rstest] +fn ai_test_limits_stop_cgroup_busy(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // A child cgroup keeps the sandbox's cgroup from being removed, so stop + // retries for a while and then gives up without failing + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let child = cgroup.join("busy"); + let sudo = + |args: &[&str]| std::process::Command::new("sudo").args(args).status(); + sudo(&["mkdir", &child.to_string_lossy()])?; + + let started = std::time::Instant::now(); + sandbox.run(&["stop"])?; + assert!(started.elapsed() >= std::time::Duration::from_secs(5)); + assert!(cgroup.exists()); + + sudo(&["rmdir", &child.to_string_lossy()])?; + sudo(&["rmdir", &cgroup.to_string_lossy()])?; + assert!(!cgroup.exists()); + Ok(()) +} diff --git a/tests/ai_test_sandbox_settings.rs b/tests/ai_test_sandbox_settings.rs index 52ec207..005f948 100644 --- a/tests/ai_test_sandbox_settings.rs +++ b/tests/ai_test_sandbox_settings.rs @@ -28,6 +28,8 @@ fn create_test_config() -> ::sandbox::config::Config { bind_mounts: Vec::new(), no_default_binds: false, audio: false, + cpus: None, + memory: None, config_files: Vec::new(), } } From e7569ce21115296e4f2933482c7c04b82a1d1afc Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Sat, 31 Oct 2026 08:34:25 -0400 Subject: [PATCH 2/5] Keep resource limit tests independent of user config files Tests that check default, environment, or changed limits now skip config files, so limits set in ~/.config/sandbox/config.toml don't change the results. --- tests/ai_test_resource_limits.rs | 39 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs index 3222d75..b83580c 100644 --- a/tests/ai_test_resource_limits.rs +++ b/tests/ai_test_resource_limits.rs @@ -56,12 +56,13 @@ fn host_cgroup(sandbox: &SandboxManager) -> Option { fn ai_test_limits_unlimited_by_default( mut sandbox: SandboxManager, ) -> Result<()> { - sandbox.run(&["config", "cpus", "memory"])?; + // Skip config files, which may set limits + sandbox.run(&["--no-config", "config", "cpus", "memory"])?; assert!(sandbox.last_stdout.contains("cpus=unlimited")); assert!(sandbox.last_stdout.contains("memory=unlimited")); // Both keys are part of the full config listing - sandbox.run(&["config"])?; + sandbox.run(&["--no-config", "config"])?; assert!(sandbox.last_stdout.contains("cpus=unlimited")); assert!(sandbox.last_stdout.contains("memory=unlimited")); Ok(()) @@ -93,19 +94,33 @@ fn ai_test_limits_cli(mut sandbox: SandboxManager) -> Result<()> { #[rstest] fn ai_test_limits_env(mut sandbox: SandboxManager) -> Result<()> { - sandbox.run_with_env(&["config", "cpus"], "SANDBOX_CPUS", "3")?; + // Config files are skipped so they can't set limits of their own + let args = ["--no-config", "config"]; + sandbox.run_with_env( + &[&args[..], &["cpus"]].concat(), + "SANDBOX_CPUS", + "3", + )?; assert_eq!(sandbox.last_stdout.trim(), "3"); - sandbox.run_with_env(&["config", "memory"], "SANDBOX_MEMORY", "2G")?; + sandbox.run_with_env( + &[&args[..], &["memory"]].concat(), + "SANDBOX_MEMORY", + "2G", + )?; assert_eq!(sandbox.last_stdout.trim(), (2u64 << 30).to_string()); // Empty values are ignored - sandbox.run_with_env(&["config", "cpus"], "SANDBOX_CPUS", "")?; + sandbox.run_with_env( + &[&args[..], &["cpus"]].concat(), + "SANDBOX_CPUS", + "", + )?; assert_eq!(sandbox.last_stdout.trim(), "unlimited"); // The CLI takes precedence over the environment sandbox.run_with_env( - &["--cpus=1", "config", "cpus"], + &["--no-config", "--cpus=1", "config", "cpus"], "SANDBOX_CPUS", "3", )?; @@ -273,8 +288,12 @@ fn ai_test_limits_changed(mut sandbox: SandboxManager) -> Result<()> { return Ok(()); } + // Restart without config files, which may set limits of their own + sandbox.run(&["stop"])?; + sandbox.run(&["--no-config", "--cpus=1", "true"])?; + // The sandbox is running with --cpus=1, so other limits are rejected - assert!(sandbox.xfail(&["true"])); + assert!(sandbox.xfail(&["--no-config", "true"])); assert!(sandbox.last_stderr.contains("Resource limits have changed")); assert!( sandbox @@ -287,7 +306,7 @@ fn ai_test_limits_changed(mut sandbox: SandboxManager) -> Result<()> { .contains("Current: cpus=unlimited memory=unlimited") ); - assert!(sandbox.xfail(&["--cpus=1", "--memory=1G", "true"])); + assert!(sandbox.xfail(&["--no-config", "--cpus=1", "--memory=1G", "true"])); assert!( sandbox .last_stderr @@ -295,9 +314,9 @@ fn ai_test_limits_changed(mut sandbox: SandboxManager) -> Result<()> { ); // The same limits are fine, and stopping lets us change them - sandbox.run(&["--cpus=1", "true"])?; + sandbox.run(&["--no-config", "--cpus=1", "true"])?; sandbox.run(&["stop"])?; - sandbox.run(&["--cpus=2", "true"])?; + sandbox.run(&["--no-config", "--cpus=2", "true"])?; Ok(()) } From 6f8de0e0f4bb22ed3b19325026f91fb5018a7e85 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Sat, 31 Oct 2026 08:49:48 -0400 Subject: [PATCH 3/5] Share the outer sandbox's limits when nested, and accept "unlimited" A sandbox started inside another sandbox can't manage cgroups, so with limits in a config file nested sandboxes refused to start. They now warn and run without a cgroup of their own, which leaves them subject to the outer sandbox's limits. Outside a sandbox, failing to set up limits is still an error. --cpus and --memory (and SANDBOX_CPUS, SANDBOX_MEMORY, and config files) now accept "unlimited", which removes a limit set by a less specific source. Also read memory.events in the same command as the allocation in the memory enforcement test, so the result doesn't depend on which process the OOM killer picks. --- README.adoc | 8 ++- src/config/cli.rs | 7 +- src/config/limits.rs | 70 +++++++++++++------- src/config/resolve.rs | 6 +- src/sandbox/cgroup.rs | 32 ++++++++- src/sandbox/get_or_create.rs | 4 +- src/sandbox/settings.rs | 5 +- tests/ai_test_resource_limits.rs | 108 ++++++++++++++++++++++++++++--- 8 files changed, 193 insertions(+), 47 deletions(-) diff --git a/README.adoc b/README.adoc index f08c554..5b230c5 100644 --- a/README.adoc +++ b/README.adoc @@ -223,8 +223,12 @@ its largest process. + Resource limits use cgroup v2 and apply to every process in the sandbox. They are set when the sandbox starts, so you need to stop the sandbox to change -them. They may not be available when `sandbox` is run inside a container or -another sandbox. +them. Use `unlimited` to remove a limit set in a config file, for example +`--cpus=unlimited` or `SANDBOX_MEMORY=unlimited`. ++ +A sandbox started inside another sandbox can't set up limits of its own. +Instead it warns and shares the limits of the sandbox it's running in. Limits +may not be available at all when `sandbox` is run inside a container. `--json`:: Formats action output as a JSON blob. Does nothing for sandboxed commands diff --git a/src/config/cli.rs b/src/config/cli.rs index 9d4e819..602d590 100644 --- a/src/config/cli.rs +++ b/src/config/cli.rs @@ -83,7 +83,8 @@ pub struct Args { /// Limit how many CPUs the sandbox can use. Fractional values such as 1.5 /// are allowed. Negative values mean all but that many CPUs, so -2 on a - /// 32 CPU machine allows the sandbox to use 30. Requires cgroup v2. + /// 32 CPU machine allows the sandbox to use 30. Use "unlimited" to remove + /// a limit set in a config file. Requires cgroup v2. #[arg( long, global = true, @@ -93,8 +94,8 @@ pub struct Args { pub cpus: Option, /// Limit how much memory the sandbox can use, for example 512M or 4G. - /// Swap is not available to sandboxes with a memory limit. Requires - /// cgroup v2. + /// Swap is not available to sandboxes with a memory limit. Use + /// "unlimited" to remove a limit set in a config file. Requires cgroup v2. #[arg(long, global = true, value_name = "SIZE")] pub memory: Option, diff --git a/src/config/limits.rs b/src/config/limits.rs index 3ab2b5c..68de8b0 100644 --- a/src/config/limits.rs +++ b/src/config/limits.rs @@ -4,6 +4,13 @@ use serde::Deserialize; /// The smallest CPU limit cgroups will accept: a 1ms quota per 100ms period const MIN_CPUS: f64 = 0.01; +/// Removes a limit, e.g. to override one set in a config file +const UNLIMITED: &str = "unlimited"; + +fn is_unlimited(spec: &str) -> bool { + spec.trim().eq_ignore_ascii_case(UNLIMITED) +} + /// Number of CPUs this process may run on. This matches `nproc` unless we /// are already running under a CPU quota, in which case the quota is used. pub fn available_cpus() -> usize { @@ -14,15 +21,19 @@ pub fn available_cpus() -> usize { /// Parses a CPU limit. Positive values are a number of CPUs and may be /// fractional. Negative values mean all but that many of the `available` -/// CPUs, so -2 on a 32 CPU machine means 30. -pub fn parse_cpus(spec: &str, available: usize) -> Result { +/// CPUs, so -2 on a 32 CPU machine means 30. "unlimited" means no limit. +pub fn parse_cpus(spec: &str, available: usize) -> Result> { let invalid = || { anyhow!( - "Invalid cpus value '{}': expected a number of CPUs such as 2 or 1.5, or a negative number such as -2 to use all but that many CPUs", + "Invalid cpus value '{}': expected a number of CPUs such as 2 or 1.5, a negative number such as -2 to use all but that many CPUs, or unlimited", spec ) }; + if is_unlimited(spec) { + return Ok(None); + } + let value: f64 = spec.trim().parse().map_err(|_| invalid())?; if !value.is_finite() || value == 0.0 { return Err(invalid()); @@ -44,19 +55,24 @@ pub fn parse_cpus(spec: &str, available: usize) -> Result { )); } - Ok(cpus) + Ok(Some(cpus)) } /// Parses a memory size such as 512M, 4G, 1.5GiB or a plain number of bytes. -/// Suffixes are binary (K = 1024) and case insensitive. -pub fn parse_memory(spec: &str) -> Result { +/// Suffixes are binary (K = 1024) and case insensitive. "unlimited" means no +/// limit. +pub fn parse_memory(spec: &str) -> Result> { let invalid = || { anyhow!( - "Invalid memory value '{}': expected a size such as 512M or 4G", + "Invalid memory value '{}': expected a size such as 512M or 4G, or unlimited", spec ) }; + if is_unlimited(spec) { + return Ok(None); + } + let trimmed = spec.trim(); let split = trimmed .find(|c: char| c.is_ascii_alphabetic()) @@ -78,7 +94,7 @@ pub fn parse_memory(spec: &str) -> Result { return Err(invalid()); } - Ok(bytes as u64) + Ok(Some(bytes as u64)) } /// Lets config files specify limits as either numbers or strings, e.g. @@ -114,13 +130,15 @@ mod tests { #[test] fn test_parse_cpus() { - assert_eq!(parse_cpus("2", 32).unwrap(), 2.0); - assert_eq!(parse_cpus(" 1.5 ", 32).unwrap(), 1.5); - assert_eq!(parse_cpus("0.01", 32).unwrap(), 0.01); - assert_eq!(parse_cpus("64", 32).unwrap(), 64.0); - assert_eq!(parse_cpus("-2", 32).unwrap(), 30.0); - assert_eq!(parse_cpus("-0.5", 4).unwrap(), 3.5); - assert_eq!(parse_cpus("-31", 32).unwrap(), 1.0); + assert_eq!(parse_cpus("2", 32).unwrap(), Some(2.0)); + assert_eq!(parse_cpus(" 1.5 ", 32).unwrap(), Some(1.5)); + assert_eq!(parse_cpus("0.01", 32).unwrap(), Some(0.01)); + assert_eq!(parse_cpus("64", 32).unwrap(), Some(64.0)); + assert_eq!(parse_cpus("-2", 32).unwrap(), Some(30.0)); + assert_eq!(parse_cpus("-0.5", 4).unwrap(), Some(3.5)); + assert_eq!(parse_cpus("-31", 32).unwrap(), Some(1.0)); + assert_eq!(parse_cpus("unlimited", 32).unwrap(), None); + assert_eq!(parse_cpus(" Unlimited ", 32).unwrap(), None); } #[test] @@ -140,16 +158,18 @@ mod tests { #[test] fn test_parse_memory() { - assert_eq!(parse_memory("1024").unwrap(), 1024); - assert_eq!(parse_memory("1024b").unwrap(), 1024); - assert_eq!(parse_memory("4k").unwrap(), 4096); - assert_eq!(parse_memory("4KiB").unwrap(), 4096); - assert_eq!(parse_memory("512M").unwrap(), 512 << 20); - assert_eq!(parse_memory("512mb").unwrap(), 512 << 20); - assert_eq!(parse_memory(" 4G ").unwrap(), 4 << 30); - assert_eq!(parse_memory("4 GiB").unwrap(), 4 << 30); - assert_eq!(parse_memory("1.5G").unwrap(), 3 << 29); - assert_eq!(parse_memory("2T").unwrap(), 2 << 40); + assert_eq!(parse_memory("1024").unwrap(), Some(1024)); + assert_eq!(parse_memory("1024b").unwrap(), Some(1024)); + assert_eq!(parse_memory("4k").unwrap(), Some(4096)); + assert_eq!(parse_memory("4KiB").unwrap(), Some(4096)); + assert_eq!(parse_memory("512M").unwrap(), Some(512 << 20)); + assert_eq!(parse_memory("512mb").unwrap(), Some(512 << 20)); + assert_eq!(parse_memory(" 4G ").unwrap(), Some(4 << 30)); + assert_eq!(parse_memory("4 GiB").unwrap(), Some(4 << 30)); + assert_eq!(parse_memory("1.5G").unwrap(), Some(3 << 29)); + assert_eq!(parse_memory("2T").unwrap(), Some(2 << 40)); + assert_eq!(parse_memory("unlimited").unwrap(), None); + assert_eq!(parse_memory("UNLIMITED").unwrap(), None); } #[test] diff --git a/src/config/resolve.rs b/src/config/resolve.rs index 6f7c0c7..56f34b4 100644 --- a/src/config/resolve.rs +++ b/src/config/resolve.rs @@ -289,7 +289,8 @@ pub fn resolve_config(cli: Args) -> Result { let cpus = partial_config .cpus .map(|cpus| parse_cpus(&cpus, available_cpus())) - .transpose()?; + .transpose()? + .flatten(); if !sources.contains_key("cpus") { sources.insert("cpus".into(), "default".into()); } @@ -297,7 +298,8 @@ pub fn resolve_config(cli: Args) -> Result { let memory = partial_config .memory .map(|memory| parse_memory(&memory)) - .transpose()?; + .transpose()? + .flatten(); if !sources.contains_key("memory") { sources.insert("memory".into(), "default".into()); } diff --git a/src/sandbox/cgroup.rs b/src/sandbox/cgroup.rs index 07cb18c..fafdf17 100644 --- a/src/sandbox/cgroup.rs +++ b/src/sandbox/cgroup.rs @@ -6,9 +6,9 @@ //! every process that later joins the sandbox is moved into it. use crate::config::Config; -use crate::sandbox::Sandbox; +use crate::sandbox::{Sandbox, SandboxSettings, describe_limits}; use anyhow::{Context, Result, anyhow}; -use log::trace; +use log::{debug, trace, warn}; use std::os::fd::OwnedFd; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -45,6 +45,34 @@ impl Sandbox { .join(self.cgroup_name()) } + /// Sets up our cgroup like create_cgroup, except that inside another + /// sandbox, where cgroups can't be managed, we warn and run without a + /// cgroup of our own. We're still in the outer sandbox's cgroup, so its + /// limits continue to apply. + pub(crate) fn create_cgroup_or_share_outer( + &self, + config: &Config, + ) -> Result> { + let error = match self.create_cgroup(config) { + Ok(cgroup) => return Ok(cgroup), + Err(e) => e.context("Failed to set up resource limits"), + }; + + // A nested sandbox's storage directory is the outer sandbox's data + // directory, which holds the outer sandbox's settings + let outer_settings = config.storage_dir.join("settings.json"); + let Ok(outer) = SandboxSettings::load_from_file(&outer_settings) else { + return Err(error); + }; + + debug!("{:#}", error); + warn!( + "Resource limits can't be set up inside another sandbox, so this sandbox shares the limits of the sandbox it's running in ({})", + describe_limits(outer.cpus, outer.memory) + ); + Ok(None) + } + /// Creates a cgroup with the resource limits from our config. Returns a /// handle to it for use with CLONE_INTO_CGROUP, or None if the config /// has no resource limits. diff --git a/src/sandbox/get_or_create.rs b/src/sandbox/get_or_create.rs index 8d21fd8..d25a586 100644 --- a/src/sandbox/get_or_create.rs +++ b/src/sandbox/get_or_create.rs @@ -194,9 +194,7 @@ impl Sandbox { /* Create a cgroup to enforce any resource limits. We do this first * so we fail before mounting anything if limits aren't available. */ - let cgroup = sandbox - .create_cgroup(&config) - .context("Failed to set up resource limits")?; + let cgroup = sandbox.create_cgroup_or_share_outer(&config)?; trace!("Mounting sandbox"); let started = sandbox diff --git a/src/sandbox/settings.rs b/src/sandbox/settings.rs index 148de3f..e68b414 100644 --- a/src/sandbox/settings.rs +++ b/src/sandbox/settings.rs @@ -169,7 +169,10 @@ impl SandboxSettings { } } -fn describe_limits(cpus: Option, memory: Option) -> String { +pub(crate) fn describe_limits( + cpus: Option, + memory: Option, +) -> String { let cpus = cpus.map_or("unlimited".to_string(), |c| c.to_string()); let memory = memory.map_or("unlimited".to_string(), |m| format!("{} bytes", m)); diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs index b83580c..21f60f3 100644 --- a/tests/ai_test_resource_limits.rs +++ b/tests/ai_test_resource_limits.rs @@ -263,22 +263,25 @@ fn ai_test_limits_memory_enforced(mut sandbox: SandboxManager) -> Result<()> { } sandbox.run(&["stop"])?; - // tail has to buffer the whole newline-free input, which won't fit - assert!(sandbox.xfail(&[ + // tail has to buffer the whole newline-free input, which won't fit. + // memory.events is read in the same command so we see this cgroup's + // counters even if the OOM killer took down more than tail. + sandbox.run(&[ "--memory=32M", "sh", "-c", - "head -c 256M /dev/zero | tail > /dev/null", - ])); - - sandbox.run(&["--memory=32M", "cat", "/sys/fs/cgroup/memory.events"])?; - let oom_kills: u64 = sandbox - .last_stdout + "head -c 256M /dev/zero | tail > /dev/null; \ + echo \"tail exited with $?\"; \ + cat /sys/fs/cgroup/memory.events", + ])?; + let output = sandbox.last_stdout.clone(); + assert!(output.contains("tail exited with 137"), "{}", output); + let oom_kills: u64 = output .lines() .find_map(|line| line.strip_prefix("oom_kill ")) .expect("memory.events should have an oom_kill entry") .parse()?; - assert!(oom_kills >= 1); + assert!(oom_kills >= 1, "{}", output); Ok(()) } @@ -510,3 +513,90 @@ fn ai_test_limits_stop_cgroup_busy(mut sandbox: SandboxManager) -> Result<()> { assert!(!cgroup.exists()); Ok(()) } + +#[rstest] +fn ai_test_limits_unlimited_override( + mut sandbox: SandboxManager, +) -> Result<()> { + // "unlimited" removes a limit set by a less specific source + let config = + write_config(&sandbox, "limited", "cpus = 2\nmemory = \"4G\"\n")?; + sandbox.run(&[ + "--config", + &config, + "--cpus=unlimited", + "--memory=Unlimited", + "config", + "cpus", + "memory", + ])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + + sandbox.run_with_env( + &["--config", &config, "config", "cpus"], + "SANDBOX_CPUS", + "unlimited", + )?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + let unlimited = + write_config(&sandbox, "unlimited", "memory = \"unlimited\"\n")?; + sandbox.run(&["--config", &unlimited, "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + // Without any limits the sandbox doesn't get a cgroup + if limits_available(&mut sandbox) { + sandbox.run(&["stop"])?; + sandbox.run(&[ + "--config", + &config, + "--cpus=unlimited", + "--memory=unlimited", + "true", + ])?; + assert!(host_cgroup(&sandbox).is_none()); + } + Ok(()) +} + +#[rstest] +fn ai_test_limits_nested(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // A sandbox inside another one can't manage cgroups, so instead of its + // own limits it shares the limits of the sandbox it's running in + let inner = get_sandbox_bin() + "-setuid"; + sandbox.run(&[ + "--no-config", + "--cpus=2", + &inner, + "--no-config", + "--cpus=1", + "cat", + "/sys/fs/cgroup/cpu.max", + ])?; + assert_eq!(sandbox.last_stdout.trim(), "200000 100000"); + assert!( + sandbox.last_stderr.contains( + "shares the limits of the sandbox it's running in (cpus=2 memory=unlimited)" + ), + "{}", + sandbox.last_stderr + ); + + // The warning makes it clear when that means no limits at all + sandbox.run(&["stop"])?; + sandbox.run(&["--no-config", &inner, "--no-config", "--cpus=1", "true"])?; + assert!( + sandbox.last_stderr.contains( + "shares the limits of the sandbox it's running in (cpus=unlimited memory=unlimited)" + ), + "{}", + sandbox.last_stderr + ); + Ok(()) +} From 95fdbd3ad675950355565d41c5162229bfd475a3 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Sat, 31 Oct 2026 08:55:12 -0400 Subject: [PATCH 4/5] Fix stop/delete leaving cgroups behind and killing waiting commands delete stops sandboxes as uid 0, so looking for the cgroup by the name we'd create (-) missed it and the cgroup was never removed. Find the cgroup from the sandbox's PID 1 instead, accepting any uid. Also stop writing cgroup.kill when removing a cgroup. By that point the only processes left in it are the `sandbox` commands waiting on the killed sandbox processes. Killing those cut off their exit status reporting (and truncated coverage profiles mid-write). They exit by themselves once their child is gone, which the removal already waits for. --- src/sandbox/cgroup.rs | 34 ++++++++++++++++---------------- tests/ai_test_resource_limits.rs | 17 ++++++++++++++++ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/sandbox/cgroup.rs b/src/sandbox/cgroup.rs index fafdf17..3907076 100644 --- a/src/sandbox/cgroup.rs +++ b/src/sandbox/cgroup.rs @@ -32,13 +32,6 @@ impl Sandbox { format!("{}-{}", self.uid, self.name) } - /// Our cgroup's path relative to the cgroup root, as shown in /proc//cgroup - fn cgroup_relative_path(&self) -> PathBuf { - Path::new("/") - .join(cgroup_parent()) - .join(self.cgroup_name()) - } - pub(crate) fn cgroup_path(&self) -> PathBuf { Path::new(CGROUP_ROOT) .join(cgroup_parent()) @@ -167,17 +160,22 @@ impl Sandbox { Ok(()) } - /// Returns the path to our cgroup if our sandbox is running in one + /// Returns the path to our cgroup if our sandbox is running in one. We + /// go by the cgroup our PID 1 is in rather than the name we'd give a new + /// cgroup, because the uid in the name is that of the user who started + /// the sandbox, which isn't always ours (e.g. `delete` doesn't know it). pub(crate) fn running_cgroup(&self) -> Option { let cgroups = std::fs::read_to_string(format!("/proc/{}/cgroup", self.pid)) .ok()?; - let current = cgroups.lines().find_map(|line| line.strip_prefix("0::")); - if current.map(Path::new) == Some(&self.cgroup_relative_path()) { - Some(self.cgroup_path()) - } else { - None - } + let current = + cgroups.lines().find_map(|line| line.strip_prefix("0::"))?; + let relative = Path::new(current).strip_prefix("/").ok()?; + let cgroup_name = + relative.strip_prefix(cgroup_parent()).ok()?.to_str()?; + let (uid, name) = cgroup_name.split_once('-')?; + (uid.parse::().is_ok() && name == self.name) + .then(|| Path::new(CGROUP_ROOT).join(relative)) } /// Moves the current process into our sandbox's cgroup, if it has one, @@ -192,11 +190,13 @@ impl Sandbox { } } -/// Kills anything left in a cgroup and removes it. Removal can't succeed -/// until killed processes finish exiting, so we wait a little while for that. +/// Removes a cgroup once the processes in it have exited, waiting a little +/// while for that. Sandbox processes have already been killed by this point. +/// The only other processes in the cgroup are the `sandbox` commands waiting +/// on them, which exit by themselves once their child is gone, so they get to +/// report its exit status instead of being killed too. pub(crate) fn remove_cgroup(path: &Path) { trace!("Removing cgroup {}", path.display()); - let _ = std::fs::write(path.join("cgroup.kill"), "1"); let deadline = Instant::now() + Duration::from_secs(5); while std::fs::remove_dir(path).is_err() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs index 21f60f3..5e8a5e2 100644 --- a/tests/ai_test_resource_limits.rs +++ b/tests/ai_test_resource_limits.rs @@ -600,3 +600,20 @@ fn ai_test_limits_nested(mut sandbox: SandboxManager) -> Result<()> { ); Ok(()) } + +#[rstest] +fn ai_test_limits_delete_removes_cgroup( + mut sandbox: SandboxManager, +) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // delete stops sandboxes without knowing which user started them, so + // it has to find the cgroup from the running sandbox + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let name = sandbox.name.clone(); + sandbox.run(&["delete", "-y", &name])?; + assert!(!cgroup.exists()); + Ok(()) +} From 730b63340191b98a9220157077a56e5f69321834 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Sat, 31 Oct 2026 08:59:37 -0400 Subject: [PATCH 5/5] Test a memory-only limit regardless of user config The swap accounting test now skips config files, so a cpus limit in ~/.config/sandbox/config.toml can't turn it into a cpus+memory test and leave the memory-only path untested. --- tests/ai_test_resource_limits.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs index 5e8a5e2..4a38b75 100644 --- a/tests/ai_test_resource_limits.rs +++ b/tests/ai_test_resource_limits.rs @@ -454,19 +454,27 @@ fn ai_test_limits_no_swap_accounting( } sandbox.run(&["stop"])?; - // Without swap accounting we can only limit memory, not swap + // Only memory is limited (config files are skipped since they may limit + // cpus too). Without swap accounting we can limit memory but not swap. + let memory_only = ["--no-config", "--memory=64M", "cat"]; sandbox.run_with_env( - &["--memory=64M", "cat", "/sys/fs/cgroup/memory.max"], + &[&memory_only[..], &["/sys/fs/cgroup/memory.max"]].concat(), "TEST_NO_SWAP_ACCOUNTING", "1", )?; assert_eq!(sandbox.last_stdout.trim(), (64u64 << 20).to_string()); - sandbox.run(&["--memory=64M", "cat", "/sys/fs/cgroup/memory.swap.max"])?; + sandbox.run(&[&memory_only[..], &["/sys/fs/cgroup/cpu.max"]].concat())?; + assert_eq!(sandbox.last_stdout.trim(), "max 100000"); + sandbox.run( + &[&memory_only[..], &["/sys/fs/cgroup/memory.swap.max"]].concat(), + )?; assert_eq!(sandbox.last_stdout.trim(), "max"); sandbox.run(&["stop"])?; // Normally swap is disabled - sandbox.run(&["--memory=64M", "cat", "/sys/fs/cgroup/memory.swap.max"])?; + sandbox.run( + &[&memory_only[..], &["/sys/fs/cgroup/memory.swap.max"]].concat(), + )?; assert_eq!(sandbox.last_stdout.trim(), "0"); Ok(()) }