diff --git a/README.adoc b/README.adoc index f862cab..5b230c5 100644 --- a/README.adoc +++ b/README.adoc @@ -208,6 +208,28 @@ microphone. Audio works with or without `--net`, and is not affected by in a config file, and `SANDBOX_AUDIO=false` will override that for a single run. +`--cpus=`:: +Limit how many CPUs the sandbox can use. Fractional values such as `1.5` are +allowed. Negative values mean all but that many CPUs, so `--cpus=-2` on a 32 +CPU machine lets the sandbox use 30. This limits the total CPU time the +sandbox gets, it does not pin it to specific CPUs, so tools like `nproc` will +still report every CPU. + +`--memory=`:: +Limit how much memory the sandbox can use, for example `512M` or `4G`. +Suffixes are binary (`1K` is 1024 bytes). Swap is not available to a sandbox +with a memory limit, and when the sandbox runs out of memory the kernel kills +its largest process. ++ +Resource limits use cgroup v2 and apply to every process in the sandbox. They +are set when the sandbox starts, so you need to stop the sandbox to change +them. Use `unlimited` to remove a limit set in a config file, for example +`--cpus=unlimited` or `SANDBOX_MEMORY=unlimited`. ++ +A sandbox started inside another sandbox can't set up limits of its own. +Instead it warns and shares the limits of the sandbox it's running in. Limits +may not be available at all when `sandbox` is run inside a container. + `--json`:: Formats action output as a JSON blob. Does nothing for sandboxed commands @@ -278,6 +300,10 @@ These config files can set most of the CLI options: mask=[ "~/.ssh", ] + + # Resource limits + cpus=-2 + memory="8G" ``` ## Environment Variables @@ -290,6 +316,8 @@ You can also set most of the options with environment variables if you wish: SANDBOX_NET="none" SANDBOX_LOG_LEVEL="info" SANDBOX_BIND="/my/dir" + SANDBOX_CPUS="-2" + SANDBOX_MEMORY="8G" ``` diff --git a/cspell.json b/cspell.json index 0df102e..a36ea8e 100644 --- a/cspell.json +++ b/cspell.json @@ -32,6 +32,7 @@ "cmdline", "codegen", "codespell", + "cpus", "CREAT", "cstring", "czvf", @@ -169,6 +170,7 @@ "PKGBUILD", "powercap", "prctl", + "procs", "profdata", "profraw", "psub", diff --git a/src/actions/config.rs b/src/actions/config.rs index 2441b7b..615c080 100644 --- a/src/actions/config.rs +++ b/src/actions/config.rs @@ -17,6 +17,8 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { "mask", "no_default_binds", "audio", + "cpus", + "memory", "storage_dir", "sandbox_dir", "upper_cwd", @@ -47,6 +49,12 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { let no_default_binds_str = format!("{}", config.no_default_binds); let audio_str = format!("{}", config.audio); + let cpus_str = config + .cpus + .map_or("unlimited".to_string(), |cpus| cpus.to_string()); + let memory_str = config + .memory + .map_or("unlimited".to_string(), |memory| memory.to_string()); let ignored_str = format!("{}", config.ignored); let config_files_str = config .config_files @@ -78,6 +86,8 @@ pub fn config(config: &Config, keys: Option>) -> Result<()> { ("no_default_binds", no_default_binds_str.as_str()) } "audio" => ("audio", audio_str.as_str()), + "cpus" => ("cpus", cpus_str.as_str()), + "memory" => ("memory", memory_str.as_str()), "name" => ("name", config.name.as_str()), "log_level" => ("log_level", config.log_level.as_str()), "ignored" => ("ignored", ignored_str.as_str()), diff --git a/src/config/cli.rs b/src/config/cli.rs index 8fd4231..602d590 100644 --- a/src/config/cli.rs +++ b/src/config/cli.rs @@ -81,6 +81,24 @@ pub struct Args { #[arg(long, global = true, action = clap::ArgAction::SetTrue)] pub audio: bool, + /// Limit how many CPUs the sandbox can use. Fractional values such as 1.5 + /// are allowed. Negative values mean all but that many CPUs, so -2 on a + /// 32 CPU machine allows the sandbox to use 30. Use "unlimited" to remove + /// a limit set in a config file. Requires cgroup v2. + #[arg( + long, + global = true, + allow_negative_numbers = true, + value_name = "CPUS" + )] + pub cpus: Option, + + /// Limit how much memory the sandbox can use, for example 512M or 4G. + /// Swap is not available to sandboxes with a memory limit. Use + /// "unlimited" to remove a limit set in a config file. Requires cgroup v2. + #[arg(long, global = true, value_name = "SIZE")] + pub memory: Option, + /// Formats action output as a JSON blob. Does nothing for sandboxed commands. #[arg(long, global = true, action = clap::ArgAction::SetTrue)] pub json: bool, diff --git a/src/config/limits.rs b/src/config/limits.rs new file mode 100644 index 0000000..68de8b0 --- /dev/null +++ b/src/config/limits.rs @@ -0,0 +1,203 @@ +use anyhow::{Result, anyhow}; +use serde::Deserialize; + +/// The smallest CPU limit cgroups will accept: a 1ms quota per 100ms period +const MIN_CPUS: f64 = 0.01; + +/// Removes a limit, e.g. to override one set in a config file +const UNLIMITED: &str = "unlimited"; + +fn is_unlimited(spec: &str) -> bool { + spec.trim().eq_ignore_ascii_case(UNLIMITED) +} + +/// Number of CPUs this process may run on. This matches `nproc` unless we +/// are already running under a CPU quota, in which case the quota is used. +pub fn available_cpus() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) +} + +/// Parses a CPU limit. Positive values are a number of CPUs and may be +/// fractional. Negative values mean all but that many of the `available` +/// CPUs, so -2 on a 32 CPU machine means 30. "unlimited" means no limit. +pub fn parse_cpus(spec: &str, available: usize) -> Result> { + let invalid = || { + anyhow!( + "Invalid cpus value '{}': expected a number of CPUs such as 2 or 1.5, a negative number such as -2 to use all but that many CPUs, or unlimited", + spec + ) + }; + + if is_unlimited(spec) { + return Ok(None); + } + + let value: f64 = spec.trim().parse().map_err(|_| invalid())?; + if !value.is_finite() || value == 0.0 { + return Err(invalid()); + } + + let cpus = if value < 0.0 { + available as f64 + value + } else { + value + }; + + if cpus < MIN_CPUS { + return Err(anyhow!( + "Invalid cpus value '{}': this leaves {} of the {} available CPUs for the sandbox, the minimum is {}", + spec, + cpus.max(0.0), + available, + MIN_CPUS + )); + } + + Ok(Some(cpus)) +} + +/// Parses a memory size such as 512M, 4G, 1.5GiB or a plain number of bytes. +/// Suffixes are binary (K = 1024) and case insensitive. "unlimited" means no +/// limit. +pub fn parse_memory(spec: &str) -> Result> { + let invalid = || { + anyhow!( + "Invalid memory value '{}': expected a size such as 512M or 4G, or unlimited", + spec + ) + }; + + if is_unlimited(spec) { + return Ok(None); + } + + let trimmed = spec.trim(); + let split = trimmed + .find(|c: char| c.is_ascii_alphabetic()) + .unwrap_or(trimmed.len()); + let (number, suffix) = trimmed.split_at(split); + + let multiplier: u64 = match suffix.to_ascii_lowercase().as_str() { + "" | "b" => 1, + "k" | "kb" | "kib" => 1 << 10, + "m" | "mb" | "mib" => 1 << 20, + "g" | "gb" | "gib" => 1 << 30, + "t" | "tb" | "tib" => 1 << 40, + _ => return Err(invalid()), + }; + + let number: f64 = number.trim().parse().map_err(|_| invalid())?; + let bytes = number * multiplier as f64; + if !bytes.is_finite() || bytes < 1.0 || bytes > u64::MAX as f64 { + return Err(invalid()); + } + + Ok(Some(bytes as u64)) +} + +/// Lets config files specify limits as either numbers or strings, e.g. +/// `cpus = -2` or `memory = "4G"` +pub(crate) fn deserialize_number_or_string<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumberOrString { + Integer(i64), + Float(f64), + String(String), + } + + Ok( + Option::::deserialize(deserializer)?.map(|value| { + match value { + NumberOrString::Integer(i) => i.to_string(), + NumberOrString::Float(f) => f.to_string(), + NumberOrString::String(s) => s, + } + }), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_cpus() { + assert_eq!(parse_cpus("2", 32).unwrap(), Some(2.0)); + assert_eq!(parse_cpus(" 1.5 ", 32).unwrap(), Some(1.5)); + assert_eq!(parse_cpus("0.01", 32).unwrap(), Some(0.01)); + assert_eq!(parse_cpus("64", 32).unwrap(), Some(64.0)); + assert_eq!(parse_cpus("-2", 32).unwrap(), Some(30.0)); + assert_eq!(parse_cpus("-0.5", 4).unwrap(), Some(3.5)); + assert_eq!(parse_cpus("-31", 32).unwrap(), Some(1.0)); + assert_eq!(parse_cpus("unlimited", 32).unwrap(), None); + assert_eq!(parse_cpus(" Unlimited ", 32).unwrap(), None); + } + + #[test] + fn test_parse_cpus_invalid() { + for spec in ["", "abc", "0", "-0", "nan", "inf", "-inf", "2 cpus"] { + let err = parse_cpus(spec, 32).unwrap_err().to_string(); + assert!(err.contains("expected a number of CPUs"), "{}", err); + } + + let err = parse_cpus("-32", 32).unwrap_err().to_string(); + assert!(err.contains("leaves 0 of the 32 available CPUs"), "{}", err); + let err = parse_cpus("-40", 32).unwrap_err().to_string(); + assert!(err.contains("leaves 0 of the 32 available CPUs"), "{}", err); + let err = parse_cpus("0.001", 32).unwrap_err().to_string(); + assert!(err.contains("the minimum is 0.01"), "{}", err); + } + + #[test] + fn test_parse_memory() { + assert_eq!(parse_memory("1024").unwrap(), Some(1024)); + assert_eq!(parse_memory("1024b").unwrap(), Some(1024)); + assert_eq!(parse_memory("4k").unwrap(), Some(4096)); + assert_eq!(parse_memory("4KiB").unwrap(), Some(4096)); + assert_eq!(parse_memory("512M").unwrap(), Some(512 << 20)); + assert_eq!(parse_memory("512mb").unwrap(), Some(512 << 20)); + assert_eq!(parse_memory(" 4G ").unwrap(), Some(4 << 30)); + assert_eq!(parse_memory("4 GiB").unwrap(), Some(4 << 30)); + assert_eq!(parse_memory("1.5G").unwrap(), Some(3 << 29)); + assert_eq!(parse_memory("2T").unwrap(), Some(2 << 40)); + assert_eq!(parse_memory("unlimited").unwrap(), None); + assert_eq!(parse_memory("UNLIMITED").unwrap(), None); + } + + #[test] + fn test_parse_memory_invalid() { + for spec in [ + "", "G", "abc", "0", "0.5", "-1G", "4X", "4GBs", "1e30T", "inf", + ] { + let err = parse_memory(spec).unwrap_err().to_string(); + assert!(err.contains("expected a size"), "{}: {}", spec, err); + } + } + + #[test] + fn test_deserialize_number_or_string() { + #[derive(Deserialize)] + struct Limits { + #[serde( + deserialize_with = "deserialize_number_or_string", + default + )] + value: Option, + } + + let parse = |s: &str| toml::from_str::(s).unwrap().value; + assert_eq!(parse("value = -2"), Some("-2".to_string())); + assert_eq!(parse("value = 1.5"), Some("1.5".to_string())); + assert_eq!(parse("value = \"4G\""), Some("4G".to_string())); + assert_eq!(parse(""), None); + assert!(toml::from_str::("value = true").is_err()); + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs index 517fcf4..bdf4dcf 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod impls; +pub mod limits; pub mod resolve; pub mod structs; diff --git a/src/config/resolve.rs b/src/config/resolve.rs index 73182f2..56f34b4 100644 --- a/src/config/resolve.rs +++ b/src/config/resolve.rs @@ -1,4 +1,5 @@ use super::PartialConfig; +use super::limits::{available_cpus, parse_cpus, parse_memory}; use crate::config::{BindMount, BindMountOptions, Network}; use crate::util::{ can_access, can_mkdir, expand_tilde_path, find_mount_point, mkdir, @@ -172,6 +173,16 @@ pub fn resolve_config(cli: Args) -> Result { } } + for (var, key, field) in [ + ("SANDBOX_CPUS", "cpus", &mut partial_config.cpus), + ("SANDBOX_MEMORY", "memory", &mut partial_config.memory), + ] { + if let Some(value) = env::var(var).ok().filter(|v| !v.is_empty()) { + *field = Some(value); + sources.insert(key.into(), "environment".into()); + } + } + // Override with CLI args if provided (highest precedence) if let Some(log_level) = cli.log_level { partial_config.log_level = Some(log_level); @@ -228,6 +239,16 @@ pub fn resolve_config(cli: Args) -> Result { sources.insert("audio".into(), "cli".into()); } + if let Some(cpus) = cli.cpus { + partial_config.cpus = Some(cpus); + sources.insert("cpus".into(), "cli".into()); + } + + if let Some(memory) = cli.memory { + partial_config.memory = Some(memory); + sources.insert("memory".into(), "cli".into()); + } + // Bind mounts and masks from the CLI form the highest precedence layer let mut cli_bind_layer = cli.bind.unwrap_or_default(); if let Some(cli_mask_paths) = cli.mask { @@ -265,6 +286,24 @@ pub fn resolve_config(cli: Args) -> Result { sources.insert("audio".into(), "default".into()); } + let cpus = partial_config + .cpus + .map(|cpus| parse_cpus(&cpus, available_cpus())) + .transpose()? + .flatten(); + if !sources.contains_key("cpus") { + sources.insert("cpus".into(), "default".into()); + } + + let memory = partial_config + .memory + .map(|memory| parse_memory(&memory)) + .transpose()? + .flatten(); + if !sources.contains_key("memory") { + sources.insert("memory".into(), "default".into()); + } + let storage_dir = resolve_sandbox_storage_dir( partial_config.storage_dir, uid_gid_home.uid, @@ -357,6 +396,8 @@ pub fn resolve_config(cli: Args) -> Result { bind_mounts, no_default_binds, audio, + cpus, + memory, config_files, }; @@ -723,6 +764,14 @@ fn merge_configs( base.audio = Some(audio); sources.insert("audio".into(), source.into()); } + if let Some(cpus) = override_config.cpus { + base.cpus = Some(cpus); + sources.insert("cpus".into(), source.into()); + } + if let Some(memory) = override_config.memory { + base.memory = Some(memory); + sources.insert("memory".into(), source.into()); + } // Bind mounts and masks from each config file form their own layer let mut bind_layer = vec![]; if let Some(bind_mounts) = override_config.bind { @@ -922,6 +971,8 @@ mod tests { bind_mounts: vec![], no_default_binds: false, audio: false, + cpus: None, + memory: None, config_files: vec![], }; assert!(validate_config(&config).is_ok()); @@ -998,6 +1049,8 @@ mod tests { mask: Some(vec!["/tmp/test3".to_string()]), no_default_binds: Some(true), audio: Some(true), + cpus: Some("-2".to_string()), + memory: Some("4G".to_string()), }; merge_configs( @@ -1022,6 +1075,10 @@ mod tests { assert!(matches!(base.net, Some(Network::Host))); assert_eq!(base.ignored, Some(true)); assert_eq!(base.audio, Some(true)); + assert_eq!(base.cpus, Some("-2".to_string())); + assert_eq!(base.memory, Some("4G".to_string())); + assert_eq!(sources.get("cpus"), Some(&"test-config".to_string())); + assert_eq!(sources.get("memory"), Some(&"test-config".to_string())); // Each merged config produces its own bind layer let expected_layer = vec![ diff --git a/src/config/structs.rs b/src/config/structs.rs index d8ee92a..94b2223 100644 --- a/src/config/structs.rs +++ b/src/config/structs.rs @@ -1,4 +1,5 @@ use super::impls::deserialize_level_filter; +use super::limits::deserialize_number_or_string; use anyhow::Result; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, path::PathBuf}; @@ -60,6 +61,10 @@ pub struct PartialConfig { pub mask: Option>, pub no_default_binds: Option, pub audio: Option, + #[serde(deserialize_with = "deserialize_number_or_string", default)] + pub cpus: Option, + #[serde(deserialize_with = "deserialize_number_or_string", default)] + pub memory: Option, } #[derive(Clone)] @@ -76,5 +81,9 @@ pub struct Config { pub bind_mounts: Vec, pub no_default_binds: bool, pub audio: bool, + /// Number of CPUs the sandbox may use, None for unlimited + pub cpus: Option, + /// Bytes of memory the sandbox may use, None for unlimited + pub memory: Option, pub config_files: Vec, } diff --git a/src/sandbox/cgroup.rs b/src/sandbox/cgroup.rs new file mode 100644 index 0000000..3907076 --- /dev/null +++ b/src/sandbox/cgroup.rs @@ -0,0 +1,243 @@ +//! cgroup v2 support for limiting the CPU and memory available to a sandbox. +//! +//! Sandboxes with resource limits get a cgroup at +//! /sys/fs/cgroup/sandbox/-. The sandbox's PID 1 is started in that +//! cgroup, which also makes it the root of the sandbox's cgroup namespace, and +//! every process that later joins the sandbox is moved into it. + +use crate::config::Config; +use crate::sandbox::{Sandbox, SandboxSettings, describe_limits}; +use anyhow::{Context, Result, anyhow}; +use log::{debug, trace, warn}; +use std::os::fd::OwnedFd; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const CGROUP_ROOT: &str = "/sys/fs/cgroup"; +const CPU_PERIOD_US: u64 = 100_000; + +/// The cgroup all sandbox cgroups are created in +fn cgroup_parent() -> String { + let parent = String::from("sandbox"); + + // Lets tests start from a parent cgroup that doesn't exist yet + #[cfg(feature = "coverage")] + let parent = std::env::var("TEST_CGROUP_PARENT").unwrap_or(parent); + + parent +} + +impl Sandbox { + fn cgroup_name(&self) -> String { + format!("{}-{}", self.uid, self.name) + } + + pub(crate) fn cgroup_path(&self) -> PathBuf { + Path::new(CGROUP_ROOT) + .join(cgroup_parent()) + .join(self.cgroup_name()) + } + + /// Sets up our cgroup like create_cgroup, except that inside another + /// sandbox, where cgroups can't be managed, we warn and run without a + /// cgroup of our own. We're still in the outer sandbox's cgroup, so its + /// limits continue to apply. + pub(crate) fn create_cgroup_or_share_outer( + &self, + config: &Config, + ) -> Result> { + let error = match self.create_cgroup(config) { + Ok(cgroup) => return Ok(cgroup), + Err(e) => e.context("Failed to set up resource limits"), + }; + + // A nested sandbox's storage directory is the outer sandbox's data + // directory, which holds the outer sandbox's settings + let outer_settings = config.storage_dir.join("settings.json"); + let Ok(outer) = SandboxSettings::load_from_file(&outer_settings) else { + return Err(error); + }; + + debug!("{:#}", error); + warn!( + "Resource limits can't be set up inside another sandbox, so this sandbox shares the limits of the sandbox it's running in ({})", + describe_limits(outer.cpus, outer.memory) + ); + Ok(None) + } + + /// Creates a cgroup with the resource limits from our config. Returns a + /// handle to it for use with CLONE_INTO_CGROUP, or None if the config + /// has no resource limits. + pub(crate) fn create_cgroup( + &self, + config: &Config, + ) -> Result> { + let mut controllers = vec![]; + if config.cpus.is_some() { + controllers.push("cpu"); + } + if config.memory.is_some() { + controllers.push("memory"); + } + if controllers.is_empty() { + return Ok(None); + } + + let root = Path::new(CGROUP_ROOT); + + #[cfg(feature = "coverage")] + let root = if std::env::var_os("TEST_NO_CGROUP2").is_some() { + Path::new("/does-not-exist") + } else { + root + }; + + if !root.join("cgroup.controllers").exists() { + return Err(anyhow!( + "Limiting cpus or memory requires cgroup v2, which is not mounted at {}", + root.display() + )); + } + + let parent = root.join(cgroup_parent()); + enable_controllers(root, &controllers)?; + if !parent.exists() { + let context = + format!("Failed to create cgroup {}", parent.display()); + std::fs::create_dir(&parent).context(context)?; + } + enable_controllers(&parent, &controllers)?; + + let path = self.cgroup_path(); + if path.exists() { + // Left behind by a sandbox that wasn't stopped cleanly, which we + // can remove because it's empty. If it's not empty it belongs to + // a running sandbox with the same name in another storage dir. + std::fs::remove_dir(&path).context(format!( + "Failed to remove existing cgroup {}, is a sandbox with the same name running from a different storage directory?", + path.display() + ))?; + } + trace!("Creating cgroup {}", path.display()); + std::fs::create_dir(&path) + .context(format!("Failed to create cgroup {}", path.display()))?; + + let result = self.write_limits(&path, config).and_then(|_| { + std::fs::File::open(&path) + .map(OwnedFd::from) + .context(format!("Failed to open cgroup {}", path.display())) + }); + if result.is_err() { + remove_cgroup(&path); + } + result.map(Some) + } + + fn write_limits(&self, path: &Path, config: &Config) -> Result<()> { + if let Some(cpus) = config.cpus { + let quota = (cpus * CPU_PERIOD_US as f64).round() as u64; + write_cgroup_file( + path, + "cpu.max", + &format!("{} {}", quota, CPU_PERIOD_US), + )?; + } + if let Some(memory) = config.memory { + write_cgroup_file(path, "memory.max", &memory.to_string())?; + // Swap would let the sandbox use more memory than its limit, but + // the file only exists when swap accounting is enabled. + let swap_accounting = path.join("memory.swap.max").exists(); + + #[cfg(feature = "coverage")] + let swap_accounting = swap_accounting + && std::env::var_os("TEST_NO_SWAP_ACCOUNTING").is_none(); + + if swap_accounting { + write_cgroup_file(path, "memory.swap.max", "0")?; + } + } + Ok(()) + } + + /// Returns the path to our cgroup if our sandbox is running in one. We + /// go by the cgroup our PID 1 is in rather than the name we'd give a new + /// cgroup, because the uid in the name is that of the user who started + /// the sandbox, which isn't always ours (e.g. `delete` doesn't know it). + pub(crate) fn running_cgroup(&self) -> Option { + let cgroups = + std::fs::read_to_string(format!("/proc/{}/cgroup", self.pid)) + .ok()?; + let current = + cgroups.lines().find_map(|line| line.strip_prefix("0::"))?; + let relative = Path::new(current).strip_prefix("/").ok()?; + let cgroup_name = + relative.strip_prefix(cgroup_parent()).ok()?.to_str()?; + let (uid, name) = cgroup_name.split_once('-')?; + (uid.parse::().is_ok() && name == self.name) + .then(|| Path::new(CGROUP_ROOT).join(relative)) + } + + /// Moves the current process into our sandbox's cgroup, if it has one, + /// so processes we launch in the sandbox are subject to its limits. + pub(crate) fn join_cgroup(&self) -> Result<()> { + if let Some(path) = self.running_cgroup() { + trace!("Joining cgroup {}", path.display()); + let pid = std::process::id().to_string(); + write_cgroup_file(&path, "cgroup.procs", &pid)?; + } + Ok(()) + } +} + +/// Removes a cgroup once the processes in it have exited, waiting a little +/// while for that. Sandbox processes have already been killed by this point. +/// The only other processes in the cgroup are the `sandbox` commands waiting +/// on them, which exit by themselves once their child is gone, so they get to +/// report its exit status instead of being killed too. +pub(crate) fn remove_cgroup(path: &Path) { + trace!("Removing cgroup {}", path.display()); + let deadline = Instant::now() + Duration::from_secs(5); + while std::fs::remove_dir(path).is_err() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn write_cgroup_file(dir: &Path, file: &str, value: &str) -> Result<()> { + let path = dir.join(file); + + #[cfg(feature = "coverage")] + let path = if std::env::var_os("TEST_CGROUP_WRITE_FAILURE") + .is_some_and(|f| f == file) + { + PathBuf::from("/does-not-exist").join(file) + } else { + path + }; + + std::fs::write(&path, value).context(format!( + "Failed to write '{}' to {}", + value, + path.display() + )) +} + +/// Enables controllers for a cgroup's children if they aren't already +fn enable_controllers(dir: &Path, controllers: &[&str]) -> Result<()> { + let subtree_control = dir.join("cgroup.subtree_control"); + let context = format!("Failed to read {}", subtree_control.display()); + let enabled = std::fs::read_to_string(&subtree_control).context(context)?; + for controller in controllers { + if !enabled.split_whitespace().any(|c| c == *controller) { + let context = format!( + "Failed to enable the {} cgroup controller. Limiting cpus or memory requires permission to manage {}, which may not be available inside containers or other sandboxes", + controller, + dir.display() + ); + let value = format!("+{}", controller); + write_cgroup_file(dir, "cgroup.subtree_control", &value) + .context(context)?; + } + } + Ok(()) +} diff --git a/src/sandbox/exec.rs b/src/sandbox/exec.rs index 80ecd29..688dcde 100644 --- a/src/sandbox/exec.rs +++ b/src/sandbox/exec.rs @@ -82,6 +82,11 @@ impl Sandbox { libc::prctl(libc::PR_SET_DUMPABLE, 1); }; + /* Join the sandbox's cgroup so we're subject to its resource limits. + * This needs to happen before we enter the sandbox's mount namespace, + * where the cgroup filesystem is read-only. */ + self.join_cgroup()?; + /* Join the sandbox namespaces */ let pid_fd = pidfd_open(self.pid, 0)?; if (unsafe { setns(pid_fd, ns_flags) }) != 0 { diff --git a/src/sandbox/get_or_create.rs b/src/sandbox/get_or_create.rs index b21486a..d25a586 100644 --- a/src/sandbox/get_or_create.rs +++ b/src/sandbox/get_or_create.rs @@ -1,3 +1,4 @@ +use super::cgroup::remove_cgroup; use super::mount_overlays::MountHash; use crate::config::{BindMount, BindMountOptions, Config, Network}; use crate::sandbox::Sandbox; @@ -7,9 +8,9 @@ use crate::util::Lock; use crate::util::get_running_sandbox_pid; use crate::util::get_sandbox_pid_path; use crate::util::{ - CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS, CLONE_NEWPID, - CLONE_NEWUTS, Clone3Args, check_path_for_mount_option_compatibility, - clone3, mkdir, mount, + CLONE_INTO_CGROUP, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, + CLONE_NEWNS, CLONE_NEWPID, CLONE_NEWUTS, Clone3Args, + check_path_for_mount_option_compatibility, clone3, mkdir, mount, }; use anyhow::Context; use anyhow::{Result, anyhow}; @@ -191,20 +192,33 @@ impl Sandbox { options: BindMountOptions::ReadWrite, }); + /* Create a cgroup to enforce any resource limits. We do this first + * so we fail before mounting anything if limits aren't available. */ + let cgroup = sandbox.create_cgroup_or_share_outer(&config)?; + trace!("Mounting sandbox"); - let overlay_mounts = sandbox + let started = sandbox .mount_overlays(&config) - .context(format!("failed to mount sandbox: {}", sandbox_name))?; - - /* Create PID 1 for our sandbox */ - sandbox.pid = match sandbox.start_sandbox( - &config, - &get_sandbox_pid_path(storage_dir, sandbox_name), - &overlay_mounts, - ) { + .context(format!("failed to mount sandbox: {}", sandbox_name)) + .and_then(|overlay_mounts| { + /* Create PID 1 for our sandbox */ + sandbox + .start_sandbox( + &config, + &get_sandbox_pid_path(storage_dir, sandbox_name), + &overlay_mounts, + cgroup.as_ref(), + ) + .map_err(|e| anyhow!("Failed to start sandbox: {}", e)) + }); + + sandbox.pid = match started { Ok(pid) => pid, Err(e) => { - return Err(anyhow!("Failed to start sandbox: {}", e)); + if cgroup.is_some() { + remove_cgroup(&sandbox.cgroup_path()); + } + return Err(e); } }; @@ -220,6 +234,7 @@ impl Sandbox { config: &Config, pid_file: &PathBuf, overlay_mounts: &Vec, + cgroup: Option<&OwnedFd>, ) -> Result { // create a pipe between ourselves and the main sandbox process so the main sandbox process can // let us know when it's done setting up the namespaces @@ -238,7 +253,10 @@ impl Sandbox { | CLONE_NEWPID | CLONE_NEWIPC | CLONE_NEWUTS - | CLONE_NEWCGROUP, + | CLONE_NEWCGROUP + // Starting in our cgroup also makes it the root of our cgroup namespace + | cgroup.map_or(0, |_| CLONE_INTO_CGROUP), + cgroup: cgroup.map_or(0, |fd| fd.as_raw_fd() as u64), ..Default::default() }; diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 9e29328..3f7a76e 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -1,3 +1,4 @@ +mod cgroup; pub(crate) mod changes; mod delete; mod exec; diff --git a/src/sandbox/settings.rs b/src/sandbox/settings.rs index 3e2069f..e68b414 100644 --- a/src/sandbox/settings.rs +++ b/src/sandbox/settings.rs @@ -10,6 +10,11 @@ pub struct SandboxSettings { pub mounts: Vec, pub network: crate::config::Network, pub bind_mounts: Vec, + // Defaulted so settings from sandboxes started by older versions load + #[serde(default)] + pub cpus: Option, + #[serde(default)] + pub memory: Option, } impl SandboxSettings { @@ -19,6 +24,8 @@ impl SandboxSettings { mounts: mounts.to_vec(), network: config.net.clone(), bind_mounts: config.bind_mounts.clone(), + cpus: config.cpus, + memory: config.memory, } } @@ -76,6 +83,16 @@ impl SandboxSettings { )); } + if self.cpus != current_settings.cpus + || self.memory != current_settings.memory + { + return Err(anyhow!( + "Resource limits have changed. Existing: {}, Current: {}. Please stop the existing sandbox to change resource limits.", + describe_limits(self.cpus, self.memory), + describe_limits(current_settings.cpus, current_settings.memory), + )); + } + if self.mounts != current_settings.mounts { let mut error_msg = String::from("Mount configuration has changed:\n"); @@ -151,3 +168,13 @@ impl SandboxSettings { Ok(()) } } + +pub(crate) fn describe_limits( + cpus: Option, + memory: Option, +) -> String { + let cpus = cpus.map_or("unlimited".to_string(), |c| c.to_string()); + let memory = + memory.map_or("unlimited".to_string(), |m| format!("{} bytes", m)); + format!("cpus={} memory={}", cpus, memory) +} diff --git a/src/sandbox/stop.rs b/src/sandbox/stop.rs index ab099f5..9e25a40 100644 --- a/src/sandbox/stop.rs +++ b/src/sandbox/stop.rs @@ -1,4 +1,5 @@ use crate::sandbox::Sandbox; +use crate::sandbox::cgroup::remove_cgroup; use anyhow::{Context, Result}; use log::trace; use nix::{fcntl::readlink, unistd::Pid}; @@ -17,6 +18,9 @@ impl Sandbox { pid ); + // Find our cgroup before our processes go away + let cgroup = self.running_cgroup(); + #[cfg(feature = "coverage")] if std::env::var_os("TEST_STOP_RACE").is_some() { // pre-kill the sandbox process to simulate a race or otherwise @@ -134,6 +138,10 @@ impl Sandbox { } } } + + if let Some(cgroup) = cgroup { + remove_cgroup(&cgroup); + } } let pid_file = PathBuf::from(format!("{}.pid", self.base.display())); diff --git a/src/util/clone3.rs b/src/util/clone3.rs index a93d060..dfc349e 100644 --- a/src/util/clone3.rs +++ b/src/util/clone3.rs @@ -62,6 +62,9 @@ pub const CLONE_NEWUSER: u64 = libc::CLONE_NEWUSER as u64; /* New user namespace pub const CLONE_NEWPID: u64 = libc::CLONE_NEWPID as u64; /* New pid namespace */ #[allow(dead_code)] pub const CLONE_NEWNET: u64 = libc::CLONE_NEWNET as u64; /* New network namespace */ +/// Start the child in the cgroup given by Clone3Args::cgroup (since Linux 5.7). +/// Not yet defined by the libc crate. +pub const CLONE_INTO_CGROUP: u64 = 0x200000000; #[allow(dead_code)] pub const CLONE_IO: u64 = libc::CLONE_IO as u64; /* Clone I/O context */ diff --git a/tests/ai_test_resource_limits.rs b/tests/ai_test_resource_limits.rs new file mode 100644 index 0000000..4a38b75 --- /dev/null +++ b/tests/ai_test_resource_limits.rs @@ -0,0 +1,627 @@ +mod fixtures; + +use anyhow::Result; +use fixtures::*; +use rstest::*; +use std::fs; +use std::path::PathBuf; + +const UNAVAILABLE: &str = "Failed to set up resource limits"; + +fn available_cpus() -> usize { + std::thread::available_parallelism().unwrap().get() +} + +fn write_config( + sandbox: &SandboxManager, + prefix: &str, + contents: &str, +) -> Result { + let path = format!("{}.toml", sandbox.test_filename(prefix)); + fs::write(&path, contents)?; + Ok(path) +} + +/// Starts the sandbox with a CPU limit to find out if this machine lets us +/// manage cgroups. When it doesn't, e.g. because we're running inside a +/// container or another sandbox, we check that we fail with a helpful error. +fn limits_available(sandbox: &mut SandboxManager) -> bool { + if sandbox.pass(&["--cpus=1", "true"]) { + return true; + } + assert!( + sandbox.last_stderr.contains(UNAVAILABLE), + "{}", + sandbox.last_stderr + ); + eprintln!("cgroup resource limits are not available, skipping"); + false +} + +/// Finds our sandbox's cgroup on the host, if it has one +fn host_cgroup(sandbox: &SandboxManager) -> Option { + let suffix = format!("-{}", sandbox.name); + fs::read_dir("/sys/fs/cgroup/sandbox") + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .find(|path| path.to_string_lossy().ends_with(&suffix)) +} + +/* + * Configuration + */ + +#[rstest] +fn ai_test_limits_unlimited_by_default( + mut sandbox: SandboxManager, +) -> Result<()> { + // Skip config files, which may set limits + sandbox.run(&["--no-config", "config", "cpus", "memory"])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + + // Both keys are part of the full config listing + sandbox.run(&["--no-config", "config"])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + Ok(()) +} + +#[rstest] +fn ai_test_limits_cli(mut sandbox: SandboxManager) -> Result<()> { + sandbox.run(&["--cpus=2", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), "2"); + + sandbox.run(&["--cpus=1.5", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), "1.5"); + + // Negative values are relative to the number of available CPUs, and + // work with or without an equals sign + let expected = (available_cpus() - 1).to_string(); + sandbox.run(&["--cpus=-1", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), expected); + sandbox.run(&["--cpus", "-1", "config", "cpus"])?; + assert_eq!(sandbox.last_stdout.trim(), expected); + + sandbox.run(&["--memory=512M", "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), (512u64 << 20).to_string()); + + sandbox.run(&["--memory", "1.5G", "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), (3u64 << 29).to_string()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_env(mut sandbox: SandboxManager) -> Result<()> { + // Config files are skipped so they can't set limits of their own + let args = ["--no-config", "config"]; + sandbox.run_with_env( + &[&args[..], &["cpus"]].concat(), + "SANDBOX_CPUS", + "3", + )?; + assert_eq!(sandbox.last_stdout.trim(), "3"); + + sandbox.run_with_env( + &[&args[..], &["memory"]].concat(), + "SANDBOX_MEMORY", + "2G", + )?; + assert_eq!(sandbox.last_stdout.trim(), (2u64 << 30).to_string()); + + // Empty values are ignored + sandbox.run_with_env( + &[&args[..], &["cpus"]].concat(), + "SANDBOX_CPUS", + "", + )?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + // The CLI takes precedence over the environment + sandbox.run_with_env( + &["--no-config", "--cpus=1", "config", "cpus"], + "SANDBOX_CPUS", + "3", + )?; + assert_eq!(sandbox.last_stdout.trim(), "1"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_config_file(mut sandbox: SandboxManager) -> Result<()> { + let config = + write_config(&sandbox, "limits", "cpus = -1\nmemory = \"4G\"\n")?; + sandbox.run(&["--config", &config, "config", "cpus", "memory"])?; + assert!( + sandbox + .last_stdout + .contains(&format!("cpus={}", available_cpus() - 1)) + ); + assert!( + sandbox + .last_stdout + .contains(&format!("memory={}", 4u64 << 30)) + ); + + // Numbers and strings are both accepted + let config = write_config( + &sandbox, + "limits-types", + "cpus = \"0.5\"\nmemory = 1073741824\n", + )?; + sandbox.run(&["--config", &config, "config", "cpus", "memory"])?; + assert!(sandbox.last_stdout.contains("cpus=0.5")); + assert!(sandbox.last_stdout.contains("memory=1073741824")); + + // The environment takes precedence over config files + sandbox.run_with_env( + &["--config", &config, "config", "cpus"], + "SANDBOX_CPUS", + "2", + )?; + assert_eq!(sandbox.last_stdout.trim(), "2"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_invalid(mut sandbox: SandboxManager) -> Result<()> { + for cpus in ["--cpus=0", "--cpus=abc", "--cpus=0.001"] { + assert!(sandbox.xfail(&[cpus, "config", "cpus"])); + assert!( + sandbox.last_stderr.contains("Invalid cpus value"), + "{}", + sandbox.last_stderr + ); + } + + let all_cpus = format!("--cpus=-{}", available_cpus()); + assert!(sandbox.xfail(&[&all_cpus, "config", "cpus"])); + assert!(sandbox.last_stderr.contains("leaves 0 of the")); + + assert!(sandbox.xfail(&["--memory=lots", "config", "memory"])); + assert!(sandbox.last_stderr.contains("Invalid memory value 'lots'")); + + let config = write_config(&sandbox, "bad-type", "cpus = true\n")?; + assert!(sandbox.xfail(&["--config", &config, "config", "cpus"])); + Ok(()) +} + +/* + * Enforcement + */ + +#[rstest] +fn ai_test_limits_applied(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + let args = ["--cpus=2", "--memory=256M"]; + let run = |sandbox: &mut SandboxManager, command: &[&str]| { + let mut full = args.to_vec(); + full.extend_from_slice(command); + sandbox + .run(&full) + .map(|_| sandbox.last_stdout.trim().to_string()) + }; + + // Our limits are visible inside the sandbox, whose cgroup namespace is + // rooted at the sandbox's cgroup + assert_eq!( + run(&mut sandbox, &["cat", "/sys/fs/cgroup/cpu.max"])?, + "200000 100000" + ); + assert_eq!( + run(&mut sandbox, &["cat", "/sys/fs/cgroup/memory.max"])?, + (256u64 << 20).to_string() + ); + assert_eq!(run(&mut sandbox, &["cat", "/proc/self/cgroup"])?, "0::/"); + + // Processes that join the running sandbox are in its cgroup too + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let mut child = + sandbox.run_in_background(&[args[0], args[1], "sleep", "30"])?; + let mut joined = false; + for _ in 0..100 { + let procs = fs::read_to_string(cgroup.join("cgroup.procs"))?; + if procs.lines().count() > 1 { + joined = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(joined, "sleep should have joined the sandbox cgroup"); + + // Stopping the sandbox removes its cgroup + sandbox.run(&["stop"])?; + let _ = child.wait(); + assert!(!cgroup.exists()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_negative_cpus(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + sandbox.run(&["--cpus=-1", "cat", "/sys/fs/cgroup/cpu.max"])?; + assert_eq!( + sandbox.last_stdout.trim(), + format!("{} 100000", (available_cpus() - 1) * 100000) + ); + Ok(()) +} + +#[rstest] +fn ai_test_limits_memory_enforced(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // tail has to buffer the whole newline-free input, which won't fit. + // memory.events is read in the same command so we see this cgroup's + // counters even if the OOM killer took down more than tail. + sandbox.run(&[ + "--memory=32M", + "sh", + "-c", + "head -c 256M /dev/zero | tail > /dev/null; \ + echo \"tail exited with $?\"; \ + cat /sys/fs/cgroup/memory.events", + ])?; + let output = sandbox.last_stdout.clone(); + assert!(output.contains("tail exited with 137"), "{}", output); + let oom_kills: u64 = output + .lines() + .find_map(|line| line.strip_prefix("oom_kill ")) + .expect("memory.events should have an oom_kill entry") + .parse()?; + assert!(oom_kills >= 1, "{}", output); + Ok(()) +} + +#[rstest] +fn ai_test_limits_changed(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // Restart without config files, which may set limits of their own + sandbox.run(&["stop"])?; + sandbox.run(&["--no-config", "--cpus=1", "true"])?; + + // The sandbox is running with --cpus=1, so other limits are rejected + assert!(sandbox.xfail(&["--no-config", "true"])); + assert!(sandbox.last_stderr.contains("Resource limits have changed")); + assert!( + sandbox + .last_stderr + .contains("Existing: cpus=1 memory=unlimited") + ); + assert!( + sandbox + .last_stderr + .contains("Current: cpus=unlimited memory=unlimited") + ); + + assert!(sandbox.xfail(&["--no-config", "--cpus=1", "--memory=1G", "true"])); + assert!( + sandbox + .last_stderr + .contains(&format!("Current: cpus=1 memory={} bytes", 1u64 << 30)) + ); + + // The same limits are fine, and stopping lets us change them + sandbox.run(&["--no-config", "--cpus=1", "true"])?; + sandbox.run(&["stop"])?; + sandbox.run(&["--no-config", "--cpus=2", "true"])?; + Ok(()) +} + +#[rstest] +fn ai_test_limits_stale_cgroup(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // Kill the sandbox without cleaning up, leaving an empty cgroup behind + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + sandbox.run(&["config", "sandbox_dir"])?; + let pid_file = format!("{}.pid", sandbox.last_stdout.trim()); + let pid = fs::read_to_string(&pid_file)?; + std::process::Command::new("sudo") + .args(["kill", "-9", pid.trim()]) + .status()?; + for _ in 0..100 { + if fs::read_to_string(cgroup.join("cgroup.procs"))?.is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(cgroup.exists()); + + // Starting again replaces the stale cgroup + sandbox.run(&["--cpus=2", "cat", "/sys/fs/cgroup/cpu.max"])?; + assert_eq!(sandbox.last_stdout.trim(), "200000 100000"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_cgroup_in_use(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // A sandbox with the same name in another storage directory would + // need the same cgroup, which is in use + let storage_dir = + std::env::current_dir()?.join(sandbox.test_filename("storage")); + fs::create_dir_all(&storage_dir)?; + let storage_arg = format!("--storage-dir={}", storage_dir.display()); + assert!(sandbox.xfail(&[&storage_arg, "--cpus=1", "true"])); + assert!( + sandbox + .last_stderr + .contains("from a different storage directory") + ); + Ok(()) +} + +#[rstest] +fn ai_test_limits_setup_failures(mut sandbox: SandboxManager) -> Result<()> { + assert!(sandbox.exfail(&["--cpus=1", "true"], "TEST_NO_CGROUP2", "1")); + assert!(sandbox.last_stderr.contains(UNAVAILABLE)); + assert!( + sandbox + .last_stderr + .contains("requires cgroup v2, which is not mounted") + ); + + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // Failing to apply limits cleans up the cgroup + for (file, args) in [ + ("cpu.max", vec!["--cpus=1", "true"]), + ("memory.max", vec!["--memory=1G", "true"]), + ] { + assert!(sandbox.exfail(&args, "TEST_CGROUP_WRITE_FAILURE", file)); + assert!(sandbox.last_stderr.contains(UNAVAILABLE)); + assert!(sandbox.last_stderr.contains("Failed to write")); + assert!(host_cgroup(&sandbox).is_none()); + } + + // As does failing to start the sandbox + assert!(sandbox.exfail( + &["--cpus=1", "true"], + "TEST_START_SANDBOX_FAILURE", + "1" + )); + assert!(sandbox.last_stderr.contains("Failed to start sandbox")); + assert!(host_cgroup(&sandbox).is_none()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_new_parent_cgroup(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // The first sandbox with limits creates the parent cgroup and enables + // the controllers we need in it + let parent = format!("{}-parent", sandbox.name); + let parent_path = PathBuf::from("/sys/fs/cgroup").join(&parent); + assert!(!parent_path.exists()); + sandbox.run_with_env( + &["--cpus=1", "--memory=64M", "cat", "/sys/fs/cgroup/cpu.max"], + "TEST_CGROUP_PARENT", + &parent, + )?; + assert_eq!(sandbox.last_stdout.trim(), "100000 100000"); + let enabled = + fs::read_to_string(parent_path.join("cgroup.subtree_control"))?; + assert!(enabled.split_whitespace().any(|c| c == "cpu")); + assert!(enabled.split_whitespace().any(|c| c == "memory")); + + sandbox.run_with_env(&["stop"], "TEST_CGROUP_PARENT", &parent)?; + assert_eq!( + fs::read_dir(&parent_path)? + .filter(|e| { e.as_ref().is_ok_and(|e| e.path().is_dir()) }) + .count(), + 0 + ); + std::process::Command::new("sudo") + .args(["rmdir", &parent_path.to_string_lossy()]) + .status()?; + Ok(()) +} + +#[rstest] +fn ai_test_limits_no_swap_accounting( + mut sandbox: SandboxManager, +) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // Only memory is limited (config files are skipped since they may limit + // cpus too). Without swap accounting we can limit memory but not swap. + let memory_only = ["--no-config", "--memory=64M", "cat"]; + sandbox.run_with_env( + &[&memory_only[..], &["/sys/fs/cgroup/memory.max"]].concat(), + "TEST_NO_SWAP_ACCOUNTING", + "1", + )?; + assert_eq!(sandbox.last_stdout.trim(), (64u64 << 20).to_string()); + sandbox.run(&[&memory_only[..], &["/sys/fs/cgroup/cpu.max"]].concat())?; + assert_eq!(sandbox.last_stdout.trim(), "max 100000"); + sandbox.run( + &[&memory_only[..], &["/sys/fs/cgroup/memory.swap.max"]].concat(), + )?; + assert_eq!(sandbox.last_stdout.trim(), "max"); + sandbox.run(&["stop"])?; + + // Normally swap is disabled + sandbox.run( + &[&memory_only[..], &["/sys/fs/cgroup/memory.swap.max"]].concat(), + )?; + assert_eq!(sandbox.last_stdout.trim(), "0"); + Ok(()) +} + +#[rstest] +fn ai_test_limits_join_failure(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // The sandbox is running with --cpus=1, joining it requires moving + // into its cgroup + assert!(sandbox.exfail( + &["--cpus=1", "true"], + "TEST_CGROUP_WRITE_FAILURE", + "cgroup.procs" + )); + assert!(sandbox.last_stderr.contains("Failed to write")); + assert!(sandbox.last_stderr.contains("cgroup.procs")); + Ok(()) +} + +#[rstest] +fn ai_test_limits_stop_cgroup_busy(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // A child cgroup keeps the sandbox's cgroup from being removed, so stop + // retries for a while and then gives up without failing + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let child = cgroup.join("busy"); + let sudo = + |args: &[&str]| std::process::Command::new("sudo").args(args).status(); + sudo(&["mkdir", &child.to_string_lossy()])?; + + let started = std::time::Instant::now(); + sandbox.run(&["stop"])?; + assert!(started.elapsed() >= std::time::Duration::from_secs(5)); + assert!(cgroup.exists()); + + sudo(&["rmdir", &child.to_string_lossy()])?; + sudo(&["rmdir", &cgroup.to_string_lossy()])?; + assert!(!cgroup.exists()); + Ok(()) +} + +#[rstest] +fn ai_test_limits_unlimited_override( + mut sandbox: SandboxManager, +) -> Result<()> { + // "unlimited" removes a limit set by a less specific source + let config = + write_config(&sandbox, "limited", "cpus = 2\nmemory = \"4G\"\n")?; + sandbox.run(&[ + "--config", + &config, + "--cpus=unlimited", + "--memory=Unlimited", + "config", + "cpus", + "memory", + ])?; + assert!(sandbox.last_stdout.contains("cpus=unlimited")); + assert!(sandbox.last_stdout.contains("memory=unlimited")); + + sandbox.run_with_env( + &["--config", &config, "config", "cpus"], + "SANDBOX_CPUS", + "unlimited", + )?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + let unlimited = + write_config(&sandbox, "unlimited", "memory = \"unlimited\"\n")?; + sandbox.run(&["--config", &unlimited, "config", "memory"])?; + assert_eq!(sandbox.last_stdout.trim(), "unlimited"); + + // Without any limits the sandbox doesn't get a cgroup + if limits_available(&mut sandbox) { + sandbox.run(&["stop"])?; + sandbox.run(&[ + "--config", + &config, + "--cpus=unlimited", + "--memory=unlimited", + "true", + ])?; + assert!(host_cgroup(&sandbox).is_none()); + } + Ok(()) +} + +#[rstest] +fn ai_test_limits_nested(mut sandbox: SandboxManager) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + sandbox.run(&["stop"])?; + + // A sandbox inside another one can't manage cgroups, so instead of its + // own limits it shares the limits of the sandbox it's running in + let inner = get_sandbox_bin() + "-setuid"; + sandbox.run(&[ + "--no-config", + "--cpus=2", + &inner, + "--no-config", + "--cpus=1", + "cat", + "/sys/fs/cgroup/cpu.max", + ])?; + assert_eq!(sandbox.last_stdout.trim(), "200000 100000"); + assert!( + sandbox.last_stderr.contains( + "shares the limits of the sandbox it's running in (cpus=2 memory=unlimited)" + ), + "{}", + sandbox.last_stderr + ); + + // The warning makes it clear when that means no limits at all + sandbox.run(&["stop"])?; + sandbox.run(&["--no-config", &inner, "--no-config", "--cpus=1", "true"])?; + assert!( + sandbox.last_stderr.contains( + "shares the limits of the sandbox it's running in (cpus=unlimited memory=unlimited)" + ), + "{}", + sandbox.last_stderr + ); + Ok(()) +} + +#[rstest] +fn ai_test_limits_delete_removes_cgroup( + mut sandbox: SandboxManager, +) -> Result<()> { + if !limits_available(&mut sandbox) { + return Ok(()); + } + + // delete stops sandboxes without knowing which user started them, so + // it has to find the cgroup from the running sandbox + let cgroup = host_cgroup(&sandbox).expect("sandbox cgroup should exist"); + let name = sandbox.name.clone(); + sandbox.run(&["delete", "-y", &name])?; + assert!(!cgroup.exists()); + Ok(()) +} diff --git a/tests/ai_test_sandbox_settings.rs b/tests/ai_test_sandbox_settings.rs index 52ec207..005f948 100644 --- a/tests/ai_test_sandbox_settings.rs +++ b/tests/ai_test_sandbox_settings.rs @@ -28,6 +28,8 @@ fn create_test_config() -> ::sandbox::config::Config { bind_mounts: Vec::new(), no_default_binds: false, audio: false, + cpus: None, + memory: None, config_files: Vec::new(), } }