diff --git a/architecture/build.md b/architecture/build.md index b6c464bbe5..a82f63c2cb 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -292,11 +292,12 @@ for direct executable installation on every environment. Release Dev and Release Tag run Ubuntu conformance through the Debian package, while Fedora continues using direct executable installation until RPM coverage is available. The Debian qualification profile keeps candidate-image overrides outside the -operator-owned gateway configuration: it writes a harness-owned file under -`/var/lib/openshell-qualification` and selects it through the packaged systemd -unit's `gateway.env` hook. Ordinary package installations continue to use the -gateway's built-in runtime-image defaults unless the operator configures an -override. +operator-owned gateway configuration. It supplies the Docker selector and exact +sandbox-runtime and supervisor references through the packaged systemd unit's +`gateway.env` hook, so config preflight and actual startup resolve the same +artifacts without generating `gateway.toml`. Ordinary package installations +continue to use the gateway's compiled runtime-image defaults unless the +operator supplies process defaults or explicit driver TOML values. ## Python Wheel Packaging diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3d8b6eda8b..b7a6ec7620 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -89,24 +89,48 @@ pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/superv /// Default OCI repository for the sandbox runtime image (no tag). pub const DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/sandbox"; -/// Return the default sandbox runtime image reference with a version-pinned tag. -#[must_use] -pub fn default_sandbox_runtime_image() -> String { +/// Process-level default for the trusted sandbox runtime image. +pub const SANDBOX_RUNTIME_IMAGE_ENV: &str = "OPENSHELL_SANDBOX_RUNTIME_IMAGE"; + +/// Process-level default for the trusted supervisor image. +pub const SUPERVISOR_IMAGE_ENV: &str = "OPENSHELL_SUPERVISOR_IMAGE"; + +fn compiled_sandbox_runtime_image() -> String { format!( "{DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO}:{}", default_supervisor_image_tag() ) } -/// Return the default supervisor image reference with a version-pinned tag. -#[must_use] -pub fn default_supervisor_image() -> String { +fn compiled_supervisor_image() -> String { format!( "{DEFAULT_SUPERVISOR_IMAGE_REPO}:{}", default_supervisor_image_tag() ) } +fn runtime_image_default(environment_value: Option, compiled_default: String) -> String { + environment_value.unwrap_or(compiled_default) +} + +/// Return the process-configured sandbox runtime image, or the compiled default. +#[must_use] +pub fn default_sandbox_runtime_image() -> String { + runtime_image_default( + std::env::var(SANDBOX_RUNTIME_IMAGE_ENV).ok(), + compiled_sandbox_runtime_image(), + ) +} + +/// Return the process-configured supervisor image, or the compiled default. +#[must_use] +pub fn default_supervisor_image() -> String { + runtime_image_default( + std::env::var(SUPERVISOR_IMAGE_ENV).ok(), + compiled_supervisor_image(), + ) +} + fn default_supervisor_image_tag() -> String { resolve_supervisor_image_tag(&[ option_env!("OPENSHELL_IMAGE_TAG").unwrap_or(""), @@ -1617,15 +1641,30 @@ mod tests { #[test] fn default_supervisor_image_is_version_pinned() { - use super::{default_sandbox_runtime_image, default_supervisor_image}; - let image = default_supervisor_image(); + use super::{compiled_sandbox_runtime_image, compiled_supervisor_image}; + let image = compiled_supervisor_image(); assert!(image.starts_with("ghcr.io/nvidia/openshell/supervisor:")); let tag = image.rsplit_once(':').unwrap().1; assert!(!tag.is_empty()); - let sandbox_image = default_sandbox_runtime_image(); + let sandbox_image = compiled_sandbox_runtime_image(); assert!(sandbox_image.starts_with("ghcr.io/nvidia/openshell/sandbox:")); let sandbox_tag = sandbox_image.rsplit_once(':').unwrap().1; assert!(!sandbox_tag.is_empty()); } + + #[test] + fn runtime_image_environment_value_replaces_compiled_default() { + use super::runtime_image_default; + + let digest = format!("registry.example.com/sandbox@sha256:{}", "a".repeat(64)); + assert_eq!( + runtime_image_default(Some(digest.clone()), "compiled:default".to_string()), + digest + ); + assert_eq!( + runtime_image_default(None, "compiled:default".to_string()), + "compiled:default" + ); + } } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 5b767979c7..838def5a56 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2133,6 +2133,23 @@ fn validate_sandbox_rejects_unknown_driver_config_fields() { assert!(err.message().contains("unknown field")); } +#[test] +fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() { + for field in ["sandbox_runtime_image", "supervisor_image"] { + let template = DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + (field): "registry.example.com/openshell/runtime:untrusted" + }))), + ..Default::default() + }; + + let error = DockerSandboxDriverConfig::from_template(&template) + .expect_err("sandbox requests must not select trusted runtime images"); + assert!(error.contains("unknown field"), "{error}"); + assert!(error.contains(field), "{error}"); + } +} + #[test] fn validate_sandbox_accepts_gpu_count_request_shape() { let mut config = runtime_config(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index cbdd5bcc7f..a02348ebdc 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -8019,6 +8019,23 @@ mod tests { assert!(err.contains("unknown field")); } + #[test] + fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() { + for field in ["sandbox_runtime_image", "supervisor_image"] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + (field): "registry.example.com/openshell/runtime:untrusted" + }))), + ..Default::default() + }; + + let error = KubernetesSandboxDriverConfig::from_template(&template) + .expect_err("sandbox requests must not select trusted runtime images"); + assert!(error.contains("unknown field"), "{error}"); + assert!(error.contains(field), "{error}"); + } + } + #[test] fn driver_config_for_spec_rejects_unknown_fields() { let sandbox = Sandbox { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index fbcaf1d5f6..6311aa8590 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -109,13 +109,13 @@ struct Args { #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] host_gateway_ip: Option, - #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")] + #[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)] sandbox_runtime_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY")] sandbox_runtime_image_pull_policy: Option, - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + #[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)] supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 572f741ac2..8f98ca87ba 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -2183,6 +2183,25 @@ mod tests { assert!(err.to_string().contains("unknown field")); } + #[test] + fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() { + use openshell_core::proto::compute::v1::DriverSandboxTemplate; + + for field in ["sandbox_runtime_image", "supervisor_image"] { + let template = DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + (field): "registry.example.com/openshell/runtime:untrusted" + }))), + ..Default::default() + }; + + let error = PodmanSandboxDriverConfig::from_template(&template) + .expect_err("sandbox requests must not select trusted runtime images"); + assert!(error.to_string().contains("unknown field"), "{error}"); + assert!(error.to_string().contains(field), "{error}"); + } + } + #[test] fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 93a3f92966..123dfb18d8 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -102,11 +102,11 @@ struct Args { health_check_interval_secs: Option, /// OCI image containing the `openshell-sandbox` runtime binary. - #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")] + #[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)] sandbox_runtime_image: Option, /// OCI image containing the `openshell-supervisor` control binary. - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + #[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)] supervisor_image: Option, /// Host path to the CA certificate for sandbox mTLS. diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index 32dcaeb24c..29f509f2dd 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -21,6 +21,7 @@ openshell-otel = { path = "../openshell-otel", optional = true } async-trait = "0.1" miette = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } [target.'cfg(not(target_os = "windows"))'.dependencies] openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } @@ -33,7 +34,6 @@ serde = { workspace = true, optional = true } rustix = { workspace = true, optional = true } tonic = { workspace = true, optional = true } tower = { workspace = true, optional = true } -tracing = { workspace = true, optional = true } [target.'cfg(target_os = "windows")'.dependencies] openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true } @@ -60,7 +60,6 @@ compute-driver-vm = [ "dep:rustix", "dep:tonic", "dep:tower", - "dep:tracing", ] telemetry = ["openshell-core/telemetry", "openshell-server/telemetry"] ## Convenience alias: every default feature except `telemetry`. Build a diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index b67e93c102..70e623cca6 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -42,6 +42,66 @@ use openshell_core::telemetry::TelemetryComputeDriver; use openshell_server::ComputeDriverRegistration; use openshell_server::ComputeDriverRegistry; +#[cfg(all( + not(target_os = "windows"), + any( + feature = "compute-driver-docker", + feature = "compute-driver-kubernetes", + feature = "compute-driver-podman" + ) +))] +fn runtime_image_source( + context: openshell_server::ComputeDriverConfigContext<'_>, + field: &str, + environment_variable: &str, +) -> &'static str { + if context.driver_config_field_is_explicit(field) { + "driver_toml" + } else if std::env::var(environment_variable).is_ok() { + "process_environment" + } else { + "compiled_default" + } +} + +#[cfg(all( + not(target_os = "windows"), + any( + feature = "compute-driver-docker", + feature = "compute-driver-kubernetes", + feature = "compute-driver-podman" + ) +))] +fn log_trusted_runtime_images( + context: openshell_server::ComputeDriverConfigContext<'_>, + driver_name: &str, + sandbox_runtime_image: &str, + supervisor_image: &str, + sandbox_runtime_active: bool, +) { + tracing::info!( + compute_driver = driver_name, + image = sandbox_runtime_image, + configuration_source = runtime_image_source( + context, + "sandbox_runtime_image", + openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV, + ), + active = sandbox_runtime_active, + "resolved trusted sandbox runtime image" + ); + tracing::info!( + compute_driver = driver_name, + image = supervisor_image, + configuration_source = runtime_image_source( + context, + "supervisor_image", + openshell_core::config::SUPERVISOR_IMAGE_ENV, + ), + "resolved trusted supervisor image" + ); +} + /// Install every first-party compute driver linked into the standard gateway. #[must_use] pub fn install_default_compute_drivers() -> ComputeDriverRegistry { @@ -253,7 +313,15 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let config = kubernetes_config(context.config_context())?; + let config_context = context.config_context(); + let config = kubernetes_config(config_context)?; + log_trusted_runtime_images( + config_context, + "kubernetes", + &config.sandbox_runtime_image, + &config.supervisor_image, + true, + ); let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( config, context.shutdown_receiver(), @@ -305,7 +373,24 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + let config_context = context.config_context(); + let mut config: openshell_driver_docker::DockerComputeConfig = + config_context.driver_config()?; + let sandbox_runtime_image = config + .sandbox_runtime_image + .clone() + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image); + let supervisor_image = config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + log_trusted_runtime_images( + config_context, + "docker", + &sandbox_runtime_image, + &supervisor_image, + config.supervisor_bin.is_none(), + ); require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls( &mut config.guest_tls_ca, @@ -351,7 +436,15 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config = podman_config(context.config_context())?; + let config_context = context.config_context(); + let mut config = podman_config(config_context)?; + log_trusted_runtime_images( + config_context, + "podman", + &config.sandbox_runtime_image, + &config.supervisor_image, + true, + ); require_guest_tls_for_local_driver(&context, "podman")?; apply_guest_tls( &mut config.guest_tls_ca, diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f264f4752f..5904db8e0a 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -151,6 +151,24 @@ where driver_config_from_file(context.file, driver_name) } +/// Return whether the selected driver's TOML table explicitly contains `field`. +/// +/// Driver config defaults cannot answer this reliably because an authored value +/// may be identical to an environment or compiled default. Callers use this +/// metadata only to apply gateway-owned precedence around the deserialized +/// driver configuration. +pub fn driver_config_field_is_explicit( + context: DriverStartupContext<'_>, + driver_name: &str, + field: &str, +) -> bool { + context + .file + .and_then(|file| file.openshell.drivers.get(driver_name)) + .and_then(toml::Value::as_table) + .is_some_and(|table| table.contains_key(field)) +} + fn driver_config_from_file( file: Option<&config_file::ConfigFile>, driver_name: &str, @@ -519,4 +537,40 @@ socket_path = "/run/openshell/kyma.sock" .contains("remote compute driver 'kyma' requires socket_path") ); } + + #[test] + fn explicit_driver_field_detection_uses_only_the_selected_toml_table() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell] +version = 2 + +[openshell.drivers.docker] +sandbox_runtime_image = "registry.example.com/openshell/sandbox:toml" +"#, + ) + .expect("valid config"); + let context = test_context(Some(&file)); + + assert!(driver_config_field_is_explicit( + context, + "docker", + "sandbox_runtime_image" + )); + assert!(!driver_config_field_is_explicit( + context, + "docker", + "supervisor_image" + )); + assert!(!driver_config_field_is_explicit( + context, + "podman", + "sandbox_runtime_image" + )); + assert!(!driver_config_field_is_explicit( + test_context(None), + "docker", + "sandbox_runtime_image" + )); + } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8cd09acca..30c1d9ad1b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1397,6 +1397,17 @@ impl ComputeDriverConfigContext<'_> { { compute::driver_config::driver_config_from_context(self.driver_startup, self.driver_name) } + + /// Return whether the selected driver's TOML table explicitly contains + /// `field`. + #[must_use] + pub fn driver_config_field_is_explicit(&self, field: &str) -> bool { + compute::driver_config::driver_config_field_is_explicit( + self.driver_startup, + self.driver_name, + field, + ) + } } pub struct ComputeDriverBuildContext<'a> { diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 9be010095a..633ffb3397 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -171,6 +171,16 @@ need a custom bundle location. The gateway then starts from built-in defaults and reads *~/.config/openshell/gateway.toml* when that file exists. +The user service also reads *~/.config/openshell/gateway.env* for both config +preflight and startup. **OPENSHELL_SANDBOX_RUNTIME_IMAGE** and +**OPENSHELL_SUPERVISOR_IMAGE** accept complete tagged or digest-pinned OCI +references for the built-in Docker, Podman, and Kubernetes drivers. Explicit +image fields in the selected driver's TOML table take precedence over these +variables; the variables take precedence over compiled release defaults. +Startup diagnostics report the effective references and their configuration +source. These are trusted gateway inputs and cannot be supplied by sandbox +requests. Do not embed registry credentials in image references. + To persist the service across logouts: sudo loginctl enable-linger $USER diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 7ae1f6f15e..e4e5f19b8b 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -51,6 +51,21 @@ editing the TOML file, add them to `~/.config/openshell/gateway.env`: OPENSHELL_BIND_ADDRESS=192.168.1.10 ``` +To select exact trusted runtime artifacts across the built-in Docker, Podman, +or Kubernetes driver, set complete tagged or digest-pinned references: + +```shell +OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256: +OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor@sha256: +``` + +An explicit `sandbox_runtime_image` or `supervisor_image` in the selected +driver's TOML table takes precedence over the environment; the environment +takes precedence over the compiled release default. Preflight and startup use +the same resolution. These values are trusted operator inputs, not sandbox +request fields. Keep registry credentials in Podman's credential store rather +than embedding them in image references. + To override the path to the TOML config file entirely: ```shell @@ -221,8 +236,8 @@ overrides that persist across package upgrades. | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | | `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. | -| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. | +| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:` | Trusted sandbox runtime image. `OPENSHELL_SANDBOX_RUNTIME_IMAGE` supplies a process default when this field is omitted. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:` | Trusted supervisor image. `OPENSHELL_SUPERVISOR_IMAGE` supplies a process default when this field is omitted. | | `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index e27d378381..1b9da3e531 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -77,6 +77,21 @@ Linux packages require glibc 2.28 or newer. The installer checks libc before dow The Linux user service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle before the gateway starts. Debian uses built-in gateway defaults unless you create a config. RPM seeds `~/.config/openshell/gateway.toml` from its packaged Podman template on first start. RPM upgrades migrate only an unchanged package-generated schema-v1 file; they preserve edited files. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) when upgrading an edited v1 configuration. +Both Debian and RPM load `~/.config/openshell/gateway.env` for configuration +preflight and gateway startup. Use it to select exact trusted runtime artifacts +without creating driver-specific TOML: + +```shell +OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256: +OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor@sha256: +``` + +Explicit image fields in the selected Docker, Podman, or Kubernetes TOML table +take precedence. Restart the user service after editing the file. Treat these +variables as trusted operator configuration; sandbox requests cannot set them, +and registry credentials must remain in the container runtime's credential +store or Kubernetes image-pull secrets. + The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. The installer starts the service for you. Use systemd user commands when you need to inspect, restart, or stop the gateway service: diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ead6f89f11..e0b426713b 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -18,6 +18,38 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa `database_url` is env-only. The loader rejects it when it appears in the file. When `OPENSHELL_DB_URL` is unset, the gateway stores its SQLite database under `$XDG_STATE_HOME/openshell/gateway/openshell.db`. +### Trusted runtime image overrides + +Docker, Podman, and Kubernetes consume two gateway-owned trusted runtime +images. Set complete tagged or digest-pinned OCI references for the gateway +process when you need to qualify exact artifacts or use a registry mirror: + +```shell +OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256: +OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor:1.2.3 +``` + +These two settings use driver-specific precedence because explicit operator +configuration must remain authoritative: + +```text +[openshell.drivers.] TOML > process environment > compiled release default +``` + +`openshell-gateway config preflight` and gateway startup use the same effective +references. Startup logs report each reference and whether it came from +`driver_toml`, `process_environment`, or `compiled_default`. Environment values +are handled exactly like the corresponding TOML fields and passed to the +selected container runtime. Configure registry credentials in Docker, Podman, +or Kubernetes image-pull secrets instead of embedding them in a reference. + +The variables affect only the built-in Docker, Podman, and Kubernetes drivers. +They do not affect VM, MXC, or extension drivers. They are trusted gateway +operator inputs and are never accepted from sandbox create requests. For +Debian and RPM user services, place them in +`~/.config/openshell/gateway.env`; the service supplies that file to both +preflight and gateway startup. + `name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `server.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`. ## Package-Managed Locations diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index fab785e91f..f5f3cbea06 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -64,6 +64,17 @@ Common gateway options: Set driver-specific values such as sandbox images, gateway endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Docker, Podman, and Kubernetes also accept gateway-process defaults for the +trusted sandbox-runtime and supervisor images through +`OPENSHELL_SANDBOX_RUNTIME_IMAGE` and `OPENSHELL_SUPERVISOR_IMAGE`. Complete +tags and digest-pinned OCI references are supported. An explicit +`sandbox_runtime_image` or `supervisor_image` in the selected driver's TOML +table overrides the corresponding environment value; the environment value +overrides the release default. Helm normally renders explicit Kubernetes +driver fields, so those chart-selected values remain authoritative. VM and +extension drivers do not use these variables. Sandbox requests cannot override +either trusted image. + Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point the gateway at the Unix socket the operator has already provisioned: diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8fccf3779d..f925157ce6 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -252,6 +252,12 @@ Common findings: - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. +- Trusted runtime image mismatch: inspect gateway startup diagnostics for the + effective sandbox-runtime and supervisor references and their source. For + Docker, Podman, and Kubernetes, selected-driver TOML overrides + `OPENSHELL_SANDBOX_RUNTIME_IMAGE` / `OPENSHELL_SUPERVISOR_IMAGE`, which + override the compiled release defaults. Package services read these variables + from `~/.config/openshell/gateway.env`. Sandbox requests cannot set them. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. - Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. If several paths share a host and port, a failure before the path is known remains in logs. Verify the actual operation when current tool availability matters. - On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages diff --git a/tasks/scripts/test-packaging-assets.sh b/tasks/scripts/test-packaging-assets.sh index fb6424ff06..b9acb6b904 100755 --- a/tasks/scripts/test-packaging-assets.sh +++ b/tasks/scripts/test-packaging-assets.sh @@ -40,10 +40,12 @@ assert_file_exists() { service="${ROOT}/deploy/deb/openshell-gateway.service" control="${ROOT}/deploy/deb/control.in" spec="${ROOT}/openshell.spec" +deb_qualification="${ROOT}/tests/ansible/playbooks/openshell-deb.yaml" assert_file_exists "$service" assert_file_exists "$control" assert_file_exists "$spec" +assert_file_exists "$deb_qualification" # Debian control files are RFC822-style metadata. Older dpkg-deb releases # reject comment lines as malformed fields, so keep SPDX metadata in the @@ -60,6 +62,8 @@ fi assert_contains \ "$service" \ 'Environment=OPENSHELL_LOCAL_TLS_DIR=%h/.local/state/openshell/tls' +assert_contains "$service" 'EnvironmentFile=-%E/openshell/gateway.env' +assert_contains "$service" 'ExecStart=/usr/bin/openshell-gateway' assert_contains \ "$service" \ 'ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal' @@ -68,6 +72,8 @@ assert_not_contains "$service" '%S/openshell/tls' assert_contains \ "$spec" \ 'Environment=OPENSHELL_LOCAL_TLS_DIR=%%h/.local/state/openshell/tls' +assert_contains "$spec" 'EnvironmentFile=-%%E/openshell/gateway.env' +assert_contains "$spec" 'ExecStart=/usr/bin/openshell-gateway' assert_contains \ "$spec" \ 'ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal' @@ -77,6 +83,16 @@ assert_contains "$spec" '%files prover' assert_contains "$spec" '%{_bindir}/%{name}-prover' assert_not_contains "$spec" '%%S/openshell/tls' +# Installed-package qualification must select the candidate trusted runtime +# images through the same environment file consumed by preflight and startup, +# without generating operator-owned gateway TOML. +assert_contains "$deb_qualification" 'dest: /home/tmachine/.config/openshell/gateway.env' +assert_contains "$deb_qualification" 'OPENSHELL_COMPUTE_DRIVER=docker' +assert_contains "$deb_qualification" 'OPENSHELL_SANDBOX_RUNTIME_IMAGE=docker.io/openshell/sandbox:tmachine' +assert_contains "$deb_qualification" 'OPENSHELL_SUPERVISOR_IMAGE=docker.io/openshell/supervisor:tmachine' +assert_not_contains "$deb_qualification" 'OPENSHELL_GATEWAY_CONFIG=' +assert_not_contains "$deb_qualification" '/var/lib/openshell-qualification/gateway.toml' + # Schema-v2 package startup wiring. snap_wrapper="${ROOT}/tasks/scripts/snap-gateway-wrapper.sh" package_deb="${ROOT}/tasks/scripts/package-deb.sh" diff --git a/tests/ansible/playbooks/openshell-deb.yaml b/tests/ansible/playbooks/openshell-deb.yaml index f1589374d2..f630c3f1bb 100644 --- a/tests/ansible/playbooks/openshell-deb.yaml +++ b/tests/ansible/playbooks/openshell-deb.yaml @@ -45,45 +45,23 @@ - openshell-sandbox - openshell-supervisor - # The package normally starts from its built-in runtime-image defaults. - # Qualification instead pins the candidate images staged by tmachine, so - # keep that override separate from the operator-owned gateway.toml. - - name: Create OpenShell qualification configuration directory - become: true - ansible.builtin.file: - path: /var/lib/openshell-qualification - state: directory - owner: root - group: root - mode: "0755" - - - name: Configure candidate OpenShell runtime images for qualification - become: true - ansible.builtin.copy: - dest: /var/lib/openshell-qualification/gateway.toml - owner: root - group: root - mode: "0644" - content: | - [openshell] - version = 2 - - [openshell.drivers.docker] - sandbox_runtime_image = "docker.io/openshell/sandbox:tmachine" - supervisor_image = "docker.io/openshell/supervisor:tmachine" - - name: Create OpenShell environment directory ansible.builtin.file: path: /home/tmachine/.config/openshell state: directory mode: "0700" - - name: Select qualification gateway configuration + # Exercise the package-owned service and its existing environment-file + # mechanism without generating operator-owned gateway TOML. Both preflight + # and daemon startup must resolve these exact candidate runtime images. + - name: Configure candidate OpenShell runtime images for qualification ansible.builtin.copy: dest: /home/tmachine/.config/openshell/gateway.env mode: "0600" content: | - OPENSHELL_GATEWAY_CONFIG=/var/lib/openshell-qualification/gateway.toml + OPENSHELL_COMPUTE_DRIVER=docker + OPENSHELL_SANDBOX_RUNTIME_IMAGE=docker.io/openshell/sandbox:tmachine + OPENSHELL_SUPERVISOR_IMAGE=docker.io/openshell/supervisor:tmachine - name: Start tmachine user manager ansible.builtin.include_role: