feat(gateway): support runtime image env overrides (#3504)

Closes #3502

Resolve trusted OCI runtime images with driver TOML, process environment, and compiled-default precedence across Docker, Podman, and Kubernetes.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

crates/openshell-core/src/config.rs

@@ -89,24 +89,48 @@ pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/superv
 /// Default OCI repository for the sandbox runtime image (no tag).
 pub const DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/sandbox";
 
-/// Return the default sandbox runtime image reference with a version-pinned tag.
-#[must_use]
-pub fn default_sandbox_runtime_image() -> String {
+/// Process-level default for the trusted sandbox runtime image.
+pub const SANDBOX_RUNTIME_IMAGE_ENV: &str = "OPENSHELL_SANDBOX_RUNTIME_IMAGE";
+
+/// Process-level default for the trusted supervisor image.
+pub const SUPERVISOR_IMAGE_ENV: &str = "OPENSHELL_SUPERVISOR_IMAGE";
+
+fn compiled_sandbox_runtime_image() -> String {
     format!(
         "{DEFAULT_SANDBOX_RUNTIME_IMAGE_REPO}:{}",
         default_supervisor_image_tag()
     )
 }
 
-/// Return the default supervisor image reference with a version-pinned tag.
-#[must_use]
-pub fn default_supervisor_image() -> String {
+fn compiled_supervisor_image() -> String {
     format!(
         "{DEFAULT_SUPERVISOR_IMAGE_REPO}:{}",
         default_supervisor_image_tag()
     )
 }
 
+fn runtime_image_default(environment_value: Option<String>, compiled_default: String) -> String {
+    environment_value.unwrap_or(compiled_default)
+}
+
+/// Return the process-configured sandbox runtime image, or the compiled default.
+#[must_use]
+pub fn default_sandbox_runtime_image() -> String {
+    runtime_image_default(
+        std::env::var(SANDBOX_RUNTIME_IMAGE_ENV).ok(),
+        compiled_sandbox_runtime_image(),
+    )
+}
+
+/// Return the process-configured supervisor image, or the compiled default.
+#[must_use]
+pub fn default_supervisor_image() -> String {
+    runtime_image_default(
+        std::env::var(SUPERVISOR_IMAGE_ENV).ok(),
+        compiled_supervisor_image(),
+    )
+}
+
 fn default_supervisor_image_tag() -> String {
     resolve_supervisor_image_tag(&[
         option_env!("OPENSHELL_IMAGE_TAG").unwrap_or(""),
@@ -1644,15 +1668,30 @@ mod tests {
 
     #[test]
     fn default_supervisor_image_is_version_pinned() {
-        use super::{default_sandbox_runtime_image, default_supervisor_image};
-        let image = default_supervisor_image();
+        use super::{compiled_sandbox_runtime_image, compiled_supervisor_image};
+        let image = compiled_supervisor_image();
         assert!(image.starts_with("ghcr.io/nvidia/openshell/supervisor:"));
         let tag = image.rsplit_once(':').unwrap().1;
         assert!(!tag.is_empty());
 
-        let sandbox_image = default_sandbox_runtime_image();
+        let sandbox_image = compiled_sandbox_runtime_image();
         assert!(sandbox_image.starts_with("ghcr.io/nvidia/openshell/sandbox:"));
         let sandbox_tag = sandbox_image.rsplit_once(':').unwrap().1;
         assert!(!sandbox_tag.is_empty());
     }
+
+    #[test]
+    fn runtime_image_environment_value_replaces_compiled_default() {
+        use super::runtime_image_default;
+
+        let digest = format!("registry.example.com/sandbox@sha256:{}", "a".repeat(64));
+        assert_eq!(
+            runtime_image_default(Some(digest.clone()), "compiled:default".to_string()),
+            digest
+        );
+        assert_eq!(
+            runtime_image_default(None, "compiled:default".to_string()),
+            "compiled:default"
+        );
+    }
 }

crates/openshell-driver-docker/src/main.rs

@@ -25,6 +25,14 @@ struct Args {
     #[arg(long, env = "OPENSHELL_DOCKER_DRIVER_CONFIG")]
     config: PathBuf,
 
+    /// OCI image containing the `openshell-sandbox` runtime binary.
+    #[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)]
+    sandbox_runtime_image: Option<String>,
+
+    /// OCI image containing the `openshell-supervisor` control binary.
+    #[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)]
+    supervisor_image: Option<String>,
+
     /// Gateway listener address used to derive the supervisor endpoint.
     #[arg(
         long,
@@ -63,6 +71,12 @@ async fn main() -> Result<()> {
         docker_config.allow_driver_config = policy.allow_driver_config;
         docker_config.resource_admission = policy.resource_admission;
     }
+    if let Some(image) = args.sandbox_runtime_image {
+        docker_config.sandbox_runtime_image = Some(image);
+    }
+    if let Some(image) = args.supervisor_image {
+        docker_config.supervisor_image = Some(image);
+    }
     let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config)
         .await
         .into_diagnostic()?;

crates/openshell-driver-docker/src/tests.rs

@@ -2503,6 +2503,23 @@ fn validate_sandbox_rejects_unknown_driver_config_fields() {
     assert!(err.message().contains("unknown field"));
 }
 
+#[test]
+fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
+    for field in ["sandbox_runtime_image", "supervisor_image"] {
+        let template = DriverSandboxTemplate {
+            driver_config: Some(json_struct(serde_json::json!({
+                (field): "registry.example.com/openshell/runtime:untrusted"
+            }))),
+            ..Default::default()
+        };
+
+        let error = DockerSandboxDriverConfig::from_template(&template)
+            .expect_err("sandbox requests must not select trusted runtime images");
+        assert!(error.contains("unknown field"), "{error}");
+        assert!(error.contains(field), "{error}");
+    }
+}
+
 #[test]
 fn validate_sandbox_accepts_gpu_count_request_shape() {
     let mut config = runtime_config();

crates/openshell-driver-kubernetes/src/driver.rs

@@ -8865,6 +8865,23 @@ mod tests {
         assert!(err.contains("unknown field"));
     }
 
+    #[test]
+    fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
+        for field in ["sandbox_runtime_image", "supervisor_image"] {
+            let template = SandboxTemplate {
+                driver_config: Some(json_struct(serde_json::json!({
+                    (field): "registry.example.com/openshell/runtime:untrusted"
+                }))),
+                ..Default::default()
+            };
+
+            let error = KubernetesSandboxDriverConfig::from_template(&template)
+                .expect_err("sandbox requests must not select trusted runtime images");
+            assert!(error.contains("unknown field"), "{error}");
+            assert!(error.contains(field), "{error}");
+        }
+    }
+
     #[test]
     fn driver_config_for_spec_rejects_unknown_fields() {
         let sandbox = Sandbox {

crates/openshell-driver-kubernetes/src/main.rs

@@ -116,13 +116,13 @@ struct Args {
     #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")]
     host_gateway_ip: Option<String>,
 
-    #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")]
+    #[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)]
     sandbox_runtime_image: Option<String>,
 
     #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY")]
     sandbox_runtime_image_pull_policy: Option<KubernetesImagePullPolicy>,
 
-    #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
+    #[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)]
     supervisor_image: Option<String>,
 
     #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")]

crates/openshell-driver-podman/src/container.rs

@@ -2313,6 +2313,25 @@ mod tests {
         assert!(err.to_string().contains("unknown field"));
     }
 
+    #[test]
+    fn sandbox_driver_config_rejects_trusted_runtime_image_overrides() {
+        use openshell_core::proto::compute::v1::DriverSandboxTemplate;
+
+        for field in ["sandbox_runtime_image", "supervisor_image"] {
+            let template = DriverSandboxTemplate {
+                driver_config: Some(json_struct(serde_json::json!({
+                    (field): "registry.example.com/openshell/runtime:untrusted"
+                }))),
+                ..Default::default()
+            };
+
+            let error = PodmanSandboxDriverConfig::from_template(&template)
+                .expect_err("sandbox requests must not select trusted runtime images");
+            assert!(error.to_string().contains("unknown field"), "{error}");
+            assert!(error.to_string().contains(field), "{error}");
+        }
+    }
+
     #[test]
     fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() {
         let sandbox = test_sandbox("test-id", "test-name");

crates/openshell-driver-podman/src/main.rs

@@ -110,11 +110,11 @@ struct Args {
     health_check_interval_secs: Option<NonZeroU64>,
 
     /// OCI image containing the `openshell-sandbox` runtime binary.
-    #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")]
+    #[arg(long, env = openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)]
     sandbox_runtime_image: Option<String>,
 
     /// OCI image containing the `openshell-supervisor` control binary.
-    #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
+    #[arg(long, env = openshell_core::config::SUPERVISOR_IMAGE_ENV)]
     supervisor_image: Option<String>,
 
     /// Host path to the CA certificate for supervisor-to-gateway TLS.

crates/openshell-gateway/src/lib.rs

@@ -42,6 +42,18 @@ use openshell_core::telemetry::TelemetryComputeDriver;
 use openshell_server::ComputeDriverRegistration;
 use openshell_server::ComputeDriverRegistry;
 
+#[cfg(all(
+    not(target_os = "windows"),
+    any(
+        feature = "compute-driver-docker",
+        feature = "compute-driver-kubernetes",
+        feature = "compute-driver-podman"
+    )
+))]
+fn prefer_environment<T>(configured: T, environment: Option<T>) -> T {
+    environment.unwrap_or(configured)
+}
+
 /// Install every first-party compute driver linked into the standard gateway.
 #[must_use]
 pub fn install_default_compute_drivers() -> ComputeDriverRegistry {
@@ -272,6 +284,14 @@ fn kubernetes_config(
 ) -> openshell_core::Result<openshell_driver_kubernetes::KubernetesComputeConfig> {
     let mut config: openshell_driver_kubernetes::KubernetesComputeConfig =
         context.driver_config()?;
+    config.sandbox_runtime_image = prefer_environment(
+        config.sandbox_runtime_image,
+        std::env::var(openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV).ok(),
+    );
+    config.supervisor_image = prefer_environment(
+        config.supervisor_image,
+        std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV).ok(),
+    );
     if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") {
         config.workspace_default_storage_size = size;
     }
@@ -296,15 +316,15 @@ impl openshell_server::ComputeDriverFactory for DockerFactory {
         &self,
         context: openshell_server::ComputeDriverConfigContext<'_>,
     ) -> openshell_core::Result<()> {
-        let config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?;
+        let config = docker_config(context)?;
         config.validate_configuration(context.gateway_bind_address())
     }
 
     async fn build(
         &self,
         context: openshell_server::ComputeDriverBuildContext<'_>,
     ) -> openshell_core::Result<openshell_server::ComputeDriverInstance> {
-        let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?;
+        let mut config = docker_config(context.config_context())?;
         require_guest_tls_for_local_driver(&context, "docker")?;
         apply_guest_tls(&mut config.guest_tls_ca, context.guest_tls_ca());
         let driver = openshell_driver_docker::DockerComputeDriver::new(
@@ -321,6 +341,26 @@ impl openshell_server::ComputeDriverFactory for DockerFactory {
     }
 }
 
+#[cfg(all(not(target_os = "windows"), feature = "compute-driver-docker"))]
+fn docker_config(
+    context: openshell_server::ComputeDriverConfigContext<'_>,
+) -> openshell_core::Result<openshell_driver_docker::DockerComputeConfig> {
+    let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?;
+    config.sandbox_runtime_image = prefer_environment(
+        config.sandbox_runtime_image,
+        std::env::var(openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV)
+            .ok()
+            .map(Some),
+    );
+    config.supervisor_image = prefer_environment(
+        config.supervisor_image,
+        std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV)
+            .ok()
+            .map(Some),
+    );
+    Ok(config)
+}
+
 #[cfg(all(not(target_os = "windows"), feature = "compute-driver-podman"))]
 #[derive(Clone, Copy)]
 struct PodmanFactory;
@@ -364,6 +404,14 @@ fn podman_config(
 ) -> openshell_core::Result<openshell_driver_podman::PodmanComputeConfig> {
     let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?;
     config.gateway_port = context.gateway_port();
+    config.sandbox_runtime_image = prefer_environment(
+        config.sandbox_runtime_image,
+        std::env::var(openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV).ok(),
+    );
+    config.supervisor_image = prefer_environment(
+        config.supervisor_image,
+        std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV).ok(),
+    );
     if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") {
         config.socket_path = Some(path.into());
     }
@@ -574,6 +622,23 @@ mod windows_tests {
 mod tests {
     use super::*;
 
+    #[test]
+    #[cfg(all(
+        not(target_os = "windows"),
+        any(
+            feature = "compute-driver-docker",
+            feature = "compute-driver-kubernetes",
+            feature = "compute-driver-podman"
+        )
+    ))]
+    fn runtime_image_environment_overrides_configured_value() {
+        assert_eq!(
+            prefer_environment("driver-toml", Some("process-environment")),
+            "process-environment"
+        );
+        assert_eq!(prefer_environment("driver-toml", None), "driver-toml");
+    }
+
     #[test]
     fn default_registry_contains_exactly_the_enabled_compute_drivers() {
         let expected: Vec<&str> = vec![

deploy/man/openshell-gateway.8.md

@@ -175,6 +175,15 @@ need a custom bundle location.
 The gateway then starts from built-in defaults and reads
 *~/.config/openshell/gateway.toml* when that file exists.
 
+The user service also reads *~/.config/openshell/gateway.env* for both config
+preflight and startup. **OPENSHELL_SANDBOX_RUNTIME_IMAGE** and
+**OPENSHELL_SUPERVISOR_IMAGE** accept complete tagged or digest-pinned OCI
+references for the built-in Docker, Podman, and Kubernetes drivers. These
+variables take precedence over explicit image fields in the selected driver's
+TOML table, which take precedence over compiled release defaults.
+These are trusted gateway inputs and cannot be supplied by sandbox requests.
+Do not embed registry credentials in image references.
+
 To persist the service across logouts:
 
     sudo loginctl enable-linger $USER

deploy/rpm/CONFIGURATION.md

@@ -51,6 +51,21 @@ editing the TOML file, add them to `~/.config/openshell/gateway.env`:
 OPENSHELL_BIND_ADDRESS=192.168.1.10
 ```
 
+To select exact trusted runtime artifacts across the built-in Docker, Podman,
+or Kubernetes driver, set complete tagged or digest-pinned references:
+
+```shell
+OPENSHELL_SANDBOX_RUNTIME_IMAGE=registry.example.com/openshell/sandbox@sha256:<digest>
+OPENSHELL_SUPERVISOR_IMAGE=registry.example.com/openshell/supervisor@sha256:<digest>
+```
+
+`OPENSHELL_SANDBOX_RUNTIME_IMAGE` and `OPENSHELL_SUPERVISOR_IMAGE` take
+precedence over explicit image fields in the selected driver's TOML table; the
+driver TOML takes precedence over the compiled release default. Preflight and
+startup use the same resolution. These values are trusted operator inputs, not
+sandbox request fields. Keep registry credentials in Podman's credential store
+rather than embedding them in image references.
+
 To override the path to the TOML config file entirely:
 
 ```shell
@@ -216,8 +231,8 @@ overrides that persist across package upgrades.
 | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. |
 | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. |
 | `[openshell.drivers.podman].default_image` | `nvcr.io/nvidia/base/ubuntu:24.04` | Default sandbox image. |
-| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. |
-| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. |
+| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Trusted sandbox runtime image. `OPENSHELL_SANDBOX_RUNTIME_IMAGE` overrides this field. |
+| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Trusted supervisor image. `OPENSHELL_SUPERVISOR_IMAGE` overrides this field. |
 | `[openshell.gateway].guest_tls_ca` | auto-generated path | Gateway CA injected into the selected local driver for supervisor-to-gateway TLS. Sandbox identity uses a bearer token. |
 | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. |
 | `disable_tls` | unset | Set to `true` to disable TLS. |