fix(docker): preserve GPU access in sandbox runtime

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

crates/openshell-driver-docker/src/isolation.rs

@@ -12,6 +12,7 @@ use std::net::IpAddr;
 use std::path::PathBuf;
 
 use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity};
+use openshell_sandbox_backend::GPU_RESOURCE_CLAIM;
 use openshell_sandbox_backend::boundary_protocol::{
     BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor,
     SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport,
@@ -28,6 +29,7 @@ pub struct DockerBoundarySpec {
     pub verification_keys: Vec<GatewayVerificationKey>,
     pub container_id: String,
     pub image_identity: String,
+    pub gpu_requested: bool,
     pub listener_socket: PathBuf,
     pub control_socket: PathBuf,
     pub sandbox_tls: SandboxTlsServerConfig,
@@ -48,10 +50,13 @@ impl DockerBoundarySpec {
     /// Docker coordinates so attach cannot bind a different container.
     #[must_use]
     pub fn provision(self) -> DockerBoundaryProvisioning {
-        let resource_claims = BTreeMap::from([
+        let mut resource_claims = BTreeMap::from([
             ("docker.container_id".to_string(), self.container_id),
             ("docker.image_identity".to_string(), self.image_identity),
         ]);
+        if self.gpu_requested {
+            resource_claims.insert(GPU_RESOURCE_CLAIM.to_string(), "true".to_string());
+        }
         let driver_fence = DriverFenceEvidence::Docker {
             container_id: resource_claims["docker.container_id"].clone(),
             network_mode: "none".to_string(),
@@ -116,6 +121,7 @@ mod tests {
             }],
             container_id: "sha256:container".to_string(),
             image_identity: "sha256:image".to_string(),
+            gpu_requested: true,
             listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"),
             control_socket: PathBuf::from("/host/control.sock"),
             sandbox_tls: SandboxTlsServerConfig {
@@ -147,6 +153,10 @@ mod tests {
             provisioned.runtime_descriptor.resource_claims["docker.container_id"],
             "sha256:container"
         );
+        assert_eq!(
+            provisioned.runtime_descriptor.resource_claims[GPU_RESOURCE_CLAIM],
+            "true"
+        );
         assert_eq!(
             provisioned.boundary_config.driver_fence,
             provisioned.runtime_descriptor.driver_fence

crates/openshell-driver-docker/src/lib.rs

@@ -1617,6 +1617,7 @@ impl DockerComputeDriver {
             &created.id,
             &image,
             &workload_identity,
+            gpu_devices.is_some(),
         )
         .await
         {
@@ -4373,6 +4374,7 @@ async fn prepare_docker_boundary_files(
     container_id: &str,
     image: &DockerImageMetadata,
     workload_identity: &ResolvedWorkloadIdentity,
+    gpu_requested: bool,
 ) -> Result<(), Status> {
     let directory = docker_boundary_state_dir(sandbox, config)?;
     let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir)
@@ -4403,6 +4405,7 @@ async fn prepare_docker_boundary_files(
         verification_keys,
         container_id: container_id.to_string(),
         image_identity: image.id.clone(),
+        gpu_requested,
         listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH),
         control_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH),
         sandbox_tls: SandboxTlsServerConfig {
@@ -5044,6 +5047,7 @@ async fn spawn_docker_control_process(
     let supervisor_id = created.id;
     let monitored_supervisor_id = supervisor_id.clone();
     let monitored_docker = failure_context.docker.clone();
+    let readiness_sandbox_id = failure_context.container_id.clone();
     let task = tokio::spawn(async move {
         let wait = async {
             let mut stream = monitored_docker.wait_container(
@@ -5134,7 +5138,9 @@ async fn spawn_docker_control_process(
         intentional_shutdown,
         task,
     };
-    if let Err(error) = wait_for_docker_supervisor_ready(docker, &supervisor_id).await {
+    if let Err(error) =
+        wait_for_docker_supervisor_ready(docker, &supervisor_id, &readiness_sandbox_id).await
+    {
         stop_docker_control_process(process).await;
         return Err(error);
     }
@@ -5144,9 +5150,23 @@ async fn spawn_docker_control_process(
 async fn wait_for_docker_supervisor_ready(
     docker: &Docker,
     supervisor_id: &str,
+    sandbox_id: &str,
 ) -> Result<(), Status> {
     let wait = async {
         loop {
+            let sandbox = docker
+                .inspect_container(sandbox_id, None)
+                .await
+                .map_err(|error| {
+                    Status::internal(format!("inspect Docker sandbox container: {error}"))
+                })?;
+            if sandbox.state.unwrap_or_default().running == Some(false) {
+                let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
+                return Err(Status::unavailable(format!(
+                    "Docker sandbox exited before supervisor became ready{}",
+                    format_named_log_tail("sandbox log tail", &sandbox_log_tail)
+                )));
+            }
             let inspected = docker
                 .inspect_container(supervisor_id, None)
                 .await
@@ -5158,16 +5178,20 @@ async fn wait_for_docker_supervisor_ready(
                 Some(HealthStatusEnum::HEALTHY) => return Ok(()),
                 Some(HealthStatusEnum::UNHEALTHY) => {
                     let log_tail = docker_container_log_tail(docker, supervisor_id).await;
+                    let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
                     return Err(Status::unavailable(format!(
-                        "Docker supervisor failed its readiness check{}",
-                        format_log_tail(&log_tail)
+                        "Docker supervisor failed its readiness check{}{}",
+                        format_log_tail(&log_tail),
+                        format_named_log_tail("sandbox log tail", &sandbox_log_tail)
                     )));
                 }
                 _ if state.running == Some(false) => {
                     let log_tail = docker_container_log_tail(docker, supervisor_id).await;
+                    let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
                     return Err(Status::unavailable(format!(
-                        "Docker supervisor exited before becoming ready{}",
-                        format_log_tail(&log_tail)
+                        "Docker supervisor exited before becoming ready{}{}",
+                        format_log_tail(&log_tail),
+                        format_named_log_tail("sandbox log tail", &sandbox_log_tail)
                     )));
                 }
                 _ => tokio::time::sleep(Duration::from_millis(100)).await,
@@ -5179,19 +5203,25 @@ async fn wait_for_docker_supervisor_ready(
         result
     } else {
         let log_tail = docker_container_log_tail(docker, supervisor_id).await;
+        let sandbox_log_tail = docker_container_log_tail(docker, sandbox_id).await;
         Err(Status::deadline_exceeded(format!(
-            "Docker supervisor did not become ready within {} seconds{}",
+            "Docker supervisor did not become ready within {} seconds{}{}",
             SUPERVISOR_READY_TIMEOUT.as_secs(),
-            format_log_tail(&log_tail)
+            format_log_tail(&log_tail),
+            format_named_log_tail("sandbox log tail", &sandbox_log_tail)
         )))
     }
 }
 
 fn format_log_tail(log_tail: &str) -> String {
+    format_named_log_tail("log tail", log_tail)
+}
+
+fn format_named_log_tail(label: &str, log_tail: &str) -> String {
     if log_tail.is_empty() {
         String::new()
     } else {
-        format!("; log tail: {log_tail}")
+        format!("; {label}: {log_tail}")
     }
 }
 

crates/openshell-sandbox-backend/src/lib.rs

@@ -18,6 +18,9 @@ pub use runtime::OpenShellRuntimeBackend;
 /// Stable isolation backend name implemented by `openshell-sandbox`.
 pub const BACKEND_NAME: &str = "openshell-sandbox";
 
+/// Resource claim set by compute drivers when the workload requests GPU access.
+pub const GPU_RESOURCE_CLAIM: &str = "openshell.gpu";
+
 /// Memory-backed parent used for supervisor CA material.
 pub const SUPERVISOR_CA_RUNTIME_ROOT: &str = "/run/openshell-supervisor-ca";
 

crates/openshell-sandbox/src/boundary_server.rs

@@ -40,6 +40,7 @@ mod linux {
         CapabilityEvidence, ExecSession, LoopbackTarget, ResolvedWorkloadIdentity,
         SandboxConfirmEvidence,
     };
+    use openshell_sandbox_backend::GPU_RESOURCE_CLAIM;
     use openshell_sandbox_backend::mediation::{
         self, DnsQueryWire, MediationFrame, MediationFrameKind,
     };
@@ -80,10 +81,86 @@ mod linux {
     const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096;
     const MAX_RETAINED_EXEC_PROCESSES: usize = 64;
 
+    const GPU_BASELINE_READ_ONLY: &[&str] = &["/run/nvidia-persistenced", "/usr/lib/wsl"];
+    const GPU_BASELINE_READ_WRITE: &[&str] = &[
+        "/dev/nvidiactl",
+        "/dev/nvidia-uvm",
+        "/dev/nvidia-uvm-tools",
+        "/dev/nvidia-modeset",
+        "/dev/dxg",
+        "/proc",
+    ];
+
     fn duration_micros(duration: Duration) -> u64 {
         u64::try_from(duration.as_micros()).unwrap_or(u64::MAX)
     }
 
+    /// Add the filesystem paths required by GPU devices visible inside the
+    /// workload container. The companion supervisor intentionally has no GPU
+    /// devices, so it cannot discover these paths on the sandbox's behalf.
+    fn enrich_gpu_filesystem_paths(
+        policy: &mut openshell_core::policy::SandboxPolicy,
+        gpu_requested: bool,
+    ) -> bool {
+        if !gpu_requested {
+            return false;
+        }
+        let has_gpu = Path::new("/dev/nvidiactl").exists() || Path::new("/dev/dxg").exists();
+        if !has_gpu {
+            return false;
+        }
+
+        let mut read_write = GPU_BASELINE_READ_WRITE
+            .iter()
+            .copied()
+            .map(std::path::PathBuf::from)
+            .collect::<Vec<_>>();
+        if let Ok(entries) = std::fs::read_dir("/dev") {
+            read_write.extend(entries.flatten().filter_map(|entry| {
+                let name = entry.file_name();
+                let suffix = name.to_str()?.strip_prefix("nvidia")?;
+                (!suffix.is_empty() && suffix.chars().all(|character| character.is_ascii_digit()))
+                    .then(|| entry.path())
+            }));
+        }
+
+        let mut modified = false;
+        for path in GPU_BASELINE_READ_ONLY.iter().copied().map(Path::new) {
+            if path.exists()
+                && !policy
+                    .filesystem
+                    .read_only
+                    .iter()
+                    .any(|allowed| allowed == path)
+                && !policy
+                    .filesystem
+                    .read_write
+                    .iter()
+                    .any(|allowed| allowed == path)
+            {
+                policy.filesystem.read_only.push(path.to_path_buf());
+                modified = true;
+            }
+        }
+        for path in read_write {
+            if !path.exists() || policy.filesystem.read_write.contains(&path) {
+                continue;
+            }
+            if policy.filesystem.read_only.contains(&path) {
+                if path != Path::new("/proc") {
+                    continue;
+                }
+                policy
+                    .filesystem
+                    .read_only
+                    .retain(|allowed| allowed != &path);
+            }
+            policy.filesystem.read_write.push(path);
+            modified = true;
+        }
+        modified
+    }
+
     struct ControlConnectionSlot(Arc<AtomicUsize>);
 
     impl Drop for ControlConnectionSlot {
@@ -2247,6 +2324,21 @@ mod linux {
                 Err(error) => return guest_error(BoundaryErrorKind::Process, error),
             };
             let mut policy = policy.into();
+            let gpu_requested = self
+                .config
+                .resource_claims
+                .get(GPU_RESOURCE_CLAIM)
+                .is_some_and(|value| value == "true");
+            if enrich_gpu_filesystem_paths(&mut policy, gpu_requested) {
+                openshell_ocsf::ocsf_emit!(
+                    openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx())
+                        .severity(openshell_ocsf::SeverityId::Informational)
+                        .status(openshell_ocsf::StatusId::Success)
+                        .state(openshell_ocsf::StateId::Enabled, "enriched")
+                        .message("Added workload-local GPU filesystem paths".to_string())
+                        .build()
+                );
+            }
             let driver_identity = DriverIdentity::Resolved {
                 uid: self.config.workload_identity.uid,
                 gid: self.config.workload_identity.gid,