fix(ssh): persist sandbox host identities (#4094)

* fix(ssh): persist sandbox host identities

Store each sandbox's Ed25519 host key in the gateway credential store and
deliver it only to the supervisor. Preserve identity across restarts,
delete owned credentials with the sandbox, and expose the public SHA256
fingerprint through sandbox and SSH-session APIs and client SDKs.

Cover credential ownership, cancellation, deletion retries, client
compatibility, and pinned SSH connections through lifecycle transitions.

Closes #3835

Signed-off-by: Mike Nguyen <miken@nvidia.com>

* fix(compute): clean up failed sandbox SSH identity creation

Signed-off-by: Mike Nguyen <miken@nvidia.com>

* test(ssh): wait for sandbox deletion before name reuse

Signed-off-by: Mike Nguyen <miken@nvidia.com>

---------

Signed-off-by: Mike Nguyen <miken@nvidia.com>

Author: quocanh261997

Cargo.lock

@@ -2995,6 +2995,7 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value {
         "id": sandbox.object_id(),
         "name": sandbox.object_name(),
         "workspace": sandbox.object_workspace(),
+        "host_key_fingerprint": sandbox.host_key_fingerprint,
         "labels": labels,
         "annotations": annotations,
         "resource_version": meta.map_or(0, |m| m.resource_version),

crates/openshell-cli/src/run.rs

@@ -229,6 +229,46 @@ mod session {
         }
     }
 
+    /// Private SSH key material is redacted from diagnostics and cleared on drop.
+    #[derive(Clone)]
+    pub struct SecretSshHostKey(Zeroizing<String>);
+
+    impl SecretSshHostKey {
+        #[must_use]
+        pub fn new(value: String) -> Self {
+            Self(Zeroizing::new(value))
+        }
+
+        #[must_use]
+        pub fn expose_secret(&self) -> &str {
+            &self.0
+        }
+    }
+
+    impl fmt::Debug for SecretSshHostKey {
+        fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+            formatter.write_str("SecretSshHostKey([REDACTED])")
+        }
+    }
+
+    impl Serialize for SecretSshHostKey {
+        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+        where
+            S: serde::Serializer,
+        {
+            serializer.serialize_str(self.expose_secret())
+        }
+    }
+
+    impl<'de> Deserialize<'de> for SecretSshHostKey {
+        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where
+            D: serde::Deserializer<'de>,
+        {
+            String::deserialize(deserializer).map(Self::new)
+        }
+    }
+
     /// Trusted launch input delivered only to `openshell-supervisor`.
     #[derive(Clone, Serialize, Deserialize)]
     #[serde(deny_unknown_fields)]
@@ -244,6 +284,9 @@ mod session {
         pub gateway_expires_at: i64,
         pub sandbox_token: SecretJwt,
         pub sandbox_expires_at: i64,
+        /// Never delivered to the workload. Missing only in older bundles.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        pub ssh_host_private_key: Option<SecretSshHostKey>,
     }
 
     /// Gateway-created authentication input trusted by a compute driver.
@@ -329,6 +372,7 @@ mod session {
                 .field("gateway_expires_at", &self.gateway_expires_at)
                 .field("sandbox_token", &"[REDACTED]")
                 .field("sandbox_expires_at", &self.sandbox_expires_at)
+                .field("ssh_host_private_key", &self.ssh_host_private_key)
                 .finish()
         }
     }
@@ -1034,6 +1078,7 @@ mod tests {
                 gateway_expires_at: pair.gateway.expires_at,
                 sandbox_token: pair.sandbox.token,
                 sandbox_expires_at: pair.sandbox.expires_at,
+                ssh_host_private_key: None,
             };
             bundle.validate().expect("non-expiring auth bundle");
         }
@@ -1060,6 +1105,9 @@ mod tests {
                 gateway_expires_at: pair.gateway.expires_at,
                 sandbox_token: pair.sandbox.token,
                 sandbox_expires_at: pair.sandbox.expires_at,
+                ssh_host_private_key: Some(SecretSshHostKey::new(
+                    "private-ssh-host-key".to_string(),
+                )),
             };
 
             let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle");
@@ -1078,7 +1126,19 @@ mod tests {
             let debug = format!("{bundle:?}");
             assert!(!debug.contains(bundle.gateway_token.expose_secret()));
             assert!(!debug.contains(bundle.sandbox_token.expose_secret()));
-            assert_eq!(debug.matches("[REDACTED]").count(), 2);
+            assert!(!debug.contains("private-ssh-host-key"));
+            assert_eq!(debug.matches("[REDACTED]").count(), 3);
+            assert_eq!(
+                decoded.ssh_host_private_key.unwrap().expose_secret(),
+                "private-ssh-host-key"
+            );
+            let mut legacy = serde_json::to_value(&bundle).unwrap();
+            legacy
+                .as_object_mut()
+                .unwrap()
+                .remove("ssh_host_private_key");
+            let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap();
+            assert!(legacy.ssh_host_private_key.is_none());
         }
 
         #[derive(Serialize)]

crates/openshell-core/src/jwt.rs

@@ -63,6 +63,7 @@ fn test_launch_authentication() -> Vec<u8> {
             gateway_expires_at: i64::MAX,
             sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
             sandbox_expires_at: i64::MAX,
+            ssh_host_private_key: None,
         },
         gateway_id: "gateway-test".to_string(),
         verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-docker/src/tests.rs

@@ -2106,6 +2106,7 @@ mod tests {
                 gateway_expires_at: i64::MAX,
                 sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
                 sandbox_expires_at: i64::MAX,
+                ssh_host_private_key: None,
             },
             gateway_id: "gateway-test".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-podman/src/driver.rs

@@ -445,6 +445,7 @@ mod tests {
                 gateway_expires_at: i64::MAX,
                 sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
                 sandbox_expires_at: i64::MAX,
+                ssh_host_private_key: None,
             },
             gateway_id: "gateway-test".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-podman/src/isolation.rs

@@ -10047,6 +10047,7 @@ mod tests {
                 gateway_expires_at: 1,
                 sandbox_token: SecretJwt::parse(format!("sandbox-{label}")).expect("sandbox token"),
                 sandbox_expires_at: 1,
+                ssh_host_private_key: None,
             },
             gateway_id: "gateway-a".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-vm/src/driver.rs

@@ -184,7 +184,7 @@ async fn local_vm_rejects_hanging_tool_with_deadline() {
     let output = fixture.run(&[]).await;
     assert!(!output.status.success());
     assert!(
-        combined(&output).contains("timed out after 5 seconds"),
+        normalized_diagnostic(&output).contains("timed out after 5 seconds"),
         "{}",
         combined(&output)
     );

crates/openshell-gateway/tests/config_preflight.rs

@@ -384,6 +384,8 @@ pub struct SandboxRef {
     pub labels: HashMap<String, String>,
     pub resource_version: u64,
     pub exit_code: Option<i32>,
+    /// Public OpenSSH SHA256 host identity; absent on older gateways.
+    pub host_key_fingerprint: Option<String>,
     pub created_from_workload_template: Option<SandboxWorkloadTemplateProvenance>,
     /// Service URLs returned by sandbox creation, keyed by service name. The
     /// empty key identifies the unnamed service. Non-create reads leave this empty.
@@ -437,6 +439,8 @@ impl SandboxRef {
             labels: meta.labels,
             resource_version: meta.resource_version,
             exit_code,
+            host_key_fingerprint: (!sandbox.host_key_fingerprint.is_empty())
+                .then_some(sandbox.host_key_fingerprint),
             created_from_workload_template,
             service_urls: HashMap::new(),
             restart_count,

crates/openshell-sdk/src/types.rs

@@ -131,6 +131,11 @@ fn sandbox_with_phase_ws(
             ..Default::default()
         }),
         created_from_workload_template,
+        host_key_fingerprint: if name == "pinned-identity" {
+            "SHA256:public-identity".to_string()
+        } else {
+            String::new()
+        },
     }
 }
 
@@ -1291,6 +1296,25 @@ async fn get_sandbox_sends_name_and_maps_phase() {
     assert_eq!(observed.as_deref(), Some("my-box"));
 }
 
+#[tokio::test]
+async fn get_sandbox_preserves_host_fingerprint_and_accepts_older_gateways() {
+    let endpoint = start_mock(Arc::new(MockState::default())).await;
+    let client = connect(&endpoint).await;
+    let sandbox = client.get_sandbox("pinned-identity").await.unwrap();
+    assert_eq!(
+        sandbox.host_key_fingerprint.as_deref(),
+        Some("SHA256:public-identity")
+    );
+    assert!(
+        client
+            .get_sandbox("legacy")
+            .await
+            .unwrap()
+            .host_key_fingerprint
+            .is_none()
+    );
+}
+
 #[tokio::test]
 async fn get_sandbox_preserves_workload_template_provenance() {
     let state = Arc::new(MockState::default());