fix(security): close credential and TLS replay paths

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

architecture/gateway.md

@@ -276,8 +276,10 @@ controlling Sandbox CR. The bootstrap path accepts
 both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox
 controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing
 deployments. Supervisors renew gateway JWTs in memory before expiry only while
-the sandbox record still exists. Older tokens are not server-revoked; shared
-deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes.
+the sandbox record still exists. Each successful refresh atomically stores the
+new gateway-token ID in that sandbox record, invalidating the consumed bearer
+across every gateway replica. Short `gateway_jwt.ttl_secs` lifetimes still bound
+the exposure of a current bearer that has not yet been refreshed.
 Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local
 single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`.
 Kubernetes and other shared deployments should set a positive TTL. Explicit

crates/openshell-core/src/jwt.rs

@@ -528,10 +528,24 @@ mod session {
         pub fn mint_pair(
             &self,
             identity: &SandboxRuntimeIdentity,
+        ) -> Result<MintedSessionTokenPair, SessionJwtError> {
+            self.mint_pair_with_gateway_token_id(identity, Uuid::new_v4())
+        }
+
+        /// Mint a token pair whose gateway-facing credential has a caller-owned
+        /// lineage identifier.
+        ///
+        /// The gateway persists this identifier before returning the token so
+        /// refresh can reject every superseded bearer across all replicas. The
+        /// Sandbox Protocol credential remains independently identified.
+        pub fn mint_pair_with_gateway_token_id(
+            &self,
+            identity: &SandboxRuntimeIdentity,
+            gateway_token_id: Uuid,
         ) -> Result<MintedSessionTokenPair, SessionJwtError> {
             Ok(MintedSessionTokenPair {
-                gateway: self.mint(SessionTokenProfile::Gateway, identity)?,
-                sandbox: self.mint(SessionTokenProfile::Sandbox, identity)?,
+                gateway: self.mint(SessionTokenProfile::Gateway, identity, gateway_token_id)?,
+                sandbox: self.mint(SessionTokenProfile::Sandbox, identity, Uuid::new_v4())?,
                 auth_epoch: identity.auth_epoch,
             })
         }
@@ -540,12 +554,12 @@ mod session {
             &self,
             profile: SessionTokenProfile,
             identity: &SandboxRuntimeIdentity,
+            token_id: Uuid,
         ) -> Result<MintedSessionToken, SessionJwtError> {
             let issued_at = self.clock.now_unix_seconds();
             let expires_at = issued_at.saturating_add(
                 i64::try_from(self.ttl.as_secs()).map_err(|_| SessionJwtError::InvalidLifetime)?,
             );
-            let token_id = Uuid::new_v4();
             let claims = SessionClaims {
                 iss: self.issuer.clone(),
                 sub: format!("{SANDBOX_SUBJECT_PREFIX}{}", identity.sandbox_id),

crates/openshell-server/src/auth/sandbox_jwt.rs

@@ -185,6 +185,7 @@ impl SandboxSessionJwtAuthority {
             sandbox_id,
             identity.runtime_generation.clone(),
             identity.auth_epoch,
+            identity.gateway_token_id,
         )
     }
 
@@ -194,17 +195,21 @@ impl SandboxSessionJwtAuthority {
         sandbox_id: &str,
         runtime_generation: SandboxGenerationId,
         auth_epoch: CredentialEpoch,
+        gateway_token_id: uuid::Uuid,
     ) -> Result<SandboxLaunchAuthentication, Status> {
         let identity = SandboxRuntimeIdentity {
             sandbox_id: SandboxId::parse(sandbox_id)
                 .map_err(|_| Status::invalid_argument("sandbox ID is invalid"))?,
             runtime_generation: runtime_generation.clone(),
             auth_epoch,
         };
-        let pair = self.issuer.mint_pair(&identity).map_err(|error| {
-            warn!(%error, "failed to mint launch-scoped sandbox credentials");
-            Status::internal("failed to mint sandbox launch credentials")
-        })?;
+        let pair = self
+            .issuer
+            .mint_pair_with_gateway_token_id(&identity, gateway_token_id)
+            .map_err(|error| {
+                warn!(%error, "failed to mint launch-scoped sandbox credentials");
+                Status::internal("failed to mint sandbox launch credentials")
+            })?;
         Ok(SandboxLaunchAuthentication {
             supervisor: SupervisorAuthBundle {
                 session_id: SandboxSessionId::new(),

crates/openshell-server/src/auth/sandbox_session.rs

@@ -9,25 +9,29 @@ use openshell_core::jwt::{AuthenticatedSandboxSession, CredentialEpoch, SessionJ
 use openshell_core::proto::{Sandbox, SandboxPhase};
 use openshell_core::sandbox_generation::SandboxGenerationId;
 use tonic::Status;
+use uuid::Uuid;
 
 use crate::persistence::Store;
 
 pub const RUNTIME_GENERATION_ANNOTATION: &str = "internal.openshell.ai/runtime-generation";
 pub const AUTH_EPOCH_ANNOTATION: &str = "internal.openshell.ai/auth-epoch";
+pub const GATEWAY_TOKEN_ID_ANNOTATION: &str = "internal.openshell.ai/gateway-token-id";
 
 /// The complete durable authorization identity for one sandbox runtime.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct PersistedSandboxIdentity {
     pub runtime_generation: SandboxGenerationId,
     pub auth_epoch: CredentialEpoch,
+    pub gateway_token_id: Uuid,
 }
 
 impl PersistedSandboxIdentity {
     pub fn new() -> Result<Self, SessionJwtError> {
         Ok(Self {
-            runtime_generation: SandboxGenerationId::parse(uuid::Uuid::new_v4().to_string())
+            runtime_generation: SandboxGenerationId::parse(Uuid::new_v4().to_string())
                 .map_err(|_| SessionJwtError::InvalidRuntimeIdentity)?,
             auth_epoch: CredentialEpoch::new(1)?,
+            gateway_token_id: Uuid::new_v4(),
         })
     }
 
@@ -45,9 +49,14 @@ impl PersistedSandboxIdentity {
             .parse::<u64>()
             .map_err(|_| SessionJwtError::InvalidCredentialEpoch)
             .and_then(CredentialEpoch::new)?;
+        let gateway_token_id = annotations
+            .get(GATEWAY_TOKEN_ID_ANNOTATION)
+            .ok_or(SessionJwtError::MissingRuntimeIdentity)
+            .and_then(|value| Uuid::parse_str(value).map_err(|_| SessionJwtError::InvalidJti))?;
         Ok(Self {
             runtime_generation,
             auth_epoch,
+            gateway_token_id,
         })
     }
 
@@ -60,18 +69,32 @@ impl PersistedSandboxIdentity {
             AUTH_EPOCH_ANNOTATION.to_string(),
             self.auth_epoch.get().to_string(),
         );
+        annotations.insert(
+            GATEWAY_TOKEN_ID_ANNOTATION.to_string(),
+            self.gateway_token_id.to_string(),
+        );
+    }
+
+    #[must_use]
+    pub fn next_gateway_token(&self) -> Self {
+        Self {
+            runtime_generation: self.runtime_generation.clone(),
+            auth_epoch: self.auth_epoch,
+            gateway_token_id: Uuid::new_v4(),
+        }
     }
 }
 
 /// Load the authoritative runtime identity and compare it with a signed JWT.
 ///
-/// Every gateway replica performs this check against shared persistence. No
-/// raw token, token ID, or refresh lineage needs to be replicated.
+/// Every gateway replica checks the persisted gateway-token ID in addition to
+/// the runtime generation and authorization epoch. Refresh replaces that ID
+/// with CAS, so an older bearer cannot rejoin the active credential lineage.
 #[allow(clippy::result_large_err)]
-pub async fn authorize_persisted(
+async fn load_authorized(
     store: &Store,
     principal: &AuthenticatedSandboxSession,
-) -> Result<PersistedSandboxIdentity, Status> {
+) -> Result<(Sandbox, PersistedSandboxIdentity), Status> {
     let sandbox = store
         .get_message::<Sandbox>(principal.sandbox_id.as_str())
         .await
@@ -98,14 +121,69 @@ pub async fn authorize_persisted(
         .ok_or_else(|| Status::unauthenticated("sandbox identity metadata is missing"))?;
     let identity = PersistedSandboxIdentity::read(&metadata.annotations)
         .map_err(|_| Status::unauthenticated("sandbox runtime identity is invalid"))?;
-    if principal.runtime_generation != identity.runtime_generation
-        || principal.auth_epoch != identity.auth_epoch
-    {
+    let presented = (
+        &principal.runtime_generation,
+        principal.auth_epoch,
+        principal.token_id,
+    );
+    let expected = (
+        &identity.runtime_generation,
+        identity.auth_epoch,
+        identity.gateway_token_id,
+    );
+    if presented != expected {
         return Err(Status::unauthenticated(
             "gateway token does not match the active sandbox identity",
         ));
     }
-    Ok(identity)
+    Ok((sandbox, identity))
+}
+
+#[allow(clippy::result_large_err)]
+pub async fn authorize_persisted(
+    store: &Store,
+    principal: &AuthenticatedSandboxSession,
+) -> Result<PersistedSandboxIdentity, Status> {
+    load_authorized(store, principal)
+        .await
+        .map(|(_, identity)| identity)
+}
+
+/// Atomically consume the presented gateway bearer and install its successor.
+///
+/// A concurrent refresh or unrelated sandbox mutation causes the CAS to fail;
+/// callers retry from freshly authenticated state. The old token is never
+/// accepted after a successful update on any gateway replica.
+#[allow(clippy::result_large_err)]
+pub async fn rotate_gateway_token(
+    store: &Store,
+    principal: &AuthenticatedSandboxSession,
+    next: &PersistedSandboxIdentity,
+) -> Result<PersistedSandboxIdentity, Status> {
+    let (sandbox, current) = load_authorized(store, principal).await?;
+    if next.runtime_generation != current.runtime_generation
+        || next.auth_epoch != current.auth_epoch
+        || next.gateway_token_id == current.gateway_token_id
+    {
+        return Err(Status::internal("gateway token successor is invalid"));
+    }
+    let metadata = sandbox
+        .metadata
+        .as_ref()
+        .ok_or_else(|| Status::unauthenticated("sandbox identity metadata is missing"))?;
+    store
+        .update_message_cas::<Sandbox, _>(
+            principal.sandbox_id.as_str(),
+            metadata.resource_version,
+            |sandbox| {
+                if let Some(metadata) = sandbox.metadata.as_mut() {
+                    next.write(&mut metadata.annotations);
+                }
+            },
+        )
+        .await
+        .map_err(|error| Status::aborted(format!("rotate gateway token: {error}")))?;
+    Ok(next.clone())
 }
 
 #[cfg(test)]
@@ -121,6 +199,7 @@ mod tests {
             runtime_generation: SandboxGenerationId::parse("generation-a")
                 .expect("runtime generation"),
             auth_epoch: CredentialEpoch::new(1).expect("auth epoch"),
+            gateway_token_id: Uuid::from_u128(1),
         };
         let mut metadata = ObjectMeta {
             id: "sandbox-a".to_string(),
@@ -140,13 +219,13 @@ mod tests {
         store.put_message(&sandbox).await.expect("persist sandbox");
     }
 
-    fn principal(auth_epoch: u64) -> AuthenticatedSandboxSession {
+    fn principal(auth_epoch: u64, token_id: Uuid) -> AuthenticatedSandboxSession {
         AuthenticatedSandboxSession {
             sandbox_id: SandboxId::parse("sandbox-a").expect("sandbox ID"),
             runtime_generation: SandboxGenerationId::parse("generation-a")
                 .expect("runtime generation"),
             auth_epoch: CredentialEpoch::new(auth_epoch).expect("auth epoch"),
-            token_id: Uuid::new_v4(),
+            token_id,
             issued_at: 1,
             expires_at: 2,
         }
@@ -157,30 +236,36 @@ mod tests {
         let store = Store::connect("sqlite::memory:").await.expect("store");
         persist_sandbox(&store, SandboxPhase::Ready).await;
 
-        authorize_persisted(&store, &principal(1))
+        authorize_persisted(&store, &principal(1, Uuid::from_u128(1)))
             .await
             .expect("first replica authorizes from persistence");
-        authorize_persisted(&store, &principal(1))
+        authorize_persisted(&store, &principal(1, Uuid::from_u128(1)))
             .await
             .expect("second replica authorizes without local state");
 
+        let error = authorize_persisted(&store, &principal(1, Uuid::from_u128(2)))
+            .await
+            .expect_err("a different token lineage must be rejected");
+        assert_eq!(error.code(), tonic::Code::Unauthenticated);
+
         store
             .update_message_cas::<Sandbox, _>("sandbox-a", 0, |sandbox| {
                 let next = PersistedSandboxIdentity {
                     runtime_generation: SandboxGenerationId::parse("generation-a")
                         .expect("runtime generation"),
                     auth_epoch: CredentialEpoch::new(2).expect("auth epoch"),
+                    gateway_token_id: Uuid::from_u128(2),
                 };
                 next.write(&mut sandbox.metadata.as_mut().expect("metadata").annotations);
             })
             .await
             .expect("advance auth epoch");
 
-        let error = authorize_persisted(&store, &principal(1))
+        let error = authorize_persisted(&store, &principal(1, Uuid::from_u128(1)))
             .await
             .expect_err("old epoch must be revoked on every replica");
         assert_eq!(error.code(), tonic::Code::Unauthenticated);
-        authorize_persisted(&store, &principal(2))
+        authorize_persisted(&store, &principal(2, Uuid::from_u128(2)))
             .await
             .expect("new epoch is active");
 
@@ -190,7 +275,7 @@ mod tests {
             })
             .await
             .expect("stop sandbox");
-        let error = authorize_persisted(&store, &principal(2))
+        let error = authorize_persisted(&store, &principal(2, Uuid::from_u128(2)))
             .await
             .expect_err("stopped runtime must reject its token");
         assert_eq!(error.code(), tonic::Code::FailedPrecondition);
@@ -202,9 +287,34 @@ mod tests {
             let store = Store::connect("sqlite::memory:").await.expect("store");
             persist_sandbox(&store, phase).await;
 
-            authorize_persisted(&store, &principal(1))
+            authorize_persisted(&store, &principal(1, Uuid::from_u128(1)))
                 .await
                 .expect("terminal runtime can finish delivering exit state");
         }
     }
+
+    #[tokio::test]
+    async fn rotating_gateway_token_revokes_its_predecessor() {
+        let store = Store::connect("sqlite::memory:").await.expect("store");
+        persist_sandbox(&store, SandboxPhase::Ready).await;
+        let first = principal(1, Uuid::from_u128(1));
+
+        let expected = PersistedSandboxIdentity {
+            runtime_generation: first.runtime_generation.clone(),
+            auth_epoch: first.auth_epoch,
+            gateway_token_id: Uuid::from_u128(2),
+        };
+        let next = rotate_gateway_token(&store, &first, &expected)
+            .await
+            .expect("rotate current gateway token");
+        assert_ne!(next.gateway_token_id, first.token_id);
+
+        let error = authorize_persisted(&store, &first)
+            .await
+            .expect_err("consumed gateway token must be rejected");
+        assert_eq!(error.code(), tonic::Code::Unauthenticated);
+        authorize_persisted(&store, &principal(1, next.gateway_token_id))
+            .await
+            .expect("successor gateway token is current");
+    }
 }

crates/openshell-server/src/compute/mod.rs

@@ -6082,6 +6082,7 @@ mod tests {
             )
             .expect("test runtime generation"),
             auth_epoch: openshell_core::jwt::CredentialEpoch::new(1).expect("test auth epoch"),
+            gateway_token_id: uuid::Uuid::new_v4(),
         }
         .write(&mut annotations);
         let mut sandbox = Sandbox {