@@ -13,7 +13,6 @@ use crate::isolation::{
1313} ;
1414use crate :: sandbox_runtime:: {
1515 BOUNDARY_CERTIFICATE_PATH , BOUNDARY_CONFIG_PATH , BOUNDARY_PRIVATE_KEY_PATH ,
16- SANDBOX_SECRET_COMPONENT , SUPERVISOR_SECRET_COMPONENT ,
1716 SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS , SandboxRuntimeNames , boundary_service,
1817 generate_proxy_ca_material, sandbox_bootstrap_secret,
1918 sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference,
@@ -3032,8 +3031,12 @@ impl KubernetesComputeDriver {
30323031 pod_is_gone,
30333032 ) ;
30343033 if stop_is_complete {
3035- self . delete_sandbox_runtime_supervisor ( sandbox_id, & namespace)
3036- . await ?;
3034+ self . delete_sandbox_runtime_supervisor (
3035+ sandbox_id,
3036+ & namespace,
3037+ sandbox_runtime_generation ( & object) . as_slice ( ) ,
3038+ )
3039+ . await ?;
30373040 patch_dynamic_object_with_resource_version_retry (
30383041 & agent_sandbox_api. api ,
30393042 & kube_name,
@@ -3216,7 +3219,9 @@ impl KubernetesComputeDriver {
32163219 let names = SandboxRuntimeNames :: for_generation ( sandbox_id, generation. as_str ( ) ) ;
32173220 self . create_sandbox_runtime_fence ( & namespace, & names)
32183221 . await ?;
3219- self . delete_sandbox_runtime_supervisor ( sandbox_id, & namespace)
3222+ let mut stale_generations = sandbox_runtime_generation ( & object) . as_slice ( ) . to_vec ( ) ;
3223+ stale_generations. push ( generation. as_str ( ) ) ;
3224+ self . delete_sandbox_runtime_supervisor ( sandbox_id, & namespace, & stale_generations)
32203225 . await ?;
32213226 let main_process_spec =
32223227 required_sandbox_annotation ( & object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC ) ?;
@@ -3499,40 +3504,28 @@ impl KubernetesComputeDriver {
34993504 & self ,
35003505 sandbox_id : & str ,
35013506 namespace : & str ,
3507+ generations : & [ & str ] ,
35023508 ) -> Result < ( ) , KubernetesDriverError > {
35033509 self . delete_sandbox_runtime_supervisor_pod ( sandbox_id, namespace)
35043510 . await ?;
3505- self . delete_sandbox_runtime_generation_secrets ( sandbox_id, namespace)
3511+ self . delete_sandbox_runtime_generation_secrets ( sandbox_id, namespace, generations )
35063512 . await
35073513 }
35083514
3515+ /// Delete the bootstrap Secrets of the given runtime generations by exact
3516+ /// name. Garbage collection through their owner Pods removes older
3517+ /// generations.
35093518 async fn delete_sandbox_runtime_generation_secrets (
35103519 & self ,
35113520 sandbox_id : & str ,
35123521 namespace : & str ,
3522+ generations : & [ & str ] ,
35133523 ) -> Result < ( ) , KubernetesDriverError > {
35143524 let secrets = Api :: < Secret > :: namespaced ( self . client . clone ( ) , namespace) ;
3515- for component in [ SANDBOX_SECRET_COMPONENT , SUPERVISOR_SECRET_COMPONENT ] {
3516- let selector =
3517- format ! ( "openshell.ai/sandbox-id={sandbox_id},openshell.ai/component={component}" ) ;
3518- let items = secrets
3519- . list ( & ListParams :: default ( ) . labels ( & selector) )
3520- . await
3521- . map_err ( KubernetesDriverError :: from_kube) ?;
3522- for secret in items {
3523- let Some ( name) = secret. metadata . name else {
3524- continue ;
3525- } ;
3526- match secrets
3527- . delete (
3528- & name,
3529- & DeleteParams :: default ( ) . preconditions ( Preconditions {
3530- uid : secret. metadata . uid ,
3531- resource_version : None ,
3532- } ) ,
3533- )
3534- . await
3535- {
3525+ for generation in generations {
3526+ let names = SandboxRuntimeNames :: for_generation ( sandbox_id, generation) ;
3527+ for name in [ & names. sandbox_secret , & names. supervisor_secret ] {
3528+ match secrets. delete ( name, & DeleteParams :: default ( ) ) . await {
35363529 Ok ( _) | Err ( KubeError :: Api ( kube:: core:: ErrorResponse { code : 404 , .. } ) ) => { }
35373530 Err ( error) => return Err ( KubernetesDriverError :: from_kube ( error) ) ,
35383531 }
@@ -3884,7 +3877,12 @@ impl KubernetesComputeDriver {
38843877 }
38853878 }
38863879 match self
3887- . reconcile_sandbox_runtime_supervisor ( & sandbox_id, namespace, desired_running)
3880+ . reconcile_sandbox_runtime_supervisor (
3881+ & sandbox_id,
3882+ namespace,
3883+ desired_running,
3884+ sandbox_runtime_generation ( & object) . as_slice ( ) ,
3885+ )
38883886 . await
38893887 {
38903888 Ok ( ( ) ) => { }
@@ -3938,7 +3936,11 @@ impl KubernetesComputeDriver {
39383936 }
39393937 }
39403938 if let Err ( error) = self
3941- . delete_sandbox_runtime_supervisor ( sandbox_id, namespace)
3939+ . delete_sandbox_runtime_supervisor (
3940+ sandbox_id,
3941+ namespace,
3942+ sandbox_runtime_generation ( object) . as_slice ( ) ,
3943+ )
39423944 . await
39433945 {
39443946 warn ! ( sandbox_id, %error, "could not finish sandbox-runtime suspension cleanup" ) ;
@@ -4208,10 +4210,11 @@ impl KubernetesComputeDriver {
42084210 sandbox_id : & str ,
42094211 namespace : & str ,
42104212 desired_running : bool ,
4213+ generations : & [ & str ] ,
42114214 ) -> Result < ( ) , KubernetesDriverError > {
42124215 if !desired_running {
42134216 return self
4214- . delete_sandbox_runtime_supervisor ( sandbox_id, namespace)
4217+ . delete_sandbox_runtime_supervisor ( sandbox_id, namespace, generations )
42154218 . await ;
42164219 }
42174220 let names = SandboxRuntimeNames :: new ( sandbox_id) ;
@@ -8440,6 +8443,7 @@ mod tests {
84408443 "resourceVersion" : "42" ,
84418444 "annotations" : {
84428445 SANDBOX_POD_NAME_ANNOTATION : "workload-pod" ,
8446+ ANNOTATION_SANDBOX_RUNTIME_GENERATION : "gen-7" ,
84438447 ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE :
84448448 SandboxRuntimeBootstrapPhase :: Suspending . as_str( )
84458449 }
@@ -8455,12 +8459,6 @@ mod tests {
84558459 serde_json:: json!( { "kind" : "Status" , "status" : "Success" , "code" : 200 } ) ,
84568460 )
84578461 } ;
8458- let no_secrets = || {
8459- kube_test_response (
8460- http:: StatusCode :: OK ,
8461- serde_json:: json!( { "apiVersion" : "v1" , "kind" : "SecretList" , "items" : [ ] } ) ,
8462- )
8463- } ;
84648462 let steps = Arc :: new ( std:: sync:: Mutex :: new ( VecDeque :: from ( [
84658463 (
84668464 http:: Method :: GET ,
@@ -8504,14 +8502,14 @@ mod tests {
85048502 kube_test_not_found ( "pods" , "os-supervisor-sandbox-1" ) ,
85058503 ) ,
85068504 (
8507- http:: Method :: GET ,
8508- "/api/v1/namespaces/openshell/secrets" ,
8509- no_secrets ( ) ,
8505+ http:: Method :: DELETE ,
8506+ "/api/v1/namespaces/openshell/secrets/os-sandbox-sandbox-1-gen7 " ,
8507+ success ( ) ,
85108508 ) ,
85118509 (
8512- http:: Method :: GET ,
8513- "/api/v1/namespaces/openshell/secrets" ,
8514- no_secrets ( ) ,
8510+ http:: Method :: DELETE ,
8511+ "/api/v1/namespaces/openshell/secrets/os-supervisor-sandbox-1-gen7 " ,
8512+ kube_test_not_found ( "secrets" , "os-supervisor-sandbox-1-gen7" ) ,
85158513 ) ,
85168514 (
85178515 http:: Method :: GET ,
0 commit comments