chore: merge upstream/main (resolve required-ci-gates.yml CARRY conflict)

.github/workflows/branch-e2e.yml

@@ -1,8 +1,6 @@
 name: Branch E2E Checks
 
 on:
-  merge_group:
-    types: [checks_requested]
   push:
     branches:
       - "pull-request/[0-9]+"
@@ -50,15 +48,6 @@ jobs:
               run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               ;;
-            merge_group)
-              # Merge groups have no PR labels. When GPU E2E is required as documented
-              # in CI.md, skipping it leaves the gate pending until the queue times out
-              # and ejects the PR. HA stays off until stable.
-              run_core_e2e=true
-              run_gpu_e2e=true
-              run_kubernetes_ha_e2e=false
-              run_kubernetes_credential_drivers_e2e=false
-              ;;
             *)
               run_core_e2e=true
               run_gpu_e2e=true
@@ -82,7 +71,7 @@ jobs:
 
   version:
     needs: [pr_metadata]
-    if: needs.pr_metadata.outputs.should_run == 'true'
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true'
     permissions:
       contents: read
     runs-on: ubuntu-latest
@@ -228,7 +217,7 @@ jobs:
   build-rpm:
     name: Build RPM packages
     needs: [pr_metadata, version, build-binaries]
-    if: needs.pr_metadata.outputs.should_run == 'true'
+    if: needs.pr_metadata.outputs.run_integration == 'true'
     permissions:
       actions: read
       contents: read

.github/workflows/required-ci-gates.yml

@@ -281,6 +281,9 @@ jobs:
           # evaluate_workflow's verify_mirror would pend these gates forever downstream,
           # since nothing creates that mirror branch here. Mark them passing directly.
           post_status "OpenShell / Branch Checks" success "Disabled in downstream fork (job requires runners unavailable downstream)" "$STATUS_TARGET_URL"
-          evaluate_workflow "OpenShell / E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e" "Core E2E result"
-          evaluate_workflow "OpenShell / GPU E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e-gpu" "GPU E2E result"
+          # E2E is opt-in on PRs and does not run for merge groups.
+          if [ "$CONTEXT_KIND" != "merge_group" ]; then
+            evaluate_workflow "OpenShell / E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e" "Core E2E result"
+            evaluate_workflow "OpenShell / GPU E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e-gpu" "GPU E2E result"
+          fi
           post_status "OpenShell / Helm Lint" success "Disabled in downstream fork (job requires runners unavailable downstream)" "$STATUS_TARGET_URL"

CI.md

@@ -72,8 +72,8 @@ Windows checks are not required for merging and do not run in merge queues.
 Main and manual runs also build release binaries, with `continue-on-error: true`
 so Windows failures do not fail the workflow.
 
-Every approved `Branch E2E Checks` run builds the RPM packages, including
-runs without optional E2E labels. Core integration qualification builds and installs
+`Branch E2E Checks` builds binaries and images only when an E2E suite is
+selected. DEB and RPM packages build only when integration qualification is selected. Core integration qualification builds and installs
 the DEB on Ubuntu with Docker and installs the CLI and gateway RPMs on Fedora with
 rootful and rootless Podman. These lanes run conformance using the matching runtime
 images. Release Dev and Release Tag use the same package installers.
@@ -433,8 +433,6 @@ Important: if a PR requires manual admission, every new commit needs another `/o
 GitHub merge queue is required for `main`. Repository administrators must enable **Require merge queue** in the branch ruleset for `main` and keep these required status contexts aligned with the PR gates:
 
 - `OpenShell / Branch Checks`
-- `OpenShell / E2E`
-- `OpenShell / GPU E2E`
 - `OpenShell / Helm Lint`
 - `OpenShell / Trivy Changes`
 
@@ -445,7 +443,7 @@ its own stable result status.
 Merge-group runs use the `merge_group` event. The event is distinct from `pull_request` and `push`, and GitHub will not report required checks for queued PRs unless the workflows include it. In this repository:
 
 - `Branch Checks` runs the standard non-E2E gates on the merge-group SHA.
-- `Branch E2E Checks` runs core E2E and GPU E2E for merge groups. Kubernetes HA E2E remains optional and label-driven on PRs.
+- `Branch E2E Checks` does not run for merge groups. E2E suites remain opt-in on PRs.
 - `Helm Lint` runs for merge groups without the PR diff optimization, because the merge-group branch is the final integration state.
 - `Trivy Changes` compares the merge-group configuration with its base and rejects new High or Critical findings.
 - `Required CI Gates` posts the same `OpenShell / ...` statuses to the merge-group SHA and does not require a `pull-request/<N>` mirror for merge-group events.
@@ -471,7 +469,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma
 | File | Role |
 |---|---|
 | `.github/workflows/branch-checks.yml` | Required non-E2E checks. Triggers on `push: pull-request/[0-9]+` for PR mirrors and `merge_group` for queued merges. |
-| `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. |
+| `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups do not run E2E. |
 | `.github/workflows/build-binaries.yml`, `build-vm-driver.yml` | Shared binary matrices used by branch and release workflows. The VM driver remains separate because its build consumes the runtime binaries. |
 | `.github/workflows/build-images.yml` | Builds and pushes multi-platform images, then uploads the same OCI images as workflow artifacts. |
 | `.github/workflows/package-release-binaries.yml` | Packages raw build artifacts into release tarballs without rebuilding them. |
@@ -507,11 +505,14 @@ These workflows run after merge to publish dev/tagged artifacts and verify them.
 Require these statuses in the branch ruleset for PR and merge-queue CI:
 
 - `OpenShell / Branch Checks`
-- `OpenShell / E2E`
-- `OpenShell / GPU E2E`
 - `OpenShell / Helm Lint`
 - `OpenShell / Trivy Changes`
 
+The following statuses are opt-in and controlled by labels:
+
+- `OpenShell / E2E`: `test:e2e`
+- `OpenShell / GPU E2E`: `test:e2e-gpu`
+
 For mirror-based workflows, require the statuses published by
 `Required CI Gates`, not their underlying jobs. `OpenShell / Trivy Changes` is
 the stable result job of the direct pull-request workflow. Together these

CONTRIBUTING.md

@@ -291,6 +291,22 @@ each worktree. Cache reuse therefore depends on the compiler inputs: outputs
 that embed absolute paths, including Rust dependencies in some builds, can
 still miss across worktrees.
 
+## Container Image Base Overrides
+
+The gateway and supervisor Dockerfiles accept `GATEWAY_BASE_IMAGE` and
+`SUPERVISOR_BASE_IMAGE` build arguments. Omitting them keeps the pinned
+distroless defaults. To build with your own bases, pass the argument through
+the corresponding task:
+
+```shell
+mise run build:docker:gateway -- --build-arg GATEWAY_BASE_IMAGE=registry.example.com/gateway-base:tag
+mise run build:docker:supervisor -- --build-arg SUPERVISOR_BASE_IMAGE=registry.example.com/supervisor-base:tag
+```
+
+Choose bases that provide the GNU runtime libraries required by the binaries
+and CA certificates for the target architecture. Validate the resulting images
+with your deployment. The sandbox image uses `scratch` and has no base override.
+
 ## Main Tasks
 
 These are the primary `mise` tasks for day-to-day development:

crates/openshell-server/src/compute/mod.rs

@@ -4434,18 +4434,23 @@ impl ComputeRuntime {
                 SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown)
             });
 
-        if existing_phase != SandboxPhase::Starting {
+        let ready_container_exit = existing_phase == SandboxPhase::Ready
+            && (driver_snapshot_confirms_stopped(&incoming)
+                || driver_snapshot_reports_runtime_restart(&incoming));
+        if existing_phase != SandboxPhase::Starting && !ready_container_exit {
             return self.apply_sandbox_update_locked(incoming, existing).await;
         }
 
         // Any snapshot can already be queued when StartSandbox moves the
         // durable phase to Starting. In particular, an old-generation Ready
         // event followed by its terminal event can otherwise promote and then
         // stop the new generation before the replacement supervisor connects.
+        // The replacement may already be Ready when the old exit arrives, so
+        // terminal container snapshots need the same check after readiness.
         // Release the global watch lock, wait for that lifecycle operation,
         // and then reread both the driver and store before applying an
         // authoritative observation. Taking the per-sandbox gate only for
-        // this ambiguous phase avoids delaying unrelated watch events behind
+        // these ambiguous snapshots avoids delaying unrelated watch events behind
         // slow lifecycle operations.
         let existing_name = existing_sandbox.as_ref().map_or_else(
             || incoming.name.clone(),
@@ -4475,7 +4480,7 @@ impl ComputeRuntime {
             {
                 warn!(
                     sandbox_id = %incoming.id,
-                    "Could not validate driver snapshot during sandbox start; retaining current sandbox state"
+                    "Could not validate driver snapshot against current runtime; retaining current sandbox state"
                 );
                 return Ok(());
             }
@@ -11604,6 +11609,65 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn container_exit_after_replacement_ready_is_revalidated() {
+        for (reason, live_exited) in [
+            ("ContainerExited", false),
+            ("ContainerExited", true),
+            ("ContainerStopped", false),
+            ("ContainerStopped", true),
+            ("ContainerRuntimeRestart", false),
+            ("ContainerRuntimeRestart", true),
+        ] {
+            let driver = ControlledDriver::new();
+            let mut sandbox =
+                sandbox_record("sb-restarted", "sandbox-restarted", SandboxPhase::Starting);
+            sandbox.status.as_mut().unwrap().restart_count = 1;
+            sandbox.status.as_mut().unwrap().exit_code = Some(17);
+            sandbox.status.as_mut().unwrap().main_process_instance_id = "previous".to_string();
+            let mut exited = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+            exited.status = Some(make_driver_status(make_driver_condition(
+                reason,
+                "container exited",
+            )));
+            let live = if live_exited {
+                exited.clone()
+            } else {
+                ready_driver_sandbox(sandbox.object_id(), sandbox.object_name())
+            };
+            driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(live)));
+            let runtime = test_runtime(driver).await;
+            runtime.store.put_message(&sandbox).await.unwrap();
+            register_test_supervisor_session(&runtime, sandbox.object_id());
+            runtime
+                .supervisor_session_connected(sandbox.object_id(), "replacement")
+                .await
+                .unwrap();
+
+            runtime.apply_sandbox_update(exited).await.unwrap();
+
+            let stored = runtime
+                .store
+                .get_message::<Sandbox>(sandbox.object_id())
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(
+                stored.phase(),
+                if live_exited {
+                    SandboxPhase::Error
+                } else {
+                    SandboxPhase::Ready
+                } as i32,
+                "only a live {reason} may fail the replacement runtime"
+            );
+            let status = stored.status.unwrap();
+            assert_eq!(status.main_process_instance_id, "replacement");
+            assert_eq!(status.restart_count, 1);
+            assert_eq!(status.exit_code, None);
+        }
+    }
+
     #[tokio::test]
     async fn stale_ready_snapshot_queued_before_start_is_revalidated() {
         let driver = ControlledDriver::new();
@@ -13375,14 +13439,18 @@ mod tests {
 
     #[tokio::test]
     async fn unexpected_term_runtime_restart_transitions_to_error() {
-        let runtime = test_runtime(Arc::new(TestDriver::default())).await;
+        let driver = ControlledDriver::new();
+        let runtime = test_runtime(driver.clone()).await;
         let sandbox = sandbox_record("sb-term-exit", "sandbox-term-exit", SandboxPhase::Ready);
         runtime.store.put_message(&sandbox).await.unwrap();
         let mut runtime_restart = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
         runtime_restart.status = Some(make_driver_status(make_driver_condition(
             "ContainerRuntimeRestart",
             "container exited with status 143",
         )));
+        driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(
+            runtime_restart.clone(),
+        )));
 
         runtime.apply_sandbox_update(runtime_restart).await.unwrap();
 

deploy/docker/Dockerfile.gateway

@@ -8,7 +8,7 @@
 #
 # Distroless Debian provides the glibc runtime required by the binary.
 
-ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:54df941ed0d06a1bd95ef5e0ce391fd8d9f94b64782dc9a60062727849ee3f97
+ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:e792ab3d241a468a4fd7519ddbbebe66b49b5f365771716ea688ad40b6c6f1c2
 FROM ${GATEWAY_BASE_IMAGE} AS gateway
 
 ARG TARGETARCH

deploy/docker/Dockerfile.supervisor

@@ -8,7 +8,8 @@
 
 # Keep the GNU runtime and CA roots without a shell or unused native TLS libraries.
 # The default variant preserves UID 0 and /; compute drivers set the runtime UID.
-FROM gcr.io/distroless/base-nossl-debian13@sha256:af5cb8dd589b8520b8c06bebb9efb73d7e16406cab58e85c51761fff49d370a0 AS supervisor
+ARG SUPERVISOR_BASE_IMAGE=gcr.io/distroless/base-nossl-debian13@sha256:af5cb8dd589b8520b8c06bebb9efb73d7e16406cab58e85c51761fff49d370a0
+FROM ${SUPERVISOR_BASE_IMAGE} AS supervisor
 
 ARG TARGETARCH
 

docs/how-it-works/sandboxes/runtimes.mdx

@@ -44,6 +44,11 @@ driver TOML overrides the compiled release defaults. Helm renders Kubernetes
 image fields into TOML, so they apply when these environment variables are
 unset. Sandbox requests cannot override either trusted image.
 
+When building your own gateway or supervisor image, the `GATEWAY_BASE_IMAGE`
+and `SUPERVISOR_BASE_IMAGE` build arguments override their pinned bases. See
+[Container Image Base Overrides](https://github.com/NVIDIA/OpenShell/blob/main/CONTRIBUTING.md#container-image-base-overrides)
+for build commands and base compatibility requirements.
+
 For TLS-enabled Docker, Podman, and MicroVM gateways, set `guest_tls_ca` in
 `[openshell.gateway]` or use the package-managed local CA. Remove the retired
 `guest_tls_cert` and `guest_tls_key` fields. Supervisors receive only the