fix(ci): retry Nix builds before executing apps once

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

.agents/skills/watch-github-actions/SKILL.md

@@ -128,7 +128,9 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 `setup-nix` retries development-shell preparation once when `prepare-shell`
 is enabled. Inspect both attempts in the job log; `setup-rust` assumes the
 shell has already been prepared. Cargo, lint, and test commands are not retried.
-Skipped dependent E2E suites indicate blocked coverage.
+Direct Nix builds and app dependency preparation also retry once; apps run
+once after preparation succeeds. Skipped dependent E2E suites indicate blocked
+coverage.
 
 For `Trivy Changes`, inspect the `Resolve PR baseline` step for the base and head
 SHAs. PR runs compare the tested merge commit with its

.github/actions/check-protobuf-compatibility/action.yml

@@ -19,4 +19,7 @@ runs:
       env:
         CHECK_REF: ${{ inputs.ref }}
       run: |
+        # Retry dependency preparation, then run the compatibility check once.
+        nix build --no-link --no-write-lock-file .#check-protobuf-compatibility ||
+          nix build --no-link --no-write-lock-file .#check-protobuf-compatibility
         nix run .#check-protobuf-compatibility --no-write-lock-file -- "$CHECK_REF"

.github/workflows/build-vm-driver.yml

@@ -82,7 +82,9 @@ jobs:
           path: vm-init
 
       - name: Build VM runtime
-        run: nix build .#vm-runtime
+        run: |
+          # HTTP 416 is not retried by Nix; restart the build once on failure.
+          nix build .#vm-runtime || nix build .#vm-runtime
 
       - name: Assemble compressed VM runtime
         run: |

.github/workflows/integration-runner.yml

@@ -85,4 +85,7 @@ jobs:
           ENVIRONMENT: ${{ matrix.environment }}
           INSTALLER: ${{ matrix.installer }}
           TESTSUITE: ${{ matrix.testsuite }}
-        run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"
+        run: |
+          # Retry dependency preparation, then execute the test suite once.
+          nix build --no-link .#tmachine || nix build --no-link .#tmachine
+          nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"

.github/workflows/prepare-integration-inputs.yml

@@ -152,13 +152,22 @@ jobs:
           docker save --output artifacts/images/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
 
       - name: Build test archives
-        run: nix run .#build-artifacts-test-archives
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-test-archives || nix build --no-link .#build-artifacts-test-archives
+          nix run .#build-artifacts-test-archives
 
       - name: Build test workload images
-        run: nix run .#build-artifacts-test-images
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-test-images || nix build --no-link .#build-artifacts-test-images
+          nix run .#build-artifacts-test-images
 
       - name: Package Helm chart
-        run: nix run .#build-artifacts-helm
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-helm || nix build --no-link .#build-artifacts-helm
+          nix run .#build-artifacts-helm
 
       - name: Upload integration inputs
         id: upload-integration-inputs

CI.md

@@ -525,3 +525,10 @@ then resume at EOF and receive HTTP 416. A fresh invocation restarts the
 operation. Both attempts appear in the job log; a second failure fails the
 step. Any preparation failure is retried once, including deterministic errors.
 Cargo, lint, and test commands are not retried.
+
+Direct `nix build` commands retry once. Before each `nix run`, CI builds the
+app's package with `nix build --no-link`, retrying preparation once, then runs
+the app once. The artifact and protobuf-check apps expose matching package
+outputs for this preparation. Runtime failures from tests, artifact generation,
+and compatibility checks are not retried. Downloads initiated inside an app
+are outside this preparation retry.

flake.nix

@@ -182,6 +182,11 @@
         };
 
         packages = {
+          # Expose app derivations so CI can prepare them before executing once.
+          check-protobuf-compatibility = checkProtobufCompatibility;
+          build-artifacts-test-archives = artifacts.testArchives;
+          build-artifacts-test-images = artifacts.testImages;
+          build-artifacts-helm = artifacts.helm;
           vm-runtime = vmRuntime;
           tmachine = testMachines.package;
           tmachine-config = testMachines.config;