test(upgrade): qualify the latest published release with default images

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

.agents/skills/test-release-canary/SKILL.md

@@ -41,7 +41,7 @@ every approved branch run, independently of optional E2E labels.
 
 The host-package jobs exercise fresh installs, not upgrades from persisted
 gateway state. Release Dev and Release Tag additionally run tmachine Ubuntu DEB
-and Fedora RPM upgrade lanes: they install the latest retained prerelease
+and Fedora RPM upgrade lanes: they install the latest published release
 packages and matching runtime images, upgrade to the candidate packages, verify
 an existing sandbox survives, and create a new sandbox. Those lanes are part of
 the release conformance matrix, not this canary. Validate Homebrew exact-default

.github/workflows/prepare-integration-inputs.yml

@@ -27,12 +27,12 @@ on:
         type: string
         default: ""
       include-deb-upgrade-source:
-        description: Include the latest retained prerelease DEB and runtime images as upgrade inputs
+        description: Include the latest published release DEB and runtime images as upgrade inputs
         required: false
         type: boolean
         default: false
       include-rpm-upgrade-source:
-        description: Include the latest retained prerelease RPMs and runtime images as upgrade inputs
+        description: Include the latest published release RPMs and runtime images as upgrade inputs
         required: false
         type: boolean
         default: false
@@ -140,113 +140,37 @@ jobs:
           mv "${gateway_rpms[0]}" artifacts/packages/rpm/openshell-gateway.rpm
           rm -rf "$download_dir"
 
-      - name: Resolve prerelease upgrade source
+      - name: Resolve released upgrade source
         if: inputs['include-deb-upgrade-source'] || inputs['include-rpm-upgrade-source']
         id: upgrade-source
         env:
           GH_TOKEN: ${{ github.token }}
-          INCLUDE_DEB: ${{ inputs['include-deb-upgrade-source'] }}
-          INCLUDE_RPM: ${{ inputs['include-rpm-upgrade-source'] }}
         run: |
           set -euo pipefail
-          successful_run_ids=$(gh api --paginate \
-            "repos/${GITHUB_REPOSITORY}/actions/workflows/release-tag.yml/runs?status=success&per_page=100" \
-            --jq '.workflow_runs[] | select(.status == "completed" and .conclusion == "success") | .id')
-          artifact_records=$(gh api --paginate \
-            "repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100" \
-            --jq '.artifacts[] | select(.expired == false) | [.workflow_run.id, .name] | @tsv')
-          source_record=$(printf '%s\n--ARTIFACTS--\n%s\n' "$successful_run_ids" "$artifact_records" | awk -F '\t' \
-            -v require_deb="$INCLUDE_DEB" -v require_rpm="$INCLUDE_RPM" '
-            $0 == "--ARTIFACTS--" {
-              reading_artifacts = 1
-              next
-            }
-            !reading_artifacts {
-              if ($1 ~ /^[0-9]+$/) successful_runs[$1] = 1
-              next
-            }
-            $1 in successful_runs {
-              run_id = $1
-              artifact = $2
-              tag = ""
-              if (artifact ~ /^openshell-v[0-9]+\.[0-9]+\.[0-9]+-pre\.[1-9][0-9]*-linux-amd64-deb$/) {
-                tag = artifact
-                sub(/^openshell-/, "", tag)
-                sub(/-linux-amd64-deb$/, "", tag)
-                deb[run_id SUBSEP tag] = artifact
-              } else if (artifact ~ /^openshell-v[0-9]+\.[0-9]+\.[0-9]+-pre\.[1-9][0-9]*-linux-x86_64-rpm$/) {
-                tag = artifact
-                sub(/^openshell-/, "", tag)
-                sub(/-linux-x86_64-rpm$/, "", tag)
-                rpm[run_id SUBSEP tag] = artifact
-              }
-              if (tag != "") {
-                candidates[run_id SUBSEP tag] = tag
-                runs[run_id SUBSEP tag] = run_id
-              }
-            }
-            END {
-              for (key in candidates) {
-                if (require_deb == "true" && !(key in deb)) continue
-                if (require_rpm == "true" && !(key in rpm)) continue
-                tag = candidates[key]
-                run_id = runs[key]
-                deb_artifact = (key in deb) ? deb[key] : "-"
-                rpm_artifact = (key in rpm) ? rpm[key] : "-"
-                version = tag
-                sub(/^v/, "", version)
-                split(version, parts, "-pre\\.")
-                split(parts[1], core, "\\.")
-                sequence = parts[2] + 0
-                if (!found || core[1] + 0 > major ||
-                    (core[1] + 0 == major && core[2] + 0 > minor) ||
-                    (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 > patch) ||
-                    (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 == patch && sequence > prerelease)) {
-                  selected = tag
-                  selected_run = run_id
-                  selected_deb = deb_artifact
-                  selected_rpm = rpm_artifact
-                  major = core[1] + 0
-                  minor = core[2] + 0
-                  patch = core[3] + 0
-                  prerelease = sequence
-                  found = 1
-                }
-              }
-              if (found) {
-                printf "%s\t%s\t%s\t%s\n", selected, selected_run, selected_deb, selected_rpm
-              }
-            }')
-          if [[ -z "$source_record" ]]; then
-            echo "no successful retained prerelease contains all requested package artifacts" >&2
-            exit 1
-          fi
-          IFS=$'\t' read -r source_tag source_run_id deb_artifact rpm_artifact <<< "$source_record"
-          {
-            echo "tag=${source_tag}"
-            echo "run-id=${source_run_id}"
-            if [[ "$deb_artifact" != "-" ]]; then echo "deb-artifact=${deb_artifact}"; fi
-            if [[ "$rpm_artifact" != "-" ]]; then echo "rpm-artifact=${rpm_artifact}"; fi
-          } >> "$GITHUB_OUTPUT"
+          source_tag=$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq '.tagName')
+          echo "tag=${source_tag}" >> "$GITHUB_OUTPUT"
 
-      - name: Download prerelease Debian upgrade source
+      - name: Download released Debian upgrade source
         if: inputs['include-deb-upgrade-source']
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: ${{ steps.upgrade-source.outputs['deb-artifact'] }}
-          path: artifacts/upgrade/deb/source/download
-          github-token: ${{ github.token }}
-          run-id: ${{ steps.upgrade-source.outputs['run-id'] }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          SOURCE_TAG: ${{ steps.upgrade-source.outputs.tag }}
+        run: |
+          set -euo pipefail
+          gh release download "$SOURCE_TAG" --repo "$GITHUB_REPOSITORY" \
+            --pattern 'openshell_*_amd64.deb' \
+            --pattern openshell-checksums-sha256.txt \
+            --dir artifacts/upgrade/deb/source/download
 
-      - name: Stage prerelease Debian upgrade source
+      - name: Stage released Debian upgrade source
         if: inputs['include-deb-upgrade-source']
         run: |
           set -euo pipefail
           source_dir=artifacts/upgrade/deb/source
           download_dir="${source_dir}/download"
           deb_path=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell_*_amd64.deb' -print -quit)
           if [[ -z "$deb_path" || ! -f "${download_dir}/openshell-checksums-sha256.txt" ]]; then
-            echo "prerelease source artifact did not contain the expected Debian package and checksums" >&2
+            echo "released source artifact did not contain the expected Debian package and checksums" >&2
             exit 1
           fi
           (
@@ -261,7 +185,7 @@ jobs:
       # Packages do not embed OCI image payloads. Their gateway binaries pin
       # this release tag, while tmachine consumes tarballs to keep guest setup
       # independent of registry access.
-      - name: Export prerelease runtime images for upgrade sources
+      - name: Export released runtime images for upgrade sources
         if: inputs['include-deb-upgrade-source'] || inputs['include-rpm-upgrade-source']
         env:
           IMAGE_TAG: ${{ steps.upgrade-source.outputs.tag }}
@@ -271,23 +195,25 @@ jobs:
           mkdir -p artifacts/upgrade/source-images
 
           docker pull "ghcr.io/nvidia/openshell/sandbox:${image_tag}"
-          docker tag "ghcr.io/nvidia/openshell/sandbox:${image_tag}" openshell/sandbox:tmachine
-          docker save --output artifacts/upgrade/source-images/openshell-sandbox-tmachine.tar openshell/sandbox:tmachine
+          docker save --output artifacts/upgrade/source-images/openshell-sandbox-tmachine.tar "ghcr.io/nvidia/openshell/sandbox:${image_tag}"
 
           docker pull "ghcr.io/nvidia/openshell/supervisor:${image_tag}"
-          docker tag "ghcr.io/nvidia/openshell/supervisor:${image_tag}" openshell/supervisor:tmachine
-          docker save --output artifacts/upgrade/source-images/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
+          docker save --output artifacts/upgrade/source-images/openshell-supervisor-tmachine.tar "ghcr.io/nvidia/openshell/supervisor:${image_tag}"
 
-      - name: Download prerelease RPM upgrade source
+      - name: Download released RPM upgrade source
         if: inputs['include-rpm-upgrade-source']
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: ${{ steps.upgrade-source.outputs['rpm-artifact'] }}
-          path: artifacts/upgrade/rpm/source/download
-          github-token: ${{ github.token }}
-          run-id: ${{ steps.upgrade-source.outputs['run-id'] }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          SOURCE_TAG: ${{ steps.upgrade-source.outputs.tag }}
+        run: |
+          set -euo pipefail
+          gh release download "$SOURCE_TAG" --repo "$GITHUB_REPOSITORY" \
+            --pattern 'openshell-[0-9]*.x86_64.rpm' \
+            --pattern 'openshell-gateway-[0-9]*.x86_64.rpm' \
+            --pattern openshell-checksums-sha256.txt \
+            --dir artifacts/upgrade/rpm/source/download
 
-      - name: Stage prerelease RPM upgrade source
+      - name: Stage released RPM upgrade source
         if: inputs['include-rpm-upgrade-source']
         run: |
           set -euo pipefail
@@ -296,7 +222,7 @@ jobs:
           cli_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-[0-9]*.x86_64.rpm' -print -quit)
           gateway_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-gateway-[0-9]*.x86_64.rpm' -print -quit)
           if [[ -z "$cli_rpm" || -z "$gateway_rpm" || ! -f "${download_dir}/openshell-checksums-sha256.txt" ]]; then
-            echo "prerelease source artifact did not contain the expected RPMs and checksums" >&2
+            echo "released source artifact did not contain the expected RPMs and checksums" >&2
             exit 1
           fi
           (

CI.md

@@ -93,7 +93,7 @@ the tmachine upgrade execution and its runtime-image preparation.
   with both managed and standalone compute drivers in `Branch E2E Checks`
 - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks`
 - `test:upgrade` runs tmachine Debian and RPM upgrade qualification in
-  `Branch E2E Checks`: the latest retained prerelease packages and matching
+  `Branch E2E Checks`: the latest published release packages and matching
   runtime images are installed first, then the PR's packages are installed and
   both existing and new sandboxes are verified
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay

tests/ansible/playbooks/openshell-deb-upgrade-source.yaml

@@ -2,15 +2,24 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Install prerelease OpenShell Debian upgrade source
+- name: Install released OpenShell Debian upgrade source
   ansible.builtin.import_playbook: openshell-deb.yaml
   vars:
-    openshell_packaged_gateway_legacy_images: true
+    openshell_packaged_gateway_released_source: true
 
 - name: Record OpenShell Debian upgrade source
   hosts: all
   gather_facts: false
   tasks:
+    - name: Create upgrade source metadata directory
+      become: true
+      ansible.builtin.file:
+        path: /var/lib/openshell-qualification
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
     - name: Copy upgrade source version metadata
       become: true
       ansible.builtin.copy:

tests/ansible/playbooks/openshell-rpm-upgrade-source.yaml

@@ -2,15 +2,24 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Install prerelease OpenShell RPM upgrade source
+- name: Install released OpenShell RPM upgrade source
   ansible.builtin.import_playbook: openshell-rpm.yaml
   vars:
-    openshell_packaged_gateway_legacy_images: true
+    openshell_packaged_gateway_released_source: true
 
 - name: Record OpenShell RPM upgrade source
   hosts: all
   gather_facts: false
   tasks:
+    - name: Create upgrade source metadata directory
+      become: true
+      ansible.builtin.file:
+        path: /var/lib/openshell-qualification
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
     - name: Record installed source package versions
       become: true
       ansible.builtin.shell: |

tests/ansible/playbooks/upgrade/deb.yaml

@@ -15,7 +15,7 @@
     - name: Wait for SSH
       ansible.builtin.wait_for_connection:
 
-    - name: Exercise prerelease baseline and upgrade candidate
+    - name: Exercise released baseline and upgrade candidate
       block:
         - name: Read expected source package version
           become: true
@@ -33,11 +33,11 @@
           register: installed_source_version
           changed_when: false
 
-        - name: Require exact prerelease source package
+        - name: Require exact released source package
           ansible.builtin.assert:
             that: installed_source_version.stdout == upgrade_source_version
             fail_msg: >-
-              Expected prerelease source {{ upgrade_source_version }}, found
+              Expected released source {{ upgrade_source_version }}, found
               {{ installed_source_version.stdout }}.
 
         - name: Check source gateway status
@@ -132,7 +132,7 @@
           ansible.builtin.assert:
             that: installed_target_version.stdout != upgrade_source_version
             fail_msg: >-
-              Candidate package did not replace prerelease source
+              Candidate package did not replace released source
               {{ upgrade_source_version }}.
 
         - name: Wait for upgraded gateway status

tests/ansible/playbooks/upgrade/rpm.yaml

@@ -16,7 +16,7 @@
     - name: Wait for SSH
       ansible.builtin.wait_for_connection:
 
-    - name: Exercise prerelease baseline and RPM upgrade candidate
+    - name: Exercise released baseline and RPM upgrade candidate
       block:
         - name: Read expected source package versions
           become: true
@@ -40,11 +40,11 @@
           register: installed_source_versions
           changed_when: false
 
-        - name: Require exact prerelease source packages
+        - name: Require exact released source packages
           ansible.builtin.assert:
             that: installed_source_versions.stdout | trim == upgrade_source_versions
             fail_msg: >-
-              Expected prerelease source packages {{ upgrade_source_versions }},
+              Expected released source packages {{ upgrade_source_versions }},
               found {{ installed_source_versions.stdout | trim }}.
 
         - name: Check source gateway status
@@ -103,7 +103,7 @@
         - name: Require candidate RPM packages to replace source packages
           ansible.builtin.assert:
             that: candidate_package_versions.stdout | trim != upgrade_source_versions
-            fail_msg: Candidate RPM versions match the retained prerelease source.
+            fail_msg: Candidate RPM versions match the released source.
 
         - name: Copy candidate runtime images
           become: true