Release from candidates: rc-* tags deploy staging, v* tags publish the crates and the local MCP

.github/scripts/build-mcpb.sh

@@ -2,7 +2,7 @@
 # Build the Claude Desktop bundle (.mcpb) for one imcp2-local release.
 #
 #   .github/scripts/build-mcpb.sh <tag> <out-dir>
-#   e.g. .github/scripts/build-mcpb.sh imcp2-local-v0.5.0 dist-mcpb
+#   e.g. .github/scripts/build-mcpb.sh v0.6.0 dist-mcpb
 #
 # The bundle is assembled from the release's own published archives, so it
 # carries exactly the binaries that release attests, each checked against the
@@ -37,10 +37,10 @@ set -euo pipefail
 tag="${1:?usage: build-mcpb.sh <tag> <out-dir>}"
 out="${2:?usage: build-mcpb.sh <tag> <out-dir>}"
 case "$tag" in
-  imcp2-local-v*) ;;
-  *) echo "not an imcp2-local release tag: $tag" >&2; exit 2 ;;
+  v[0-9]*) ;;
+  *) echo "not a vX.Y.Z release tag: $tag" >&2; exit 2 ;;
 esac
-version="${tag#imcp2-local-v}"
+version="${tag#v}"
 base="https://github.com/dfinity/imcp2/releases/download/$tag"
 repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
 lipo="${LIPO:-lipo}"
@@ -74,7 +74,7 @@ fi
 # refuses even a genuinely attested archive from an older release.
 verify_provenance() {
   gh attestation verify "$1" --repo dfinity/imcp2 \
-    --signer-workflow dfinity/imcp2/.github/workflows/imcp2-local-release.yml \
+    --signer-workflow dfinity/imcp2/.github/workflows/v-release.yml \
     --source-ref "refs/tags/$tag" --deny-self-hosted-runners >/dev/null
 }
 

.github/workflows/deploy-candidate.yml

@@ -0,0 +1,261 @@
+# Deploy a release candidate to staging.
+#
+# Staging is the one host this repository deploys, and it moves only when someone
+# cuts a candidate: an `rc-X.Y.Z-N` tag on a commit of `main` whose manifests
+# already say X.Y.Z. A merge to `main` deploys nothing, so what is under test never
+# changes underneath the tester. Production is not deployed from here — it embeds
+# the crate that the matching `vX.Y.Z` tag publishes (publish-crate.yml) — so the
+# flow is candidate, test on staging, promote. See deploy/native/README.md.
+#
+# Cutting a candidate (the version bump has already landed on main):
+#   git tag rc-0.6.0-1 <commit on main>
+#   git push origin rc-0.6.0-1
+# A failed candidate is followed by a fix on main and rc-0.6.0-2. Nothing is
+# bumped after a candidate, so the commit that is promoted is the one tested here.
+#
+# Rolling back, or redeploying: run this workflow from the Actions tab with an
+# earlier rc-* tag (or a full commit SHA) as `ref`.
+#
+# Required repository secrets (Settings -> Secrets and variables -> Actions). A job
+# calling a reusable workflow with `uses:` cannot declare an `environment:`, so the
+# secrets passed below resolve at repository and organization scope only. The
+# converse is the trap: deploy-native.yml's ship job does set `environment:`, so a
+# secret defined on the `staging` environment resolves there and OVERRIDES what is
+# passed here. Define each secret in exactly one place.
+#   DEPLOY_SSH_KEY    private SSH key (PEM/OpenSSH) for a sudo-capable user on the host
+#   DEPLOY_HOST       user@host, the host's PRIVATE address: the ship runner reaches
+#                     it over the VPN (see the ship_runs_on comment below)
+#   DEPLOY_DOMAIN     public FQDN served over HTTPS, e.g. mcp.example.com
+#   DEPLOY_ACME_EMAIL email for Let's Encrypt / ACME
+# Optional:
+#   DEPLOY_KNOWN_HOSTS  host public keys (output of `ssh-keyscan <host>`). If unset,
+#                       the host key is fetched at run time via ssh-keyscan (TOFU).
+name: Deploy candidate
+
+on:
+  push:
+    tags: ['rc-*']
+  # Manual re-run, and the rollback path: pass an earlier candidate tag as `ref`.
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: >-
+          rc-* tag (e.g. rc-0.6.0-1, or the fully-qualified refs/tags/rc-0.6.0-1),
+          or a full 40-character commit SHA.
+        required: true
+        type: string
+
+# Never run two deploys at once; let an in-flight deploy finish rather than cancel it
+# half-way through replacing the binary on the host.
+concurrency:
+  group: deploy-staging
+  cancel-in-progress: false
+
+jobs:
+  # An `rc-*` tag push is constrained by the trigger. workflow_dispatch is not: its
+  # `ref` is free text, so `main` would deploy whatever that branch points at right
+  # now, which is the property this workflow exists to prevent. Accept only refs
+  # that cannot move: an rc-* tag, or a full 40-character commit SHA for the case
+  # where the commit to roll back to never had a tag cut for it.
+  validate-ref:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-24.04
+    outputs:
+      # The accepted ref in unambiguous form, consumed by deploy below. Empty when
+      # this job is skipped (tag push), which is what the fallback there handles.
+      ref: ${{ steps.normalize.outputs.ref }}
+    steps:
+      - name: Reject refs that can move
+        id: normalize
+        env:
+          REF: ${{ inputs.ref }}
+        run: |
+          set -euo pipefail
+
+          # Constrain the entire input to one line of refname characters BEFORE
+          # looking at its shape. This step writes an input-derived value to
+          # $GITHUB_OUTPUT, and the dispatch input is free text that the REST API
+          # will happily accept newlines in. A newline turns the write below into
+          # multiple key=value lines, a second `ref=` line wins, and the deploy
+          # checks out whatever was appended while the run's inputs still show the
+          # innocuous value. Anchors must bind the whole string, so this uses bash
+          # [[ =~ ]] rather than grep, which is line-oriented.
+          #
+          # $REF is deliberately not echoed here: it has not been vetted yet, and
+          # stdout is parsed for ::workflow commands::.
+          if [[ ! "$REF" =~ ^[A-Za-z0-9._/-]+$ ]]; then
+            echo "::error::Refusing the dispatch ref: it must be a single line of [A-Za-z0-9._/-] characters. Pass an rc-* tag or a full 40-character commit SHA."
+            exit 1
+          fi
+
+          # Strip a fully-qualified tag prefix so refs/tags/rc-X is treated the same
+          # as rc-X. Only refs/tags/ is stripped: refs/heads/rc-X must stay
+          # rejected, and does, because after no stripping it no longer starts
+          # with "rc-".
+          ref="${REF#refs/tags/}"
+
+          if [[ "$ref" =~ ^rc-[A-Za-z0-9._/-]+$ ]]; then
+            # git's own refname grammar catches what a character class cannot.
+            if ! git check-ref-format "refs/tags/$ref"; then
+              echo "::error::Refusing to deploy '$REF': not a valid git tag name."
+              exit 1
+            fi
+            # Emit the tag fully qualified: a bare rc-X is ambiguous if a *branch*
+            # of that name also exists, and refs/tags/rc-X can only resolve to the
+            # tag, failing outright if no such tag exists.
+            echo "ref=refs/tags/$ref" >> "$GITHUB_OUTPUT"
+          elif [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then
+            echo "ref=$ref" >> "$GITHUB_OUTPUT"
+          else
+            echo "::error::Refusing to deploy '$REF': it is neither an rc-* tag (bare or refs/tags/-qualified) nor a full 40-character commit SHA. Branches and abbreviated SHAs can move or become ambiguous, so the deployed revision would not be reproducible."
+            exit 1
+          fi
+          echo "accepted '$REF'"
+
+  # A candidate names the version it is a candidate for, and the manifests must
+  # already say so: the vX.Y.Z tag that promotes it is checked against every
+  # Cargo.toml by publish-crate.yml and against imcp2-local's by dist, and neither
+  # can be satisfied after the fact without changing the commit that was tested.
+  # The commit must also be on main, like the release tag's.
+  verify:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+      # Third-party actions are pinned to a commit SHA (supply-chain hardening);
+      # the trailing comment records the human-readable version.
+      - name: Checkout
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          # Full history: the ancestry check below needs the commits between this
+          # tag and main, which the default shallow clone does not fetch.
+          fetch-depth: 0
+
+      - name: Check the tag names the version in every Cargo.toml
+        run: |
+          set -euo pipefail
+          if [[ ! "$GITHUB_REF_NAME" =~ ^rc-([0-9]+\.[0-9]+\.[0-9]+)-([0-9]+)$ ]]; then
+            echo "::error::${GITHUB_REF_NAME} is not an rc-X.Y.Z-N tag" >&2
+            exit 1
+          fi
+          version="${BASH_REMATCH[1]}"
+          for pkg in imcp2-core imcp2 imcp2-local; do
+            manifest="$(cargo metadata --format-version 1 --no-deps |
+              jq -er --arg pkg "$pkg" '.packages[] | select(.name == $pkg) | .version')"
+            echo "candidate=$version $pkg=$manifest"
+            if [ "$version" != "$manifest" ]; then
+              echo "::error::${GITHUB_REF_NAME} is a candidate for ${version} but ${pkg}'s Cargo.toml says ${manifest}; land the version bump on main first" >&2
+              exit 1
+            fi
+          done
+
+      - name: Check the tag is on main
+        run: |
+          git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
+          if ! git merge-base --is-ancestor "${GITHUB_SHA}" origin/main; then
+            echo "::error::${GITHUB_REF_NAME} points at a commit that is not on main" >&2
+            exit 1
+          fi
+
+  deploy:
+    needs: [validate-ref, verify]
+    # One of the two is skipped on every path (validate-ref on a tag push, verify on
+    # a dispatch), and a skipped dependency would skip this job too under the
+    # default `success()` gate. Gate on "nothing failed" instead.
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/deploy-native.yml
+    with:
+      environment: staging
+      # A bare scale-set name, not a label list: the org's self-hosted capacity is
+      # ARC-managed pools, and a runner scale set is selected by its name alone,
+      # carrying none of the default labels classic runners get. dind-small is one
+      # of those org-wide, Kubernetes-backed pools; its pods reach this host on its
+      # private address over the VPN. The pool spans more than one cluster, so the
+      # source address the host sees is not fixed.
+      ship_runs_on: '"dind-small"'
+      # The staging host is arm64 (Graviton). Build natively for it.
+      arch: arm64
+      build_runs_on: '"ubuntu-24.04-arm"'
+      # validate-ref's normalized output on a dispatch; on a tag push it is skipped,
+      # its output is empty, and github.ref (already refs/tags/rc-*) is used.
+      ref: ${{ needs.validate-ref.outputs.ref || github.ref }}
+    secrets:
+      DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+      DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+      DEPLOY_DOMAIN: ${{ secrets.DEPLOY_DOMAIN }}
+      DEPLOY_ACME_EMAIL: ${{ secrets.DEPLOY_ACME_EMAIL }}
+      DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
+      # Submission-specific, not host-specific (see deploy-native.yml).
+      OPENAI_APPS_CHALLENGE_TOKEN: ${{ secrets.OPENAI_APPS_CHALLENGE_TOKEN }}
+
+  # Attach the binary that was actually deployed to a GitHub prerelease named after
+  # the candidate. Without this the only copy is a workflow artifact, which expires
+  # after 7 days, so the revision under test would stop being retrievable while it
+  # is still being tested. A prerelease, so GitHub's `latest` release stays a
+  # promoted vX.Y.Z (the release dist creates), never a candidate. It is also the
+  # marker publish-crate.yml checks before promoting: the asset exists only if the
+  # deploy succeeded, so keep its name (imcp2-linux-arm64) in step with that check.
+  #
+  # Runs only after deploy succeeds, because an asset here asserts "this is what
+  # staging is running". And only on the tag-push path: a dispatch redeploys an
+  # existing candidate whose prerelease already carries these assets, and
+  # github.ref_name on a dispatch is the branch rather than the tag.
+  publish:
+    needs: deploy
+    # Combined with a status function on purpose: a bare event check gets an
+    # implicit success() over the whole upstream chain, which contains a skipped
+    # job on every path, and would never pass.
+    if: ${{ !failure() && !cancelled() && github.event_name == 'push' }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write  # create the prerelease and upload assets to it
+      actions: read    # read this run's artifacts (the download below)
+    env:
+      GH_TOKEN: ${{ github.token }}
+      # No checkout in this job, so gh cannot infer the repository from a remote.
+      GH_REPO: ${{ github.repository }}
+    steps:
+      # GITHUB_RUN_ID scopes the download to this run, so it can only ever pick up
+      # the artifact the deploy above built.
+      - name: Download the deployed binary
+        run: gh run download "$GITHUB_RUN_ID" --name imcp2-binary --dir dist
+
+      - name: Name and checksum the asset
+        env:
+          ARCH: arm64
+        run: |
+          set -euo pipefail
+          mv dist/imcp2 "dist/imcp2-linux-${ARCH}"
+          ( cd dist && sha256sum "imcp2-linux-${ARCH}" > "imcp2-linux-${ARCH}.sha256" )
+          cat "dist/imcp2-linux-${ARCH}.sha256"
+
+      - name: Publish to the prerelease page
+        env:
+          TAG: ${{ github.ref_name }}
+          # From the reusable workflow, not github.sha: it records what was built.
+          BUILT_SHA: ${{ needs.deploy.outputs.sha }}
+        run: |
+          set -euo pipefail
+          notes="$(cat <<EOF
+          Release candidate, deployed to staging.
+
+          | | |
+          |---|---|
+          | Commit | \`$BUILT_SHA\` |
+          | Target | \`linux/arm64\` (Amazon Linux 2023, Graviton) |
+          | Run | [$GITHUB_RUN_ID](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |
+
+          The attached binary is the one that was shipped: the deploy asserts
+          \`GET /version\` on the host reports this commit before the run is allowed
+          to pass. Verify with \`sha256sum -c imcp2-linux-arm64.sha256\`. Promoting
+          this candidate (\`git tag vX.Y.Z $TAG^{}\`) publishes the crates and the
+          local MCP binaries from this same commit.
+          EOF
+          )"
+          # Idempotent so a re-run repairs a partial publish rather than failing.
+          if gh release view "$TAG" >/dev/null 2>&1; then
+            gh release upload "$TAG" dist/* --clobber
+          else
+            gh release create "$TAG" --prerelease --title "$TAG" --notes "$notes" dist/*
+          fi

.github/workflows/deploy-native.yml

@@ -1,9 +1,8 @@
 # Reusable native deploy: cross-build the binary for inputs.arch (arm64 or amd64),
-# then ship it over SSH and (re)start the systemd services. Called by deploy.yml
-# (main -> staging) and deploy-release.yml (release-* tags -> production). Each
-# caller supplies its own environment, runner labels and architecture; the release
-# caller additionally pins an explicit ref, so a rollback can name an earlier tag.
-# See deploy/native/README.md.
+# then ship it over SSH and (re)start the systemd services. Called by
+# deploy-candidate.yml (rc-* tags -> staging), which supplies the environment,
+# runner labels and architecture, and on a manual run pins an explicit ref so a
+# rollback can name an earlier tag. See deploy/native/README.md.
 #
 # Split into two jobs on purpose:
 #