From 36d390a4d97836ba9f604d885279a56dc62c8110 Mon Sep 17 00:00:00 2001 From: Neyts Zupan Date: Wed, 18 Nov 2026 22:14:05 +0000 Subject: [PATCH] chore: Streamline Passlet releases Keep releases focused on the already-tested candidate and its signed artifacts. Add generated metadata to the release PR automatically and verify it without repeating product tests. Move website audits out of the release gate and report progress at milestones. --- .agents/skills/release-passlet/SKILL.md | 31 +++++------- .github/workflows/ci.yml | 59 ++++++++++++++++++++-- .github/workflows/release-metadata.yml | 30 ++++++++++++ docs/RELEASING.md | 17 +++---- packaging/README.md | 2 +- scripts/check-release-metadata.py | 42 ++++++++++++++++ scripts/release.sh | 1 - scripts/test-release.py | 52 ++++++++++++++++++++ scripts/update-release-pr.py | 65 +++++++++++++++++++++++++ scripts/verify-release.py | 45 +++++++++-------- 10 files changed, 290 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/release-metadata.yml create mode 100644 scripts/check-release-metadata.py create mode 100644 scripts/update-release-pr.py diff --git a/.agents/skills/release-passlet/SKILL.md b/.agents/skills/release-passlet/SKILL.md index fed9b9a..7e15ca1 100644 --- a/.agents/skills/release-passlet/SKILL.md +++ b/.agents/skills/release-passlet/SKILL.md @@ -9,20 +9,15 @@ Work from the Passlet repository. A release is installable only after the signed Creating or editing this skill does not authorize a release. For a readiness request, perform checks and report outstanding work. For a release request, establish the intended unused `vX.Y.Z` version and target commit. If publication is not already authorized, prepare the version, exact SHA, release notes, and check results before asking for approval. Do not publish, merge PRs, or activate someone's machine merely to test the workflow. -## Visible release checklist +## Progress reports -At the start of every release, show the user a checklist covering every checkbox in the numbered phases below, in order. Shorten the wording for readability, but keep each action, approval, and wait as a separate item. This conversation checklist is the release's progress record, not an edit to this skill. - -Reprint the checklist as each item completes and whenever work starts waiting for CI, approval, or another blocker. Tick `[x]` only after verifying completion, with the relevant PR, workflow, release, or post URL as evidence. Keep running, awaiting approval, blocked, failed, and unperformed items unchecked and label their status. An existing authorization can satisfy an approval item without asking again, but starting a run never satisfies its completion item. If the candidate changes, reopen checks tied to the old SHA. Mark inapplicable items or explicit user skips with a reason instead of silently dropping them or claiming they passed. - -Keep the checklist and exact next action in handoffs and resume from verified progress. Finish with the full checklist and any remaining blockers, never a completion claim while required items remain unresolved. +Report milestones: candidate checked, artifacts prepared, metadata ready for review, published, and availability verified. Include the relevant link and any action needed from the user. Keep the candidate SHA and outstanding steps in handoffs. Do not repeat the full checklist on every update. ## Sources and access Read these files from the release candidate, rather than relying on historical release results: -- `docs/RELEASING.md`: signing credentials, environment protection, and production validation. -- `docs/VALIDATION.md`: manual checks for protected Keychain and native approval. +- `docs/RELEASING.md`: signing credentials and environment protection. - `.github/workflows/release.yml` and `scripts/release.sh`: actual trigger, signing, asset creation, and upload behavior. - `packaging/README.md` and `scripts/package-release.py`: package metadata generation and validation. @@ -37,11 +32,9 @@ Pass all authenticated `gh` and Git operations through `passlet exec --grant `, build the site, and run Hakuto's `seo-audit` skill against that build, including its opt-in external-link liveness checks. Verify broken-link reports before changing destinations, and record any links that could not be checked. Fix findings, rebuild, and rerun the audit. Record the candidate SHA, website revision, and results. -- [ ] Run Hakuto's `pagespeed-audit` skill against the publicly reachable website preview for that candidate, covering the homepage and affected documentation pages on mobile and desktop. Fix actionable regressions and rerun affected audits. Record the URLs and results. If the preview is inaccessible to PageSpeed, report the check as blocked rather than substituting the current production site or treating it as passed. -- [ ] Run applicable checks in `nix-shell`: `make check-all`, `nix-build nix/packaging-tests.nix --no-out-link`, and the relevant macOS/integration checks. Existing successful CI for the exact candidate can supply platform checks that cannot run locally. Distinguish passed, failed, and unperformed checks. -- [ ] Verify production signing and manual validation prerequisites in `docs/RELEASING.md` and `docs/VALIDATION.md`. Do not treat preview signatures, fixtures, or older releases as proof of production Keychain behavior. +- [ ] Confirm the documentation preview build and link checks pass for the candidate. Website design, SEO and PageSpeed audits are outside the release gate. Production PageSpeed checks run weekly through [the ops updater](https://github.com/teamniteo/ops/tree/main/.agents/skills-local/updater). +- [ ] Use the successful CI run for the exact candidate as product-test evidence. Run local checks when developing fixes, not again merely to release the same SHA. Source changes require fresh CI. +- [ ] Verify production signing prerequisites in `docs/RELEASING.md`. - [ ] Wait for the latest PR CI run for the final candidate SHA to finish successfully. Verify that SHA is still the release PR head before preparing signed artifacts. The Release workflow has two phases. `vX.Y.Z-prepare` builds and signs the CI-verified head of an open same-repository release PR without publishing. After its generated package metadata is added to that same PR, reviewed, and squash-merged, `vX.Y.Z` publishes those exact prepared artifacts. Preparation checks the exact PR head and passing PR CI; the final tag must be on main. Both phases retain the protected `release` environment. Review the exact preparation commit before approving signing, especially changes to workflows and build scripts. Never move tags, rebuild approved artifacts during publication, or publish an empty release. @@ -52,9 +45,9 @@ The Release workflow has two phases. `vX.Y.Z-prepare` builds and signs the CI-ve - [ ] Push the unused `vX.Y.Z-prepare` tag at the reviewed full candidate SHA through the scoped grant. - [ ] Identify the Release workflow run for that exact tag and SHA. Link the run and verify its tag and SHA. - [ ] Wait for the preparation run’s protected `release` environment approval, linking the run when user action is needed. Never weaken or bypass environment protection. -- [ ] Wait for preparation CI to finish successfully. It tests, signs, notarizes and staples the macOS app, builds Linux archives, and generates checksums and metadata. It does not create a GitHub release. +- [ ] Wait for preparation CI to finish successfully. It signs, notarizes and staples the macOS app, builds Linux archives, and generates checksums and metadata. It does not create a GitHub release. - [ ] Download `passlet-release-assets` and `passlet-package-updates` from that successful run into a private task directory. Retain the run ID and candidate SHA. -- [ ] Verify the exact three archives and `passlet_X.Y.Z_SHA256SUMS`, all checksums, the macOS signature, and its stapled ticket. Perform applicable production validation before final publication. Report manual checks not performed; do not substitute previous releases or fixture tests for them. +- [ ] Verify the exact three archives and `passlet_X.Y.Z_SHA256SUMS`, all checksums, the macOS signature, and its stapled ticket. Apple's `stapler` attaches and validates notarization tickets so Gatekeeper can verify notarization offline. A Nix shell can set `DEVELOPER_DIR` to an SDK that lacks this tool, making `xcrun stapler` report it missing even when Apple Command Line Tools are installed. Check `/Library/Developer/CommandLineTools/usr/bin/stapler` before installing anything. If present, validate the downloaded bundle directly: @@ -69,8 +62,8 @@ For a failed run, report the failing stage and any draft or published assets. If ## 3. Add metadata to the release PR - [ ] Inspect all five generated files: `Casks/passlet.rb`, `Formula/passlet-agent.rb`, `packaging/release.json`, `plugin.json`, and `.claude-plugin/plugin.json`. Verify exact archive URLs, hashes, matching versions, and the preparation run ID and SHA. -- [ ] Copy them onto the existing release PR branch. Do not open a second metadata PR. If regeneration is necessary, run `python3 scripts/package-release.py X.Y.Z --run-id --source-sha ` in the development shell. Never invent provenance or hashes. -- [ ] Run `make check` and `nix-build nix/packaging-tests.nix --no-out-link`. Before publication, the final archive URLs are not public, so use the downloaded signed archives for installation validation. Never replace the maintainer's working installation or credentials as an implicit test. +- [ ] Wait for the Release metadata workflow to add the generated files and docs labels to the same PR and dispatch CI. If the PR head moved, prepare the new candidate rather than overwriting changes. For recovery, regenerate with `python3 scripts/package-release.py X.Y.Z --run-id --source-sha `, inspect and commit the result to the same PR. +- [ ] Confirm metadata CI verifies preparation provenance, unchanged source, archive checksums and exact generated metadata. It reuses the preparation's product-test evidence, without running the platform matrix again. Any source change takes the full CI path. - [ ] Update the existing PR to describe the implementation and release together. Keep one final commit where practical. - [ ] Wait for the latest PR CI run for the updated metadata commit to finish successfully. - [ ] Obtain code and generated metadata review and confirm merge/publication authorization, requesting only approvals not already given. @@ -98,6 +91,6 @@ Include generated `docs/` files from `passlet-package-updates` alongside the pac ## 6. Hand off -- [ ] Report preparation and final SHAs, release URL, workflow results, release PR state, checks performed, and any remaining blockers. Include the final checklist. Do not call the release complete before its package/plugin metadata and installation have been verified. +- [ ] Report preparation and final SHAs, release URL, workflow results, release PR state, checks performed, and any remaining blockers. Do not call the release complete before its package/plugin metadata and installation have been verified. -If work stops awaiting environment approval, metadata review, or manual validation, preserve those exact next steps. +If work stops awaiting environment approval or metadata review, preserve those exact next steps. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a3a51e..db21f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,57 @@ defaults: run: shell: bash -eo pipefail {0} jobs: + metadata: + name: Verify release metadata + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + verified: ${{ steps.verify.outputs.metadata }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - id: verify + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/check-release-metadata.py >> "$GITHUB_OUTPUT" + + required: + name: ${{ matrix.check }} + needs: [integration, macos-package, metadata, test] + if: always() + runs-on: ubuntu-26.04 + strategy: + matrix: + check: + - NixOS SSH integration + - Test (macos-26) + - Test (ubuntu-26.04) + - Test macOS Nix package + steps: + - env: + METADATA_RESULT: ${{ needs.metadata.result }} + VERIFIED: ${{ needs.metadata.outputs.verified }} + TEST_RESULT: ${{ needs.test.result }} + PACKAGE_RESULT: ${{ needs.macos-package.result }} + INTEGRATION_RESULT: ${{ needs.integration.result }} + run: | + test "$METADATA_RESULT" = success + if [ "$VERIFIED" != true ]; then + test "$VERIFIED" = false + test "$TEST_RESULT" = success + test "$PACKAGE_RESULT" = success + test "$INTEGRATION_RESULT" = success + fi + test: - name: Test (${{ matrix.os }}) + needs: metadata + if: needs.metadata.outputs.verified != 'true' + name: Full tests (${{ matrix.os }}) strategy: fail-fast: false matrix: @@ -99,7 +148,9 @@ jobs: retention-days: 14 macos-package: - name: Test macOS Nix package + needs: metadata + if: needs.metadata.outputs.verified != 'true' + name: Full macOS Nix package runs-on: macos-26 timeout-minutes: 20 steps: @@ -118,7 +169,9 @@ jobs: run: ./scripts/check-package-artifact.sh integration: - name: NixOS SSH integration + needs: metadata + if: needs.metadata.outputs.verified != 'true' + name: Full NixOS SSH integration runs-on: ubuntu-26.04 timeout-minutes: 40 steps: diff --git a/.github/workflows/release-metadata.yml b/.github/workflows/release-metadata.yml new file mode 100644 index 0000000..c4f7f47 --- /dev/null +++ b/.github/workflows/release-metadata.yml @@ -0,0 +1,30 @@ +name: Release metadata +on: + workflow_run: + workflows: [Release] + types: [completed] +permissions: + contents: read +concurrency: + group: passlet-release-metadata + cancel-in-progress: false +jobs: + update: + if: github.event.workflow_run.conclusion == 'success' && endsWith(github.event.workflow_run.head_branch, '-prepare') + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + actions: write + contents: write + pull-requests: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Add verified metadata to the release PR + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PREPARATION_RUN: ${{ github.event.workflow_run.id }} + run: python3 scripts/update-release-pr.py "$PREPARATION_RUN" diff --git a/docs/RELEASING.md b/docs/RELEASING.md index be7c716..2edfe41 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -33,8 +33,7 @@ uploads, or automatic dependency merging into this CLI project. 5. The production bundle ID has a separate Keychain access group. Import user credentials through the production signed helper; do not silently broaden the prototype's Keychain ACL or copy stale bootstrap tokens. Fresh GitHub user - authorization may be needed. Confirm approval, denial, cancellation, and refresh - storage using the production identity before release. + authorization may be needed. ## Local checks @@ -52,17 +51,19 @@ the existing prototype identity. The icon source is `assets/passlet.png`; ## Prepare and validate a release -Before pushing the preparation tag, update documentation for the release and complete the website checks in the [release checklist](../.agents/skills/release-passlet/SKILL.md). Build `teamniteo/site-passlet` with `PASSLET_DOCS_REF` set to the full candidate SHA, run Hakuto's `hakuto-review` on the website source and configuration, run `seo-audit` on that build including external-link liveness checks, and run `pagespeed-audit` against its public preview on mobile and desktop. Cover the homepage and affected documentation pages, fix findings, and record the audited revisions, URLs, and results. An unavailable audit is outstanding work, not a passing check. Finalize documentation changes before signing, since publication only permits the generated release-label replacements afterward. +Before pushing the preparation tag, update documentation and confirm the candidate's documentation preview build and link checks pass. Finalize documentation before signing, since publication only permits generated release-label replacements afterward. Website design and SEO audits are outside the release gate. Production PageSpeed audits run weekly through [the ops updater](https://github.com/teamniteo/ops/tree/main/.agents/skills-local/updater). Open the release PR in this repository against main. Include concise, user-facing notes in `docs/releases/vX.Y.Z.md`, describing changes since the previous published release and linking to the full changelog. Both workflow phases require this nonempty file. Finalize the notes before preparation, because source verification rejects later changes to them. After its CI passes, push an unused preparation tag such as `v0.4.2-prepare` at its exact reviewed head commit. Preparation verifies the PR is still open, the tag points to its current head, and its latest PR CI run passed. Fork PRs are rejected. Review that commit, including workflow and build-script changes, before approving signing. Both preparation and final publication require approval under the existing `release` environment, restricted to `v*` tags. No signing credentials are available to PR workflows. -Preparation runs the pinned tests, builds Apple Silicon macOS binaries, verifies Developer ID signing, notarizes, staples, assesses Gatekeeper, and builds Linux archives and SHA-256 sums. It saves `passlet-release-assets` and `passlet-package-updates` as workflow artifacts. It does not create a GitHub release or the final version tag. +Preparation reuses the exact candidate’s passing product CI, builds Apple Silicon macOS binaries, verifies Developer ID signing, notarizes, staples, assesses Gatekeeper, and builds Linux archives and SHA-256 sums. It saves `passlet-release-assets` and `passlet-package-updates` as workflow artifacts. It does not create a GitHub release or the final version tag. Preparation then tests the Linux archives themselves. Each archive runs natively on an amd64 and an arm64 runner, which checks its version and that `passlet approver` refuses to run. The amd64 archive also replaces the source build as the Agent in the NixOS integration test, so the binary we publish pairs, requests grants and uses Git end to end. Publication only accepts a preparation run where all of these passed. To rerun them against an earlier preparation, start the `Test Linux release archives` workflow with that run ID and version. Stapling and ticket validation retry the observed Apple connection-loss error (`NSURLErrorDomain Code=-1005`, exit 68) up to four attempts, waiting 15, 30, and 45 seconds between attempts. Other failures stop immediately. These retries reuse the signed, notarized app without rebuilding or resubmitting it. -Download both artifacts from that successful run. Verify checksums and production signatures, then add to that same PR `Casks/passlet.rb`, `Formula/passlet-agent.rb`, `packaging/release.json`, `plugin.json`, and `.claude-plugin/plugin.json`. The release JSON identifies the exact preparation run and source SHA. Do not edit generated files, invent checksums, or bump plugin versions separately. +After successful preparation, the Release metadata workflow regenerates and verifies package metadata and docs version labels, commits them to the same PR, and dispatches CI. It refuses to update a PR whose head moved since preparation. The release JSON identifies the exact preparation run and source SHA. Do not edit generated files, invent checksums, or bump plugin versions separately. + +CI verifies the successful preparation, unchanged source, archive checksums and exact regenerated metadata before reusing the preparation's product-test evidence. Generated metadata updates use this lightweight path. Source changes run the full platform tests. Required check names remain the same, and verification failures block them. Review the generated metadata and squash-merge the release PR, then push the final `vX.Y.Z` tag at that commit on main. Only those five generated metadata files and the generated `AddedIn` version replacements may differ from the prepared source. Verification compares file contents rather than requiring the pre-squash commit to be an ancestor, so code and metadata can land as one commit. If a rebase or merge resolution changes other files, prepare a new release rather than publishing binaries for different code. Merge immediately before final publication: consumers following main can see the new package URLs before the assets are public, so do not leave this intermediate state deployed. @@ -77,10 +78,6 @@ Keep the CLI and helper together inside the signed bundle. entitlement and provisioning profile only there. The bundle contains the CLI and approval helper only. The build signs the CLI before the outer helper bundle. -On a clean Mac, verify Gatekeeper acceptance and the real protected-Keychain -approval/denial/cancellation flow. Confirm Mac-offline use after token delivery. - -Run production signing/Keychain validation before announcing the release. Preparation and publication have separate tag triggers and environment approvals; publication reuses the verified signed artifacts. Keep GitHub parent credentials, refresh tokens, and app client secrets out of all CI secrets, artifacts, logs, and screenshots. @@ -116,7 +113,7 @@ skills are not modified by editing the release source. ## v0.2 controls -See [REVIEW-POLICY.md](REVIEW-POLICY.md) for built-in PR/release approval settings, protected paths, and remaining administrator limitations. A matching CODEOWNERS list or roadmap statement is not enforcement. PR CI receives no signing credentials or cache-write secret. Release signing remains behind its protected environment. Do not publish v0.2 until the manual [validation checklist](VALIDATION.md) and administrative settings review are complete. The historical v0.1 results above do not establish v0.2 validation. +See [REVIEW-POLICY.md](REVIEW-POLICY.md) for built-in PR/release approval settings, protected paths, and remaining administrator limitations. A matching CODEOWNERS list or roadmap statement is not enforcement. PR CI receives no signing credentials or cache-write secret. Release signing remains behind its protected environment. The historical v0.1 results above do not establish v0.2 validation. ## Documentation release labels diff --git a/packaging/README.md b/packaging/README.md index c3d3d71..8925dad 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -6,7 +6,7 @@ Homebrew and Nix/Home Manager are the supported installation paths. They own ser The `vX.Y.Z-prepare` workflow builds and signs the CI-verified head of an open release PR, then uploads `passlet-release-assets` and `passlet-package-updates`. The latter contains `Casks/passlet.rb`, `Formula/passlet-agent.rb`, `packaging/release.json`, `plugin.json`, and `.claude-plugin/plugin.json`, all generated from the same archives. The release JSON records the preparation run ID and source SHA, and both plugin manifests carry the release version. -Download the artifacts from the successful preparation run, verify the signed archives, and add all five generated files to the same release PR. These URLs deliberately refer to the pending release. Squash-merge the reviewed code and metadata as one commit immediately before pushing the final `vX.Y.Z` tag. Do not deploy the intermediate main revision before publication, because its new artifact URLs are not public yet. +After successful preparation, the Release metadata workflow adds verified generated metadata and docs version labels to the same PR and dispatches metadata CI. It stops if the PR head moved. Review the changes before squash-merging. Metadata CI verifies preparation provenance, checksums and unchanged source instead of repeating product tests. Use the generator below only for local fixtures or recovery. Publication downloads the recorded preparation artifacts, verifies that the merged source tree matches the prepared PR apart from generated metadata, verifies checksums, and regenerates the metadata to check exact equality with the final tag. It then uploads and publishes the immutable release. Do not rebuild between preparation and publication, use `latest` or `:no_check`, invent hashes, or change either tag. A final release tag therefore contains matching Nix/Homebrew metadata and plugin versions. diff --git a/scripts/check-release-metadata.py b/scripts/check-release-metadata.py new file mode 100644 index 0000000..283d953 --- /dev/null +++ b/scripts/check-release-metadata.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Reuse preparation tests only after verifying the generated release metadata.""" +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location("verify_release", ROOT / "scripts/verify-release.py") +release = importlib.util.module_from_spec(spec) +spec.loader.exec_module(release) + + +def candidate(root=ROOT): + manifest = json.loads((root / "packaging/release.json").read_text()) + version = manifest.get("version", "") + preparation = manifest.get("preparation", {}) + source = preparation.get("source_sha", "") + run_id = preparation.get("run_id") + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) or not re.fullmatch(r"[0-9a-f]{40}", source) or type(run_id) is not int or run_id <= 0: + return None + try: + release.verify_source(version, source, root) + except (ValueError, subprocess.CalledProcessError): + return None + return version, run_id + + +if __name__ == "__main__": + metadata = candidate() + if metadata is None: + print("metadata=false") + else: + version, run_id = metadata + with tempfile.TemporaryDirectory() as directory: + artifacts = Path(directory) / "assets" + run = json.loads(subprocess.check_output(["gh", "api", f"repos/ParetoSecurity/passlet/actions/runs/{run_id}"])) + subprocess.run(["gh", "run", "download", str(run_id), "--repo", "ParetoSecurity/passlet", "--name", "passlet-release-assets", "--dir", str(artifacts)], check=True) + release.verify(version, artifacts, run) + print("metadata=true") diff --git a/scripts/release.sh b/scripts/release.sh index 8df1722..e215ed6 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -17,7 +17,6 @@ fi /usr/bin/xcrun --find notarytool >/dev/null /usr/bin/xcrun --find stapler >/dev/null export PASSLET_BUILD_MODE=release -make check ./scripts/build-macos.sh mkdir -p dist app=build/release/Passlet.app diff --git a/scripts/test-release.py b/scripts/test-release.py index 0bd256a..45bd208 100644 --- a/scripts/test-release.py +++ b/scripts/test-release.py @@ -19,6 +19,13 @@ preparation = importlib.util.module_from_spec(spec) spec.loader.exec_module(preparation) +spec = importlib.util.spec_from_file_location("metadata", ROOT / "scripts/check-release-metadata.py") +metadata = importlib.util.module_from_spec(spec) +spec.loader.exec_module(metadata) +spec = importlib.util.spec_from_file_location("update_pr", ROOT / "scripts/update-release-pr.py") +update_pr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(update_pr) + class StaplerTests(unittest.TestCase): def run_stapler(self, failures): @@ -130,6 +137,27 @@ def test_notes_are_set_when_creating_the_draft(self): self.assertIn("--draft=false", events[-1]["args"]) +class RequiredChecksTests(unittest.TestCase): + def test_required_checks_fail_when_verification_or_full_tests_fail(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text() + section = workflow.split(" required:\n", 1)[1].split("\n test:", 1)[0] + script = textwrap.dedent(section.split(" run: |\n", 1)[1]) + env = dict(os.environ, METADATA_RESULT="success", VERIFIED="false", TEST_RESULT="success", PACKAGE_RESULT="success", INTEGRATION_RESULT="success") + for verified in ["true", "false"]: + result = subprocess.run(["bash", "-euc", script], env=dict(env, VERIFIED=verified)) + self.assertEqual(result.returncode, 0) + for field in ["METADATA_RESULT", "TEST_RESULT", "PACKAGE_RESULT", "INTEGRATION_RESULT"]: + for value in ["failure", "cancelled", "skipped", ""]: + with self.subTest(field=field, value=value): + result = subprocess.run(["bash", "-euc", script], env=dict(env, **{field: value})) + self.assertNotEqual(result.returncode, 0) + for value in ["failure", "cancelled", "skipped", ""]: + result = subprocess.run(["bash", "-euc", script], env=dict(env, VERIFIED="true", METADATA_RESULT=value)) + self.assertNotEqual(result.returncode, 0) + result = subprocess.run(["bash", "-euc", script], env=dict(env, VERIFIED="")) + self.assertNotEqual(result.returncode, 0) + + class PreparationTests(unittest.TestCase): def setUp(self): self.sha = "a" * 40 @@ -163,6 +191,19 @@ def test_rejects_forks_closed_prs_wrong_bases_and_stale_heads(self): with self.assertRaisesRegex(ValueError, "open PR"): preparation.verify(self.sha, [], {"workflow_runs": [self.run]}) + def test_metadata_update_rejects_moved_heads_and_foreign_runs(self): + run = {"head_sha": self.sha, "head_branch": "v1.2.3-prepare", "event": "push", "path": ".github/workflows/release.yml", "repository": {"full_name": preparation.REPOSITORY}, "status": "completed", "conclusion": "success"} + self.pr["head"]["ref"] = "chore/release" + self.assertEqual(update_pr.select_pr(run, [self.pr]), ("1.2.3", "chore/release")) + with self.assertRaises(ValueError): + update_pr.select_pr(run, [self.pr, self.pr]) + for key, value in [("event", "pull_request"), ("conclusion", "failure"), ("head_branch", "main"), ("repository", {"full_name": "other/repo"})]: + with self.subTest(field=key), self.assertRaises(ValueError): + update_pr.select_pr(dict(run, **{key: value}), [self.pr]) + for key, value in [("sha", "b" * 40), ("ref", "main"), ("repo", {"full_name": "other/repo"})]: + with self.subTest(field=key), self.assertRaises(ValueError): + update_pr.select_pr(run, [dict(self.pr, head=dict(self.pr["head"], **{key: value}))]) + def test_rejects_missing_failed_pending_or_unrelated_ci(self): for field, value in [("head_sha", "b" * 40), ("path", ".github/workflows/other.yml"), ("event", "push"), ("status", "in_progress"), ("conclusion", "failure")]: with self.subTest(field=field): @@ -253,6 +294,17 @@ def test_rejects_unconverted_labels(self): with self.assertRaisesRegex(ValueError, "documentation differs"): self.verify() + def test_metadata_fast_path_requires_unchanged_source(self): + self.assertEqual(metadata.candidate(self.root), ("1.2.3", 123)) + (self.root / "app.go").write_text("changed product code\n") + self.commit() + self.assertIsNone(metadata.candidate(self.root)) + + def test_metadata_fast_path_does_not_accept_arbitrary_doc_changes(self): + (self.root / "docs/feature.mdx").write_text("changed documentation\n") + self.commit() + self.assertIsNone(metadata.candidate(self.root)) + def test_final_tag_matches_prepared_assets_and_both_plugins(self): self.verify() diff --git a/scripts/update-release-pr.py b/scripts/update-release-pr.py new file mode 100644 index 0000000..829eba6 --- /dev/null +++ b/scripts/update-release-pr.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Add verified preparation metadata to its unchanged release PR.""" +import argparse +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parent.parent +REPOSITORY = "ParetoSecurity/passlet" +spec = importlib.util.spec_from_file_location("verify_release", ROOT / "scripts/verify-release.py") +release = importlib.util.module_from_spec(spec) +spec.loader.exec_module(release) + + +def select_pr(run, pulls): + match = re.fullmatch(r"v([0-9]+\.[0-9]+\.[0-9]+)-prepare", run.get("head_branch", "")) + if not (match and run.get("event") == "push" and run.get("path") == ".github/workflows/release.yml" + and run.get("repository", {}).get("full_name") == REPOSITORY + and run.get("status") == "completed" and run.get("conclusion") == "success" + and re.fullmatch(r"[0-9a-f]{40}", run.get("head_sha", ""))): + raise ValueError("Expected a successful release preparation in this repository") + matches = [pr for pr in pulls if pr.get("state") == "open" + and pr.get("base", {}).get("ref") == "main" + and pr.get("base", {}).get("repo", {}).get("full_name") == REPOSITORY + and (pr.get("head", {}).get("repo") or {}).get("full_name") == REPOSITORY + and pr.get("head", {}).get("ref") != "main" + and pr.get("head", {}).get("sha") == run["head_sha"]] + if len(matches) != 1: + raise ValueError("Expected one open release PR still at the preparation SHA") + return match[1], matches[0]["head"]["ref"] + + +def update(run_id): + def api(endpoint): + return json.loads(subprocess.check_output(["gh", "api", f"repos/{REPOSITORY}/{endpoint}"])) + + run = api(f"actions/runs/{run_id}") + source = run["head_sha"] + version, branch = select_pr(run, api(f"commits/{source}/pulls")) + subprocess.run(["git", "check-ref-format", "--branch", branch], check=True) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + checkout, assets = root / "checkout", root / "assets" + subprocess.run(["git", "worktree", "add", "--detach", str(checkout), source], check=True) + try: + subprocess.run(["gh", "run", "download", str(run_id), "--repo", REPOSITORY, "--name", "passlet-release-assets", "--dir", str(assets)], check=True) + release.package.generate(version, assets, checkout, run_id, source, source=checkout) + subprocess.run(["git", "-C", str(checkout), "add", "--", *sorted(release.METADATA), "docs"], check=True) + subprocess.run(["git", "-C", str(checkout), "-c", "user.name=github-actions[bot]", "-c", "user.email=41898282+github-actions[bot]@users.noreply.github.com", "-c", "commit.gpgsign=false", "commit", "-m", f"chore: Prepare v{version} packages", "-m", "Pin packages and plugins to the signed release artifacts."], check=True) + release.verify(version, assets, run, checkout) + # A normal push rejects concurrent changes instead of overwriting them. + subprocess.run(["gh", "auth", "setup-git"], check=True) + subprocess.run(["git", "-C", str(checkout), "push", f"https://github.com/{REPOSITORY}.git", f"HEAD:refs/heads/{branch}"], check=True) + subprocess.run(["gh", "workflow", "run", "ci.yml", "--repo", REPOSITORY, "--ref", branch], check=True) + finally: + subprocess.run(["git", "worktree", "remove", "--force", str(checkout)], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("run_id", type=int) + update(parser.parse_args().run_id) diff --git a/scripts/verify-release.py b/scripts/verify-release.py index 09912de..926c5e2 100644 --- a/scripts/verify-release.py +++ b/scripts/verify-release.py @@ -23,26 +23,7 @@ } -def verify(version, artifacts, run, root=ROOT): - if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): - raise ValueError("Expected X.Y.Z release version") - release = json.loads((root / "packaging/release.json").read_text()) - preparation = release.get("preparation", {}) - source = preparation.get("source_sha", "") - run_id = preparation.get("run_id") - if not isinstance(run_id, int) or run_id <= 0 or not re.fullmatch(r"[0-9a-f]{40}", source): - raise ValueError("Release metadata must identify its preparation run and source SHA") - if not ( - run.get("id") == run_id - and run.get("head_sha") == source - and run.get("head_branch") == f"v{version}-prepare" - and run.get("event") == "push" - and run.get("path") == ".github/workflows/release.yml" - and run.get("repository", {}).get("full_name") == "ParetoSecurity/passlet" - and run.get("status") == "completed" - and run.get("conclusion") == "success" - ): - raise ValueError("Artifacts must come from the successful preparation workflow for this release") +def verify_source(version, source, root=ROOT): changed = subprocess.check_output(["git", "diff", "--name-only", source, "HEAD"], cwd=root, text=True) docs = subprocess.check_output(["git", "ls-tree", "-r", "--name-only", source, "--", "docs"], cwd=root, text=True).splitlines() expected_docs = {} @@ -64,6 +45,30 @@ def verify(version, artifacts, run, root=ROOT): if json.loads((root / name).read_text()) != original: raise ValueError(f"Tagged metadata changes the prepared plugin beyond its version: {name}") + + +def verify(version, artifacts, run, root=ROOT): + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError("Expected X.Y.Z release version") + release = json.loads((root / "packaging/release.json").read_text()) + preparation = release.get("preparation", {}) + source = preparation.get("source_sha", "") + run_id = preparation.get("run_id") + if not isinstance(run_id, int) or run_id <= 0 or not re.fullmatch(r"[0-9a-f]{40}", source): + raise ValueError("Release metadata must identify its preparation run and source SHA") + if not ( + run.get("id") == run_id + and run.get("head_sha") == source + and run.get("head_branch") == f"v{version}-prepare" + and run.get("event") == "push" + and run.get("path") == ".github/workflows/release.yml" + and run.get("repository", {}).get("full_name") == "ParetoSecurity/passlet" + and run.get("status") == "completed" + and run.get("conclusion") == "success" + ): + raise ValueError("Artifacts must come from the successful preparation workflow for this release") + verify_source(version, source, root) + names = [f"passlet_{version}_{platform}.{suffix}" for platform, suffix in [ ("macos", "zip"), ("linux_amd64", "tar.gz"), ("linux_arm64", "tar.gz"), ]]