diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml new file mode 100644 index 00000000000..f1d49e5c795 --- /dev/null +++ b/.github/workflows/release-notes.yml @@ -0,0 +1,148 @@ +name: Release Notes — Publish GitHub Release + +# Automates the release-notes half of Release Guide Step 1, which is currently +# manual and has been skipped more often than not: +# +# v1.1.3 -> Release exists +# v1.1.1 -> tag exists, NO Release object +# v1.1.3-spark4.0 / -spark4.1 / -python3.* -> no Release object +# +# The gap is not cosmetic. GitHub anchors auto-generated notes to the previous +# *Release*, not the previous tag, so v1.1.1 having no Release made v1.1.3's +# notes span all of v1.1.1 + v1.1.3. This workflow computes the previous +# primary tag itself and pins the diff base, so notes stay correct even when +# an older Release object is missing. +# +# Only the primary vX.Y.Z tag gets a Release. The -spark*/-python* tags are +# build pointers consumed by ADO pipelines, not separate products. + +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +permissions: + contents: write + +jobs: + publish-release: + name: Publish GitHub Release + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + # The tag filter above already excludes suffixed tags, but a tag such as + # v1.1.3-spark4.0 can still reach a `push` event through some ref + # rewrites, and minting a full Release for a build pointer would be wrong. + - name: Resolve version + id: v + env: + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + if [[ ! "$REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Not a primary release tag: $REF_NAME — nothing to do." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" + echo "tag=$REF_NAME" >> "$GITHUB_OUTPUT" + + # A release tag must describe a commit that is actually on master. + # Tagging a stale or side branch vX.Y.Z would publish notes for code that + # was never reviewed into the mainline. + - name: Verify the tag is on master + if: steps.v.outputs.skip == 'false' + env: + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + git fetch --quiet origin master + if ! git merge-base --is-ancestor "refs/tags/${TAG}" origin/master; then + echo "::error::${TAG} does not point at a commit contained in master. \ + Refusing to publish a release for an off-mainline commit." + exit 1 + fi + + # Pick the highest primary tag strictly below this one. `sort -V` gives + # correct numeric ordering (v1.1.10 > v1.1.9), which a lexical sort does not. + - name: Determine previous release tag + if: steps.v.outputs.skip == 'false' + id: prev + env: + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + PREV=$(git tag --list 'v[0-9]*.[0-9]*.[0-9]*' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | awk -v cur="$TAG" '$0 == cur {exit} {last=$0} END {print last}') + if [ -z "$PREV" ]; then + echo "No earlier release tag found — notes will cover full history." + else + echo "Previous release: $PREV" + fi + echo "prev=$PREV" >> "$GITHUB_OUTPUT" + + - name: Skip if the Release already exists + if: steps.v.outputs.skip == 'false' + id: exists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "Release $TAG already exists — leaving it untouched." + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Generate and publish release notes + if: steps.v.outputs.skip == 'false' && steps.exists.outputs.found == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.v.outputs.tag }} + PREV: ${{ steps.prev.outputs.prev }} + run: | + set -euo pipefail + + ARGS=(-f tag_name="$TAG" -f target_commitish=master) + if [ -n "$PREV" ]; then + ARGS+=(-f previous_tag_name="$PREV") + fi + + NOTES=$(gh api "repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + -X POST "${ARGS[@]}" --jq '.body') + + VERSION="${TAG#v}" + { + echo "## Installation" + echo + echo '```bash' + echo "pip install synapseml==${VERSION}" + echo '```' + echo + echo "Maven coordinate: \`com.microsoft.azure:synapseml_2.12:${VERSION}\`" + echo + echo "| Spark | Python | Tag |" + echo "| --- | --- | --- |" + echo "| 3.5 | 3.11 | \`${TAG}-spark3.5\` |" + echo "| 4.0 | 3.12 | \`${TAG}-spark4.0\` |" + echo "| 4.1 | 3.13 | \`${TAG}-spark4.1\` |" + echo + echo "$NOTES" + } > notes.md + + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$TAG" \ + --notes-file notes.md \ + --verify-tag + + echo "Published release $TAG (diff base: ${PREV:-})" diff --git a/.github/workflows/release-prepare.yml b/.github/workflows/release-prepare.yml new file mode 100644 index 00000000000..33bc0b67980 --- /dev/null +++ b/.github/workflows/release-prepare.yml @@ -0,0 +1,191 @@ +name: Release Prepare — Version Bump PR + +# Automates Release Guide Step 1.1-1.3: bump every version string in the repo, +# regenerate the versioned docs snapshot, and open the release PR. +# +# Two of the last four version bumps (v1.1.0, v1.1.3) landed as unsigned direct +# pushes to master with no PR and no review. This workflow makes the reviewed +# PR the only path, so a release can never again be an unreviewed force-push. +# +# See also: release-tag.yml (tags master once this PR merges). + +on: + workflow_dispatch: + inputs: + version: + description: "New OSS version, three components (e.g. 1.1.4)" + required: true + type: string + skip_docs: + description: "Skip the versioned-docs snapshot (faster; PR will be incomplete)" + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + +jobs: + prepare: + name: Bump versions & open release PR + runs-on: ubuntu-latest + + steps: + # A release must be cut from master. Running this from a feature branch + # would open a PR that bumps versions against unreleased code. + - name: Validate ref + env: + REF: ${{ github.ref }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/master" ]; then + echo "::error::Release prepare must run on master, got '$REF'." + exit 1 + fi + + - name: Validate version format + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Version '$VERSION' must be exactly X.Y.Z. SynapseML OSS \ + does not use a fourth component; the .N super-patch belongs to SynapseML-Internal." + exit 1 + fi + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + # Refuse to re-prepare a version that is already published. Without this + # the workflow would happily open a PR that "bumps" to a shipped version. + - name: Guard against an already-released version + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if git rev-parse -q --verify "refs/tags/v${VERSION}" >/dev/null; then + echo "::error::Tag v${VERSION} already exists. That version is already released." + exit 1 + fi + if git ls-remote --exit-code --heads origin "release/prepare-v${VERSION}" >/dev/null 2>&1; then + echo "::error::Branch release/prepare-v${VERSION} already exists on origin. \ + Delete it or finish the existing release PR first." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Set up JDK 11 + if: ${{ !inputs.skip_docs }} + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: temurin + java-version: 11 + cache: sbt + + - name: Install sbt + if: ${{ !inputs.skip_docs }} + run: | + SBT_VERSION="$(sed -n 's/^sbt.version *= *//p' project/build.properties | tr -d ' ')" + mkdir -p "$HOME/.local/bin" + curl -L -o "$HOME/.local/bin/sbt-launch.jar" \ + "https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" + cat > "$HOME/.local/bin/sbt" <> "$GITHUB_PATH" + + - name: Set up Node + if: ${{ !inputs.skip_docs }} + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: website/package-lock.json + + - name: Install website dependencies + if: ${{ !inputs.skip_docs }} + working-directory: website + run: npm ci + + # bump-version.py is context-anchored: it refuses to replace a bare version + # number that has no SynapseML-identifying text near it, and exits non-zero + # if any anchored occurrence of the old version survives the run. Both + # behaviours are load-bearing here, so the exit code is not suppressed. + - name: Bump version strings + id: bump + env: + VERSION: ${{ inputs.version }} + SKIP_DOCS: ${{ inputs.skip_docs }} + run: | + set -euo pipefail + ARGS=(--to "$VERSION") + if [ "$SKIP_DOCS" = "true" ]; then + ARGS+=(--skip-docs) + fi + python scripts/bump-version.py "${ARGS[@]}" + + - name: Verify the working tree actually changed + run: | + set -euo pipefail + if git diff --quiet; then + echo "::error::bump-version.py reported success but changed nothing. \ + Refusing to open an empty release PR." + exit 1 + fi + echo "Files touched: $(git diff --name-only | wc -l)" + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Commit and push + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + BRANCH="release/prepare-v${VERSION}" + git checkout -b "$BRANCH" + git add -A + git commit -m "chore: Bump version to v${VERSION}" + git push origin "$BRANCH" + + - name: Open release PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + SKIP_DOCS: ${{ inputs.skip_docs }} + run: | + set -euo pipefail + BODY=$(cat < dash, rebuild counter + SYNAPSEML_INTERNAL_VERSION=1.1.1-0-spark4.0 # spark dot PRESERVED, no counter + +Getting either wrong produces a VHD that fails at image-build time, hours later +and far from the typo. This script derives both from release_matrix, so the two +conventions are applied by the same code that the tests pin against production. + +version.txt is a VHD component revision, unrelated to the SynapseML version; it +is bumped by exactly one patch to force the image to rebuild. + +Usage: + python bump_bbcvhd.py --repo --version 1.1.4 \\ + --internal-patch 0 --target spark4.0 +""" + +import argparse +import os +import re +import sys +from pathlib import Path +from typing import Optional, Tuple + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from release_matrix import TARGETS_BY_KEY, build_plan # noqa: E402 + +# BBC-VHD names component directories without the dot: spark4.0 -> spark40. +COMPONENT_DIR = {"master": "spark35", "spark4.0": "spark40", "spark4.1": "spark41"} + +OSS_VAR = "SYNAPSEML_VERSION" +INTERNAL_VAR = "SYNAPSEML_INTERNAL_VERSION" + + +def bump_component_revision(text: str) -> Tuple[str, str, str]: + """Bump the trailing patch of a version.txt revision (1.4.26 -> 1.4.27).""" + old = text.strip() + m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", old) + if not m: + raise ValueError( + f"version.txt must contain a bare X.Y.Z revision, found {old!r}" + ) + new = f"{m.group(1)}.{m.group(2)}.{int(m.group(3)) + 1}" + # Preserve the original trailing-newline style rather than normalising it, + # so the diff shown to a BBC-VHD reviewer is exactly one line. + return text.replace(old, new, 1), old, new + + +def set_shell_var(text: str, var: str, value: str) -> Tuple[str, Optional[str]]: + """Replace `VAR=...` on its own line. Returns (new_text, old_value).""" + pat = re.compile(rf"^(?P{re.escape(var)}=)(?P.*)$", re.MULTILINE) + found = pat.search(text) + if not found: + return text, None + old = found.group("val") + # A single anchored assignment is expected. Two would mean the later one + # silently wins at runtime while this script rewrites only the first. + if len(pat.findall(text)) > 1: + raise ValueError(f"{var} is assigned more than once; refusing to guess") + return pat.sub(lambda _: f"{found.group('lead')}{value}", text, count=1), old + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="Apply a SynapseML release to BBC-VHD.") + p.add_argument("--repo", required=True, type=Path, help="BBC-VHD checkout root") + p.add_argument("--version", required=True, help="OSS version, e.g. 1.1.4") + p.add_argument("--internal-patch", default="0", help="Internal super-patch digit") + p.add_argument( + "--target", + required=True, + choices=sorted(COMPONENT_DIR), + help="Which spark component to update", + ) + p.add_argument("--upack-iteration", type=int, default=0, help="OSS rebuild counter") + p.add_argument( + "--internal-upack-iteration", + type=int, + default=0, + help="Internal rebuild counter", + ) + p.add_argument("--dry-run", action="store_true") + args = p.parse_args(argv) + + if args.target not in TARGETS_BY_KEY: + print(f"error: unknown target {args.target}", file=sys.stderr) + return 2 + + try: + plan = build_plan( + args.version, + args.internal_patch, + [args.target], + {args.target: args.upack_iteration} if args.upack_iteration else None, + ( + {args.target: args.internal_upack_iteration} + if args.internal_upack_iteration + else None + ), + ) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + tp = plan.targets[0] + comp = args.repo / "Components" / "MMLSpark" / COMPONENT_DIR[args.target] + setup_sh, version_txt = comp / "setup.sh", comp / "version.txt" + + for f in (setup_sh, version_txt): + if not f.is_file(): + print( + f"error: {f} not found. Is --repo a BBC-VHD checkout?", file=sys.stderr + ) + return 2 + + setup_text = setup_sh.read_text(encoding="utf-8") + try: + setup_text, old_oss = set_shell_var(setup_text, OSS_VAR, tp.oss_upack_version) + setup_text, old_int = set_shell_var( + setup_text, INTERNAL_VAR, tp.internal_upack_version + ) + except ValueError as e: + print(f"error: {setup_sh}: {e}", file=sys.stderr) + return 2 + + # Both assignments must exist. A missing one means the component layout + # changed and a silent no-op would ship the previous release's artifacts. + for var, old in ((OSS_VAR, old_oss), (INTERNAL_VAR, old_int)): + if old is None: + print(f"error: {var} not found in {setup_sh}", file=sys.stderr) + return 2 + + try: + version_text, old_rev, new_rev = bump_component_revision( + version_txt.read_text(encoding="utf-8") + ) + except ValueError as e: + print(f"error: {version_txt}: {e}", file=sys.stderr) + return 2 + + label = "[DRY RUN] " if args.dry_run else "" + print(f"{label}{comp.relative_to(args.repo).as_posix()}") + print(f" {OSS_VAR} {old_oss} -> {tp.oss_upack_version}") + print(f" {INTERNAL_VAR} {old_int} -> {tp.internal_upack_version}") + print(f" version.txt {old_rev} -> {new_rev}") + + if args.dry_run: + return 0 + + setup_sh.write_text(setup_text, encoding="utf-8", newline="") + version_txt.write_text(version_text, encoding="utf-8", newline="") + + # Post-condition: re-read and confirm. The whole point of this script is to + # remove doubt about what landed in the file. + check = setup_sh.read_text(encoding="utf-8") + for var, want in ( + (OSS_VAR, tp.oss_upack_version), + (INTERNAL_VAR, tp.internal_upack_version), + ): + if not re.search(rf"^{re.escape(var)}={re.escape(want)}$", check, re.MULTILINE): + print(f"error: post-condition failed for {var}", file=sys.stderr) + return 1 + print(" verified on disk") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/release_matrix.py b/scripts/release/release_matrix.py new file mode 100644 index 00000000000..91472ceaa50 --- /dev/null +++ b/scripts/release/release_matrix.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +""" +SynapseML Release Matrix - the single source of truth for a release. + +Given one decision (the OSS version) and one optional decision (the internal +super-patch), this module derives EVERY downstream identifier a release needs: +git tags in both repos, Universal Package versions, pip wheel versions, and the +BBC-VHD variable values. + +Why this exists +--------------- +These identifiers are NOT consistently derivable by eye. Verified against the +live feeds for v1.1.1/v1.1.3: + + * OSS UPack mangles dots to dashes: synapseml 1.1.3-spark4-0 + * Internal UPack preserves dots: synapseml_internal 1.1.3-0-spark4.0 + * Pip uses a PEP 440 local segment: synapseml 1.1.3+python3.12 + * Internal pip folds the super-patch in: synapseml-internal 1.1.3.0+python3.12 + * master carries THREE tags: v1.1.3, v1.1.3-spark3.5, v1.1.3-python3.11 + +Every one of those asymmetries has caused, or can cause, a hand-typed mistake. +Encode them once, here, and have every other tool read from this. + +Usage: + python scripts/release/release_matrix.py --version 1.1.4 + python scripts/release/release_matrix.py --version 1.1.4 --json + python scripts/release/release_matrix.py --version 1.1.4 --targets master,spark4.0 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, asdict, field +from typing import Dict, List, Optional + +OSS_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") + +UPACK_FEED = "BBC-VHD_PublicPackages" +PIP_FEED = "Synapse-Conda" +ADO_ORG = "https://msdata.visualstudio.com" +ADO_PROJECT = "A365" + + +@dataclass(frozen=True) +class Target: + """One Spark/Python build target and the branch that produces it.""" + + key: str + branch: str + spark: str + python: str + base_branch: Optional[str] + # master is the anchor: it alone carries the bare `vX.Y.Z` tag. + is_anchor: bool = False + + +TARGETS: List[Target] = [ + Target("master", "master", "3.5", "3.11", None, is_anchor=True), + Target("spark4.0", "spark4.0", "4.0", "3.12", "master"), + Target("spark4.1", "spark4.1", "4.1", "3.13", "spark4.0"), +] + +TARGETS_BY_KEY = {t.key: t for t in TARGETS} + + +def _upack_oss_suffix(target: Target) -> str: + """OSS UPack suffix. Spark dots become dashes: 4.0 -> spark4-0.""" + if target.is_anchor: + return "" + return "-spark" + target.spark.replace(".", "-") + + +def _upack_internal_suffix(target: Target) -> str: + """Internal UPack suffix. Spark dots are PRESERVED: 4.0 -> spark4.0.""" + if target.is_anchor: + return "" + return f"-spark{target.spark}" + + +@dataclass +class TargetPlan: + key: str + branch: str + base_branch: Optional[str] + spark: str + python: str + oss_tags: List[str] + internal_tags: List[str] + oss_upack_version: str + internal_upack_version: str + oss_pip_version: str + internal_pip_version: str + + +@dataclass +class ReleasePlan: + oss_version: str + internal_version: str + internal_patch: str + upack_feed: str = UPACK_FEED + pip_feed: str = PIP_FEED + targets: List[TargetPlan] = field(default_factory=list) + + @property + def all_oss_tags(self) -> List[str]: + return [t for tp in self.targets for t in tp.oss_tags] + + @property + def all_internal_tags(self) -> List[str]: + return [t for tp in self.targets for t in tp.internal_tags] + + +def build_plan( + oss_version: str, + internal_patch: str = "0", + target_keys: Optional[List[str]] = None, + upack_iteration: Optional[Dict[str, int]] = None, + internal_upack_iteration: Optional[Dict[str, int]] = None, +) -> ReleasePlan: + """Derive the full release plan. + + `upack_iteration` maps a target key to a rebuild counter. Azure Artifacts + UPack versions are immutable per version string, so a re-publish after a bad + build must append `-N`. This is the `-1` in the real `1.1.1-spark4-0-1`. + + OSS and Internal are published as separate packages and are rebuilt + independently, so they carry independent counters. Production proves it: + v1.1.1 shipped `synapseml=1.1.1-spark4-0-1` alongside + `synapseml_internal=1.1.1-0-spark4.0` (no counter). + """ + if not OSS_VERSION_RE.match(oss_version): + raise ValueError(f"OSS version must be X.Y.Z (got {oss_version!r})") + if not internal_patch.isdigit(): + raise ValueError(f"internal patch must be a digit (got {internal_patch!r})") + + keys = target_keys or [t.key for t in TARGETS] + unknown = [k for k in keys if k not in TARGETS_BY_KEY] + if unknown: + raise ValueError(f"unknown target(s): {unknown}. Known: {list(TARGETS_BY_KEY)}") + + upack_iteration = upack_iteration or {} + internal_upack_iteration = internal_upack_iteration or {} + internal_version = f"{oss_version}.{internal_patch}" + plan = ReleasePlan( + oss_version=oss_version, + internal_version=internal_version, + internal_patch=internal_patch, + ) + + for key in keys: + t = TARGETS_BY_KEY[key] + v, iv = oss_version, internal_version + + oss_tags = [f"v{v}-spark{t.spark}", f"v{v}-python{t.python}"] + internal_tags = [f"v{iv}-spark{t.spark}", f"v{iv}-python{t.python}"] + if t.is_anchor: + oss_tags.insert(0, f"v{v}") + internal_tags.insert(0, f"v{iv}") + + it = upack_iteration.get(key) + iter_suffix = f"-{it}" if it else "" + it_int = internal_upack_iteration.get(key) + iter_suffix_internal = f"-{it_int}" if it_int else "" + + plan.targets.append( + TargetPlan( + key=t.key, + branch=t.branch, + base_branch=t.base_branch, + spark=t.spark, + python=t.python, + oss_tags=oss_tags, + internal_tags=internal_tags, + oss_upack_version=f"{v}{_upack_oss_suffix(t)}{iter_suffix}", + internal_upack_version=( + f"{v}-{internal_patch}{_upack_internal_suffix(t)}{iter_suffix_internal}" + ), + oss_pip_version=f"{v}+python{t.python}", + internal_pip_version=f"{iv}+python{t.python}", + ) + ) + return plan + + +def render_text(plan: ReleasePlan) -> str: + out: List[str] = [] + out.append( + f"SynapseML release plan OSS v{plan.oss_version} Internal v{plan.internal_version}" + ) + out.append("") + out.append("GIT TAGS") + for tp in plan.targets: + out.append( + f" [{tp.key}] branch={tp.branch} spark={tp.spark} python={tp.python}" + ) + out.append(f" github/microsoft/SynapseML : {', '.join(tp.oss_tags)}") + out.append(f" ado/SynapseML-Internal : {', '.join(tp.internal_tags)}") + out.append("") + out.append(f"UPACK ({plan.upack_feed})") + for tp in plan.targets: + out.append( + f" [{tp.key}] synapseml={tp.oss_upack_version} synapseml_internal={tp.internal_upack_version}" + ) + out.append("") + out.append(f"PIP ({plan.pip_feed})") + for tp in plan.targets: + out.append( + f" [{tp.key}] synapseml={tp.oss_pip_version} synapseml-internal={tp.internal_pip_version}" + ) + out.append("") + out.append("BBC-VHD setup.sh values") + for tp in plan.targets: + comp = "spark35" if tp.key == "master" else "spark" + tp.spark.replace(".", "") + out.append(f" Components/MMLSpark/{comp}/setup.sh") + out.append(f" SYNAPSEML_VERSION={tp.oss_upack_version}") + out.append(f" SYNAPSEML_INTERNAL_VERSION={tp.internal_upack_version}") + return "\n".join(out) + + +def main(argv: Optional[List[str]] = None) -> int: + p = argparse.ArgumentParser(description="Derive the SynapseML release matrix.") + p.add_argument("--version", required=True, help="OSS version, e.g. 1.1.4") + p.add_argument( + "--internal-patch", default="0", help="Internal super-patch digit (default 0)" + ) + p.add_argument( + "--targets", default="", help="Comma-separated subset, e.g. master,spark4.0" + ) + p.add_argument( + "--upack-iteration", + default="", + metavar="KEY=N", + help="OSS UPack rebuild counters, e.g. spark4.0=1. Repeat with commas. " + "Azure Artifacts versions are immutable, so a re-publish needs -N.", + ) + p.add_argument( + "--internal-upack-iteration", + default="", + metavar="KEY=N", + help="Internal UPack rebuild counters. Independent of --upack-iteration, " + "because the two packages are published and rebuilt separately.", + ) + p.add_argument("--json", action="store_true", help="Emit JSON instead of text") + args = p.parse_args(argv) + + keys = [k.strip() for k in args.targets.split(",") if k.strip()] or None + + def parse_iterations(raw: str, flag: str) -> Optional[Dict[str, int]]: + out: Dict[str, int] = {} + for item in (x.strip() for x in raw.split(",") if x.strip()): + if "=" not in item: + print(f"error: {flag} expects KEY=N, got {item!r}", file=sys.stderr) + return None + k, _, n = item.partition("=") + if not n.isdigit(): + print( + f"error: iteration for {k!r} must be a number, got {n!r}", + file=sys.stderr, + ) + return None + out[k.strip()] = int(n) + return out + + iterations = parse_iterations(args.upack_iteration, "--upack-iteration") + if iterations is None: + return 2 + internal_iterations = parse_iterations( + args.internal_upack_iteration, "--internal-upack-iteration" + ) + if internal_iterations is None: + return 2 + + try: + plan = build_plan( + args.version, args.internal_patch, keys, iterations, internal_iterations + ) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + print(json.dumps(asdict(plan), indent=2) if args.json else render_text(plan)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/test_prev_tag.sh b/scripts/release/test_prev_tag.sh new file mode 100755 index 00000000000..8e428e3294e --- /dev/null +++ b/scripts/release/test_prev_tag.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Validates the "previous primary release tag" algorithm used by +# .github/workflows/release-notes.yml against the repository's real tag list. +set -euo pipefail + +prev_tag() { + local cur="$1" + git tag --list 'v[0-9]*.[0-9]*.[0-9]*' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | awk -v cur="$cur" '$0 == cur {exit} {last=$0} END {print last}' +} + +fail=0 +check() { + local tag="$1" want="$2" got + got="$(prev_tag "$tag")" + if [ "$got" = "$want" ]; then + printf 'PASS %-12s prev=%s\n' "$tag" "${got:-}" + else + printf 'FAIL %-12s want=%s got=%s\n' "$tag" "${want:-}" "${got:-}" + fail=1 + fi +} + +# Expectations transcribed from the live tag list. +check v1.1.3 v1.1.1 # v1.1.2 was abandoned; must skip the gap +check v1.1.1 v1.1.0 +check v1.1.0 v1.0.15 +check v1.0.15 v1.0.14 +check v1.0.14 v1.0.13 +check v1.0.10 v1.0.9 # numeric, not lexical: v1.0.10 must follow v1.0.9 +check v0.9.0 "" # oldest tag has no predecessor + +# Suffixed tags must never be selected as a predecessor. +if prev_tag v1.1.3 | grep -q -- '-'; then + echo "FAIL suffixed tag leaked into predecessor selection" + fail=1 +else + echo "PASS suffixed tags excluded" +fi + +exit "$fail" diff --git a/scripts/release/test_release_matrix.py b/scripts/release/test_release_matrix.py new file mode 100644 index 00000000000..148f4909dca --- /dev/null +++ b/scripts/release/test_release_matrix.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Tests for release_matrix. Expected values are transcribed from the LIVE +v1.1.3 / v1.1.1 releases (github tags + BBC-VHD_PublicPackages + Synapse-Conda), +so a regression here means the matrix has drifted from reality.""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from release_matrix import build_plan # noqa: E402 + + +def _by_key(plan): + return {tp.key: tp for tp in plan.targets} + + +def test_rejects_bad_versions(): + for bad in ["1.1", "v1.1.3", "1.1.3.0", "abc", ""]: + with pytest.raises(ValueError): + build_plan(bad) + + +def test_rejects_non_numeric_internal_patch(): + with pytest.raises(ValueError): + build_plan("1.1.3", internal_patch="x") + + +def test_rejects_unknown_target(): + with pytest.raises(ValueError): + build_plan("1.1.3", target_keys=["spark9.9"]) + + +def test_master_carries_three_oss_tags(): + """Verified live: v1.1.3, v1.1.3-spark3.5 and v1.1.3-python3.11 all point + at commit a833941704b5. A release that creates only two of them is broken.""" + m = _by_key(build_plan("1.1.3"))["master"] + assert m.oss_tags == ["v1.1.3", "v1.1.3-spark3.5", "v1.1.3-python3.11"] + assert m.internal_tags == ["v1.1.3.0", "v1.1.3.0-spark3.5", "v1.1.3.0-python3.11"] + + +def test_non_anchor_targets_have_no_bare_tag(): + t = _by_key(build_plan("1.1.3"))["spark4.0"] + assert t.oss_tags == ["v1.1.3-spark4.0", "v1.1.3-python3.12"] + assert "v1.1.3" not in t.oss_tags + + +def test_upack_dot_dash_asymmetry_is_preserved(): + """The single most error-prone fact in the whole release: + OSS UPack mangles the dot, internal UPack does not.""" + t = _by_key(build_plan("1.1.3"))["spark4.0"] + assert t.oss_upack_version == "1.1.3-spark4-0" + assert t.internal_upack_version == "1.1.3-0-spark4.0" + + t41 = _by_key(build_plan("1.1.3"))["spark4.1"] + assert t41.oss_upack_version == "1.1.3-spark4-1" + assert t41.internal_upack_version == "1.1.3-0-spark4.1" + + +def test_master_upack_has_no_spark_suffix(): + m = _by_key(build_plan("1.1.3"))["master"] + assert m.oss_upack_version == "1.1.3" + assert m.internal_upack_version == "1.1.3-0" + + +def test_pip_uses_pep440_local_segment(): + m = _by_key(build_plan("1.1.3")) + assert m["master"].oss_pip_version == "1.1.3+python3.11" + assert m["spark4.0"].oss_pip_version == "1.1.3+python3.12" + assert m["spark4.1"].internal_pip_version == "1.1.3.0+python3.13" + + +def test_internal_superpatch_flows_everywhere(): + """v1.1.3.1 was a real internal-only hotfix: UPack 1.1.3-1, pip 1.1.3.1+python3.11.""" + m = _by_key(build_plan("1.1.3", internal_patch="1"))["master"] + assert m.internal_tags[0] == "v1.1.3.1" + assert m.internal_upack_version == "1.1.3-1" + assert m.internal_pip_version == "1.1.3.1+python3.11" + assert ( + m.oss_upack_version == "1.1.3" + ), "OSS artifacts must not move on an internal-only hotfix" + + +def test_upack_rebuild_iteration_suffix(): + """1.1.1-spark4-0-1 exists in the live feed: a republish after a bad build.""" + m = _by_key(build_plan("1.1.1", upack_iteration={"spark4.0": 1}))["spark4.0"] + assert m.oss_upack_version == "1.1.1-spark4-0-1" + assert m.internal_upack_version == "1.1.1-0-spark4.0", ( + "OSS and Internal are separate packages with independent rebuild " + "counters; an OSS republish must not renumber the Internal package" + ) + + +def test_internal_rebuild_iteration_is_independent(): + m = _by_key(build_plan("1.1.1", internal_upack_iteration={"spark4.0": 2}))[ + "spark4.0" + ] + assert m.oss_upack_version == "1.1.1-spark4-0" + assert m.internal_upack_version == "1.1.1-0-spark4.0-2" + + +def test_reproduces_production_bbcvhd_spark40_setup_sh(): + """Byte-for-byte round-trip against the live BBC-VHD dev/spark40 file: + + SYNAPSEML_VERSION=1.1.1-spark4-0-1 + SYNAPSEML_INTERNAL_VERSION=1.1.1-0-spark4.0 + + Note the asymmetry that makes hand-editing this file so error-prone: the + OSS package mangles the spark dot to a dash and carries a rebuild counter, + while the Internal package preserves the dot and carries none. + """ + m = _by_key( + build_plan("1.1.1", internal_patch="0", upack_iteration={"spark4.0": 1}) + )["spark4.0"] + assert m.oss_upack_version == "1.1.1-spark4-0-1" + assert m.internal_upack_version == "1.1.1-0-spark4.0" + + +def test_target_subset_is_respected(): + plan = build_plan("1.1.4", target_keys=["master", "spark4.0"]) + assert [tp.key for tp in plan.targets] == ["master", "spark4.0"] + + +def test_base_branch_chain_matches_rebase_order(): + b = {tp.key: tp.base_branch for tp in build_plan("1.1.4").targets} + assert b == {"master": None, "spark4.0": "master", "spark4.1": "spark4.0"} + + +def test_all_tag_helpers_are_unique_and_complete(): + plan = build_plan("1.1.4") + assert len(plan.all_oss_tags) == len(set(plan.all_oss_tags)) == 7 + assert len(plan.all_internal_tags) == len(set(plan.all_internal_tags)) == 7 diff --git a/scripts/release/verify_release.py b/scripts/release/verify_release.py new file mode 100644 index 00000000000..cb45d330ad5 --- /dev/null +++ b/scripts/release/verify_release.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +""" +Verify a SynapseML release end-to-end against live sources of truth. + +Replaces the manual "Step 4 - Verify Artifacts" checklist, and doubles as a +regression test for `release_matrix.py`: run it against an already-shipped +version and every row must be PRESENT. + +NOTE: the wiki currently tells you to run `az artifacts universal show`. +That command does not exist in the Azure CLI. This uses the Azure Artifacts +REST API instead, which works. + +Auth: needs an Azure DevOps bearer token. Either + --token +or leave it out and the script shells out to + az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 + +Usage: + python scripts/release/verify_release.py --version 1.1.3 + python scripts/release/verify_release.py --version 1.1.4 --internal-patch 0 --json + python scripts/release/verify_release.py --version 1.1.4 --skip ado # GitHub only +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import subprocess +import sys +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Tuple + +sys.path.insert(0, __file__.rsplit("/", 1)[0].rsplit("\\", 1)[0]) +from release_matrix import ADO_ORG, ADO_PROJECT, build_plan # noqa: E402 + +ADO_RESOURCE = "499b84ac-1321-427f-aa17-267ca6975798" +ORG_SHORT = "msdata" +GITHUB_REPO = "microsoft/SynapseML" +INTERNAL_REPO = "SynapseML-Internal" + +OK, MISSING, SKIPPED = "PRESENT", "MISSING", "SKIPPED" + + +def _get_ado_token(explicit: Optional[str]) -> str: + if explicit: + return explicit + out = subprocess.run( + [ + "az", + "account", + "get-access-token", + "--resource", + ADO_RESOURCE, + "--query", + "accessToken", + "-o", + "tsv", + ], + capture_output=True, + text=True, + shell=(sys.platform == "win32"), + ) + if out.returncode != 0: + raise RuntimeError(f"could not get ADO token: {out.stderr.strip()}") + return out.stdout.strip() + + +def _json_get(url: str, headers: Dict[str, str]) -> Optional[dict]: + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read().decode("utf-8")) + except urllib.error.HTTPError as e: + if e.code in (401, 403): + raise RuntimeError(f"auth failed ({e.code}) for {url}") from e + return None + except urllib.error.URLError: + return None + + +class Checker: + def __init__(self, token: Optional[str], gh_token: Optional[str], skip: List[str]): + self.skip = set(skip) + self._ado_headers = None + if not {"ado", "upack", "pip", "internal"} <= self.skip: + self._ado_headers = {"Authorization": f"Bearer {_get_ado_token(token)}"} + self._gh_headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "synapseml-release-verify", + } + if gh_token: + self._gh_headers["Authorization"] = f"Bearer {gh_token}" + self._pkg_cache: Dict[Tuple[str, str], Dict[str, List[str]]] = {} + + # --- git tags --------------------------------------------------------- + def github_tag(self, tag: str) -> str: + if "github" in self.skip: + return SKIPPED + url = f"https://api.github.com/repos/{GITHUB_REPO}/git/ref/tags/{tag}" + return OK if _json_get(url, self._gh_headers) else MISSING + + def ado_tag(self, tag: str) -> str: + if "ado" in self.skip or "internal" in self.skip: + return SKIPPED + url = ( + f"https://dev.azure.com/{ORG_SHORT}/{ADO_PROJECT}/_apis/git/repositories/" + f"{INTERNAL_REPO}/refs?filter=tags/{tag}&api-version=7.1" + ) + data = _json_get(url, self._ado_headers) + if not data: + return MISSING + wanted = f"refs/tags/{tag}" + return ( + OK + if any(v.get("name") == wanted for v in data.get("value", [])) + else MISSING + ) + + # --- artifact feeds --------------------------------------------------- + def _feed_versions(self, feed: str, protocol: str, package: str) -> List[str]: + key = (feed, protocol) + if key not in self._pkg_cache: + url = ( + f"https://feeds.dev.azure.com/{ORG_SHORT}/{ADO_PROJECT}/_apis/packaging/Feeds/" + f"{feed}/packages?protocolType={protocol}&includeAllVersions=true&api-version=7.1-preview.1" + ) + data = _json_get(url, self._ado_headers) or {} + self._pkg_cache[key] = { + p["name"].lower(): [v["version"] for v in p.get("versions", [])] + for p in data.get("value", []) + } + return self._pkg_cache[key].get(package.lower(), []) + + def upack(self, package: str, version: str) -> str: + if "upack" in self.skip or "ado" in self.skip: + return SKIPPED + return ( + OK + if version + in self._feed_versions("BBC-VHD_PublicPackages", "upack", package) + else MISSING + ) + + def pip(self, package: str, version: str) -> str: + if "pip" in self.skip or "ado" in self.skip: + return SKIPPED + # Azure Artifacts normalises pypi names: synapseml_internal -> synapseml-internal + return ( + OK + if version + in self._feed_versions("Synapse-Conda", "pypi", package.replace("_", "-")) + else MISSING + ) + + +def run( + version: str, internal_patch: str, target_keys, token, gh_token, skip +) -> Tuple[List[dict], bool]: + plan = build_plan(version, internal_patch, target_keys) + c = Checker(token, gh_token, skip) + rows: List[dict] = [] + + def add(kind, target, name, ident, status): + rows.append( + { + "kind": kind, + "target": target, + "name": name, + "identifier": ident, + "status": status, + } + ) + + for tp in plan.targets: + for tag in tp.oss_tags: + add("git-tag", tp.key, "github/" + GITHUB_REPO, tag, c.github_tag(tag)) + for tag in tp.internal_tags: + add("git-tag", tp.key, "ado/" + INTERNAL_REPO, tag, c.ado_tag(tag)) + add( + "upack", + tp.key, + "synapseml", + tp.oss_upack_version, + c.upack("synapseml", tp.oss_upack_version), + ) + add( + "upack", + tp.key, + "synapseml_internal", + tp.internal_upack_version, + c.upack("synapseml_internal", tp.internal_upack_version), + ) + add( + "pip", + tp.key, + "synapseml", + tp.oss_pip_version, + c.pip("synapseml", tp.oss_pip_version), + ) + add( + "pip", + tp.key, + "synapseml-internal", + tp.internal_pip_version, + c.pip("synapseml_internal", tp.internal_pip_version), + ) + + ok = not any(r["status"] == MISSING for r in rows) + return rows, ok + + +def main(argv=None) -> int: + p = argparse.ArgumentParser( + description="Verify a SynapseML release's artifacts and tags." + ) + p.add_argument("--version", required=True) + p.add_argument("--internal-patch", default="0") + p.add_argument("--targets", default="") + p.add_argument( + "--skip", default="", help="Comma-separated: github,ado,upack,pip,internal" + ) + p.add_argument( + "--token", + default=None, + help="ADO bearer token (default: az account get-access-token)", + ) + p.add_argument("--github-token", default=None) + p.add_argument("--json", action="store_true") + args = p.parse_args(argv) + + keys = [k.strip() for k in args.targets.split(",") if k.strip()] or None + skip = [s.strip() for s in args.skip.split(",") if s.strip()] + + try: + rows, ok = run( + args.version, args.internal_patch, keys, args.token, args.github_token, skip + ) + except (ValueError, RuntimeError) as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + if args.json: + print( + json.dumps( + {"version": args.version, "complete": ok, "rows": rows}, indent=2 + ) + ) + else: + print( + f"{'STATUS':<8} {'KIND':<8} {'TARGET':<9} {'PACKAGE/REPO':<30} IDENTIFIER" + ) + for r in rows: + print( + f"{r['status']:<8} {r['kind']:<8} {r['target']:<9} {r['name']:<30} {r['identifier']}" + ) + n_missing = sum(1 for r in rows if r["status"] == MISSING) + print("") + print( + f"{len(rows)} checks, {n_missing} missing -> {'COMPLETE' if ok else 'INCOMPLETE'}" + ) + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_bump_version.py b/scripts/test_bump_version.py index 35571a72120..09790a4280b 100644 --- a/scripts/test_bump_version.py +++ b/scripts/test_bump_version.py @@ -912,7 +912,7 @@ def live_results(self): docusaurus = REPO_ROOT / "website" / "docusaurus.config.js" if not docusaurus.exists(): pytest.skip("Not running inside SynapseML repo") - content = docusaurus.read_text() + content = docusaurus.read_text(encoding="utf-8") m = re.search(r'let version\s*=\s*"([^"]+)"', content) assert m, "Cannot detect version from docusaurus.config.js" old_v = m.group(1) @@ -927,9 +927,9 @@ def live_results(self): continue r = analyze(fp, rel, c, old_v, bare_re, self_a, line_a, file_a) if r.matches: - results[str(rel)] = len(r.matches) + results[rel.as_posix()] = len(r.matches) if r.unanchored: - unanchored[str(rel)] = r.unanchored + unanchored[rel.as_posix()] = r.unanchored return { "files": results, "total_files": len(results),