From 2566b8d9ece007e5e289cda4148f4d3437a56be8 Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Wed, 19 Aug 2026 16:57:17 +0000 Subject: [PATCH 1/5] ci: run copies of the Linux CI lanes on ix machines (preview) Non-required copies of cargo-tests, size-gate, wasm-pack-tests and webview-tests, routed to ix machines by family label (with the hosted runner as the fork-PR fallback), plus a path-gated router. Deltas vs the originals are confined to runs-on routing, warm-seed checkouts, parallelism honesty pins for the 64-vCPU elastic guests, fork/repo gates, and timeout headroom; the router forwards no secrets and the size-gate report keeps its permissions, jq probe and guard fixes. Deleting the ix-* files removes every ix-scheduled job. --- .github/actionlint.yaml | 11 + .../workflows/ix-cargo-tests.reusable.yaml | 1638 +++++++++++++++++ .github/workflows/ix-ci.yml | 142 ++ .github/workflows/ix-size-gate.reusable.yaml | 633 +++++++ .../ix-wasm-pack-tests.reusable.yaml | 90 + .../workflows/ix-webview-tests.reusable.yaml | 194 ++ 6 files changed, 2708 insertions(+) create mode 100644 .github/workflows/ix-cargo-tests.reusable.yaml create mode 100644 .github/workflows/ix-ci.yml create mode 100644 .github/workflows/ix-size-gate.reusable.yaml create mode 100644 .github/workflows/ix-wasm-pack-tests.reusable.yaml create mode 100644 .github/workflows/ix-webview-tests.reusable.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 8e2eb2804fa..0fe7dcfe68f 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -9,3 +9,14 @@ self-hosted-runner: - blacksmith-4vcpu-ubuntu-2204-arm - blacksmith-8vcpu-windows-2025 - blacksmith-4vcpu-windows-2025 + # self-hosted ix runner pool + - ix + # ix pool job-family affinity labels (pool definition in the + # ix-runners repo) + - ix-msrv + - ix-musl + - ix-gnu + - ix-sdk + - ix-web + - ix-wasm + - ix-light diff --git a/.github/workflows/ix-cargo-tests.reusable.yaml b/.github/workflows/ix-cargo-tests.reusable.yaml new file mode 100644 index 00000000000..b9f8053f844 --- /dev/null +++ b/.github/workflows/ix-cargo-tests.reusable.yaml @@ -0,0 +1,1638 @@ +# ix preview copy of .github/workflows/cargo-tests.reusable.yaml - the +# same jobs pointed at the ix runner pool (runs-on labels, parallelism +# env pins, and per-lane timeout adjustments). It runs side by side with +# the original via ix-ci.yml, is never a required check, and changes +# nothing about how the original runs. Promote or delete after the +# trial; do not evolve both copies. +# +# sccache strategy +# ================ +# +# This workflow uses sccache for Rust compilation artifacts and +# Swatinem/rust-cache for Cargo registry/git state — i.e. everything cargo +# downloads before it compiles: the crates.io index, the `.crate` tarballs, and +# the git checkouts of our forks (minijinja, aws-sdk-rust, google-cloud-rust). +# `cache-all-crates: true` keeps all of those (the git forks in particular, +# which are large and otherwise re-cloned on every runner); `cache-targets: +# false` leaves the compiled `target/` tree to sccache so the two caches don't +# compete. +# +# Because `target/` is excluded, rust-cache's payload is small (registry + git, +# no build output). As on canary, only canary writes the cache +# (`save-if: ${{ github.ref == 'refs/heads/canary' }}`); PR branches read the +# shared entry instead of each saving a fresh one, so cache usage doesn't blow +# out. rust-cache keys are content-addressed on Cargo.lock, so jobs sharing a +# `shared-key` reuse the same entry. +# +# mise installs sccache and direnv, then each cargo job loads .envrc via +# `direnv export gha` so CI uses the exact same sccache config as local shells: +# .envrc is the single source of truth. The R2 credentials are passed through +# GitHub Actions secrets using the same BAML_SCCACHE_R2_* names that the +# RUSTC_WRAPPER maps to AWS_* before exec'ing sccache — the `tools/baml-sccache` +# script on POSIX, and the native `tools_sccache` crate's binary on Windows +# (built per-job; see the "Build native sccache wrapper" step). +# +# Secretless runs (e.g. fork PRs) still install sccache, but .envrc sees no R2 +# credentials and falls back to the runner-local cache. +name: Cargo Tests (Reusable) + +on: + workflow_call: + secrets: + BAML_SCCACHE_R2_ACCESS_KEY_ID: + required: false + BAML_SCCACHE_R2_SECRET_ACCESS_KEY: + required: false + +# Read-only by design: these jobs run PR-controlled cargo build/test code, so +# they should not need elevated repository permissions. +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + # Inputs read by .envrc; everything else (RUSTC_WRAPPER, SCCACHE_*) is set by + # .envrc itself so it stays the single source of truth across CI and local. + # The R2 credentials .envrc maps to AWS_* for sccache (mapped to AWS_*). + # Native Cargo linker overrides live in baml_language/.cargo/config.toml. + BAML_SCCACHE_R2_ACCESS_KEY_ID: ${{ secrets.BAML_SCCACHE_R2_ACCESS_KEY_ID }} + BAML_SCCACHE_R2_SECRET_ACCESS_KEY: ${{ secrets.BAML_SCCACHE_R2_SECRET_ACCESS_KEY }} + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + CARGO_TERM_COLOR: always + # CI executes the compiler thousands of times per job — in-process in + # `baml_tests`, and as a spawned debug `baml-cli` in the CLI e2e suites — so + # test runtime dwarfs compile time here (opt-level 0 test execution was + # ~55min of CPU per run across the linux jobs alone). opt-level 1 keeps + # debug-assertions on and costs one sccache-cold rebuild after this change + # lands; per-package overrides in Cargo.toml (e.g. salsa) still apply on + # top. Pinned on both `dev` (dependencies) and `test` (the `--test` + # binaries) so the setting is unambiguous, mirroring the DEBUG pins in the + # windows job. + CARGO_PROFILE_DEV_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + # Honesty pin for the elastic ix guests: cargo defaults -j to the guest's + # full 64 cores, and every concurrent machine doing that oversubscribes + # the host it shares (measured: the same cold build was 317s at full-wave + # contention vs 120s alone). Hosted fallback runners have <=16 vCPUs, so + # this changes nothing there. + CARGO_BUILD_JOBS: "16" + # Set at the workflow level so the value is deterministic per commit + # regardless of what any machine image carries. Pairs with + # the one-slot typescript-web-workers nextest group in + # baml_language/.config/nextest.toml, so peak workerd equals this dial; + # if the lane ever OOMs, step down to 4. + VITEST_MAX_WORKERS: "12" + RUSTUP_MAX_RETRIES: 10 + +jobs: + + cargo-test-linux: + name: "cargo test (x86_64-unknown-linux-gnu)" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + # 40, not 25: a run landing on a cold ix machine recompiles from local + # sccache and has been measured at 1519s+ under + # family contention (cancelled at the old 25m cap); warm runs are 92s. + # The cap is a hang backstop, not a perf gate. + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + # mise installs sccache/direnv AND cargo-nextest (cargo: backend -> + # cargo-binstall prebuilt), replacing taiki-e/install-action. + - name: "Install tools (mise)" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv cargo:cargo-nextest" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/canary' }} + shared-key: "linux-cargo" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + # `baml_tests`, `baml_cli`, and `baml_lsp2_actions` already run on linux + # with `--all-features` in the dedicated `snapshot-tests` job, so this + # job neither builds nor runs them — their test binaries are also the + # heaviest links in the workspace. (A nextest `-E` filter would only + # skip *running* them; the cargo-level `--exclude` skips building them + # too.) The `sdk_test_*` crates need uv and run in the dedicated + # `sdk-tests` matrix below. + - name: "Build tests with --timings" + run: cargo test --no-run --all-features --timings --workspace --exclude baml_tests --exclude baml_cli --exclude baml_lsp2_actions --exclude "sdk_test_*" --exclude baml_bridge + working-directory: baml_language + + - name: "Run tests with nextest" + run: cargo nextest run --all-features --workspace --exclude baml_tests --exclude baml_cli --exclude baml_lsp2_actions --exclude "sdk_test_*" --exclude baml_bridge + working-directory: baml_language + + - name: "Upload test results on failure" + if: failure() + uses: actions/upload-artifact@v7 + with: + name: test-failures-linux + path: | + baml_language/**/*.snap.new + baml_language/**/target/nextest/ + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-test-linux.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-test-linux + path: sccache-stats-test-linux.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-test-linux.html + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-test-linux + path: baml_language/target/cargo-timings/cargo-timing-test-linux.html + if-no-files-found: ignore + + cargo-test-linux-musl: + name: "cargo test (x86_64-unknown-linux-musl)" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-musl"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + # 45, not 35: same reasoning as the gnu lane above; measured 1400s under + # contention against the old 35m cap (23 of 35 minutes used). + timeout-minutes: 45 + env: + CARGO_BUILD_TARGET: x86_64-unknown-linux-musl + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: | + set -euo pipefail + rustup show + rustup target add x86_64-unknown-linux-musl + working-directory: baml_language + + - name: "Setup musl cross toolchain" + uses: ./.github/actions/setup-musl-cross + with: + target: x86_64-unknown-linux-musl + + - name: "Install tools (mise)" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv cargo:cargo-nextest" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/canary' }} + shared-key: "linux-cargo" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + # Enabling every feature turns on the optional native-tls backend, which + # needs a separate target-specific OpenSSL build. Default features match + # the rustls-based musl artifacts we ship. + - name: "Build tests with --timings" + run: cargo test --no-run --timings --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge + working-directory: baml_language + + - name: "Run tests with nextest" + run: cargo nextest run --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge -E 'not binary(=pack_e2e)' + working-directory: baml_language + + - name: "Run pack end-to-end tests" + run: cargo test -p baml_cli --test pack_e2e + working-directory: baml_language + + - name: "Upload test results on failure" + if: failure() + uses: actions/upload-artifact@v7 + with: + name: test-failures-linux-musl + path: | + baml_language/**/*.snap.new + baml_language/**/target/nextest/ + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-test-linux-musl.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-test-linux-musl + path: sccache-stats-test-linux-musl.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-test-linux-musl.html + working-directory: baml_language + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-test-linux-musl + path: baml_language/target/cargo-timings/cargo-timing-test-linux-musl.html + if-no-files-found: ignore + + # TODO(B-1545): Temporarily disabled due to flakiness; re-enable via https://linear.app/boundaryml2/issue/B-1545/investigate-flakiness-in-cargo-test-aarch64-apple-darwin-ci-job + # cargo-test-macos: + # name: "cargo test (aarch64-apple-darwin)" + # # Skip in the merge queue: macos already ran on the PR. The merge_group + # # event is inherited from ci.yaml through the workflow_call boundary. + # if: ${{ github.event_name != 'merge_group' }} + # runs-on: blacksmith-6vcpu-macos-latest + # timeout-minutes: 35 + # steps: + # - uses: actions/checkout@v6 + # with: + # persist-credentials: false + + # - name: "Install Rust toolchain" + # run: rustup show + # working-directory: baml_language + + # - name: "Install tools (mise)" + # uses: ./.github/actions/setup-mise + # with: + # install_args: "sccache direnv cargo:cargo-nextest" + + # - name: "Verify sccache" + # run: sccache --version + + # - uses: Swatinem/rust-cache@v2 + # with: + # workspaces: "baml_language -> target" + # cache-all-crates: true + # cache-targets: false + # save-if: ${{ github.ref == 'refs/heads/canary' }} + # shared-key: "macos-cargo" + + # - name: "Load .envrc with direnv" + # run: | + # set -euo pipefail + + # direnv allow .envrc + # direnv export gha >> "$GITHUB_ENV" + + # # Separate step so if fetch is slow, build is fast. + # - name: "Fetch cargo dependencies" + # run: cargo fetch + # working-directory: baml_language + + # # Snapshot tests live in `baml_tests` and are exercised end-to-end by + # # the dedicated `snapshot-tests` job. The `sdk_test_*` crates need uv + # # and run in the dedicated `sdk-tests` matrix below. Skip both here — + # # at the cargo level (`--exclude`), not a nextest `-E` filter, so their + # # test binaries (the heaviest links in the workspace) aren't built. + # - name: "Build tests with --timings" + # run: cargo test --no-run --all-features --timings --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge + # working-directory: baml_language + + # - name: "Run tests with nextest" + # run: cargo nextest run --all-features --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge + # working-directory: baml_language + + # - name: "Show sccache stats" + # if: always() + # run: sccache --show-stats + + # - name: "Dump sccache stats (json)" + # if: always() + # run: sccache --show-stats --stats-format json > sccache-stats-test-macos.json + + # - name: "Upload sccache stats" + # if: always() + # uses: actions/upload-artifact@v7 + # with: + # name: sccache-stats-test-macos + # path: sccache-stats-test-macos.json + # if-no-files-found: ignore + + # - name: "Rename cargo timings" + # if: always() + # run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-test-macos.html + # working-directory: baml_language + # - name: "Upload cargo timings" + # if: always() + # uses: actions/upload-artifact@v7 + # with: + # name: cargo-timings-test-macos + # path: baml_language/target/cargo-timings/cargo-timing-test-macos.html + # if-no-files-found: ignore + + cargo-test-windows: + name: "cargo test (x86_64-pc-windows-msvc)" + # Skip in the merge queue: windows already ran on the PR. In this copy + # the caller is ix-ci.yml, which has no merge_group trigger, so the + # merge_group arm is inert in the preview. + # Also skip outside the upstream repo (no Blacksmith Windows runners). + if: ${{ github.event_name != 'merge_group' && github.repository == 'BoundaryML/baml' }} + # 8-vCPU + CARGO_BUILD_JOBS=16 (2x cores) was the value sweet spot for the + # native Windows Rust builds (build-caching/04 Exp 7/9). + runs-on: blacksmith-8vcpu-windows-2025 + env: + CARGO_BUILD_JOBS: "16" + # Windows debug builds emit very large PDBs; full debuginfo across the + # whole `--all-features` test build overruns the runner's ~130 GB disk + # ("no space on device" during codegen, before any test runs). + # `line-tables-only` keeps file:line backtraces for test failures while + # dropping the bulk of the debug-info size (variable/type DIEs). Pinned on + # both `dev` (dependencies) and `test` (the `--test` binaries, which + # inherit `dev` but are pinned explicitly so the reduction is unambiguous). + # Drop to "0" if this is still too tight. + CARGO_PROFILE_DEV_DEBUG: "line-tables-only" + CARGO_PROFILE_TEST_DEBUG: "line-tables-only" + # TODO: Reduce this once our changes to aws-sdk-rust are upstreamed + # Right now it needs to download our repo every time which is very slow + timeout-minutes: 50 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sccache, direnv, and cargo-nextest" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv cargo:cargo-nextest" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/canary' }} + shared-key: "windows-cargo" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Build native sccache wrapper" + run: | + set -euo pipefail + # The native RUSTC_WRAPPER (tools_sccache crate) maps the R2 creds and + # execs sccache; cargo launches it via CreateProcess (32767-char + # limit), avoiding cmd.exe's ~8191-char limit that windows-sys' huge + # `--check-cfg cfg(feature, values(...))` line blows past. The binary + # lives in the build tree, so build it first — .envrc leaves + # RUSTC_WRAPPER unset until it exists, so this bootstrap compiles with + # plain rustc — then point RUSTC_WRAPPER at it for the cargo steps + # below. + cargo build -p tools_sccache + echo "RUSTC_WRAPPER=$(pwd -W)/target/debug/baml-sccache.exe" >> "$GITHUB_ENV" + working-directory: baml_language + + # Snapshot tests live in `baml_tests` and are covered by the dedicated + # `snapshot-tests` job. The `sdk_test_*` crates need uv and run in the + # dedicated `sdk-tests` matrix below. Skip both here — at the cargo + # level (`--exclude`), not a nextest `-E` filter, so their test binaries + # (the heaviest links, and on Windows the largest PDBs) aren't built. + - name: "Build tests with --timings" + run: cargo test --no-run --all-features --timings --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge + working-directory: baml_language + + # Profiling-specific test binaries opt in explicitly; keeping + # BAML_PROFILE off for the general suite avoids filling the Windows + # runner with incidental `.bamlprof` files. Run the two binaries that + # need `baml-pack-host` separately so their process-global OnceLocks + # build and share it instead of relinking it for every nextest case. + - name: "Run tests" + run: cargo nextest run --all-features --workspace --exclude baml_tests --exclude "sdk_test_*" --exclude baml_bridge -E 'not binary(=pack_e2e) and not binary(=exit_code_e2e)' + working-directory: baml_language + env: + BAML_PROFILE: "0" + + - name: "Free disk for CLI binary e2e tests" + run: | + set -euo pipefail + mv target/cargo-timings/cargo-timing.html cargo-timing-test-windows.html + cp target/debug/baml-sccache.exe baml-sccache.exe + echo "RUSTC_WRAPPER=$(pwd -W)/baml-sccache.exe" >> "$GITHUB_ENV" + cargo clean + working-directory: baml_language + + - name: "Run CLI binary e2e tests (windows)" + run: cargo test -p baml_cli --test pack_e2e --test exit_code_e2e --all-features + working-directory: baml_language + env: + BAML_PROFILE: "0" + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-test-windows.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-test-windows + path: sccache-stats-test-windows.json + if-no-files-found: ignore + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-test-windows + path: baml_language/cargo-timing-test-windows.html + if-no-files-found: ignore + + sdk-test-matrix: + name: "sdk test matrix" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + sdk-dirs: ${{ steps.matrix.outputs.sdk-dirs }} + temporarily-disabled-sdk-dirs: ${{ steps.matrix.outputs.temporarily-disabled-sdk-dirs }} + steps: + - name: "Build matrix" + id: matrix + uses: actions/github-script@v8 + with: + script: | + // Fork PRs never run on ix machines; untrusted events fall back + // to the GitHub-hosted runner these jobs used before the + // migration. Read off `context` rather than interpolated by + // the workflow templater, so no event data is spliced into this + // script. + // Same-repo test, not head.repo.fork: `fork` answers "is the head + // repo a fork of anything", which is true for every PR on a fork + // of upstream and would route same-repo PRs to runner labels that + // do not exist there. + const poolOk = + (context.eventName !== 'pull_request' && context.eventName !== 'pull_request_target') || + context.payload.pull_request?.head?.repo?.full_name === + `${context.repo.owner}/${context.repo.repo}`; + // One constant covers every linux SDK entry, so the whole + // generated matrix lands in the sdk family: these jobs share the + // gnu Rust build underneath and differ only in which language + // toolchain each one leaves warm in the seed lineage. + const LINUX_RUNNER = poolOk ? ["self-hosted", "ix", "ix-sdk"] : "blacksmith-16vcpu-ubuntu-2404"; + + const baseMatrix = [ + { + "sdk-label": "go", + "sdk-dir": "go", + "package-name": "sdk_test_go", + "sdk-path": "go", + "install-args": "sccache direnv go cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "python-pydantic2", + "sdk-dir": "python", + "package-name": "sdk_test_python_pydantic2", + "sdk-path": "python_pydantic2", + "install-args": "sccache direnv python uv cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "ruby-sorbet", + "sdk-dir": "ruby", + "package-name": "sdk_test_ruby_sorbet", + "sdk-path": "ruby_sorbet", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "typescript", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript", + "sdk-path": "typescript", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "typescript-web", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript_web", + "sdk-path": "typescript_web", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest github:rustwasm/wasm-pack", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + // The C++ sdk-test crate needs the host C++ toolchain and + // CMake (preinstalled on the runners) plus nextest; its + // setup.sh builds the bridge_cffi cdylib with cargo and + // pre-clones the pinned protobuf sources. Windows joins + // once the MSVC test driver lands (test.sh is bash). + "sdk-label": "cpp", + "sdk-dir": "cpp", + "package-name": "sdk_test_cpp", + "sdk-path": "cpp", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "csharp", + "sdk-dir": "csharp", + "package-name": "sdk_test_csharp", + "sdk-path": "csharp", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "rust", + "sdk-dir": "rust", + "package-name": "sdk_test_rust", + "sdk-path": "rust", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + // The Java sdk-test crate needs the mise-pinned JDK and + // Gradle (setup script builds the bridge_java cdylib and + // the baml_bridge jar; test time spawns `gradle` from + // PATH, so `java` + `gradle` must be in install-args). + "sdk-label": "java", + "sdk-dir": "java", + "package-name": "sdk_test_java", + "sdk-path": "java", + "install-args": "sccache direnv java gradle cargo:cargo-nextest", + "os-label": "linux", + "runs-on": LINUX_RUNNER, + "rust-cache-key": "linux-cargo", + "timeout": 30, + }, + { + "sdk-label": "go", + "sdk-dir": "go", + "package-name": "sdk_test_go", + "sdk-path": "go", + "install-args": "sccache direnv go cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "python-pydantic2", + "sdk-dir": "python", + "package-name": "sdk_test_python_pydantic2", + "sdk-path": "python_pydantic2", + "install-args": "sccache direnv python uv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "ruby-sorbet", + "sdk-dir": "ruby", + "package-name": "sdk_test_ruby_sorbet", + "sdk-path": "ruby_sorbet", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "cpp", + "sdk-dir": "cpp", + "package-name": "sdk_test_cpp", + "sdk-path": "cpp", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "typescript", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript", + "sdk-path": "typescript", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "typescript-web", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript_web", + "sdk-path": "typescript_web", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest github:rustwasm/wasm-pack", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "csharp", + "sdk-dir": "csharp", + "package-name": "sdk_test_csharp", + "sdk-path": "csharp", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "rust", + "sdk-dir": "rust", + "package-name": "sdk_test_rust", + "sdk-path": "rust", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "java", + "sdk-dir": "java", + "package-name": "sdk_test_java", + "sdk-path": "java", + "install-args": "sccache direnv java gradle cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + // Swift is macOS-only: the fixtures need SwiftPM/XCTest, + // and the nextest-bound setup script assembles the + // BamlBridgeFFI xcframework (host slice) before the run. + "sdk-label": "swift", + "sdk-dir": "swift", + "package-name": "sdk_test_swift", + "sdk-path": "swift", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "macos", + "runs-on": "blacksmith-6vcpu-macos-latest", + "rust-cache-key": "macos-cargo", + "timeout": 40, + }, + { + "sdk-label": "go", + "sdk-dir": "go", + "package-name": "sdk_test_go", + "sdk-path": "go", + "install-args": "sccache direnv go cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "python-pydantic2", + "sdk-dir": "python", + "package-name": "sdk_test_python_pydantic2", + "sdk-path": "python_pydantic2", + "install-args": "sccache direnv python uv cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "ruby-sorbet", + "sdk-dir": "ruby", + "package-name": "sdk_test_ruby_sorbet", + "sdk-path": "ruby_sorbet", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "typescript", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript", + "sdk-path": "typescript", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "typescript-web", + "sdk-dir": "typescript", + "package-name": "sdk_test_typescript_web", + "sdk-path": "typescript_web", + "install-args": "sccache direnv node npm:pnpm cargo:cargo-nextest github:rustwasm/wasm-pack", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "csharp", + "sdk-dir": "csharp", + "package-name": "sdk_test_csharp", + "sdk-path": "csharp", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "rust", + "sdk-dir": "rust", + "package-name": "sdk_test_rust", + "sdk-path": "rust", + "install-args": "sccache direnv cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + { + "sdk-label": "java", + "sdk-dir": "java", + "package-name": "sdk_test_java", + "sdk-path": "java", + "install-args": "sccache direnv java gradle cargo:cargo-nextest", + "os-label": "windows", + "runs-on": "blacksmith-8vcpu-windows-2025", + "rust-cache-key": "windows-cargo", + "timeout": 60, + }, + ]; + + // TODO(B-1545): Re-enable macOS SDK tests after the SIGABRT flakiness is fixed: https://linear.app/boundaryml2/issue/B-1545/investigate-flakiness-in-cargo-test-aarch64-apple-darwin-ci-job + const enabledMatrix = baseMatrix.filter((entry) => entry["os-label"] !== "macos"); + // Linux-only in the merge queue (mac/win already ran on the PR) + // and on forks (no Blacksmith mac/win runners - jobs would queue + // forever). GITHUB_REPOSITORY is always set by Actions and the + // compare is case-insensitive, so an absent or differently-cased + // payload cannot silently drop the mac/win legs upstream. The + // merge_group arm is inert in this copy: the caller ix-ci.yml + // has no merge_group trigger. + const linuxOnly = + context.eventName === "merge_group" || + process.env.GITHUB_REPOSITORY.toLowerCase() !== "boundaryml/baml"; + const matrix = linuxOnly + ? enabledMatrix.filter((entry) => entry["os-label"] === "linux") + : enabledMatrix; + const enabledSdkDirs = new Set(enabledMatrix.map((entry) => entry["sdk-dir"])); + const temporarilyDisabledSdkDirs = [...new Set( + baseMatrix + .map((entry) => entry["sdk-dir"]) + .filter((sdkDir) => !enabledSdkDirs.has(sdkDir)), + )].sort(); + + core.setOutput("matrix", JSON.stringify(matrix, null, 2)); + core.setOutput("sdk-dirs", JSON.stringify([...enabledSdkDirs].sort())); + core.setOutput("temporarily-disabled-sdk-dirs", JSON.stringify(temporarilyDisabledSdkDirs)); + + sdk-test-coverage: + name: "sdk test coverage" + runs-on: ubuntu-latest + needs: sdk-test-matrix + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: "Assert every SDK directory has a matrix entry" + uses: actions/github-script@v8 + env: + MATRIX_SDK_DIRS: ${{ needs.sdk-test-matrix.outputs.sdk-dirs }} + TEMPORARILY_DISABLED_SDK_DIRS: ${{ needs.sdk-test-matrix.outputs.temporarily-disabled-sdk-dirs }} + with: + script: | + const fs = require("fs"); + const path = require("path"); + + const sdkRoot = path.join(process.env.GITHUB_WORKSPACE, "baml_language", "sdks"); + const sdkDirs = fs.readdirSync(sdkRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .filter((sdkDir) => sdkDir !== "agent-docs") + .sort(); + const coveredSdkDirs = new Set(JSON.parse(process.env.MATRIX_SDK_DIRS)); + const temporarilyDisabledSdkDirs = new Set(JSON.parse(process.env.TEMPORARILY_DISABLED_SDK_DIRS)); + const missingSdkDirs = sdkDirs.filter( + (sdkDir) => !coveredSdkDirs.has(sdkDir) && !temporarilyDisabledSdkDirs.has(sdkDir), + ); + + await core.summary + .addHeading("SDK test matrix coverage") + .addTable([ + [{ data: "SDK directory", header: true }, { data: "Matrix entry", header: true }], + ...sdkDirs.map((sdkDir) => [ + sdkDir, + coveredSdkDirs.has(sdkDir) + ? "yes" + : temporarilyDisabledSdkDirs.has(sdkDir) + ? "temporarily disabled (B-1545)" + : "no", + ]), + ]) + .write(); + + if (missingSdkDirs.length > 0) { + core.setFailed(`SDK directories without an sdk-test-matrix entry: ${missingSdkDirs.join(", ")}`); + } + + # SDK tests (codegen end-to-end). The `sdk_test_*` crates run bridge and + # generated-language checks under nextest. Most emit fixture projects from + # `.baml` sources via build.rs; the initial Ruby/Sorbet entry exercises its + # loader directly until its generator lands. Their setup scripts do the + # expensive package-manager/native-build work (`uv sync` + `maturin` for + # Python, Bundler + bridge_cffi for Ruby, a NAPI build for Node TypeScript, + # and a Web/Wasm build plus Chromium for Web TypeScript). Run the generated + # setup guards first so CI exposes setup cost/failures as a separate step; the + # full test step re-runs the setup scripts, but the warm path should be cheap. + # Keep each SDK as a separate matrix entry so setup/test timing + # and failures are attributed to the specific SDK generator. + sdk-tests: + name: "sdk tests (${{ matrix.sdk-label }}, ${{ matrix.os-label }})" + needs: sdk-test-matrix + strategy: + fail-fast: false + # In the merge queue (`merge_group`) only the linux leg runs — the + # macos/windows legs already ran on the PR. Keep these entries in sync + # with the standalone platform cargo-test jobs above. + matrix: + include: ${{ fromJSON(needs.sdk-test-matrix.outputs.matrix) }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: ${{ matrix.timeout }} + env: + # Overrides the image's conservative 16 default; 32 measured clean on + # the 64-vCPU guests. If the rust sdk init-race or loom soak flakes + # return, revert to 16 - those were the reason for the original pin. + NEXTEST_TEST_THREADS: "32" + RUST_TEST_THREADS: "32" + # Same honesty pin family as CARGO_BUILD_JOBS: the go fixture gates run + # `go test ./...` on a 64-vCPU elastic guest shared with concurrent runners. + GOMAXPROCS: "16" + # Skip DWARF generation on fixture test-binary links; the gates only + # ever run the binaries, nothing debugs them. (-s too would be nicer, + # but GOFLAGS cannot carry a flag value containing a space; -w alone is + # most of the link-time win.) + GOFLAGS: "-ldflags=-w" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sdk test tools" + uses: ./.github/actions/setup-mise + with: + install_args: ${{ matrix.install-args }} + + - name: "Install Ruby" + if: matrix.sdk-label == 'ruby-sorbet' + uses: ./.github/actions/setup-ruby + with: + ruby-version: "3.2.2" + + - name: "Install .NET SDK" + if: matrix.sdk-label == 'csharp' + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.301 + + - name: "Test C# NuGet payload comparison" + if: matrix.sdk-label == 'csharp' + run: | + dotnet run --project \ + baml_language/sdks/csharp/bridge_csharp/tools/Baml.NuGetNormalizer/Baml.NuGetNormalizer.csproj \ + -- --self-test + + - name: "Test nightly C# package version" + if: matrix.sdk-label == 'csharp' && matrix.os-label == 'linux' + env: + BAML_LANGUAGE_RELEASED_AT: "2026-07-23T12:34:56Z" + BAML_LANGUAGE_VERSION_DATE: "20260723" + BAML_LANGUAGE_VERSION_NIGHTLY_LETTER: "h" + run: | + set -euo pipefail + work="$(mktemp -d)" + trap 'rm -rf "$work"' EXIT + scripts/baml-language-version plan \ + --channel nightly \ + --out "$work/release-plan.json" + canonical="$(jq -r .canonical_version "$work/release-plan.json")" + nuget="$(jq -r .registry_versions.nuget "$work/release-plan.json")" + test "$nuget" = "$canonical" + + scripts/baml-csharp-release-contract matrix \ + | jq -r '.include[] + | "runtimes/\(.rid)/native/\(.canonical)"' \ + | while IFS= read -r runtime_path; do + mkdir -p "$work/native/$(dirname "$runtime_path")" + printf 'version-only fixture\n' \ + > "$work/native/$runtime_path" + done + printf '\n' > "$work/baml-bridge.targets" + expected_count="$(scripts/baml-csharp-release-contract matrix \ + | jq '.include | length')" + project=baml_language/sdks/csharp/bridge_csharp/src/Baml.Bridge.csproj + dotnet build "$project" --configuration Release --nologo \ + -p:NuGetAudit=false \ + -p:Version="$canonical" \ + -p:InformationalVersion="$canonical" \ + -p:PackageVersion="$nuget" + dotnet pack "$project" --configuration Release --nologo \ + --no-build --no-restore \ + --output "$work/package" \ + -p:NuGetAudit=false \ + -p:Version="$canonical" \ + -p:InformationalVersion="$canonical" \ + -p:PackageVersion="$nuget" \ + -p:BamlExpectedNativeAssetCount="$expected_count" \ + -p:BamlNativeAssetRoot="$work/native" \ + -p:BamlGeneratedTargetsPath="$work/baml-bridge.targets" + package="$work/package/baml-bridge.$nuget.nupkg" + test -f "$package" + test "$(unzip -p "$package" baml-bridge.nuspec \ + | sed -n 's#.*\([^<]*\).*#\1#p')" = "$nuget" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: false + shared-key: ${{ matrix.rust-cache-key }} + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + # Windows-only: swap the .cmd RUSTC_WRAPPER for the native tools_sccache + # .exe. See the matching step in `cargo-test-windows` for why. + - name: "Build native sccache wrapper" + if: matrix.os-label == 'windows' + run: | + set -euo pipefail + cargo build -p tools_sccache + echo "RUSTC_WRAPPER=$(pwd -W)/target/debug/baml-sccache.exe" >> "$GITHUB_ENV" + # CARGO_BUILD_JOBS oversubscription only matters for the Windows build. + echo "CARGO_BUILD_JOBS=16" >> "$GITHUB_ENV" + working-directory: baml_language + + # Windows Python only: setup.ps1 builds one baml_bridge wheel and strips + # the editable source block from each fixture so `uv sync` installs that + # prebuilt wheel instead of rebuilding baml_bridge per fixture. The later + # test-time `uv run` processes also need this wheelhouse. + - name: "Configure Python wheelhouse" + if: matrix.os-label == 'windows' && matrix.sdk-label == 'python-pydantic2' + run: echo "UV_FIND_LINKS=$(pwd -W)/target/wheels" >> "$GITHUB_ENV" + working-directory: baml_language + + - name: "Build tests with --timings" + run: cargo test --no-run -p ${{ matrix.package-name }} --all-features --timings + working-directory: baml_language + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Run sdk test setup" + # The C# crate uses nextest's setup-script hook directly and has no + # generated setup_guard test. Its setup still runs before the full suite. + if: matrix.sdk-label != 'csharp' + run: cargo nextest run -p ${{ matrix.package-name }} --all-features -E 'package(=${{ matrix.package-name }}) and test(setup_guard::ran)' + working-directory: baml_language + + - name: "Run sdk tests" + if: matrix.sdk-label != 'csharp' + run: cargo nextest run -p ${{ matrix.package-name }} --all-features -E 'package(=${{ matrix.package-name }})' + working-directory: baml_language + + - name: "Run C# sdk tests" + if: matrix.sdk-label == 'csharp' + run: cargo nextest run -p ${{ matrix.package-name }} --all-features --no-fail-fast -E 'package(=${{ matrix.package-name }})' + working-directory: baml_language + + # The Rust bridge runtime crate (baml_bridge). Its tests load the engine + # cdylib at run time, and its lib test binary mixes pure (loader/encode) + # with engine-dependent (completion, live-engine) tests, so the whole + # package runs here — where sdk_test_rust's setup already built the engine + # into the main target dir — instead of in the general cargo-test lane + # (where the non-cacheable cdylib build ran as a silent nextest setup + # script and stalled the Windows job). + - name: "Run rust bridge tests" + if: matrix.sdk-label == 'rust' + run: cargo nextest run -p baml_bridge --all-features + working-directory: baml_language + + - name: "Upload test results on failure" + if: failure() + uses: actions/upload-artifact@v7 + with: + name: sdk-test-failures-${{ matrix.sdk-label }}-${{ matrix.os-label }} + path: | + baml_language/sdk_tests/crates/${{ matrix.sdk-path }}/*/generated/ + baml_language/target/nextest/ + if-no-files-found: ignore + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }}.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }} + path: sccache-stats-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }}.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }}.html + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }} + path: baml_language/target/cargo-timings/cargo-timing-sdk-tests-${{ matrix.sdk-label }}-${{ matrix.os-label }}.html + if-no-files-found: ignore + + cargo-test-wasm: + name: "cargo test (wasm32-unknown-unknown)" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: | + set -euo pipefail + rustup show + # `rustup target add wasm32` fetches a small (~20MB) std component that + # normally installs in seconds. A multi-minute duration means a stalled + # download (rustup has no idle timeout), which otherwise burns the whole + # job timeout and dequeues the PR from the merge queue. Cap each attempt + # well above the normal time and retry on a fresh connection; surface a + # genuine outage after 3 tries instead of hanging. + for attempt in 1 2 3; do + if timeout 300 rustup target add wasm32-unknown-unknown; then + break + fi + echo "::warning::rustup target add wasm32 attempt $attempt stalled/failed; retrying" >&2 + [ "$attempt" = 3 ] && { echo "::error::rustup target add wasm32 failed after 3 attempts"; exit 1; } + sleep 5 + done + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/canary' }} + shared-key: "linux-wasm" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + - name: "Build for WASM" + run: | + set -euo pipefail + readarray -t wasm_packages < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | select(.source==null) | select(if .metadata.ci.wasm_support == null then true else .metadata.ci.wasm_support end) | .name') + if [[ ${#wasm_packages[@]} -eq 0 ]]; then + echo "Error: no WASM packages discovered; cargo metadata or jq may have failed" >&2 + exit 1 + fi + package_args=() + for package in "${wasm_packages[@]}"; do + package_args+=(-p "$package") + done + cargo build "${package_args[@]}" --target wasm32-unknown-unknown --no-default-features --release --timings + working-directory: baml_language + + - name: "List WASM artifacts" + run: ls -lh target/wasm32-unknown-unknown/release/*.wasm || true + working-directory: baml_language + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-test-wasm.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-test-wasm + path: sccache-stats-test-wasm.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: | + if [[ -f target/cargo-timings/cargo-timing.html ]]; then + mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-test-wasm.html + fi + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-test-wasm + path: baml_language/target/cargo-timings/cargo-timing-test-wasm.html + if-no-files-found: ignore + + # Size-gate jobs live in size-gate.reusable.yaml — they are intentionally + # decoupled from cargo-tests so a violation never blocks the test workflow. + + cargo-build-msrv: + name: "cargo build (msrv)" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-msrv"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + # 40 (was 25): the MSRV toolchain is unique to this job, so a run on a + # machine without it rebuilds with a cold compile cache and measured + # 21-25+ min under wave contention (three straight 25-min timeouts on + # cold machines, 2026-08-15). Warm/steady-state runs are far inside the + # old budget; the headroom only exists for cold landings. + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - uses: SebRollen/toml-action@v1.2.0 + id: msrv + with: + file: "baml_language/Cargo.toml" + field: "workspace.package.rust-version" + + - name: "Install Rust toolchain (MSRV)" + env: + MSRV: ${{ steps.msrv.outputs.value }} + run: | + rustup toolchain install "${MSRV}" + rustup default "${MSRV}" + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/canary' }} + shared-key: "linux-msrv" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + env: + MSRV: ${{ steps.msrv.outputs.value }} + run: cargo "+${MSRV}" fetch + working-directory: baml_language + + - name: "Build with MSRV" + env: + MSRV: ${{ steps.msrv.outputs.value }} + run: cargo "+${MSRV}" test --no-run --all-features --timings + working-directory: baml_language + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-msrv.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-msrv + path: sccache-stats-msrv.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-msrv.html + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-msrv + path: baml_language/target/cargo-timings/cargo-timing-msrv.html + if-no-files-found: ignore + + cargo-doc: + name: "cargo doc" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 25 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: false + shared-key: "linux-cargo" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Generate documentation" + run: cargo doc --all --no-deps --timings + working-directory: baml_language + env: + RUSTDOCFLAGS: "-D warnings" + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-doc.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-doc + path: sccache-stats-doc.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-doc.html + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-doc + path: baml_language/target/cargo-timings/cargo-timing-doc.html + if-no-files-found: ignore + + snapshot-tests: + name: "snapshot tests" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + # 45, not 30: on ix machines this lane runs 7-16 min quiet but has + # been measured at 30m+ when two full waves stack on the gnu family + # (three timeout-cancels on 2026-08-16, all contention, all green on + # rerun). 30m leaves zero headroom for the contended regime. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install tools (mise)" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv python uv cargo:cargo-insta cargo:cargo-nextest" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + save-if: false + shared-key: "linux-cargo" + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Build tests with --timings" + run: cargo test --no-run -p baml_tests -p baml_cli -p baml_lsp2_actions --all-features --timings + working-directory: baml_language + + # This job provides glibc coverage of `baml_tests`, `baml_cli`, and + # `baml_lsp2_actions`. The x86_64 GNU job excludes all three to avoid + # running them twice; keep its `--exclude` list in sync with the package + # list here. The musl job separately runs `baml_cli`, including + # `pack_e2e`, so both Linux libc variants exercise the ELF pack path. + - name: "Run snapshot tests" + run: cargo insta test --test-runner nextest -p baml_tests -p baml_cli -p baml_lsp2_actions --all-features --unreferenced=reject + working-directory: baml_language + + - name: "Check for snapshot changes" + run: | + pathspecs=( + ':(glob)baml_language/**/*.snap' + ':(glob)baml_language/**/*.snap.new' + ) + if [ -n "$(git status --porcelain -- "${pathspecs[@]}")" ]; then + echo "::error::Snapshot tests have uncommitted changes" + git diff -- "${pathspecs[@]}" + exit 1 + fi + + - name: "Upload snapshot failures" + if: failure() + uses: actions/upload-artifact@v7 + with: + name: snapshot-failures + path: | + baml_language/**/*.snap + baml_language/**/*.snap.new + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-snapshot.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-snapshot + path: sccache-stats-snapshot.json + if-no-files-found: ignore + + - name: "Rename cargo timings" + if: always() + run: mv target/cargo-timings/cargo-timing.html target/cargo-timings/cargo-timing-snapshot.html + working-directory: baml_language + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-snapshot + path: baml_language/target/cargo-timings/cargo-timing-snapshot.html + if-no-files-found: ignore + + cargo-timings: + name: "cargo timings (merged)" + runs-on: ubuntu-latest + needs: + - cargo-test-linux + - cargo-test-linux-musl + - cargo-test-windows + - sdk-tests + - cargo-test-wasm + - cargo-build-msrv + - cargo-doc + - snapshot-tests + steps: + - name: "Merge cargo timings artifacts" + uses: actions/upload-artifact/merge@v4 + with: + name: cargo-timings + pattern: cargo-timings-* + delete-merged: true + sccache-stats: + name: "sccache stats (merged)" + runs-on: ubuntu-latest + needs: + - cargo-test-linux + - cargo-test-linux-musl + - cargo-test-windows + - sdk-tests + - cargo-test-wasm + - cargo-build-msrv + - cargo-doc + - snapshot-tests + steps: + - name: "Merge sccache stats artifacts" + uses: actions/upload-artifact/merge@v4 + with: + name: sccache-stats + pattern: sccache-stats-* + delete-merged: true diff --git a/.github/workflows/ix-ci.yml b/.github/workflows/ix-ci.yml new file mode 100644 index 00000000000..25de43970ba --- /dev/null +++ b/.github/workflows/ix-ci.yml @@ -0,0 +1,142 @@ +# CI on ix (preview): the Linux CI running side by side on an ix runner +# pool, next to the gating Blacksmith checks. A trial artifact: none of +# these checks is required, no existing workflow is changed, and deleting +# the ix-* workflow files (plus the IX_TOKEN secret) removes every +# ix-scheduled job; nothing outside .github/ is touched. +# +# The jobs are copies of the reusable workflows that carry the Linux +# spend, pointed at the pool - each ix-*.reusable.yaml header names its +# source. Fork PRs are skipped by design: untrusted code never schedules +# onto ix machines; the hosted lanes remain fork PRs' CI. +name: CI on ix (preview) + +permissions: + contents: read + +on: + push: + branches: [canary] + pull_request: + workflow_dispatch: + +concurrency: + group: ix-preview-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + determine_changes: + name: "Determine changes" + # A free hosted runner on purpose: events that path-gate to nothing + # never wake the pool. The original workflow's code filters, minus its + # release/publish workflow paths (this preview runs none of them), plus + # the ix copies. + runs-on: ubuntu-latest + # Repo-gated so a fork of this repo never queues jobs against runner + # labels it does not have; IX_CI_PREVIEW=true (a repo variable) opts a + # fork in deliberately - it is also the kill switch here. + if: (github.repository == 'BoundaryML/baml' || vars.IX_CI_PREVIEW == 'true') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + outputs: + code: ${{ steps.check_code.outputs.changed }} + webview: ${{ steps.check_webview.outputs.changed }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Determine merge base + id: merge_base + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'canary' }} + run: | + sha=$(git merge-base HEAD "origin/${BASE_REF}") + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + + - name: Check if code changed + id: check_code + env: + MERGE_BASE: ${{ steps.merge_base.outputs.sha }} + run: | + if git diff --quiet "${MERGE_BASE}...HEAD" -- \ + ':baml_language/**' \ + ':typescript2/app-promptfiddle/src/playground/default.baml' \ + ':scripts/baml-language-version' \ + ':scripts/baml-wrapper-version' \ + ':scripts/baml-release-manifests' \ + ':scripts/baml-package-manager-artifacts' \ + ':scripts/baml-release-platforms' \ + ':scripts/baml_release_platforms.py' \ + ':scripts/baml-bridge-cffi-hygiene' \ + ':scripts/tests/**' \ + ':release/platforms.json' \ + ':scripts/install.sh' \ + ':scripts/install.ps1' \ + ':packaging/aur/**' \ + ':tools/pkg_boundaryml_com/**' \ + ':.github/workflows/ix-ci.yml' \ + ':.github/workflows/ix-cargo-tests.reusable.yaml' \ + ':.github/workflows/ix-size-gate.reusable.yaml' \ + ':.github/workflows/ix-wasm-pack-tests.reusable.yaml' \ + ':.github/actions/setup-musl-cross/**' \ + ':.github/actions/setup-mise/**' \ + ; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Check if webview changed + id: check_webview + env: + MERGE_BASE: ${{ steps.merge_base.outputs.sha }} + run: | + if git diff --quiet "${MERGE_BASE}...HEAD" -- \ + ':typescript2/app-vscode-webview/**' \ + ':typescript2/pkg-playground/**' \ + ':typescript2/pkg-proto/**' \ + ':baml_language/crates/bridge_ctypes/types/**' \ + ':.github/workflows/ix-webview-tests.reusable.yaml' \ + ':.github/actions/setup-node2/**' \ + ; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + cargo-tests: + name: "Cargo Tests" + needs: determine_changes + if: needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/canary' + # No secrets, deliberately (maintainer-requested boundary): these jobs + # run PR-controlled code on ix machines, so nothing secret may + # flow to them - .envrc sees no R2 credentials and sccache falls back to + # the runner-local cache, which is the regime the pool numbers were + # measured under anyway. + uses: ./.github/workflows/ix-cargo-tests.reusable.yaml + + size-gate: + name: "Size Gate" + needs: determine_changes + if: needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/canary' + uses: ./.github/workflows/ix-size-gate.reusable.yaml + # The ix copy drops the report's PR-comment step outright, so the + # gating workflow's comment stays the only one; the report is still in + # the job summary. + permissions: + contents: read + + wasm-pack-tests: + name: "WASM Pack Tests" + needs: determine_changes + if: needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/canary' + uses: ./.github/workflows/ix-wasm-pack-tests.reusable.yaml + + webview-tests: + name: "Webview Tests" + needs: determine_changes + if: needs.determine_changes.outputs.webview == 'true' || github.ref == 'refs/heads/canary' + uses: ./.github/workflows/ix-webview-tests.reusable.yaml diff --git a/.github/workflows/ix-size-gate.reusable.yaml b/.github/workflows/ix-size-gate.reusable.yaml new file mode 100644 index 00000000000..a4841a33fbe --- /dev/null +++ b/.github/workflows/ix-size-gate.reusable.yaml @@ -0,0 +1,633 @@ +# ix preview copy of .github/workflows/size-gate.reusable.yaml - the +# same jobs pointed at the ix runner pool (runs-on labels and per-lane +# adjustments). It runs side by side with the original via ix-ci.yml, is +# never a required check, and changes nothing about how the original +# runs. Promote or delete after the trial; do not evolve both copies. +# +# sccache strategy +# ================ +# +# These jobs compile the same release artifacts the cargo tests do, so they +# share the sccache + Swatinem/rust-cache setup documented at the top of +# cargo-tests.reusable.yaml: sccache holds the compiled `target/` tree (R2 +# backend), while rust-cache keeps the registry/git state (`cache-all-crates: +# true`, `cache-targets: false`). They are read-only consumers of the shared +# rust-cache entries written by canary in cargo-tests.reusable.yaml, and read +# the same sccache R2 cache via .envrc. +# +# Secretless runs (e.g. fork PRs) still install sccache, but .envrc sees no R2 +# credentials and falls back to the runner-local cache. +name: Size Gate (Reusable) + +on: + workflow_call: + secrets: + BAML_SCCACHE_R2_ACCESS_KEY_ID: + required: false + BAML_SCCACHE_R2_SECRET_ACCESS_KEY: + required: false + +# Read-only by design, same as the sibling preview workflows: these jobs run +# PR-controlled build code and only upload artifacts, so they must not inherit +# whatever the caller grants. +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + # Inputs read by .envrc; everything else (RUSTC_WRAPPER, SCCACHE_*) is set by + # .envrc itself so it stays the single source of truth across CI and local. + # The R2 credentials .envrc maps to AWS_* for sccache. + BAML_SCCACHE_R2_ACCESS_KEY_ID: ${{ secrets.BAML_SCCACHE_R2_ACCESS_KEY_ID }} + BAML_SCCACHE_R2_SECRET_ACCESS_KEY: ${{ secrets.BAML_SCCACHE_R2_SECRET_ACCESS_KEY }} + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + CARGO_TERM_COLOR: always + RUSTUP_MAX_RETRIES: 10 + +# Individual measure-baml-size jobs are intentionally never allowed to fail. +# Each platform job has `continue-on-error: true` so a violation, runner +# death, or build hiccup does not fail the calling workflow. The +# enforce-baml-size job aggregates all available reports and is the one +# place where a real failure (policy violation or a missing platform +# report) can block CI. + +jobs: + size-gate-linux: + name: "measure-baml-size (linux)" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 20 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + shared-key: "linux-cargo" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Run size-gate check" + id: check + working-directory: baml_language + run: | + cargo run --timings -p cargo-size-gate -- size-gate check \ + --only baml-cli,packed-program \ + --format json > size-gate-linux.json 2> size-gate-stderr.txt || exit_code=$? + cat size-gate-stderr.txt >&2 + echo "exit_code=${exit_code:-0}" >> "$GITHUB_OUTPUT" + + - name: "Upload size-gate report" + if: always() + uses: actions/upload-artifact@v7 + with: + name: size-gate-linux + path: baml_language/size-gate-linux.json + if-no-files-found: ignore + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-size-gate-linux.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-size-gate-linux + path: sccache-stats-size-gate-linux.json + if-no-files-found: ignore + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-size-gate-linux + path: baml_language/target/cargo-timings/ + if-no-files-found: ignore + + size-gate-macos: + name: "measure-baml-size (macos)" + # Forks have no Blacksmith mac/win runners; the job would queue forever. + if: ${{ github.repository == 'BoundaryML/baml' }} + runs-on: blacksmith-6vcpu-macos-latest + timeout-minutes: 20 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + shared-key: "macos-cargo" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Run size-gate check" + id: check + working-directory: baml_language + run: | + cargo run --timings -p cargo-size-gate -- size-gate check \ + --only baml-cli,packed-program \ + --format json > size-gate-macos.json 2> size-gate-stderr.txt || exit_code=$? + cat size-gate-stderr.txt >&2 + echo "exit_code=${exit_code:-0}" >> "$GITHUB_OUTPUT" + + - name: "Upload size-gate report" + if: always() + uses: actions/upload-artifact@v7 + with: + name: size-gate-macos + path: baml_language/size-gate-macos.json + if-no-files-found: ignore + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-size-gate-macos.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-size-gate-macos + path: sccache-stats-size-gate-macos.json + if-no-files-found: ignore + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-size-gate-macos + path: baml_language/target/cargo-timings/ + if-no-files-found: ignore + + size-gate-windows: + name: "measure-baml-size (windows)" + # Forks have no Blacksmith mac/win runners; the job would queue forever. + if: ${{ github.repository == 'BoundaryML/baml' }} + runs-on: blacksmith-8vcpu-windows-2025 + # TODO: Reduce this once our changes to aws-sdk-rust are upstreamed + # Right now it needs to download our repo every time which is very slow + timeout-minutes: 45 + continue-on-error: true + env: + CARGO_BUILD_JOBS: "16" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + # Cargo linking uses rust-lld from the Rust toolchain on Windows. The + # extra clang tool is installed through mise for llvm-strip, replacing + # the previous Chocolatey LLVM install. + - name: "Install sccache, direnv, and clang" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv clang" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + shared-key: "windows-cargo" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Build native sccache wrapper" + run: | + set -euo pipefail + # The native RUSTC_WRAPPER (tools_sccache crate) maps the R2 creds and + # execs sccache; cargo launches it via CreateProcess (32767-char + # limit), avoiding cmd.exe's ~8191-char limit that windows-sys' huge + # `--check-cfg cfg(feature, values(...))` line blows past. The binary + # lives in the build tree, so build it first — .envrc leaves + # RUSTC_WRAPPER unset until it exists, so this bootstrap compiles with + # plain rustc — then point RUSTC_WRAPPER at it for the cargo steps + # below. + cargo build -p tools_sccache + echo "RUSTC_WRAPPER=$(pwd -W)/target/debug/baml-sccache.exe" >> "$GITHUB_ENV" + working-directory: baml_language + + - name: "Run size-gate check" + id: check + working-directory: baml_language + run: | + cargo run --timings -p cargo-size-gate -- size-gate check \ + --only baml-cli,packed-program \ + --format json > size-gate-windows.json 2> size-gate-stderr.txt || exit_code=$? + cat size-gate-stderr.txt >&2 + echo "exit_code=${exit_code:-0}" >> "$GITHUB_OUTPUT" + + - name: "Upload size-gate report" + if: always() + uses: actions/upload-artifact@v7 + with: + name: size-gate-windows + path: baml_language/size-gate-windows.json + if-no-files-found: ignore + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-size-gate-windows.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-size-gate-windows + path: sccache-stats-size-gate-windows.json + if-no-files-found: ignore + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-size-gate-windows + path: baml_language/target/cargo-timings/ + if-no-files-found: ignore + + size-gate-wasm: + name: "measure-baml-size (wasm)" + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 25 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Install Rust toolchain" + run: | + rustup show + rustup target add wasm32-unknown-unknown + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + shared-key: "linux-wasm" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + - name: "Run size-gate check" + id: check + working-directory: baml_language + run: | + cargo run --timings -p cargo-size-gate -- size-gate check \ + --only bridge_wasm \ + --format json > size-gate-wasm.json 2> size-gate-stderr.txt || exit_code=$? + cat size-gate-stderr.txt >&2 + echo "exit_code=${exit_code:-0}" >> "$GITHUB_OUTPUT" + + - name: "Upload size-gate report" + if: always() + uses: actions/upload-artifact@v7 + with: + name: size-gate-wasm + path: baml_language/size-gate-wasm.json + if-no-files-found: ignore + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Dump sccache stats (json)" + if: always() + run: sccache --show-stats --stats-format json > sccache-stats-size-gate-wasm.json + + - name: "Upload sccache stats" + if: always() + uses: actions/upload-artifact@v7 + with: + name: sccache-stats-size-gate-wasm + path: sccache-stats-size-gate-wasm.json + if-no-files-found: ignore + + - name: "Upload cargo timings" + if: always() + uses: actions/upload-artifact@v7 + with: + name: cargo-timings-size-gate-wasm + path: baml_language/target/cargo-timings/ + if-no-files-found: ignore + + size-gate-report: + name: "enforce-baml-size" + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-light"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + if: ${{ always() && !cancelled() }} + needs: [size-gate-linux, size-gate-macos, size-gate-windows, size-gate-wasm] + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Download size-gate reports" + uses: actions/download-artifact@v8 + with: + pattern: size-gate-* + merge-multiple: true + path: reports + + # Build cargo-size-gate from source. We share the linux size-gate + # job's rust-cache (same shared-key, same runner OS), so this is a + # near-instant cache hit in the common case — no upload/download of + # the binary required. + - name: "Install Rust toolchain" + run: rustup show + working-directory: baml_language + + - name: "Install sccache and direnv" + uses: ./.github/actions/setup-mise + with: + install_args: "sccache direnv" + + - name: "Verify sccache" + run: sccache --version + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + cache-all-crates: true + cache-targets: false + shared-key: "linux-cargo" + # The linux size-gate job already populated this cache; we just + # consume it here. + save-if: false + + - name: "Load .envrc with direnv" + run: | + set -euo pipefail + + direnv allow .envrc + direnv export gha >> "$GITHUB_ENV" + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch + working-directory: baml_language + + - name: "Build cargo-size-gate" + run: cargo build -p cargo-size-gate + working-directory: baml_language + + - name: "Show sccache stats" + if: always() + run: sccache --show-stats + + - name: "Compose unified report" + env: + RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + BIN: baml_language/target/debug/cargo-size-gate + RESULT_LINUX: ${{ needs.size-gate-linux.result }} + RESULT_MACOS: ${{ needs.size-gate-macos.result }} + RESULT_WINDOWS: ${{ needs.size-gate-windows.result }} + RESULT_WASM: ${{ needs.size-gate-wasm.result }} + FORK_GATED: ${{ github.repository != 'BoundaryML/baml' }} + run: | + # Track which expected platforms produced a JSON report. + missing=() + json_files=() + + declare -A platform_results=( + [linux]="$RESULT_LINUX" + [macos]="$RESULT_MACOS" + [windows]="$RESULT_WINDOWS" + [wasm]="$RESULT_WASM" + ) + + for platform in linux macos windows wasm; do + f="reports/size-gate-${platform}.json" + if [ -s "$f" ]; then + json_files+=("$f") + elif [ "${platform_results[$platform]:-unknown}" = "skipped" ] \ + && [ "$FORK_GATED" = "true" ] \ + && { [ "$platform" = "macos" ] || [ "$platform" = "windows" ]; }; then + # The only sanctioned skip: the mac/win legs are gated to the + # upstream repo, which owns the Blacksmith runners. A gated-off + # platform is not a missing report. Any OTHER skip - either leg + # skipped upstream, or linux/wasm skipped anywhere - is + # unexpected and falls through to the missing callout. + echo "note: size-gate ${platform} skipped by the fork gate" + else + missing+=("$platform (job result: ${platform_results[$platform]:-unknown})") + fi + done + + # Aggregate the JSONs we did receive. + "$BIN" size-gate agg \ + "${json_files[@]}" \ + --run-url "$RUN_URL" \ + > unified-report.md || true + + # Prepend a "Missing platforms" callout if any expected platform + # didn't produce a report. This fires whether the platform's job + # was cancelled, errored before writing JSON, or never ran. + if [ "${#missing[@]}" -gt 0 ]; then + { + echo "> ⚠️ **Missing size-gate report(s):**" + for entry in "${missing[@]}"; do + echo "> - \`${entry}\`" + done + echo ">" + echo "> The unified report below only reflects platforms that produced output." + echo "" + cat unified-report.md + } > unified-report.with-missing.md + mv unified-report.with-missing.md unified-report.md + fi + + # Persist for the failure step to consult. Gate the write on + # the array length: `printf '%s\n' "${missing[@]}"` with zero + # arguments still emits one trailing newline, which would make + # the file non-empty (1 byte) and trip the `[ -s ... ]` check + # in the failure step even when nothing is missing. + : > missing-platforms.txt + if [ "${#missing[@]}" -gt 0 ]; then + printf '%s\n' "${missing[@]}" > missing-platforms.txt + fi + + # No PR-comment step in this copy: the gating workflow's comment stays + # the only one, and the caller grants contents: read only. + - name: "Print report + write job summary" + # Publish even when a preparation step failed - the report (or its + # absence) is most valuable on exactly those runs. + if: ${{ !cancelled() }} + run: | + if [ ! -f unified-report.md ]; then + echo "size-gate report was not produced (a preparation step failed before report assembly)" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # Print to the workflow console log (so the status is visible in + # the job output, not just the summary tab)... + cat unified-report.md + # ...and to the run's summary tab. + cat unified-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: "Fail if any platform is missing or any check failed" + env: + RESULT_LINUX: ${{ needs.size-gate-linux.result }} + RESULT_MACOS: ${{ needs.size-gate-macos.result }} + RESULT_WINDOWS: ${{ needs.size-gate-windows.result }} + RESULT_WASM: ${{ needs.size-gate-wasm.result }} + run: | + fail=0 + + if [ -s missing-platforms.txt ]; then + echo "::error::Missing size-gate report(s):" + while IFS= read -r entry; do + [ -n "$entry" ] && echo "::error:: - ${entry}" + done < missing-platforms.txt + fail=1 + fi + + # Each platform's job is `continue-on-error: true`, so a job-level + # 'failure' here means the job died before producing a report + # (covered above) OR the job succeeded but uploaded a JSON marked + # as a violation. Re-check the JSONs for the latter. + # + # jq missing must fail LOUDLY here: the per-file check redirects + # stderr, so a 127 would otherwise read as "no violation" and turn + # this gate fail-open (it did, on runners whose shell lacked jq). + command -v jq >/dev/null || { echo "::error::jq is not on PATH; the violation re-check cannot run"; exit 1; } + for platform in linux macos windows wasm; do + f="reports/size-gate-${platform}.json" + if [ -f "$f" ] && jq -e '.ok == false' "$f" > /dev/null 2>&1; then + echo "::error::size-gate (${platform}) reported a policy violation" + fail=1 + fi + done + + exit "$fail" diff --git a/.github/workflows/ix-wasm-pack-tests.reusable.yaml b/.github/workflows/ix-wasm-pack-tests.reusable.yaml new file mode 100644 index 00000000000..f7bc272301f --- /dev/null +++ b/.github/workflows/ix-wasm-pack-tests.reusable.yaml @@ -0,0 +1,90 @@ +# ix preview copy of .github/workflows/wasm-pack-tests.reusable.yaml - +# the same job pointed at the ix runner pool (runs-on label and the +# warm-seed checkout). It runs side by side with the original +# via ix-ci.yml, is never a required check, and changes nothing about +# how the original runs. Promote or delete after the trial; do not +# evolve both copies. +name: WASM Pack Tests (Reusable) + +on: + workflow_call: {} + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-wasm-pack-tests + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + CARGO_TERM_COLOR: always + RUSTUP_MAX_RETRIES: 10 + +jobs: + wasm-pack-test: + name: "wasm-pack test" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + # Install Rust toolchain BEFORE rust-cache so cache key uses correct Rust version + - name: "Install Rust toolchain" + run: | + set -euo pipefail + rustup show + # `rustup target add wasm32` fetches a small (~20MB) std component that + # normally installs in seconds. A multi-minute duration means a stalled + # download (rustup has no idle timeout), which otherwise burns the whole + # 20-min job timeout and dequeues the PR from the merge queue. Cap each + # attempt well above the normal time and retry on a fresh connection; + # surface a genuine outage after 3 tries instead of hanging. + for attempt in 1 2 3; do + if timeout 300 rustup target add wasm32-unknown-unknown; then + break + fi + echo "::warning::rustup target add wasm32 attempt $attempt stalled/failed; retrying" >&2 + [ "$attempt" = 3 ] && { echo "::error::rustup target add wasm32 failed after 3 attempts"; exit 1; } + sleep 5 + done + working-directory: baml_language + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + # Share cache across similar Linux WASM jobs for better hit rates + shared-key: "linux-wasm" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + # `sys_llm` (this job's original wasm-pack test target) was deleted when + # the provider clients went native-BAML; no crate carries wasm-gated + # tests today. Until one does, this gate builds the wasm bridge — the + # check that catches wasm-only dependency breakage (e.g. a missing + # `getrandom/js` feature), which is what this job's failures actually + # meant in practice. + # NOTE: if you add a crate with wasm-gated tests, restore a + # `wasm-pack test --node crates/` step here. + - name: "Check wasm build (bridge_wasm)" + run: cargo check -p bridge_wasm --target wasm32-unknown-unknown + working-directory: baml_language diff --git a/.github/workflows/ix-webview-tests.reusable.yaml b/.github/workflows/ix-webview-tests.reusable.yaml new file mode 100644 index 00000000000..7aaaba95ed8 --- /dev/null +++ b/.github/workflows/ix-webview-tests.reusable.yaml @@ -0,0 +1,194 @@ +# ix preview copy of .github/workflows/webview-tests.reusable.yaml - +# the same jobs pointed at the ix runner pool (runs-on labels and the +# warm-seed checkout). It runs side by side with the original +# via ix-ci.yml, is never a required check, and changes nothing about +# how the original runs. Promote or delete after the trial; do not +# evolve both copies. +name: Webview Tests (Reusable) + +on: + workflow_call: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-webview-tests + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + typecheck: + name: "Typecheck" + # Fork PRs never run on ix machines; untrusted events fall back to the + # GitHub-hosted runner this job used before the migration. + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Setup Node.js for typescript2" + uses: ./.github/actions/setup-node2 + + # Install Rust toolchain BEFORE rust-cache so cache key uses correct Rust version + - name: "Install Rust toolchain" + run: | + rustup show + rustup target add wasm32-unknown-unknown + working-directory: baml_language + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + # Share cache across similar Linux WASM jobs for better hit rates + shared-key: "linux-wasm" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + # node + pnpm come from setup-node2 above; we only need mise for + # wasm-pack here. + - name: "Install mise" + uses: ./.github/actions/setup-mise + with: + install_args: "github:rustwasm/wasm-pack" + + # Path-mapped imports from app-vscode-webview resolve into pkg-playground/wasm + # and pkg-proto/src/generated, so both must be present before tsc runs. + - name: "Build WASM" + run: pnpm --filter pkg-playground build:wasm + working-directory: typescript2 + + - name: "Generate proto types" + run: pnpm --filter @b/pkg-proto generate + working-directory: typescript2 + + - name: "Typecheck pkg-proto" + run: pnpm --filter @b/pkg-proto typecheck + working-directory: typescript2 + + - name: "Typecheck app-vscode-webview" + run: pnpm --filter app-vscode-webview typecheck + working-directory: typescript2 + + unit-tests: + name: "Unit Tests (jsdom)" + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Setup Node.js for typescript2" + uses: ./.github/actions/setup-node2 + + # Install Rust toolchain BEFORE rust-cache so cache key uses correct Rust version + - name: "Install Rust toolchain" + run: | + rustup show + rustup target add wasm32-unknown-unknown + working-directory: baml_language + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + # Share cache across similar Linux WASM jobs for better hit rates + shared-key: "linux-wasm" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + # node + pnpm come from setup-node2 above; we only need mise for + # wasm-pack here. + - name: "Install mise" + uses: ./.github/actions/setup-mise + with: + install_args: "github:rustwasm/wasm-pack" + + - name: "Build WASM" + run: pnpm --filter pkg-playground build:wasm + working-directory: typescript2 + + - name: "Run unit tests" + run: pnpm --filter app-vscode-webview test:unit:run + working-directory: typescript2 + + browser-tests: + name: "Browser Tests (Playwright)" + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Warm-seed checkout: the ix machine is forked from a warm seed, so + # keep its IGNORED files (target/, node_modules); tracked files + # still hard-reset to the new SHA. A no-op on hosted runners. + clean: false + + - name: "Setup Node.js for typescript2" + uses: ./.github/actions/setup-node2 + + # Install Rust toolchain BEFORE rust-cache so cache key uses correct Rust version + - name: "Install Rust toolchain" + run: | + rustup show + rustup target add wasm32-unknown-unknown + working-directory: baml_language + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "baml_language -> target" + # Share cache across similar Linux WASM jobs for better hit rates + shared-key: "linux-wasm" + # Read-only consumer. The canary saver lives in + # cargo-tests.reusable.yaml; see the cache strategy comment there. + save-if: false + + # Separate step so if fetch is slow, build is fast. + - name: "Fetch cargo dependencies" + run: cargo fetch --target wasm32-unknown-unknown + working-directory: baml_language + + # node + pnpm come from setup-node2 above; we only need mise for + # wasm-pack here. + - name: "Install mise" + uses: ./.github/actions/setup-mise + with: + install_args: "github:rustwasm/wasm-pack" + + - name: "Build WASM" + run: pnpm --filter pkg-playground build:wasm + working-directory: typescript2 + + - name: "Install Playwright browsers" + run: npx playwright install chromium + working-directory: typescript2/app-vscode-webview + + - name: "Run browser tests" + run: pnpm --filter app-vscode-webview test:browser:run + working-directory: typescript2 From 0f7a6b53264390a93ef63748faf9378d68a9ef87 Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Wed, 19 Aug 2026 16:57:30 +0000 Subject: [PATCH 2/5] ci: reconcile the ix runner fleet Hosted-only control plane, one step: the action carries the pool definition and template itself, versioned by its pin, so no checkout of this repo is needed. Each queued ix-labeled job gets a fresh machine forked from a per-lane seed. The pin is a placeholder substituted before merge. --- .github/workflows/ix-runners.yml | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/ix-runners.yml diff --git a/.github/workflows/ix-runners.yml b/.github/workflows/ix-runners.yml new file mode 100644 index 00000000000..2cc68a5f150 --- /dev/null +++ b/.github/workflows/ix-runners.yml @@ -0,0 +1,68 @@ +# Reconciles the ix runner fleet serving the ix-* preview lanes. +# +# Setup (the whole of it): set IX_TOKEN in the repo's Actions secrets, +# install the ix-runners GitHub App, and merge. Machines exist per job: +# each queued ix-labeled job gets a fresh runner forked from a per-lane +# seed, and seeds are minted from green canary runs. The pool definition +# (lanes, sizes, region, every dial) lives in the ix-runners repo, +# versioned by the action pin below - nothing in this repo configures it, +# so the reconcile needs no checkout of this repo at all. +# +# `schedule` and `workflow_run` triggers register from the default branch +# only; until this file is on canary, reconcile by hand (workflow_dispatch, +# likewise default-branch only). +name: ix runners + +on: + workflow_dispatch: + schedule: + - cron: "*/15 * * * *" + workflow_run: + workflows: ["**"] + types: [requested] + +# Overlapping ticks can double-spawn; this group is the lock. +# cancel-in-progress: false also coalesces a burst: GitHub keeps at most +# one run pending per group, so many workflow_run wakes arriving together +# become one reconcile after the current one. +concurrency: + group: ix-runners + cancel-in-progress: false + +# Every GitHub mutation goes through a token ix vends for this run: the +# reconcile proves which repository it is with the run's OIDC identity and +# gets back an installation token scoped to this repo (the ix-runners +# GitHub App must stay installed here). GITHUB_TOKEN reads the job queue +# for the demand signal and nothing else. +permissions: + contents: read + actions: read + # How ix knows which repository is asking for a runner token. + id-token: write + +jobs: + reconcile: + name: Reconcile runners + # Control plane is hosted-only; secrets never reach the fleet. + runs-on: ubuntu-latest + steps: + # Repos without IX_TOKEN configured (forks, and this repo until the + # secret is set) skip quietly instead of failing every wave. + - name: Check opt-in + id: optin + env: + IX_TOKEN: ${{ secrets.IX_TOKEN }} + run: | + if [ -z "$IX_TOKEN" ]; then + echo "IX_TOKEN not configured; skipping" + else + echo "ok=true" >> "$GITHUB_OUTPUT" + fi + + - name: Reconcile + if: steps.optin.outputs.ok == 'true' + uses: indexable-inc/ix-runners@1da294a29ca9dcc20eec21f3f5c9856dc10645eb + with: + ix-token: ${{ secrets.IX_TOKEN }} + token-source: ix + pool: baml From b8d48a06aecf966dcd31986dd1dc931e12e0064b Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Wed, 19 Aug 2026 17:12:42 +0000 Subject: [PATCH 3/5] ci(ix): drop self-hosted from the ix runs-on label sets The v2 runner credential is minted with the job's exact labels minus self-hosted, so a job demanding [self-hosted, ix, ix-gnu] never matches the runner registered [ix, ix-gnu] (observed live: online idle runner, job queued indefinitely). The ix marker + family label is the whole lineage identity; self-hosted adds nothing but the mismatch. --- .github/workflows/ix-cargo-tests.reusable.yaml | 14 +++++++------- .github/workflows/ix-size-gate.reusable.yaml | 6 +++--- .github/workflows/ix-wasm-pack-tests.reusable.yaml | 2 +- .github/workflows/ix-webview-tests.reusable.yaml | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ix-cargo-tests.reusable.yaml b/.github/workflows/ix-cargo-tests.reusable.yaml index b9f8053f844..8ac7ee233f8 100644 --- a/.github/workflows/ix-cargo-tests.reusable.yaml +++ b/.github/workflows/ix-cargo-tests.reusable.yaml @@ -94,7 +94,7 @@ jobs: name: "cargo test (x86_64-unknown-linux-gnu)" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} # 40, not 25: a run landing on a cold ix machine recompiles from local # sccache and has been measured at 1519s+ under # family contention (cancelled at the old 25m cap); warm runs are 92s. @@ -199,7 +199,7 @@ jobs: name: "cargo test (x86_64-unknown-linux-musl)" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-musl"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-musl"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} # 45, not 35: same reasoning as the gnu lane above; measured 1400s under # contention against the old 35m cap (23 of 35 minutes used). timeout-minutes: 45 @@ -558,7 +558,7 @@ jobs: // generated matrix lands in the sdk family: these jobs share the // gnu Rust build underneath and differ only in which language // toolchain each one leaves warm in the seed lineage. - const LINUX_RUNNER = poolOk ? ["self-hosted", "ix", "ix-sdk"] : "blacksmith-16vcpu-ubuntu-2404"; + const LINUX_RUNNER = poolOk ? ["ix", "ix-sdk"] : "blacksmith-16vcpu-ubuntu-2404"; const baseMatrix = [ { @@ -1194,7 +1194,7 @@ jobs: name: "cargo test (wasm32-unknown-unknown)" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 45 steps: - uses: actions/checkout@v6 @@ -1310,7 +1310,7 @@ jobs: name: "cargo build (msrv)" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-msrv"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-msrv"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} # 40 (was 25): the MSRV toolchain is unique to this job, so a run on a # machine without it rebuilds with a cold compile cache and measured # 21-25+ min under wave contention (three straight 25-min timeouts on @@ -1408,7 +1408,7 @@ jobs: name: "cargo doc" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 25 steps: - uses: actions/checkout@v6 @@ -1489,7 +1489,7 @@ jobs: name: "snapshot tests" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-gnu"]')) || 'blacksmith-16vcpu-ubuntu-2404' }} # 45, not 30: on ix machines this lane runs 7-16 min quiet but has # been measured at 30m+ when two full waves stack on the gnu family # (three timeout-cancels on 2026-08-16, all contention, all green on diff --git a/.github/workflows/ix-size-gate.reusable.yaml b/.github/workflows/ix-size-gate.reusable.yaml index a4841a33fbe..7c3bc985f74 100644 --- a/.github/workflows/ix-size-gate.reusable.yaml +++ b/.github/workflows/ix-size-gate.reusable.yaml @@ -60,7 +60,7 @@ jobs: name: "measure-baml-size (linux)" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-gnu"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 20 continue-on-error: true steps: @@ -348,7 +348,7 @@ jobs: size-gate-wasm: name: "measure-baml-size (wasm)" - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 25 continue-on-error: true steps: @@ -440,7 +440,7 @@ jobs: size-gate-report: name: "enforce-baml-size" - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-light"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-light"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} if: ${{ always() && !cancelled() }} needs: [size-gate-linux, size-gate-macos, size-gate-windows, size-gate-wasm] permissions: diff --git a/.github/workflows/ix-wasm-pack-tests.reusable.yaml b/.github/workflows/ix-wasm-pack-tests.reusable.yaml index f7bc272301f..9ef7d987b6f 100644 --- a/.github/workflows/ix-wasm-pack-tests.reusable.yaml +++ b/.github/workflows/ix-wasm-pack-tests.reusable.yaml @@ -31,7 +31,7 @@ jobs: name: "wasm-pack test" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-wasm"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 20 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/ix-webview-tests.reusable.yaml b/.github/workflows/ix-webview-tests.reusable.yaml index 7aaaba95ed8..a16ab09eb4d 100644 --- a/.github/workflows/ix-webview-tests.reusable.yaml +++ b/.github/workflows/ix-webview-tests.reusable.yaml @@ -25,7 +25,7 @@ jobs: name: "Typecheck" # Fork PRs never run on ix machines; untrusted events fall back to the # GitHub-hosted runner this job used before the migration. - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 15 steps: - uses: actions/checkout@v6 @@ -87,7 +87,7 @@ jobs: unit-tests: name: "Unit Tests (jsdom)" - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 15 steps: - uses: actions/checkout@v6 @@ -139,7 +139,7 @@ jobs: browser-tests: name: "Browser Tests (Playwright)" - runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ (((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["ix", "ix-web"]')) || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 20 steps: - uses: actions/checkout@v6 From 0f6c59fcd71455242331bde5af046fe644b501b0 Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Wed, 19 Aug 2026 17:24:55 +0000 Subject: [PATCH 4/5] ci(ix): the machine image provides musl and ruby toolchains setup-musl-cross uses sudo (absent on the image) to install what the image already bakes; setup-ruby refuses self-hosted non-Ubuntu-image runners by design. Both steps stay for the hosted fallback and skip on self-hosted, where the image ships the toolchain. Observed live on the first v2 wave: musl exit 127 on sudo, ruby hard refusal. --- .github/workflows/ix-cargo-tests.reusable.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ix-cargo-tests.reusable.yaml b/.github/workflows/ix-cargo-tests.reusable.yaml index 8ac7ee233f8..476231bae5c 100644 --- a/.github/workflows/ix-cargo-tests.reusable.yaml +++ b/.github/workflows/ix-cargo-tests.reusable.yaml @@ -222,6 +222,9 @@ jobs: working-directory: baml_language - name: "Setup musl cross toolchain" + # The ix machine image bakes the musl toolchain (and has no sudo for + # this action's package installs); hosted fallback still needs it. + if: runner.environment != 'self-hosted' uses: ./.github/actions/setup-musl-cross with: target: x86_64-unknown-linux-musl @@ -1004,7 +1007,9 @@ jobs: install_args: ${{ matrix.install-args }} - name: "Install Ruby" - if: matrix.sdk-label == 'ruby-sorbet' + # setup-ruby refuses self-hosted non-Ubuntu-image runners by design; + # the ix machine image ships ruby. Hosted fallback still installs it. + if: matrix.sdk-label == 'ruby-sorbet' && runner.environment != 'self-hosted' uses: ./.github/actions/setup-ruby with: ruby-version: "3.2.2" From f5407712d865bc03aaf24be75489345662ca3c74 Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Wed, 19 Aug 2026 17:43:46 +0000 Subject: [PATCH 5/5] ci(ix): wire the musl cargo linker; vitest workers to the measured value - musl: setup-musl-cross is skipped on ix (image bakes the toolchain) but the action was also what exported the cargo linker; without it rustc linked with glibc cc and the -ldl lookup failed. Set CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc explicitly. - VITEST_MAX_WORKERS 12 -> 4: function_calls::vitest_workers failed at 76s on the first v2 wave, the known memory-burst class; 4 is the A/B-measured production value on 64-vCPU elastic machines. --- .github/workflows/ix-cargo-tests.reusable.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ix-cargo-tests.reusable.yaml b/.github/workflows/ix-cargo-tests.reusable.yaml index 476231bae5c..5017c0f89e7 100644 --- a/.github/workflows/ix-cargo-tests.reusable.yaml +++ b/.github/workflows/ix-cargo-tests.reusable.yaml @@ -85,7 +85,10 @@ env: # the one-slot typescript-web-workers nextest group in # baml_language/.config/nextest.toml, so peak workerd equals this dial; # if the lane ever OOMs, step down to 4. - VITEST_MAX_WORKERS: "12" + # 4 is the measured production value on 64-vCPU elastic machines: peak + # workerd count tracks this dial 1:1, and 12 was A/B-marginal under the + # guest memory-reclaim race. Raise only after the platform memory floor. + VITEST_MAX_WORKERS: "4" RUSTUP_MAX_RETRIES: 10 jobs: @@ -205,6 +208,10 @@ jobs: timeout-minutes: 45 env: CARGO_BUILD_TARGET: x86_64-unknown-linux-musl + # The ix machine image ships musl-gcc (a real musl cross compiler); + # setup-musl-cross is skipped there, so wire the cargo linker + # explicitly. Identical to what the action exports on hosted runners. + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc steps: - uses: actions/checkout@v6 with: