Keep only local FineWeb and PolarSignals beyond the core SQL PR set

Move statpopgen and FineWeb on S3 to the extended label only, and test
that `pr-full` covers every `pr` target.

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KFeuTV21uQCX8GcNyZbKwf

Author: claude

bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py

@@ -15,7 +15,7 @@
     "develop": "Every regular SQL benchmark at full target coverage.",
     "pr": "The quicker pull-request SQL benchmark matrix (action/bench-sql).",
     "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.",
-    "pr-all": "The PR and PR Compact benchmark matrices plus Clickbench Sorted (action/bench-all).",
+    "pr-all": "The PR and PR Compact benchmark matrices plus extended-only default targets (action/bench-all).",
     "pr-full": "Every regular SQL benchmark at full PR target coverage (action/bench-sql-extended).",
     "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
 }
@@ -202,7 +202,6 @@
         scale_factor=100,
         local_dir="vortex-bench/data/statpopgen",
         runs={
-            "pr": DUCKDB_DEFAULT,
             "pr-compact": COMPACT_DUCKDB,
             "pr-all": DUCKDB_STANDARD,
             "pr-full": DUCKDB_STANDARD,
@@ -231,7 +230,6 @@
         local_dir="vortex-bench/data/fineweb",
         remote_key="fineweb",
         runs={
-            "pr": DEFAULT,
             "pr-compact": COMPACT,
             "pr-all": STANDARD,
             "pr-full": STANDARD,

bench-orchestrator/tests/test_matrix.py

@@ -42,7 +42,15 @@
     "pr": tuple(
         benchmark_id
         for benchmark_id in REGULAR_IDS
-        if benchmark_id not in {"clickbench-sorted-nvme", "tpch-s3-10", "appian-nvme", "vortex-queries"}
+        if benchmark_id
+        not in {
+            "clickbench-sorted-nvme",
+            "tpch-s3-10",
+            "statpopgen",
+            "fineweb-s3",
+            "appian-nvme",
+            "vortex-queries",
+        }
     ),
     "pr-compact": COMPACT_IDS,
     "pr-all": PR_ALL_IDS,
@@ -105,6 +113,14 @@ def test_pr_all_covers_focused_presets() -> None:
                 assert set(cast("list[str]", source["data_formats"])) <= set(cast("list[str]", entry["data_formats"]))
 
 
+def test_pr_full_covers_pr() -> None:
+    pr = {entry["id"]: entry for entry in _entries("pr")}
+    pr_full = {entry["id"]: entry for entry in _entries("pr-full")}
+
+    for benchmark_id, entry in pr.items():
+        assert _targets(entry) <= _targets(pr_full[benchmark_id])
+
+
 def test_resolver_rejects_empty_targets() -> None:
     benchmark = replace(
         CATALOG.benchmarks[0],

docs/developer-guide/benchmarking.md

@@ -262,14 +262,16 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman
 - **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression
   cases on a GPU runner.
 - **SQL** -- `action/bench-sql` runs the `pr` preset, a quick core subset of the SQL benchmarks
-  that excludes `vortex-compact`, Clickbench Sorted, Appian, TPC-H SF=10 on S3, and Vortex queries.
+  that excludes `vortex-compact`, Clickbench Sorted, Appian, statpopgen, FineWeb on S3, TPC-H SF=10
+  on S3, and Vortex queries.
 - **SQL Extended** -- `action/bench-sql-extended` runs the `pr-full` preset: every regular SQL
   benchmark, including `vortex-compact` and DuckDB-format targets.
 - **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which benchmarks
   `vortex-compact` plus Parquet control rows used to distinguish code changes from runner drift.
 - **All CPU benchmarks** -- `action/bench-all` runs random access, compression, string encoding,
-  and the `pr-all` SQL preset, which combines the `pr` and `pr-compact` coverage, plus
-  Clickbench Sorted, without repeating shared jobs. Do not combine it with other benchmark labels;
+  and the `pr-all` SQL preset, which combines the `pr` and `pr-compact` coverage, plus the
+  default targets of Clickbench Sorted, statpopgen, and FineWeb on S3, without repeating shared
+  jobs. Do not combine it with other benchmark labels;
   GPU compression is the only exception.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and