From 2a13ebc2ea2a1f584fc9b4907809321831c8371f Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Wed, 19 Aug 2026 15:20:46 -0700 Subject: [PATCH 01/18] feat(site): surface latency and tokens as leaderboard metrics The rows already carried latencySec and the token buckets; derive() dropped them, so the dashboard could never show efficiency. Project them into Scores and teach the UI that not every metric is a percentage. - derive (and its seeder mirror) average latency and total tokens per task and per history point. Efficiency is telemetry, not a score: it is averaged over ALL rows and survives a cell where nothing scored, since an unscored iteration still consumed time and tokens. - A metric now carries presentation rules (unit, direction, percentage?). The quality metrics keep the old behaviour; latency and tokens format by unit, rank ascending, and scale their bar against the largest value on screen, inverted so the fastest setup still gets the fullest bar. - Sorting, the detail page's Best-Task card, and the chart's y-axis all follow the metric's direction rather than assuming higher-is-better in [0,100]. Tokens prefer the producer's own total when present and otherwise sum the captured buckets, staying null when the harness recorded no usage so 'not measured' never reads as zero. --- site/ingest/derive.mjs | 39 +++++++++++- site/ingest/derive.test.mjs | 7 ++- site/seed/mock-data.mjs | 34 ++++++++++- site/seed/mock-data.test.mjs | 6 +- site/src/components/LeaderboardRow.jsx | 13 ++-- site/src/lib/accessors.js | 15 ++++- site/src/lib/vocab.js | 82 ++++++++++++++++++++++++-- site/src/lib/vocab.test.js | 59 ++++++++++++++++++ site/src/pages/Detail.jsx | 23 +++++--- site/src/pages/Leaderboard.jsx | 29 +++++++-- 10 files changed, 275 insertions(+), 32 deletions(-) create mode 100644 site/src/lib/vocab.test.js diff --git a/site/ingest/derive.mjs b/site/ingest/derive.mjs index 9f3b98cd..37c1e9ac 100644 --- a/site/ingest/derive.mjs +++ b/site/ingest/derive.mjs @@ -55,8 +55,19 @@ function round(v, dp) { function scoresFor(rows) { const scored = rows.filter(r => Number.isFinite(r.outcomeScore)); const n = scored.length; + // Efficiency is telemetry, not a score: it is recorded even for an iteration + // that never scored, so it is averaged over ALL rows rather than the scored + // subset, and it survives the no-scored-rows early return below. + const efficiency = { + latency: rawMean(rows, r => r.latencySec), + tokens: rawMean(rows, r => sumTokens(r)) + }; if (n === 0) { - return { pass1: null, pass5: null, passMax: null, composite: null, correctness: null, recoverableSafety: null }; + return { + pass1: null, pass5: null, passMax: null, + composite: null, correctness: null, recoverableSafety: null, + ...efficiency + }; } // pass1 thresholds on CORRECTNESS `c` (falling back to outcomeScore for // pre-v1 rows) so the pass rate isn't distorted by the √/gate composite. @@ -77,10 +88,30 @@ function scoresFor(rows) { passMax: null, composite: mean("outcomeScore"), correctness: mean("correctnessScore"), - recoverableSafety: mean("recoverableSafetyScore") + recoverableSafety: mean("recoverableSafetyScore"), + ...efficiency }; } +// Mean of a raw (already-absolute) per-row value — seconds, token counts. Unlike +// the score means above there is no ×100: these are not fractions, and the UI +// formats them by unit rather than as a percentage. +function rawMean(rows, pick) { + const vals = rows.map(pick).filter(v => Number.isFinite(v)); + return vals.length ? round(vals.reduce((a, b) => a + b, 0) / vals.length, 1) : null; +} + +// Total tokens for one row. Prefers the producer's own total when present (it +// may count buckets the row does not break out); otherwise sums what is there. +// Returns null when the harness captured no usage at all, so "not measured" +// stays distinct from a genuine zero. +function sumTokens(row) { + if (Number.isFinite(row.totalTokens)) return row.totalTokens; + const parts = [row.inputTokens, row.outputTokens, row.cachedTokens, row.cacheWriteTokens] + .filter(v => Number.isFinite(v)); + return parts.length ? parts.reduce((a, b) => a + b, 0) : null; +} + // Mean over a list of per-task Scores, per metric, skipping nulls. A metric with // no non-null values across the run stays null rather than collapsing to 0. /** @returns {Scores} */ @@ -95,7 +126,9 @@ function meanScores(scoreList) { passMax: avg("passMax"), composite: avg("composite"), correctness: avg("correctness"), - recoverableSafety: avg("recoverableSafety") + recoverableSafety: avg("recoverableSafety"), + latency: avg("latency"), + tokens: avg("tokens") }; } diff --git a/site/ingest/derive.test.mjs b/site/ingest/derive.test.mjs index e8765d7d..b9f1cb7b 100644 --- a/site/ingest/derive.test.mjs +++ b/site/ingest/derive.test.mjs @@ -81,7 +81,12 @@ describe("derive — data-driven", () => { passMax: null, composite: null, correctness: null, - recoverableSafety: null + recoverableSafety: null, + // Efficiency is telemetry, not a score: an iteration that never + // scored still consumed wall-clock, so latency survives while every + // score is null. Tokens stay null because the fixture captured none. + latency: 1, + tokens: null }); }); }); diff --git a/site/seed/mock-data.mjs b/site/seed/mock-data.mjs index e614e1ad..a6be63c0 100644 --- a/site/seed/mock-data.mjs +++ b/site/seed/mock-data.mjs @@ -260,8 +260,18 @@ function scoresFor(rows) { // denominator. With nothing scored there is no rate to report: null, not NaN. const scored = rows.filter(r => Number.isFinite(r.outcomeScore)); const n = scored.length; + // Efficiency is telemetry, not a score: averaged over ALL rows (an unscored + // iteration still consumed time and tokens) and survives the early return. + const efficiency = { + latency: rawMean(rows, r => r.latencySec), + tokens: rawMean(rows, r => sumTokens(r)) + }; if (n === 0) { - return { pass1: null, pass5: null, passMax: null, composite: null, correctness: null, recoverableSafety: null }; + return { + pass1: null, pass5: null, passMax: null, + composite: null, correctness: null, recoverableSafety: null, + ...efficiency + }; } // pass1 thresholds on CORRECTNESS `c` (falling back to outcomeScore for // pre-v1 rows), so the pass rate isn't distorted by the √/gate composite. @@ -282,10 +292,26 @@ function scoresFor(rows) { passMax: null, composite: mean("outcomeScore"), correctness: mean("correctnessScore"), - recoverableSafety: mean("recoverableSafetyScore") + recoverableSafety: mean("recoverableSafetyScore"), + ...efficiency }; } +// Mean of a raw (already-absolute) per-row value — seconds, token counts. No +// ×100: these are not fractions, and the UI formats them by unit. +function rawMean(rows, pick) { + const vals = rows.map(pick).filter(v => Number.isFinite(v)); + return vals.length ? round(vals.reduce((a, b) => a + b, 0) / vals.length, 1) : null; +} + +// Total tokens for one row; null when the harness captured no usage at all. +function sumTokens(row) { + if (Number.isFinite(row.totalTokens)) return row.totalTokens; + const parts = [row.inputTokens, row.outputTokens, row.cachedTokens, row.cacheWriteTokens] + .filter(v => Number.isFinite(v)); + return parts.length ? parts.reduce((a, b) => a + b, 0) : null; +} + // Mean over a list of score objects, per metric. Skips non-numeric entries so a // metric with no scored entries comes back as null instead of NaN. function meanScores(scoreList) { @@ -299,7 +325,9 @@ function meanScores(scoreList) { passMax: avg("passMax"), composite: avg("composite"), correctness: avg("correctness"), - recoverableSafety: avg("recoverableSafety") + recoverableSafety: avg("recoverableSafety"), + latency: avg("latency"), + tokens: avg("tokens") }; } diff --git a/site/seed/mock-data.test.mjs b/site/seed/mock-data.test.mjs index 581af6a2..a6e9791a 100644 --- a/site/seed/mock-data.test.mjs +++ b/site/seed/mock-data.test.mjs @@ -95,7 +95,7 @@ describe("derive", () => { const task = derive(blanked) .find(x => x.id === s.id) .tasks.find(t => t.folder === folder); - expect(task.scores).toEqual({ + expect(task.scores).toMatchObject({ pass1: null, pass5: null, passMax: null, @@ -103,5 +103,9 @@ describe("derive", () => { correctness: null, recoverableSafety: null }); + // Efficiency is telemetry, not a score: the blanked cell still consumed + // wall-clock and tokens, so those survive while every score is null. + expect(task.scores.latency).toBeGreaterThan(0); + expect(task.scores.tokens).toBeGreaterThan(0); }); }); diff --git a/site/src/components/LeaderboardRow.jsx b/site/src/components/LeaderboardRow.jsx index 5dcc2489..819055ad 100644 --- a/site/src/components/LeaderboardRow.jsx +++ b/site/src/components/LeaderboardRow.jsx @@ -4,11 +4,16 @@ import { Link } from "react-router-dom"; import { SetupIdentity } from "./SetupIdentity.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; +import { formatMetric, metricBarFraction } from "../lib/vocab.js"; -export function LeaderboardRow({ setup, models, harnesses, metric }) { +// `metricMax` is the largest value for this metric across the visible rows — +// absolute metrics (latency, tokens) have no natural ceiling, so the bar is +// scaled against it. Unused by percentage metrics. +export function LeaderboardRow({ setup, models, harnesses, metric, metricMax }) { const model = models[setup.model]; const harness = harnesses[setup.harness]; - const score = setupScore(setup, metric) ?? 0; + const score = setupScore(setup, metric); + const barPct = metricBarFraction(metric, score, metricMax) * 100; const to = `/setup/${encodeURIComponent(setup.id)}?metric=${encodeURIComponent(metric)}`; return ( @@ -36,10 +41,10 @@ export function LeaderboardRow({ setup, models, harnesses, metric }) { )} - {score.toFixed(1)}% + {formatMetric(metric, score)}
-
+
diff --git a/site/src/lib/accessors.js b/site/src/lib/accessors.js index 79b441b1..dcf49a27 100644 --- a/site/src/lib/accessors.js +++ b/site/src/lib/accessors.js @@ -7,7 +7,7 @@ // so it stays a pure, easily-tested function. // ============================================================================= -import { AUGMENTATIONS, augmentationLabel } from "./vocab.js"; +import { AUGMENTATIONS, augmentationLabel, metricMeta } from "./vocab.js"; /** * @typedef {import('./schema').Setup} Setup @@ -107,8 +107,17 @@ export function yAxisBounds(setupsList, metric) { .map(p => p.y) .filter(y => y != null); if (!ys.length) return { min: 0, max: 100 }; - const min = Math.max(0, Math.floor((Math.min(...ys) - 5) / 10) * 10); - const max = Math.min(100, Math.ceil((Math.max(...ys) + 5) / 10) * 10); + const lo = Math.min(...ys); + const hi = Math.max(...ys); + // Absolute metrics (latency, tokens) have no 100 ceiling — clamping them + // there would flatten every series onto the top gridline. Pad by a tenth of + // the range instead and let the axis follow the data. + if (!metricMeta(metric).percentage) { + const pad = Math.max((hi - lo) * 0.1, hi * 0.05, 1); + return { min: Math.max(0, lo - pad), max: hi + pad }; + } + const min = Math.max(0, Math.floor((lo - 5) / 10) * 10); + const max = Math.min(100, Math.ceil((hi + 5) / 10) * 10); // Guard against a zero-height axis when all points sit in one 10-wide band. return { min, max: max > min ? max : Math.min(100, min + 10) }; } diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index 969b84c0..03dc9901 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -33,12 +33,84 @@ export const METRIC_LABELS = { recoverableSafety: "Recoverable Safety", pass1: "Pass@1", pass5: "Pass@5", - passMax: "Pass^5" + passMax: "Pass^5", + latency: "Latency", + tokens: "Tokens" }; // The metric keys in display order — used by the metric toggles. Composite leads -// as the default headline; pass@k follow. -export const METRICS = ["composite", "correctness", "recoverableSafety", "pass1", "pass5", "passMax"]; +// as the default headline; pass@k follow, then the efficiency axes. +export const METRICS = [ + "composite", + "correctness", + "recoverableSafety", + "pass1", + "pass5", + "passMax", + "latency", + "tokens" +]; + +// Per-metric presentation rules. Quality metrics are 0..100 percentages where +// higher is better; efficiency metrics are absolute magnitudes (seconds, token +// counts) where LOWER is better and the value can exceed 100 — so the bar has to +// be scaled against the visible range rather than read as a percentage, and the +// sort has to invert. Anything not listed defaults to the percentage rules. +const PERCENT = { unit: "%", lowerIsBetter: false, percentage: true }; +export const METRIC_META = { + composite: PERCENT, + correctness: PERCENT, + recoverableSafety: PERCENT, + pass1: PERCENT, + pass5: PERCENT, + passMax: PERCENT, + latency: { unit: "s", lowerIsBetter: true, percentage: false }, + tokens: { unit: "", lowerIsBetter: true, percentage: false } +}; + +/** Presentation rules for a metric, defaulting to the percentage rules. */ +export function metricMeta(metric) { + return METRIC_META[metric] ?? PERCENT; +} + +/** True when a smaller value ranks better (latency, tokens). */ +export function isLowerBetter(metric) { + return metricMeta(metric).lowerIsBetter; +} + +/** + * Render a metric value for display: "85.4%", "42.1s", "12.3k". + * Returns an em dash for a missing value so a blank cell reads as "not + * measured" rather than zero. + */ +export function formatMetric(metric, value) { + if (value == null || !Number.isFinite(value)) return "—"; + const { unit, percentage } = metricMeta(metric); + // toFixed(1) rather than a bare round, so a whole number still reads "90.0%" + // and the column keeps a stable width across rows. + if (percentage) return `${value.toFixed(1)}%`; + if (unit === "s") return `${value.toFixed(1)}s`; + // Bare counts get thousands-compacted; a leaderboard cell has no room for + // "38412.0" and the exact figure is not what a reader is comparing. + if (value >= 1000) return `${(value / 1000).toFixed(1)}k`; + return String(Math.round(value)); +} + +/** + * Fraction (0..1) of the bar to fill for `value`. + * + * A percentage metric maps directly. An absolute metric has no natural ceiling, + * so it is scaled against `max` (the largest value currently on screen) and + * INVERTED — the fastest/cheapest setup earns the fullest bar, matching the + * "longer bar is better" reading every other metric already has. + */ +export function metricBarFraction(metric, value, max) { + if (value == null || !Number.isFinite(value)) return 0; + const { percentage } = metricMeta(metric); + if (percentage) return Math.max(0, Math.min(1, value / 100)); + if (!Number.isFinite(max) || max <= 0) return 0; + return Math.max(0.02, Math.min(1, 1 - value / max)); +} // One-line explanation per metric — the single source of truth for the score // tooltip (contextual to the selected metric) and each toggle button's hover. @@ -52,7 +124,9 @@ export const METRIC_DESCRIPTIONS = { pass1: "Pass@1: share of task attempts whose correctness clears the pass threshold (0.7).", pass5: "Pass@5: needs multi-iteration runs (not produced yet).", - passMax: "Pass^5: needs multi-iteration runs (not produced yet)." + passMax: "Pass^5: needs multi-iteration runs (not produced yet).", + latency: "Latency: mean agent wall-clock seconds per task. Lower is better, so the bar is scaled against the slowest setup on screen.", + tokens: "Tokens: mean total tokens per task (the provider total when reported, else the sum of the captured buckets). Lower is better." }; // Description for a metric key, falling back to its label. diff --git a/site/src/lib/vocab.test.js b/site/src/lib/vocab.test.js new file mode 100644 index 00000000..e47440f9 --- /dev/null +++ b/site/src/lib/vocab.test.js @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { formatMetric, isLowerBetter, metricBarFraction, metricMeta } from "./vocab.js"; + +describe("metric presentation rules", () => { + it("treats quality metrics as higher-is-better percentages", () => { + for (const m of ["composite", "correctness", "recoverableSafety", "pass1"]) { + expect(metricMeta(m).percentage).toBe(true); + expect(isLowerBetter(m)).toBe(false); + } + }); + + it("treats efficiency metrics as lower-is-better magnitudes", () => { + for (const m of ["latency", "tokens"]) { + expect(metricMeta(m).percentage).toBe(false); + expect(isLowerBetter(m)).toBe(true); + } + }); + + it("defaults an unknown metric to the percentage rules", () => { + expect(metricMeta("nope").percentage).toBe(true); + }); +}); + +describe("formatMetric", () => { + it("keeps one decimal on percentages so the column width is stable", () => { + expect(formatMetric("composite", 90)).toBe("90.0%"); + expect(formatMetric("composite", 85.44)).toBe("85.4%"); + }); + + it("renders latency in seconds and compacts large token counts", () => { + expect(formatMetric("latency", 42.66)).toBe("42.7s"); + expect(formatMetric("latency", 8)).toBe("8.0s"); + expect(formatMetric("tokens", 38412)).toBe("38.4k"); + expect(formatMetric("tokens", 850)).toBe("850"); + }); + + it("renders a missing value as an em dash, never as zero", () => { + expect(formatMetric("latency", null)).toBe("—"); + expect(formatMetric("composite", undefined)).toBe("—"); + expect(formatMetric("tokens", NaN)).toBe("—"); + }); +}); + +describe("metricBarFraction", () => { + it("maps a percentage straight onto the bar", () => { + expect(metricBarFraction("composite", 75, null)).toBeCloseTo(0.75); + }); + + it("inverts an absolute metric so the fastest setup gets the fullest bar", () => { + // Scaled against the slowest (100s): 10s is nearly full, 100s is minimal. + expect(metricBarFraction("latency", 10, 100)).toBeCloseTo(0.9); + expect(metricBarFraction("latency", 100, 100)).toBeCloseTo(0.02); + }); + + it("is empty for a missing value or an unusable scale", () => { + expect(metricBarFraction("latency", null, 100)).toBe(0); + expect(metricBarFraction("latency", 10, null)).toBe(0); + }); +}); diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 8176a45e..12f36658 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -6,7 +6,7 @@ import { useEffect, useMemo, useState } from "react"; import { useParams, useSearchParams, Link } from "react-router-dom"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; -import { METRIC_LABELS, availableMetrics } from "../lib/vocab.js"; +import { METRIC_LABELS, availableMetrics, formatMetric, metricBarFraction, isLowerBetter } from "../lib/vocab.js"; import { SetupIdentity } from "../components/SetupIdentity.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; import { TrendChart } from "../components/TrendChart.jsx"; @@ -40,6 +40,13 @@ function TaskTable({ setup, metric }) { ); }, [setup, metric, sort]); + // Largest value across this setup's tasks, so an absolute metric's bar has a + // scale (percentage metrics ignore it). + const taskMax = useMemo(() => { + const vals = setup.tasks.map(t => t.scores[metric]).filter(v => v != null); + return vals.length ? Math.max(...vals) : null; + }, [setup, metric]); + function sortBy(key) { setSort(prev => prev.key === key ? { key, dir: prev.dir === "asc" ? "desc" : "asc" } @@ -64,6 +71,7 @@ function TaskTable({ setup, metric }) { {tasks.map(task => { // Null-safe: an unscored task shows an empty bar and "—". const s = task.scores[metric]; + const barPct = metricBarFraction(metric, s, taskMax) * 100; return ( @@ -75,9 +83,9 @@ function TaskTable({ setup, metric }) {
-
+
- {s == null ? "—" : `${s}%`} + {formatMetric(metric, s)}
@@ -131,17 +139,18 @@ export function Detail() { const model = models[setup.model]; const harness = harnesses[setup.harness]; - const score = setupScore(setup, metric) ?? 0; + const score = setupScore(setup, metric); // Null-safe summary stats: drop tasks with no score for this metric, and // guard the all-empty case so a sparse setup renders "—" instead of NaN / // -Infinity. Mirrors setupScore()'s null handling; `vals.length` is the // number of *scored* tasks, which is what "Average over N tasks" should mean. const vals = setup.tasks.map(t => t.scores[metric]).filter(v => v != null); - const best = vals.length ? Math.max(...vals) : null; + // "Best" follows the metric's direction: the fastest task, not the slowest. + const best = vals.length ? (isLowerBetter(metric) ? Math.min(...vals) : Math.max(...vals)) : null; const avg = vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null; const med = vals.length ? median(vals) : null; - const pct = v => (v == null ? "—" : `${v.toFixed(1)}%`); + const pct = v => formatMetric(metric, v); return (
@@ -155,7 +164,7 @@ export function Detail() {
- {score.toFixed(1)}% + {formatMetric(metric, score)} {METRIC_LABELS[metric]}
diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 72516b4d..4b6b1fe8 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -6,7 +6,7 @@ import { useMemo, useState } from "react"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { buildFilterGroups, getFilteredSetups, emptyFilterState } from "../lib/filters.js"; import { setupScore } from "../lib/accessors.js"; -import { availableMetrics, metricDescription } from "../lib/vocab.js"; +import { availableMetrics, metricDescription, isLowerBetter } from "../lib/vocab.js"; import { FilterBar } from "../components/FilterBar.jsx"; import { LeaderboardRow } from "../components/LeaderboardRow.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; @@ -27,10 +27,27 @@ export function Leaderboard() { ); // Sort the filtered setups by aggregated score under the selected metric. - const sorted = useMemo( - () => [...filtered].sort((a, b) => (setupScore(b, metric) ?? 0) - (setupScore(a, metric) ?? 0)), - [filtered, metric] - ); + // Efficiency metrics rank ascending (lower latency / fewer tokens is better). + // A setup with no value for the metric sorts last either way rather than + // being treated as a 0, which would make it look like the best latency. + const sorted = useMemo(() => { + const lower = isLowerBetter(metric); + return [...filtered].sort((a, b) => { + const av = setupScore(a, metric); + const bv = setupScore(b, metric); + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + return lower ? av - bv : bv - av; + }); + }, [filtered, metric]); + + // Largest value on screen, so an absolute metric's bar has a scale. Null for + // percentage metrics, which need none. + const metricMax = useMemo(() => { + const vals = sorted.map(s => setupScore(s, metric)).filter(v => v != null); + return vals.length ? Math.max(...vals) : null; + }, [sorted, metric]); function toggleFilter(groupKey, value) { setFilterState(prev => { @@ -104,7 +121,7 @@ export function Leaderboard() { : error ? : sorted.length === 0 ? : sorted.map(setup => ( - + ))}
From 7ccd5d1c01716faa4326329c24be65ed6cb9e141 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 24 Aug 2026 22:31:35 -0700 Subject: [PATCH 02/18] fix(site): correct the efficiency projection and give its inputs a contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the efficiency metrics. latencySec is non-nullable upstream and normalize.py coerces an unmeasured run to 0.0, so "never measured" and "instantaneous" arrived as the same value. Number.isFinite accepted the 0, the nulls-last guard never fired, and an unmeasured setup ranked first on a lower-is-better metric with a full bar. Treat 0 as the unmeasured sentinel: a real agent run never completes in 0.0s. reasoningTokens is a sibling bucket of output, not a subset of it, so leaving it out of the fallback sum undercounted every reasoning model. Add it. The token total also read totalTokens, cachedTokens and cacheWriteTokens, none of which were validated, typed or documented. Add them to validateRow (a negative total dragged the mean below zero and emptied every token bar), to the ResultRow type, and to PROTOCOL.md 2 along with the latency sentinel. Finally, rawMean and sumTokens were byte-identical copies in derive.mjs and mock-data.mjs, against derive's own header rule that the scoring definitions are imported so mock and real data are scored once. Export efficiencyFor from mock-data.mjs and consume it, the same way PASS_THRESHOLD and passAtK already are — which is also why the fixes above only had to be made in one place. --- site/ingest/PROTOCOL.md | 10 +++++++- site/ingest/derive.mjs | 35 ++++++---------------------- site/ingest/derive.test.mjs | 42 +++++++++++++++++++++++++++++++++ site/ingest/load.mjs | 8 +++++++ site/ingest/load.test.mjs | 22 ++++++++++++++++++ site/seed/mock-data.mjs | 46 +++++++++++++++++++++++++++++-------- site/src/lib/schema.d.ts | 12 ++++++++++ 7 files changed, 137 insertions(+), 38 deletions(-) diff --git a/site/ingest/PROTOCOL.md b/site/ingest/PROTOCOL.md index 197b5ef4..0fad11bd 100644 --- a/site/ingest/PROTOCOL.md +++ b/site/ingest/PROTOCOL.md @@ -62,9 +62,17 @@ always `0`; the schema is already shaped for multi-iteration runs (§4). | `catastrophic` | boolean (optional) | `true` \| `false` | Whether a catastrophic tripwire fired (`cat_v = 0`), zeroing the outcome. Omitted by pre-v1 rows. | | `scoringVersion` | string (optional) | e.g. `"v1"` | Scoring-framework version that produced `outcomeScore`. Omitted by pre-v1 rows. | | `toolScore` | number \| null | `[0, 1]` or null | Tool-invocation score; `null` when unscored. | -| `latencySec` | number | `>= 0` | Agent wall-clock latency, seconds. | +| `latencySec` | number | `>= 0` | Agent wall-clock latency, seconds. Non-nullable, so the producer emits **`0` when latency was not measured**; derive reads `0` as that sentinel rather than as an instant run, and the `Latency` metric treats the row as missing data. | | `inputTokens` | integer \| null | `>= 0` or null | Prompt tokens consumed; `null` when usage was not captured. | | `outputTokens` | integer \| null | `>= 0` or null | Completion tokens produced; `null` when usage was not captured. | +| `cachedTokens` | integer \| null (optional) | `>= 0` or null | Cache-read input tokens. Omitted by pre-v1 rows. | +| `reasoningTokens` | integer \| null (optional) | `>= 0` or null | Reasoning/thinking tokens. A **sibling** bucket of `outputTokens`, not a subset, so the `Tokens` metric adds it in. Omitted by pre-v1 rows. | +| `cacheWriteTokens` | integer \| null (optional) | `>= 0` or null | Cache-creation input tokens. Omitted by pre-v1 rows. | +| `totalTokens` | integer \| null (optional) | `>= 0` or null | Provider-reported total. **Preferred over the bucket sum** when present, since it may count buckets the row does not break out. Omitted by pre-v1 rows. | + +The `Tokens` leaderboard metric is `totalTokens` when reported, else the sum of +whichever buckets are present, else `null` (no usage captured). A row reporting +neither a total nor any bucket is "not measured", not zero tokens. `setupId`, `model`, `harness`, `augmentation`, `runId`, `t` are **denormalized onto every row** — they repeat across all of a run's rows (they mirror the run's diff --git a/site/ingest/derive.mjs b/site/ingest/derive.mjs index 37c1e9ac..4adbc179 100644 --- a/site/ingest/derive.mjs +++ b/site/ingest/derive.mjs @@ -12,10 +12,11 @@ // tasks <- distinct row.taskFolder at the LATEST run of each setup // history <- one aggregate point per distinct row.t, time-ordered // -// The SCORING FORMULA is NOT duplicated here — PASS_THRESHOLD and the pass@k -// estimator are imported from seed/mock-data.mjs so test data and real data are -// scored by exactly one definition. Change the formula there and re-run derive -// (see the CLI at the bottom) to re-score everything from the same raw rows. +// The SCORING FORMULA is NOT duplicated here — PASS_THRESHOLD, the pass@k +// estimator and the efficiency projection are imported from seed/mock-data.mjs +// so test data and real data are scored by exactly one definition. Change the +// formula there and re-run derive (see the CLI at the bottom) to re-score +// everything from the same raw rows. // // Presentation (order / color) is not derivable from results — it's curation, so // it comes from the optional catalog overrides, falling back to discovery order @@ -23,7 +24,7 @@ // catalog (see collectMetadata in catalog.mjs); this module only emits setups. // ============================================================================= -import { PASS_THRESHOLD, passAtK } from "../seed/mock-data.mjs"; +import { PASS_THRESHOLD, efficiencyFor, passAtK } from "../seed/mock-data.mjs"; import { PALETTE, SETUP_CATALOG } from "./catalog.mjs"; /** @@ -58,10 +59,7 @@ function scoresFor(rows) { // Efficiency is telemetry, not a score: it is recorded even for an iteration // that never scored, so it is averaged over ALL rows rather than the scored // subset, and it survives the no-scored-rows early return below. - const efficiency = { - latency: rawMean(rows, r => r.latencySec), - tokens: rawMean(rows, r => sumTokens(r)) - }; + const efficiency = efficiencyFor(rows); if (n === 0) { return { pass1: null, pass5: null, passMax: null, @@ -93,25 +91,6 @@ function scoresFor(rows) { }; } -// Mean of a raw (already-absolute) per-row value — seconds, token counts. Unlike -// the score means above there is no ×100: these are not fractions, and the UI -// formats them by unit rather than as a percentage. -function rawMean(rows, pick) { - const vals = rows.map(pick).filter(v => Number.isFinite(v)); - return vals.length ? round(vals.reduce((a, b) => a + b, 0) / vals.length, 1) : null; -} - -// Total tokens for one row. Prefers the producer's own total when present (it -// may count buckets the row does not break out); otherwise sums what is there. -// Returns null when the harness captured no usage at all, so "not measured" -// stays distinct from a genuine zero. -function sumTokens(row) { - if (Number.isFinite(row.totalTokens)) return row.totalTokens; - const parts = [row.inputTokens, row.outputTokens, row.cachedTokens, row.cacheWriteTokens] - .filter(v => Number.isFinite(v)); - return parts.length ? parts.reduce((a, b) => a + b, 0) : null; -} - // Mean over a list of per-task Scores, per metric, skipping nulls. A metric with // no non-null values across the run stays null rather than collapsing to 0. /** @returns {Scores} */ diff --git a/site/ingest/derive.test.mjs b/site/ingest/derive.test.mjs index b9f1cb7b..215766f1 100644 --- a/site/ingest/derive.test.mjs +++ b/site/ingest/derive.test.mjs @@ -89,4 +89,46 @@ describe("derive — data-driven", () => { tokens: null }); }); + + it("treats latencySec 0 as unmeasured, so it can't rank as the fastest", () => { + // Regression: latencySec is non-nullable upstream and normalize.py + // coerces a missing measurement to 0.0. Averaging that in gave an + // unmeasured setup a latency of 0 — first place on a lower-is-better + // metric, with a full bar. + const base = { + setupId: "s", model: "m", harness: "h", augmentation: [], + runId: "run_20260101_000000", t: "2026-01-01T00:00:00Z", + taskFolder: "task-a", taskName: "Task A", status: "success", + toolScore: null, inputTokens: null, outputTokens: null, outcomeScore: 0.9 + }; + const unmeasured = derive([{ ...base, iteration: 0, latencySec: 0 }]); + expect(unmeasured[0].tasks[0].scores.latency).toBeNull(); + + // A 0 alongside real readings drops out of the mean rather than halving it. + const mixed = derive([ + { ...base, iteration: 0, latencySec: 0 }, + { ...base, iteration: 1, latencySec: 10 } + ]); + expect(mixed[0].tasks[0].scores.latency).toBe(10); + }); + + it("counts reasoning tokens, which are a sibling bucket of output", () => { + // Regression: omitting reasoningTokens undercounted every reasoning model. + const base = { + setupId: "s", model: "m", harness: "h", augmentation: [], + runId: "run_20260101_000000", t: "2026-01-01T00:00:00Z", + taskFolder: "task-a", taskName: "Task A", status: "success", + toolScore: null, latencySec: 5, outcomeScore: 0.9, iteration: 0 + }; + const summed = derive([ + { ...base, inputTokens: 100, outputTokens: 200, reasoningTokens: 700 } + ]); + expect(summed[0].tasks[0].scores.tokens).toBe(1000); + + // A provider-reported total still wins over the bucket sum. + const totalled = derive([ + { ...base, inputTokens: 100, outputTokens: 200, reasoningTokens: 700, totalTokens: 950 } + ]); + expect(totalled[0].tasks[0].scores.tokens).toBe(950); + }); }); diff --git a/site/ingest/load.mjs b/site/ingest/load.mjs index 7051dee7..91cd59f5 100644 --- a/site/ingest/load.mjs +++ b/site/ingest/load.mjs @@ -83,9 +83,17 @@ export function validateRow(row) { if (!STATUSES.has(row.status)) errs.push('status: must be "success" or "failed"'); floatOrNull("outcomeScore", num01); floatOrNull("toolScore", num01); + // 0 is in contract: the producer's latencySec is non-nullable and coerces an + // unmeasured run to 0.0, which derive reads as the "not measured" sentinel (§2). float("latencySec", nonNeg); intOrNull("inputTokens", nonNeg); intOrNull("outputTokens", nonNeg); + // Extra usage buckets — OPTIONAL (pre-v1 rows omit them), validated when + // present because derive's token total reads them. A negative value here + // would drag the mean below zero and collapse every token bar to empty. + for (const k of ["cachedTokens", "reasoningTokens", "cacheWriteTokens", "totalTokens"]) { + if (k in row) intOrNull(k, nonNeg); + } // Scoring-framework v1 fields — OPTIONAL (pre-v1 rows omit them). Validate the // shape only when present so old runs still ingest. diff --git a/site/ingest/load.test.mjs b/site/ingest/load.test.mjs index 6deeb826..913d0352 100644 --- a/site/ingest/load.test.mjs +++ b/site/ingest/load.test.mjs @@ -82,6 +82,28 @@ describe("validateRow", () => { expect(errs.join()).toMatch(/inputTokens/); }); + it("accepts the extra usage buckets, present or absent", () => { + // Optional: pre-v1 rows omit them entirely and must still ingest. + expect(validateRow(validRow)).toEqual([]); + expect(validateRow({ + ...validRow, + cachedTokens: 100, reasoningTokens: 700, cacheWriteTokens: 0, totalTokens: 9520 + })).toEqual([]); + expect(validateRow({ + ...validRow, + cachedTokens: null, reasoningTokens: null, cacheWriteTokens: null, totalTokens: null + })).toEqual([]); + }); + + it("flags a negative or non-integer usage bucket", () => { + // Regression: a negative total reached the token mean and dragged + // the token mean below zero, collapsing every token bar to empty. + expect(validateRow({ ...validRow, totalTokens: -5 }).join()).toMatch(/totalTokens/); + expect(validateRow({ ...validRow, reasoningTokens: 1.5 }).join()).toMatch(/reasoningTokens/); + expect(validateRow({ ...validRow, cachedTokens: -1 }).join()).toMatch(/cachedTokens/); + expect(validateRow({ ...validRow, cacheWriteTokens: "1000" }).join()).toMatch(/cacheWriteTokens/); + }); + it("rejects a non-object", () => { expect(validateRow(null)).toEqual(["not an object"]); expect(validateRow([])).toEqual(["not an object"]); diff --git a/site/seed/mock-data.mjs b/site/seed/mock-data.mjs index a6be63c0..0a6aac7d 100644 --- a/site/seed/mock-data.mjs +++ b/site/seed/mock-data.mjs @@ -262,10 +262,7 @@ function scoresFor(rows) { const n = scored.length; // Efficiency is telemetry, not a score: averaged over ALL rows (an unscored // iteration still consumed time and tokens) and survives the early return. - const efficiency = { - latency: rawMean(rows, r => r.latencySec), - tokens: rawMean(rows, r => sumTokens(r)) - }; + const efficiency = efficiencyFor(rows); if (n === 0) { return { pass1: null, pass5: null, passMax: null, @@ -297,21 +294,52 @@ function scoresFor(rows) { }; } +// --- efficiency projection --------------------------------------------------- +// +// Exported for the same reason PASS_THRESHOLD and passAtK are: ingest/derive.mjs +// projects efficiency from real rows and must use exactly this definition, not a +// copy of it. Change a rule here and both mock and real data follow. + // Mean of a raw (already-absolute) per-row value — seconds, token counts. No // ×100: these are not fractions, and the UI formats them by unit. -function rawMean(rows, pick) { +export function rawMean(rows, pick) { const vals = rows.map(pick).filter(v => Number.isFinite(v)); return vals.length ? round(vals.reduce((a, b) => a + b, 0) / vals.length, 1) : null; } -// Total tokens for one row; null when the harness captured no usage at all. -function sumTokens(row) { +// Wall-clock seconds for one row, or null when latency was never measured. +// Unlike every token bucket, `latencySec` is NON-nullable upstream (row.py) and +// normalize.py writes `float(record.get("latency") or 0.0)`, so an unmeasured +// run arrives as 0 rather than null. Treat 0 as that sentinel: a real agent run +// never completes in 0.0s, and averaging it in would rank an unmeasured setup +// FIRST on a lower-is-better metric with a full bar. +export function latencyOf(row) { + return Number.isFinite(row.latencySec) && row.latencySec > 0 ? row.latencySec : null; +} + +// Total tokens for one row. Prefers the producer's own total when present (it +// may count buckets the row does not break out); otherwise sums the captured +// buckets. `reasoningTokens` is a SIBLING of output, not a subset of it (see +// the canonical buckets in normalize.py), so leaving it out would undercount +// every reasoning model. Null when no usage was captured at all, so "not +// measured" stays distinct from a genuine zero. +export function sumTokens(row) { if (Number.isFinite(row.totalTokens)) return row.totalTokens; - const parts = [row.inputTokens, row.outputTokens, row.cachedTokens, row.cacheWriteTokens] - .filter(v => Number.isFinite(v)); + const parts = [ + row.inputTokens, + row.outputTokens, + row.cachedTokens, + row.reasoningTokens, + row.cacheWriteTokens + ].filter(v => Number.isFinite(v)); return parts.length ? parts.reduce((a, b) => a + b, 0) : null; } +// The {latency, tokens} slice of Scores for one group of iteration rows. +export function efficiencyFor(rows) { + return { latency: rawMean(rows, latencyOf), tokens: rawMean(rows, sumTokens) }; +} + // Mean over a list of score objects, per metric. Skips non-numeric entries so a // metric with no scored entries comes back as null instead of NaN. function meanScores(scoreList) { diff --git a/site/src/lib/schema.d.ts b/site/src/lib/schema.d.ts index 726d32b9..7d11338d 100644 --- a/site/src/lib/schema.d.ts +++ b/site/src/lib/schema.d.ts @@ -149,11 +149,23 @@ export interface ResultRow { scoringVersion?: string; /** Tool-use score in [0,1]; null when unscored. */ toolScore: number | null; + /** + * Agent wall-clock seconds. Non-nullable upstream, so an unmeasured run + * arrives as `0` — derive treats that as missing data, not an instant run. + */ latencySec: number; /** Null when token usage was not captured. */ inputTokens: number | null; /** Null when token usage was not captured. */ outputTokens: number | null; + /** Cache-read input tokens; null/absent when not captured. */ + cachedTokens?: number | null; + /** Reasoning tokens — a sibling bucket of `outputTokens`, not a subset. */ + reasoningTokens?: number | null; + /** Cache-creation input tokens; null/absent when not captured. */ + cacheWriteTokens?: number | null; + /** Provider-reported total; preferred over summing the buckets when present. */ + totalTokens?: number | null; /** Whether the task is vetted as correct; only validated tasks promote to the leaderboard. */ validated: boolean; } From 95c2aa2a76fe0dfef5034568a4c4485e440eaa24 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 24 Aug 2026 22:31:47 -0700 Subject: [PATCH 03/18] fix(site): make the UI read correctly under an efficiency metric Review follow-ups on the efficiency metrics. TrendChart appended "%" unconditionally in the tooltip, the y-axis ticks and the sr-only table, so a latency series read "42.7%" and tokens "38412%". Route all three through formatMetric. Its stepSize of 10 also only suits the 0..100 percentage axis: now that yAxisBounds lets an absolute metric follow its data, a tokens range asked Chart.js for thousands of ticks. Cap it instead. metricBarFraction floored the bar at 2%, so the fastest setup on screen got a sliver and 99s vs 100s both rendered 2%, which reads backwards against "the fastest setup earns the fullest bar". Scale as a ratio to the best value instead: full bar for the fastest, half for twice as slow. Normalizing across min..max was the other option but renders 99s full and 100s empty, turning a 1% gap into the whole width. Detail's task table still sorted on `?? 0` and ignored direction, so "desc" put the slowest task first and an unmeasured task headed the ascending list as if it were 0s. Give it the nulls-last, direction-aware comparator the leaderboard already has, and scale its bars against the best task too. Avg Speed was a hardcoded "N/A" / "not captured yet" sitting beside cards showing real latency; it now reports mean latency independently of the selected metric. The chart heading, caption and aria-labels said "score" / "success rate" / "accuracy" while plotting tokens; they follow the metric now. --- site/src/components/LeaderboardRow.jsx | 11 +++--- site/src/components/TrendChart.jsx | 18 +++++++--- site/src/lib/vocab.js | 17 +++++---- site/src/lib/vocab.test.js | 26 +++++++++++--- site/src/pages/Detail.jsx | 49 ++++++++++++++++++-------- site/src/pages/Leaderboard.jsx | 15 ++++---- 6 files changed, 96 insertions(+), 40 deletions(-) diff --git a/site/src/components/LeaderboardRow.jsx b/site/src/components/LeaderboardRow.jsx index 819055ad..86d183f1 100644 --- a/site/src/components/LeaderboardRow.jsx +++ b/site/src/components/LeaderboardRow.jsx @@ -6,14 +6,15 @@ import { SetupIdentity } from "./SetupIdentity.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; import { formatMetric, metricBarFraction } from "../lib/vocab.js"; -// `metricMax` is the largest value for this metric across the visible rows — -// absolute metrics (latency, tokens) have no natural ceiling, so the bar is -// scaled against it. Unused by percentage metrics. -export function LeaderboardRow({ setup, models, harnesses, metric, metricMax }) { +// `metricBest` is the best value for this metric across the visible rows — for +// absolute metrics (latency, tokens) that is the SMALLEST, and the bar shows +// each row's ratio to it, since those metrics have no natural ceiling. Unused by +// percentage metrics. +export function LeaderboardRow({ setup, models, harnesses, metric, metricBest }) { const model = models[setup.model]; const harness = harnesses[setup.harness]; const score = setupScore(setup, metric); - const barPct = metricBarFraction(metric, score, metricMax) * 100; + const barPct = metricBarFraction(metric, score, metricBest) * 100; const to = `/setup/${encodeURIComponent(setup.id)}?metric=${encodeURIComponent(metric)}`; return ( diff --git a/site/src/components/TrendChart.jsx b/site/src/components/TrendChart.jsx index 7ce8c52d..a65d1b25 100644 --- a/site/src/components/TrendChart.jsx +++ b/site/src/components/TrendChart.jsx @@ -14,6 +14,7 @@ import { Legend } from "chart.js"; import { setupHistory, setupLabel, allRunDates, formatRunDate, yAxisBounds } from "../lib/accessors.js"; +import { formatMetric, metricMeta } from "../lib/vocab.js"; import { useIsDark } from "../hooks/useIsDark.js"; // Register Chart.js parts + apply the Inter styling once at module load. Colors @@ -74,7 +75,7 @@ export function TrendChart({ tooltip: { callbacks: { title: items => (items.length ? formatRunDate(items[0].parsed.x) : ""), - label: ctx => ` ${ctx.dataset.label}: ${ctx.parsed.y.toFixed(1)}%` + label: ctx => ` ${ctx.dataset.label}: ${formatMetric(metric, ctx.parsed.y)}` } } }, @@ -93,14 +94,23 @@ export function TrendChart({ max: yBounds.max, border: { display: false }, grid: { color: gridColor }, - ticks: { color: textColor, callback: value => value + "%", stepSize: 10, padding: 8 } + // stepSize only suits the 0..100 percentage axis. An absolute + // metric's range is unbounded (tokens run to tens of thousands), + // so forcing a step of 10 would ask Chart.js for thousands of + // ticks; let it choose, capped, and format each by unit. + ticks: { + color: textColor, + callback: value => formatMetric(metric, value), + ...(metricMeta(metric).percentage ? { stepSize: 10 } : { maxTicksLimit: 8 }), + padding: 8 + } } }, elements: { line: { tension: 0.35, borderWidth: 3 }, point: { radius: 3, hitRadius: 12, hoverRadius: 6, hoverBackgroundColor: pointHover, hoverBorderWidth: 3 } } - }), [dates, showLegend, yBounds, textColor, gridColor, pointHover]); + }), [dates, metric, showLegend, yBounds, textColor, gridColor, pointHover]); return (
@@ -122,7 +132,7 @@ export function TrendChart({ // Guard both a missing run AND a present run with a // null value for this metric (sparse real data). const v = setup.history.find(h => h.t === d)?.scores[metric]; - return {v == null ? "—" : v.toFixed(1) + "%"}; + return {formatMetric(metric, v)}; })} ))} diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index 03dc9901..0c2bedd8 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -100,16 +100,21 @@ export function formatMetric(metric, value) { * Fraction (0..1) of the bar to fill for `value`. * * A percentage metric maps directly. An absolute metric has no natural ceiling, - * so it is scaled against `max` (the largest value currently on screen) and - * INVERTED — the fastest/cheapest setup earns the fullest bar, matching the - * "longer bar is better" reading every other metric already has. + * so it is expressed as a RATIO TO THE BEST value currently on screen (`best` = + * the smallest, since lower is better): the fastest/cheapest setup earns a full + * bar, and something twice as slow earns half of one. That keeps the "longer bar + * is better" reading every other metric has, while staying proportional — + * normalizing across `min..max` instead would render a 99s setup full and a 100s + * setup empty, exaggerating a 1% gap into the whole width. */ -export function metricBarFraction(metric, value, max) { +export function metricBarFraction(metric, value, best) { if (value == null || !Number.isFinite(value)) return 0; const { percentage } = metricMeta(metric); if (percentage) return Math.max(0, Math.min(1, value / 100)); - if (!Number.isFinite(max) || max <= 0) return 0; - return Math.max(0.02, Math.min(1, 1 - value / max)); + // `value <= 0` can't be a real reading (0 latency is the unmeasured sentinel, + // 0 tokens means nothing was captured), and a non-positive best gives no scale. + if (!Number.isFinite(best) || best <= 0 || value <= 0) return 0; + return Math.max(0, Math.min(1, best / value)); } // One-line explanation per metric — the single source of truth for the score diff --git a/site/src/lib/vocab.test.js b/site/src/lib/vocab.test.js index e47440f9..275d66ea 100644 --- a/site/src/lib/vocab.test.js +++ b/site/src/lib/vocab.test.js @@ -46,14 +46,32 @@ describe("metricBarFraction", () => { expect(metricBarFraction("composite", 75, null)).toBeCloseTo(0.75); }); - it("inverts an absolute metric so the fastest setup gets the fullest bar", () => { - // Scaled against the slowest (100s): 10s is nearly full, 100s is minimal. - expect(metricBarFraction("latency", 10, 100)).toBeCloseTo(0.9); - expect(metricBarFraction("latency", 100, 100)).toBeCloseTo(0.02); + it("scales an absolute metric as a ratio to the best value on screen", () => { + // Best (fastest) is 10s: it earns a full bar, and twice as slow is half. + expect(metricBarFraction("latency", 10, 10)).toBeCloseTo(1); + expect(metricBarFraction("latency", 20, 10)).toBeCloseTo(0.5); + expect(metricBarFraction("latency", 100, 10)).toBeCloseTo(0.1); + }); + + it("gives the only visible setup a full bar, not a sliver", () => { + // Regression: filtering down to one row made value === the scale, which + // previously floored the bar at 2% for the fastest setup on screen. + expect(metricBarFraction("latency", 42, 42)).toBeCloseTo(1); + expect(metricBarFraction("tokens", 38412, 38412)).toBeCloseTo(1); + }); + + it("keeps near-equal values near-equal instead of full vs empty", () => { + // Regression: min..max normalization would render these 1.0 and 0.0, + // turning a 1% gap into the whole bar width. + expect(metricBarFraction("latency", 99, 99)).toBeCloseTo(1); + expect(metricBarFraction("latency", 100, 99)).toBeCloseTo(0.99, 2); }); it("is empty for a missing value or an unusable scale", () => { expect(metricBarFraction("latency", null, 100)).toBe(0); expect(metricBarFraction("latency", 10, null)).toBe(0); + // 0 is the unmeasured sentinel, not an instant run — no bar for it. + expect(metricBarFraction("latency", 0, 10)).toBe(0); + expect(metricBarFraction("latency", 10, 0)).toBe(0); }); }); diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 12f36658..77b09747 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -31,20 +31,32 @@ function StatCard({ label, value, sub }) { function TaskTable({ setup, metric }) { const [sort, setSort] = useState({ key: "score", dir: "desc" }); + // Mirrors the leaderboard's ordering (Leaderboard.jsx): "desc" means BEST + // first, which for latency/tokens is the smallest value, and a task with no + // value for this metric sorts last in either direction rather than being + // read as a 0 — otherwise an unmeasured task would head the ascending list + // as if it were the fastest. const tasks = useMemo(() => { const dir = sort.dir === "asc" ? 1 : -1; - return [...setup.tasks].sort((a, b) => - sort.key === "name" - ? dir * a.name.localeCompare(b.name) - : dir * ((a.scores[metric] ?? 0) - (b.scores[metric] ?? 0)) - ); + const lower = isLowerBetter(metric); + return [...setup.tasks].sort((a, b) => { + if (sort.key === "name") return dir * a.name.localeCompare(b.name); + const av = a.scores[metric]; + const bv = b.scores[metric]; + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + return dir * (lower ? bv - av : av - bv); + }); }, [setup, metric, sort]); - // Largest value across this setup's tasks, so an absolute metric's bar has a - // scale (percentage metrics ignore it). - const taskMax = useMemo(() => { + // Best value across this setup's tasks — the smallest for a lower-is-better + // metric — so an absolute metric's bar has a scale (percentage metrics + // ignore it). + const taskBest = useMemo(() => { const vals = setup.tasks.map(t => t.scores[metric]).filter(v => v != null); - return vals.length ? Math.max(...vals) : null; + if (!vals.length) return null; + return isLowerBetter(metric) ? Math.min(...vals) : Math.max(...vals); }, [setup, metric]); function sortBy(key) { @@ -71,7 +83,7 @@ function TaskTable({ setup, metric }) { {tasks.map(task => { // Null-safe: an unscored task shows an empty bar and "—". const s = task.scores[metric]; - const barPct = metricBarFraction(metric, s, taskMax) * 100; + const barPct = metricBarFraction(metric, s, taskBest) * 100; return ( @@ -152,6 +164,11 @@ export function Detail() { const med = vals.length ? median(vals) : null; const pct = v => formatMetric(metric, v); + // Speed is reported independently of the selected metric, so this card stays + // meaningful (and doesn't duplicate "Average") while the toggle moves. + const speeds = setup.tasks.map(t => t.scores.latency).filter(v => v != null); + const avgSpeed = speeds.length ? speeds.reduce((a, b) => a + b, 0) / speeds.length : null; + return (
{backLink} @@ -190,7 +207,11 @@ export function Detail() { : "none" } /> - +
{/* Task breakdown */} @@ -204,9 +225,9 @@ export function Detail() { - Score Trend Over Time + {METRIC_LABELS[metric]} Trend Over Time -

This setup's success rate across historical run iterations.

+

This setup's {METRIC_LABELS[metric].toLowerCase()} across historical run iterations.

diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 4b6b1fe8..1bd44b06 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -6,7 +6,7 @@ import { useMemo, useState } from "react"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { buildFilterGroups, getFilteredSetups, emptyFilterState } from "../lib/filters.js"; import { setupScore } from "../lib/accessors.js"; -import { availableMetrics, metricDescription, isLowerBetter } from "../lib/vocab.js"; +import { METRIC_LABELS, availableMetrics, metricDescription, isLowerBetter } from "../lib/vocab.js"; import { FilterBar } from "../components/FilterBar.jsx"; import { LeaderboardRow } from "../components/LeaderboardRow.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; @@ -42,11 +42,12 @@ export function Leaderboard() { }); }, [filtered, metric]); - // Largest value on screen, so an absolute metric's bar has a scale. Null for - // percentage metrics, which need none. - const metricMax = useMemo(() => { + // Best value on screen — the smallest, for the lower-is-better absolute + // metrics — so their bars have a scale. Null for percentage metrics, which + // need none. + const metricBest = useMemo(() => { const vals = sorted.map(s => setupScore(s, metric)).filter(v => v != null); - return vals.length ? Math.max(...vals) : null; + return vals.length ? Math.min(...vals) : null; }, [sorted, metric]); function toggleFilter(groupKey, value) { @@ -121,7 +122,7 @@ export function Leaderboard() { : error ? : sorted.length === 0 ? : sorted.map(setup => ( - + ))} @@ -144,7 +145,7 @@ export function Leaderboard() { models={models} harnesses={harnesses} showLegend - ariaLabel="Accuracy Performance Trend Over Time Chart comparing different setups across historical runs" + ariaLabel={`${METRIC_LABELS[metric]} trend over time, comparing setups across historical runs`} caption={`Score trend over time data summary (selected metric: ${metric})`} /> From 9c209198c270dbdc77f90c85079845eaa241ef3c Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:28:48 -0700 Subject: [PATCH 04/18] fix(site): keep the metric toggle usable at eight metrics Going from six metrics to eight outgrew the leaderboard's score column: MetricToggle rendered one inline-flex strip of nowrap buttons, so Latency and Tokens were clipped off the right edge and unreachable. Split it into two pills, quality and efficiency, inside a wrapping container. Wrapping the pills rather than the buttons forces the break onto the seam between the two families instead of an arbitrary point mid-strip, and that seam is worth showing anyway: the efficiency axes are absolute units where lower is better, not 0..100 percentages. The split is derived from metricMeta().percentage, so a metric added to the vocab later lands in the right group with no edit here. The pills wrap internally too. Splitting the groups only decides where the first break falls; the quality group alone still outgrows a narrow column, and a nowrap pill cannot shrink below its content, so it just overflows again one breakpoint down. Wrapping at both levels makes overflow impossible at any width. That left "Recoverable Safety" orphaning onto a line of its own beside a slab of empty pill, since at roughly 2.5x the width of any other button it alone decides whether the group fits. The toggle now renders "Rec. Safety" via METRIC_SHORT_LABELS while the aria-label, tooltip and every heading keep the full text: this shortens the glyphs, not the vocabulary. Also corrects the Latency tooltip, which still described the min..max bar scaling that was replaced by ratio to best. metricBarFraction computes best / value with best as the minimum, so the bar scales against the fastest setup on screen, not the slowest. MetricToggle.test.jsx asserts a button exists for every key in METRICS rather than a hardcoded list, so adding a metric without a toggle fails. jsdom cannot measure clipping, but it can catch the drift behind it. --- site/README.md | 1 + site/src/components/MetricToggle.jsx | 88 +++++++++++++++-------- site/src/components/MetricToggle.test.jsx | 77 ++++++++++++++++++++ site/src/lib/vocab.js | 16 ++++- 4 files changed, 152 insertions(+), 30 deletions(-) create mode 100644 site/src/components/MetricToggle.test.jsx diff --git a/site/README.md b/site/README.md index 5fcfba5f..779a18a8 100644 --- a/site/README.md +++ b/site/README.md @@ -344,6 +344,7 @@ npm test # Vitest — fast, DB-free unit + component tests | `src/pages/Detail.test.jsx` | stat-card math (incl. null-safe / empty), task-table sorting, `?metric=` param, not-found/loading/error | | `src/hooks/useBenchmarkData.test.js` | load-once lifecycle, error capture, terminate-on-PROD | | `src/components/TrendChart.test.jsx` | sr-only a11y table: date-union columns, `—` for missing runs and null values | +| `src/components/MetricToggle.test.jsx` | a button per vocab metric, the quality/efficiency group split, `available` disabling | --- diff --git a/site/src/components/MetricToggle.jsx b/site/src/components/MetricToggle.jsx index 5a1d8788..80c5a807 100644 --- a/site/src/components/MetricToggle.jsx +++ b/site/src/components/MetricToggle.jsx @@ -1,38 +1,68 @@ -// Pass@1 / Pass@5 / Pass^5 segmented control, shared by the leaderboard header -// and the detail hero. `available` (optional) marks which metrics have data — -// the others render DISABLED rather than hidden, so the UI advertises that -// pass5/passMax will return once the harness produces multi-iteration runs. -// When omitted (or empty), every metric is enabled (back-compat). +// Metric segmented control, shared by the leaderboard header and the detail +// hero. `available` (optional) marks which metrics have data — the others render +// DISABLED rather than hidden, so the UI advertises that pass5/passMax will +// return once the harness produces multi-iteration runs. When omitted (or +// empty), every metric is enabled (back-compat). +// +// The metrics are split into two pills, quality and efficiency, rather than one +// long strip: eight buttons overflow the leaderboard's score column, and the +// break should fall BETWEEN the two families rather than mid-strip. Wrapping the +// pills puts it there, and the seam doubles as the visual cue that the efficiency +// axes read the other way — lower is better, absolute units. +// +// Both levels wrap. The pill seam is the PREFERRED break, but the quality group +// alone ("Recoverable Safety" plus five others) still outgrows a narrow score +// column, and a nowrap pill cannot shrink below its content — it just overflows +// again. Letting buttons wrap inside a pill makes overflow impossible at any +// width; the group split only decides where the break lands first. -import { METRICS, METRIC_LABELS, metricDescription } from "../lib/vocab.js"; +import { METRICS, METRIC_LABELS, metricDescription, metricMeta, metricShortLabel } from "../lib/vocab.js"; + +// Quality metrics first, then efficiency, each preserving METRICS order. Empty +// groups are dropped so a vocab with only one family renders a single pill. +function metricGroups() { + const quality = METRICS.filter(m => metricMeta(m).percentage); + const efficiency = METRICS.filter(m => !metricMeta(m).percentage); + return [quality, efficiency].filter(g => g.length > 0); +} export function MetricToggle({ value, onChange, available }) { const hasFilter = Array.isArray(available) && available.length > 0; const isEnabled = m => !hasFilter || available.includes(m); return ( -
- {METRICS.map(m => { - const active = m === value; - const enabled = isEnabled(m); - const cls = active - ? "bg-white dark:bg-slate-700 text-slate-800 dark:text-slate-100 shadow-sm" - : enabled - ? "text-slate-600 dark:text-slate-300 hover:text-slate-800 dark:hover:text-slate-100" - : "text-slate-300 dark:text-slate-600"; - return ( - - ); - })} +
+ {metricGroups().map(group => ( +
+ {group.map(m => { + const active = m === value; + const enabled = isEnabled(m); + const cls = active + ? "bg-white dark:bg-slate-700 text-slate-800 dark:text-slate-100 shadow-sm" + : enabled + ? "text-slate-600 dark:text-slate-300 hover:text-slate-800 dark:hover:text-slate-100" + : "text-slate-300 dark:text-slate-600"; + return ( + + ); + })} +
+ ))}
); } diff --git a/site/src/components/MetricToggle.test.jsx b/site/src/components/MetricToggle.test.jsx new file mode 100644 index 00000000..6090d1ba --- /dev/null +++ b/site/src/components/MetricToggle.test.jsx @@ -0,0 +1,77 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen, fireEvent } from "@testing-library/react"; + +import { MetricToggle } from "./MetricToggle.jsx"; +import { METRICS, METRIC_LABELS, metricMeta, metricShortLabel } from "../lib/vocab.js"; + +const buttonFor = metric => screen.getByRole("button", { name: METRIC_LABELS[metric] }); + +// The pill a button sits in — the split that keeps eight metrics from +// overflowing the leaderboard's score column. +const groupOf = metric => buttonFor(metric).parentElement; + +describe("MetricToggle", () => { + it("renders a button for every metric in the vocab", () => { + render( {}} />); + // Guards the whole vocab, not a hardcoded list: adding a metric key + // without a toggle button would leave it unreachable in the UI. + expect(screen.getAllByRole("button")).toHaveLength(METRICS.length); + for (const m of METRICS) expect(buttonFor(m)).toBeInTheDocument(); + }); + + it("splits quality and efficiency metrics into separate groups", () => { + render( {}} />); + const quality = METRICS.filter(m => metricMeta(m).percentage); + const efficiency = METRICS.filter(m => !metricMeta(m).percentage); + + // Every member of a family shares one pill, and the two pills differ — + // so a wrap falls between the families rather than mid-strip. + expect(new Set(quality.map(groupOf)).size).toBe(1); + expect(new Set(efficiency.map(groupOf)).size).toBe(1); + expect(groupOf(quality[0])).not.toBe(groupOf(efficiency[0])); + }); + + it("keeps METRICS order within each group", () => { + render( {}} />); + const rendered = screen.getAllByRole("button").map(b => b.textContent); + const expected = [ + ...METRICS.filter(m => metricMeta(m).percentage), + ...METRICS.filter(m => !metricMeta(m).percentage) + ].map(metricShortLabel); + expect(rendered).toEqual(expected); + }); + + it("abbreviates the visible text but keeps the full accessible name", () => { + render( {}} />); + // Shortening is a fit concern, not a vocabulary change: a screen reader + // and every query below still address the metric by its real label. + const button = buttonFor("recoverableSafety"); + expect(button).toHaveTextContent("Rec. Safety"); + expect(button).toHaveAccessibleName("Recoverable Safety"); + }); + + it("marks the active metric and reports a click", () => { + const onChange = vi.fn(); + render(); + expect(buttonFor("latency")).toHaveAttribute("aria-pressed", "true"); + expect(buttonFor("composite")).toHaveAttribute("aria-pressed", "false"); + + fireEvent.click(buttonFor("tokens")); + expect(onChange).toHaveBeenCalledWith("tokens"); + }); + + it("disables metrics missing from `available` rather than hiding them", () => { + const onChange = vi.fn(); + render(); + + expect(buttonFor("pass5")).toBeDisabled(); + expect(buttonFor("latency")).toBeEnabled(); + fireEvent.click(buttonFor("pass5")); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("enables every metric when `available` is omitted or empty", () => { + render( {}} available={[]} />); + for (const m of METRICS) expect(buttonFor(m)).toBeEnabled(); + }); +}); diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index 0c2bedd8..2132c2b1 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -38,6 +38,20 @@ export const METRIC_LABELS = { tokens: "Tokens" }; +// Abbreviated labels for the metric toggle only, where eight buttons compete for +// the width of one table column. "Recoverable Safety" is ~2.5x the width of any +// other button, so it alone decides whether the group fits on one line. Headings, +// tooltips and the accessible name of the button all keep the full METRIC_LABELS +// text — this shortens the visible glyphs, not the vocabulary. +const METRIC_SHORT_LABELS = { + recoverableSafety: "Rec. Safety" +}; + +/** Toggle-button text for a metric, falling back to the full label. */ +export function metricShortLabel(metric) { + return METRIC_SHORT_LABELS[metric] ?? METRIC_LABELS[metric] ?? metric; +} + // The metric keys in display order — used by the metric toggles. Composite leads // as the default headline; pass@k follow, then the efficiency axes. export const METRICS = [ @@ -130,7 +144,7 @@ export const METRIC_DESCRIPTIONS = { "Pass@1: share of task attempts whose correctness clears the pass threshold (0.7).", pass5: "Pass@5: needs multi-iteration runs (not produced yet).", passMax: "Pass^5: needs multi-iteration runs (not produced yet).", - latency: "Latency: mean agent wall-clock seconds per task. Lower is better, so the bar is scaled against the slowest setup on screen.", + latency: "Latency: mean agent wall-clock seconds per task. Lower is better, so the bar is scaled against the fastest setup on screen — a full bar is the fastest, half a bar is twice as slow.", tokens: "Tokens: mean total tokens per task (the provider total when reported, else the sum of the captured buckets). Lower is better." }; From 0ed985c2a078403dd142c82ab967077a458d62bd Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:02:16 -0700 Subject: [PATCH 05/18] fix(site): name the selected metric in the leaderboard trend heading The section was hardcoded "Accuracy Performance Trend Over Time" with the caption "Comparing agent configuration success rates". Under Latency or Tokens the series is seconds or token counts, not a success rate, so both strings read as a falsehood. The aria-label directly below them was already metric-aware, so the visible text and the screen-reader text disagreed about what the chart showed. Both now read from METRIC_LABELS, matching how the detail page already titles the same chart. The sr-only table captions on both pages drop "Score" for the same reason. --- site/src/pages/Detail.jsx | 2 +- site/src/pages/Leaderboard.jsx | 6 +++--- site/src/pages/Leaderboard.test.jsx | 12 ++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 77b09747..51e71d44 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -237,7 +237,7 @@ export function Detail() { showLegend={false} fill ariaLabel={`${METRIC_LABELS[metric]} trend over time for this setup`} - caption={`Score trend for ${setupLabel(setup, models, harnesses)} (metric: ${metric})`} + caption={`${METRIC_LABELS[metric]} trend for ${setupLabel(setup, models, harnesses)}`} /> diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 1bd44b06..6772cb85 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -135,9 +135,9 @@ export function Leaderboard() { - Accuracy Performance Trend Over Time + {METRIC_LABELS[metric]} Trend Over Time -

Comparing agent configuration success rates across historical run iterations.

+

Comparing agent configuration {METRIC_LABELS[metric].toLowerCase()} across historical run iterations.

)} diff --git a/site/src/pages/Leaderboard.test.jsx b/site/src/pages/Leaderboard.test.jsx index f48f7a60..46232456 100644 --- a/site/src/pages/Leaderboard.test.jsx +++ b/site/src/pages/Leaderboard.test.jsx @@ -67,4 +67,16 @@ describe("Leaderboard", () => { fireEvent.click(pass5); expect(pass5).toHaveAttribute("aria-pressed", "true"); }); + + it("names the selected metric in the trend heading and caption", () => { + // The heading used to be hardcoded "Accuracy Performance Trend Over + // Time", which reads as a falsehood under an efficiency metric where + // the series is seconds or tokens rather than a success rate. + renderPage(); + expect(screen.getByRole("heading", { name: /Outcome Trend Over Time/i })).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Pass@5" })); + expect(screen.getByRole("heading", { name: /Pass@5 Trend Over Time/i })).toBeInTheDocument(); + expect(screen.queryByText(/success rates/i)).not.toBeInTheDocument(); + }); }); From 963c7bcf70e76c889ccd5cd1347fd590e244c412 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:10:37 -0700 Subject: [PATCH 06/18] fix(site): snap the efficiency y-axis to round endpoints The absolute branch of yAxisBounds returned its raw padded bounds, so the latency chart drew a "54.8s" endpoint label hard against the "54.0s" gridline, and tokens ran from 22.1k to 26.5k between round interior ticks. The percentage branch already snapped to tens for exactly this reason; the absolute branch just could not reuse the constant, since its range spans seconds to tens of thousands of tokens. Derive a round step per range (1, 2, 2.5 or 5 times a power of ten, the smallest keeping the axis under about eight gridlines) and floor/ceil the bounds to it. Latency now runs 44s to 56s, tokens 21k to 27k. Padding is unchanged, so the vertical framing of the series is the same. --- site/src/lib/accessors.js | 29 ++++++++++++++++++++++- site/src/lib/accessors.test.js | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/site/src/lib/accessors.js b/site/src/lib/accessors.js index dcf49a27..94217dd5 100644 --- a/site/src/lib/accessors.js +++ b/site/src/lib/accessors.js @@ -92,6 +92,25 @@ export function formatRunDate(t) { return new Date(t).toLocaleDateString("en-CA", { timeZone: "UTC", year: "numeric", month: "2-digit", day: "2-digit" }); } +// A round tick step for an absolute range: 1, 2, 2.5 or 5 times a power of ten, +// the smallest that keeps the axis under ~8 gridlines. The percentage branch can +// hardcode 10 because its range is always 0..100; an absolute range spans +// seconds to tens of thousands of tokens, so the step has to follow the data. +function niceStep(range) { + const magnitude = 10 ** Math.floor(Math.log10(range / 4)); + for (const m of [1, 2, 2.5, 5]) { + if (range / (magnitude * m) <= 8) return magnitude * m; + } + return magnitude * 10; +} + +// Round `value` to a multiple of `step` with `round` (Math.floor / Math.ceil). +// The trailing toFixed(6) drops float dust so a 0.1 step yields 24.3, not +// 24.299999999999997, which would print as a long decimal on the axis. +function snap(value, step, round) { + return Number((round(value / step) * step).toFixed(6)); +} + // Trend-chart y-axis [min, max] for the given setups + metric. Fits the plotted // scores instead of a fixed 60–100 window so low scorers aren't clipped off the // bottom: pad by 5, snap to tens (keeps the 10-step ticks clean), clamp to @@ -114,7 +133,15 @@ export function yAxisBounds(setupsList, metric) { // the range instead and let the axis follow the data. if (!metricMeta(metric).percentage) { const pad = Math.max((hi - lo) * 0.1, hi * 0.05, 1); - return { min: Math.max(0, lo - pad), max: hi + pad }; + // Snap to a round step for the same reason the percentage branch snaps + // to tens: Chart.js labels the endpoints as well as the interior ticks, + // so a raw padded bound prints a stray "54.8s" hard against the "54.0s" + // gridline above it. + const step = niceStep(hi + pad - Math.max(0, lo - pad)); + return { + min: Math.max(0, snap(lo - pad, step, Math.floor)), + max: snap(hi + pad, step, Math.ceil) + }; } const min = Math.max(0, Math.floor((lo - 5) / 10) * 10); const max = Math.min(100, Math.ceil((hi + 5) / 10) * 10); diff --git a/site/src/lib/accessors.test.js b/site/src/lib/accessors.test.js index 8431989c..2b3330be 100644 --- a/site/src/lib/accessors.test.js +++ b/site/src/lib/accessors.test.js @@ -128,4 +128,46 @@ describe("yAxisBounds", () => { const allNull = makeSetup({ history: [{ t: "2026-01-15T00:00:00Z", scores: { pass1: null } }] }); expect(yAxisBounds([allNull], "pass1")).toEqual({ min: 0, max: 100 }); }); + + describe("absolute metrics", () => { + const withHistory = (metric, values) => makeSetup({ + history: values.map((v, i) => ({ + t: `2026-0${i + 1}-15T00:00:00Z`, + scores: { [metric]: v } + })) + }); + + it("follows the data instead of clamping to 100", () => { + // Token means run to five figures; a [0, 100] clamp would push every + // series off the top of the chart. + const b = yAxisBounds([withHistory("tokens", [23200, 24100, 25200])], "tokens"); + expect(b.min).toBeGreaterThan(100); + expect(b.min).toBeLessThan(23200); + expect(b.max).toBeGreaterThan(25200); + }); + + it("snaps the endpoints to a round step so no stray label crowds a gridline", () => { + // Raw padding gave [45.3, 54.7], printing "54.8s" against "54.0s". + expect(yAxisBounds([withHistory("latency", [47.9, 50.1, 52.2])], "latency")) + .toEqual({ min: 44, max: 56 }); + expect(yAxisBounds([withHistory("tokens", [23200, 24100, 25200])], "tokens")) + .toEqual({ min: 21000, max: 27000 }); + }); + + it("scales the step to the magnitude rather than assuming tens", () => { + // Sub-minute latencies need a fractional step; tokens need thousands. + const small = yAxisBounds([withHistory("latency", [2.1, 2.4])], "latency"); + expect(small).toEqual({ min: 1, max: 3.5 }); + }); + + it("keeps a non-zero height for a single data point", () => { + const b = yAxisBounds([withHistory("latency", [50])], "latency"); + expect(b.max).toBeGreaterThan(b.min); + }); + + it("never drops the axis below zero", () => { + const b = yAxisBounds([withHistory("latency", [0.5])], "latency"); + expect(b.min).toBeGreaterThanOrEqual(0); + }); + }); }); From 4b7f7af250f5e7bc142222d5bd27bba0e91b935f Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:29:07 -0700 Subject: [PATCH 07/18] fix(site): stop calling the efficiency metrics scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selecting an efficiency metric left several bits of the UI describing it as a score, and one card printing the same number twice. The detail page's Avg Speed card predated latency being selectable, so it was written on the assumption it could never collide with Average. It can now: under Latency both cards are the mean latency, side by side. It reports whichever efficiency axis the toggle is NOT showing instead, picked off metricMeta() rather than hardcoded, so it stays a number the reader doesn't already have. The task table header said "Score (Tokens)" and the leaderboard column said "SCORE" above a token count. Neither latency nor tokens is a score; the table now heads the column with the metric itself, which is what the parenthetical was compensating for, and the leaderboard says "METRIC" rather than echoing the highlighted toggle button an inch below it. The sort arrow reported the internal flag rather than the data. "desc" means best-first, and best-first under a lower-is-better metric is ascending values, so the column ran 22.3k to 28.0k beneath a ▼. It now derives the glyph from the direction the values actually run. --- site/src/pages/Detail.jsx | 51 ++++++++++++++++++++-------- site/src/pages/Detail.test.jsx | 62 +++++++++++++++++++++++++++++++--- site/src/pages/Leaderboard.jsx | 8 +++-- 3 files changed, 100 insertions(+), 21 deletions(-) diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 51e71d44..8a9e0834 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -6,7 +6,7 @@ import { useEffect, useMemo, useState } from "react"; import { useParams, useSearchParams, Link } from "react-router-dom"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; -import { METRIC_LABELS, availableMetrics, formatMetric, metricBarFraction, isLowerBetter } from "../lib/vocab.js"; +import { METRICS, METRIC_LABELS, availableMetrics, formatMetric, metricBarFraction, isLowerBetter, metricMeta } from "../lib/vocab.js"; import { SetupIdentity } from "../components/SetupIdentity.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; import { TrendChart } from "../components/TrendChart.jsx"; @@ -65,9 +65,17 @@ function TaskTable({ setup, metric }) { : { key, dir: key === "name" ? "asc" : "desc" }); } - const Arrow = ({ k }) => sort.key === k - ? {sort.dir === "asc" ? "▲" : "▼"} - : ↕; + // The arrow reports which way the VALUES run, not the internal sort flag. + // "desc" means best-first, and best-first under latency/tokens is ascending + // numbers — so the glyph has to invert or the column reads 22.3k → 28.0k + // under a ▼. + const Arrow = ({ k }) => { + if (sort.key !== k) return ↕; + const ascending = k === "name" + ? sort.dir === "asc" + : (sort.dir === "asc") !== isLowerBetter(metric); + return {ascending ? "▲" : "▼"}; + }; return (
@@ -76,7 +84,10 @@ function TaskTable({ setup, metric }) { sortBy("name")}>Task - sortBy("score")}>Score ({METRIC_LABELS[metric]}) + {/* The metric names the column on its own: "Score (Tokens)" + calls a token count a score, and the parenthetical was + only ever there because "Score" couldn't carry which one. */} + sortBy("score")}>{METRIC_LABELS[metric]} @@ -164,10 +175,20 @@ export function Detail() { const med = vals.length ? median(vals) : null; const pct = v => formatMetric(metric, v); - // Speed is reported independently of the selected metric, so this card stays - // meaningful (and doesn't duplicate "Average") while the toggle moves. - const speeds = setup.tasks.map(t => t.scores.latency).filter(v => v != null); - const avgSpeed = speeds.length ? speeds.reduce((a, b) => a + b, 0) / speeds.length : null; + // The fifth card reports an efficiency axis the toggle is NOT showing, so it + // adds a number instead of repeating one. This card used to be a hardcoded + // "Avg Speed", which was independent of the metric back when latency wasn't + // selectable; now that it is, selecting Latency makes "Average" the mean + // latency and the two cards print the same figure side by side. Taking the + // first efficiency metric other than the selected one gives Tokens under + // Latency and Latency everywhere else, without naming either key here. + const companion = METRICS.find(m => !metricMeta(m).percentage && m !== metric); + const companionVals = companion + ? setup.tasks.map(t => t.scores[companion]).filter(v => v != null) + : []; + const companionAvg = companionVals.length + ? companionVals.reduce((a, b) => a + b, 0) / companionVals.length + : null; return (
@@ -207,11 +228,13 @@ export function Detail() { : "none" } /> - + {companion ? ( + + ) : null}
{/* Task breakdown */} diff --git a/site/src/pages/Detail.test.jsx b/site/src/pages/Detail.test.jsx index c18fe861..c9e187f7 100644 --- a/site/src/pages/Detail.test.jsx +++ b/site/src/pages/Detail.test.jsx @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; -import { render, screen, fireEvent, within } from "@testing-library/react"; +import { render, screen, fireEvent, within, cleanup } from "@testing-library/react"; import { MemoryRouter, Routes, Route } from "react-router-dom"; // Stub the chart (jsdom has no canvas); the context is mocked per-test below. @@ -26,10 +26,13 @@ function makeBenchmark(overrides = {}) { { id: SETUP_ID, order: 0, model: "alpha-pro", harness: "gemini-cli", augmentation: [], color: "#3b82f6", + // Latency/tokens are chosen so their means are round (50.0s, + // 20.0k) and their best is the SMALLEST, which is the opposite + // end from the percentage metrics above. tasks: [ - { folder: "a", name: "Apple", scores: { composite: 60, pass1: 60, pass5: 65, passMax: 70 } }, - { folder: "b", name: "Banana", scores: { composite: 90, pass1: 90, pass5: 95, passMax: 100 } }, - { folder: "c", name: "Cherry", scores: { composite: 80, pass1: 80, pass5: 85, passMax: 90 } } + { folder: "a", name: "Apple", scores: { composite: 60, pass1: 60, pass5: 65, passMax: 70, latency: 40, tokens: 10000 } }, + { folder: "b", name: "Banana", scores: { composite: 90, pass1: 90, pass5: 95, passMax: 100, latency: 50, tokens: 20000 } }, + { folder: "c", name: "Cherry", scores: { composite: 80, pass1: 80, pass5: 85, passMax: 90, latency: 60, tokens: 30000 } } ], history: [ { t: "2026-01-15T00:00:00Z", scores: { composite: 70, pass1: 70, pass5: 75, passMax: 80 } }, @@ -155,6 +158,39 @@ describe("Detail", () => { expect(within(card("Catastrophic")).getByText("outcomes zeroed")).toBeInTheDocument(); }); + it("reports the efficiency axis the toggle is not showing", () => { + const card = label => screen.getByText(label).closest("div"); + + // Under a quality metric the spare card is latency, as it always was. + renderAt(`/setup/${SETUP_ID}`); + expect(within(card("Avg Latency")).getByText("50.0s")).toBeInTheDocument(); + + // Under Latency it has to switch, or it prints the same figure as + // "Average" in the card next to it. + cleanup(); + renderAt(`/setup/${SETUP_ID}?metric=latency`); + expect(within(card("Average")).getByText("50.0s")).toBeInTheDocument(); + expect(screen.queryByText("Avg Latency")).not.toBeInTheDocument(); + expect(within(card("Avg Tokens")).getByText("20.0k")).toBeInTheDocument(); + }); + + it("orients the stat cards by the metric's direction", () => { + // Best is the FASTEST task under latency — the minimum, where every + // percentage metric takes the maximum. + renderAt(`/setup/${SETUP_ID}?metric=latency`); + const card = label => screen.getByText(label).closest("div"); + expect(within(card("Best Task")).getByText("40.0s")).toBeInTheDocument(); + expect(within(card("Median")).getByText("50.0s")).toBeInTheDocument(); + }); + + it("heads the task column with the metric rather than calling it a score", () => { + // A token count is telemetry, not a score; the old "Score (Tokens)" + // header said otherwise. + renderAt(`/setup/${SETUP_ID}?metric=tokens`); + expect(screen.getByRole("columnheader", { name: /Tokens/ })).toBeInTheDocument(); + expect(screen.queryByRole("columnheader", { name: /Score/ })).not.toBeInTheDocument(); + }); + it("honors the ?metric= query param", () => { renderAt(`/setup/${SETUP_ID}?metric=pass5`); expect(screen.getByRole("button", { name: "Pass@5" })).toHaveAttribute("aria-pressed", "true"); @@ -181,10 +217,26 @@ describe("Detail", () => { it("toggles score sort direction on repeated header clicks", () => { renderAt(`/setup/${SETUP_ID}`); - fireEvent.click(screen.getByRole("columnheader", { name: /Score/ })); + fireEvent.click(screen.getByRole("columnheader", { name: /Outcome/ })); expect(taskOrder()).toEqual(["Apple", "Cherry", "Banana"]); // now ascending }); + it("points the sort arrow the way the values actually run", () => { + // Both default to best-first, but "best" is the largest percentage and + // the smallest token count, so the same sort state has to draw opposite + // arrows — otherwise a column ascending 10.0k → 30.0k sits under a ▼. + const header = name => screen.getByRole("columnheader", { name }); + + renderAt(`/setup/${SETUP_ID}`); + expect(header(/Outcome/)).toHaveTextContent("▼"); // 90 → 60, descending + + cleanup(); + renderAt(`/setup/${SETUP_ID}?metric=tokens`); + expect(header(/Tokens/)).toHaveTextContent("▲"); // 10.0k → 30.0k, ascending + fireEvent.click(header(/Tokens/)); + expect(header(/Tokens/)).toHaveTextContent("▼"); + }); + it("shows a NotFound state for an unknown setup id", () => { renderAt("/setup/does-not-exist"); expect(screen.getByText(/No setup found/i)).toBeInTheDocument(); diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 6772cb85..24f51a20 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -102,8 +102,12 @@ export function Leaderboard() {
- SCORE -
+ {/* "METRIC", not "SCORE": the toggle below can select + latency or tokens, and neither is a score. Naming the + selected metric here instead would just echo the + highlighted button an inch beneath it. */} + METRIC +
From 4394f7dbbece201af11552945913f66867cacfbb Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:34:59 -0700 Subject: [PATCH 08/18] fix(site): stop the detail hero truncating the model name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hero's metric block was shrink-0, so it always claimed its full max-content width and the identity beside it, which is min-w-0 with a truncate, silently absorbed the shortfall. Two more metrics made that block two buttons wider, enough to clip the longest model name to "Gamma Co…" at full window width. The column can shrink now, so the toggle's own wrapping takes the pressure instead. The headline figure keeps shrink-0 and nowrap, so the number and its unit never break across lines. --- site/src/pages/Detail.jsx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 8a9e0834..c4ce5dab 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -200,8 +200,19 @@ export function Detail() {
-
-
+ {/* At eight metrics the toggle is two buttons wider than it + was, which is enough to clip the longest model name to + "Gamma Co…" — the one string on the page that must not be + abbreviated. Merely making this column shrinkable isn't + enough: flex splits the shortfall in proportion to content + width, so the identity still gives up pixels it can only + pay for by truncating, while the toggle beside it could + have wrapped for free. The lopsided shrink factor says + which item yields — this one, all the way down to the + headline figure (shrink-0, so it never breaks), and only + then does the name start to shorten. */} +
+
{formatMetric(metric, score)} {METRIC_LABELS[metric]}
From c2d330403b001d1ec33779a11454c43b75235d23 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Fri, 14 Aug 2026 15:22:46 -0700 Subject: [PATCH 09/18] fix(site): default the emulator project to the one the app reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seeder and ingest defaulted to project 'devops-bench-demo' under the emulator, but the browser always reads VITE_FIREBASE_PROJECT_ID ('devops-bench-shared') with no emulator override. The emulator namespaces data per project id, so out of the box the writers and the reader used different namespaces and the dashboard came up silently empty — no error, just no rows. Default both writers to the same project the app reads, and correct the README, which was passing the mismatched id explicitly and so reproduced the bug. --- site/README.md | 15 +++++++++++++-- site/ingest/firestore.mjs | 2 +- site/seed/seed.mjs | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/site/README.md b/site/README.md index 779a18a8..6dc66a89 100644 --- a/site/README.md +++ b/site/README.md @@ -293,15 +293,26 @@ Needs **Java** for the Firestore emulator ```bash # 1. start the emulator (Firestore :8080, UI :4000) -npx -y firebase-tools emulators:start --only firestore --project devops-bench-demo +npx -y firebase-tools emulators:start --only firestore --project devops-bench-shared # 2. in another shell: seed the emulator (writes to the leaderboard-test DB) -cd seed && FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-demo npm run seed && cd .. +cd seed && FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-shared npm run seed && cd .. # 3. run the app — firebase.js auto-connects to the emulator on localhost npm run dev # http://localhost:5173 ``` +> **The project id must match `VITE_FIREBASE_PROJECT_ID` in `.env`** (that is +> `devops-bench-shared`, the same project named at the top of this section). The +> emulator keeps each project id in a **separate namespace**, so seeding one id +> while the app reads another leaves the dashboard **silently empty** — no error, +> just no rows. If that happens, check the id in all three places above, or query +> the emulator directly to see which namespace the data landed in: +> +> ```bash +> curl -s "http://127.0.0.1:8080/v1/projects/devops-bench-shared/databases/leaderboard-test/documents/setups" | head +> ``` + ### B) Staging (real cloud DB, fabricated data) > ✅ `leaderboard-test` is **already created and seeded** — just run the dev server: diff --git a/site/ingest/firestore.mjs b/site/ingest/firestore.mjs index 5bba2aeb..4a69e9e6 100644 --- a/site/ingest/firestore.mjs +++ b/site/ingest/firestore.mjs @@ -35,7 +35,7 @@ export function openDb() { const projectId = process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT || - (emulator ? "devops-bench-demo" : "devops-bench-shared"); + "devops-bench-shared"; const databaseId = process.env.FIRESTORE_DATABASE_ID || "leaderboard-test"; if (databaseId === PROD_DATABASE_ID && process.env.ALLOW_PROD_INGEST !== "true") { diff --git a/site/seed/seed.mjs b/site/seed/seed.mjs index b5a49bb6..5ed15f39 100644 --- a/site/seed/seed.mjs +++ b/site/seed/seed.mjs @@ -32,7 +32,7 @@ const EMULATOR = !!process.env.FIRESTORE_EMULATOR_HOST; const PROJECT_ID = process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT || - (EMULATOR ? "devops-bench-demo" : "devops-bench-shared"); + "devops-bench-shared"; // Named Firestore database (must match the client's VITE_FIRESTORE_DATABASE_ID). // Defaults to the test DB; never silently defaults to prod. const DATABASE_ID = process.env.FIRESTORE_DATABASE_ID || "leaderboard-test"; From dbb2011686cbc76444a01dcc3bd16f34fcd1d20d Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Fri, 14 Aug 2026 15:27:53 -0700 Subject: [PATCH 10/18] docs(site): purge the remaining devops-bench-demo references The ingest README and two usage comments still passed the mismatched project id, which reproduces the same silently-empty-dashboard failure the previous commit fixed in the defaults. --- site/ingest/README.md | 6 +++--- site/ingest/derive.mjs | 2 +- site/seed/seed.mjs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/site/ingest/README.md b/site/ingest/README.md index 0fb397f7..5bf693fd 100644 --- a/site/ingest/README.md +++ b/site/ingest/README.md @@ -88,11 +88,11 @@ brew install openjdk export PATH="/opt/homebrew/opt/openjdk/bin:$PATH" # 2. Start the emulator (from site/, leave it running in another terminal): -cd .. && npx -y firebase-tools emulators:start --only firestore --project devops-bench-demo +cd .. && npx -y firebase-tools emulators:start --only firestore --project devops-bench-shared # Firestore on :8080, emulator UI on :4000 # 3. Ingest against it (from site/ingest/): -FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-demo \ +FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-shared \ node ingest.mjs fixtures/ ``` @@ -131,7 +131,7 @@ or `catalog.mjs` presentation? Re-score every setup from the **existing** raw rows — no re-upload (emulator from Option A still running, or ADC for real Firestore): ```bash -FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-demo \ +FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-shared \ node derive.mjs ``` diff --git a/site/ingest/derive.mjs b/site/ingest/derive.mjs index 4adbc179..ec12bef1 100644 --- a/site/ingest/derive.mjs +++ b/site/ingest/derive.mjs @@ -213,7 +213,7 @@ export function derive(rows, opts = {}) { // re-score every setup from the existing raw rows WITHOUT re-uploading. The // normal path (ingest.mjs) runs derive automatically after each upload. // -// FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-demo \ +// FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-shared \ // node derive.mjs // GCLOUD_PROJECT=devops-bench-shared FIRESTORE_DATABASE_ID=leaderboard-test \ // node derive.mjs diff --git a/site/seed/seed.mjs b/site/seed/seed.mjs index 5ed15f39..1faa273d 100644 --- a/site/seed/seed.mjs +++ b/site/seed/seed.mjs @@ -11,7 +11,7 @@ // Two targets, selected by whether FIRESTORE_EMULATOR_HOST is set: // // EMULATOR (default, no credentials needed): -// FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-demo \ +// FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 GCLOUD_PROJECT=devops-bench-shared \ // node seed.mjs # → DB leaderboard-test // // REAL Firestore (the shared TEST database — uses Application Default Creds; From 936ff25df5b5b150611f44405132d84bf8d3161e Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 14:45:59 -0700 Subject: [PATCH 11/18] fix(site): point .firebaserc at the project the app reads Addresses review feedback on #241: the emulator falls back to .firebaserc when --project is omitted, so leaving "default" as devops-bench-demo reproduced the exact bug that PR set out to fix, just one command shorter. The UI on :4000 came up with no docs even though the seed reported success. Also names all four places the project id appears, rather than saying 'all three places' while .firebaserc quietly made it a fourth. --- site/.firebaserc | 2 +- site/README.md | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/site/.firebaserc b/site/.firebaserc index 1f8f65a9..adb750c7 100644 --- a/site/.firebaserc +++ b/site/.firebaserc @@ -1,5 +1,5 @@ { "projects": { - "default": "devops-bench-demo" + "default": "devops-bench-shared" } } diff --git a/site/README.md b/site/README.md index 6dc66a89..7508cd37 100644 --- a/site/README.md +++ b/site/README.md @@ -306,8 +306,17 @@ npm run dev # http://localhost:5173 > `devops-bench-shared`, the same project named at the top of this section). The > emulator keeps each project id in a **separate namespace**, so seeding one id > while the app reads another leaves the dashboard **silently empty** — no error, -> just no rows. If that happens, check the id in all three places above, or query -> the emulator directly to see which namespace the data landed in: +> just no rows. Four places name the project and all four have to agree: +> +> - the `--project` flag in step 1 +> - `GCLOUD_PROJECT` in step 2 +> - `VITE_FIREBASE_PROJECT_ID` in `.env`, which is what the browser reads +> - `"default"` in `.firebaserc`, which is where the emulator falls back if you +> drop `--project` — get this one wrong and the emulator UI on :4000 shows no +> docs even though the seed reported success +> +> If that happens, query the emulator directly to see which namespace the data +> actually landed in: > > ```bash > curl -s "http://127.0.0.1:8080/v1/projects/devops-bench-shared/databases/leaderboard-test/documents/setups" | head From 1bbc6c0e9d8442a2b06970898a762f25fca82351 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Tue, 25 Aug 2026 15:34:08 -0700 Subject: [PATCH 12/18] fix(site): explain a disabled metric by why it is actually missing Every disabled toggle button shared one hardcoded tooltip, "Available once multi-iteration runs land". That was accurate while pass5 and passMax were the only metrics that could be absent. This PR added latency and tokens to METRICS, and availableMetrics() filters over all of them, so a harness that reports no timings now greys out Latency and explains it by multi-iteration runs, which have nothing to do with it. The reason is per-metric: pass@k waits on repeated iterations, the efficiency axes on harness telemetry. --- site/src/components/MetricToggle.jsx | 11 ++++++----- site/src/components/MetricToggle.test.jsx | 17 ++++++++++++++++- site/src/lib/vocab.js | 17 +++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/site/src/components/MetricToggle.jsx b/site/src/components/MetricToggle.jsx index 80c5a807..68d12eb4 100644 --- a/site/src/components/MetricToggle.jsx +++ b/site/src/components/MetricToggle.jsx @@ -1,8 +1,9 @@ // Metric segmented control, shared by the leaderboard header and the detail // hero. `available` (optional) marks which metrics have data — the others render -// DISABLED rather than hidden, so the UI advertises that pass5/passMax will -// return once the harness produces multi-iteration runs. When omitted (or -// empty), every metric is enabled (back-compat). +// DISABLED rather than hidden, so the UI advertises the axis exists and says why +// it's empty. The reason is per-metric (see metricUnavailableReason): pass@k is +// waiting on multi-iteration runs, latency/tokens on harness telemetry. When +// `available` is omitted (or empty), every metric is enabled (back-compat). // // The metrics are split into two pills, quality and efficiency, rather than one // long strip: eight buttons overflow the leaderboard's score column, and the @@ -16,7 +17,7 @@ // again. Letting buttons wrap inside a pill makes overflow impossible at any // width; the group split only decides where the break lands first. -import { METRICS, METRIC_LABELS, metricDescription, metricMeta, metricShortLabel } from "../lib/vocab.js"; +import { METRICS, METRIC_LABELS, metricDescription, metricMeta, metricShortLabel, metricUnavailableReason } from "../lib/vocab.js"; // Quality metrics first, then efficiency, each preserving METRICS order. Empty // groups are dropped so a vocab with only one family renders a single pill. @@ -54,7 +55,7 @@ export function MetricToggle({ value, onChange, available }) { // The visible text may be abbreviated to fit; the // accessible name stays the full metric label. aria-label={METRIC_LABELS[m]} - title={enabled ? metricDescription(m) : "Available once multi-iteration runs land"} + title={enabled ? metricDescription(m) : metricUnavailableReason(m)} className={`px-2 py-1 font-medium rounded-md whitespace-nowrap transition-all duration-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-indigo-500 focus-visible:ring-offset-2 dark:focus-visible:ring-offset-slate-900 disabled:cursor-not-allowed ${cls}`} > {metricShortLabel(m)} diff --git a/site/src/components/MetricToggle.test.jsx b/site/src/components/MetricToggle.test.jsx index 6090d1ba..30334750 100644 --- a/site/src/components/MetricToggle.test.jsx +++ b/site/src/components/MetricToggle.test.jsx @@ -2,7 +2,7 @@ import { describe, it, expect, vi } from "vitest"; import { render, screen, fireEvent } from "@testing-library/react"; import { MetricToggle } from "./MetricToggle.jsx"; -import { METRICS, METRIC_LABELS, metricMeta, metricShortLabel } from "../lib/vocab.js"; +import { METRICS, METRIC_LABELS, metricDescription, metricMeta, metricShortLabel } from "../lib/vocab.js"; const buttonFor = metric => screen.getByRole("button", { name: METRIC_LABELS[metric] }); @@ -74,4 +74,19 @@ describe("MetricToggle", () => { render( {}} available={[]} />); for (const m of METRICS) expect(buttonFor(m)).toBeEnabled(); }); + + it("explains a disabled metric by the reason it is actually missing", () => { + // One hardcoded sentence used to cover every disabled button, so a + // harness that reports no timings put "Available once multi-iteration + // runs land" under Latency, which has nothing to do with iterations. + render( {}} available={["composite"]} />); + expect(buttonFor("pass5")).toHaveAttribute("title", "Available once multi-iteration runs land"); + expect(buttonFor("latency")).toHaveAttribute("title", "Not reported by these runs"); + expect(buttonFor("tokens")).toHaveAttribute("title", "Not reported by these runs"); + }); + + it("describes an enabled metric instead of explaining its absence", () => { + render( {}} available={["composite", "latency"]} />); + expect(buttonFor("latency")).toHaveAttribute("title", metricDescription("latency")); + }); }); diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index 2132c2b1..9a54dfd9 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -153,6 +153,23 @@ export function metricDescription(metric) { return METRIC_DESCRIPTIONS[metric] ?? METRIC_LABELS[metric] ?? metric; } +// Why a metric has no data, shown on its disabled toggle button. A metric can +// go missing for unrelated reasons — pass@k needs the same task run repeatedly, +// while an efficiency axis just needs the harness to report telemetry — so one +// hardcoded sentence can't cover both. It used to, which put "Available once +// multi-iteration runs land" under a greyed-out Latency button. +const METRIC_UNAVAILABLE_REASONS = { + pass5: "Available once multi-iteration runs land", + passMax: "Available once multi-iteration runs land", + latency: "Not reported by these runs", + tokens: "Not reported by these runs" +}; + +/** Tooltip for a metric with no data in the current dataset. */ +export function metricUnavailableReason(metric) { + return METRIC_UNAVAILABLE_REASONS[metric] ?? "Not reported by these runs"; +} + // Which metrics actually have any non-null value across the given setups. Used // by the metric toggle so pass@k buttons stay hidden until the harness // produces the multi-iteration runs that populate them. From e8446a2c051f610b3f974b03a5adf823e7684b4e Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Fri, 28 Aug 2026 14:28:25 -0700 Subject: [PATCH 13/18] fix(site): treat a zeroed token total as unmeasured, not as zero tokens Three fixes from review, all variants of the 0-vs-null sentinel problem already handled for latency. sumTokens returned a real 0 for an all-zero usage record. antigravity's parser emits {input: 0, output: 0, total: 0, cached: 0} for an empty session log and normalize.py coerces those through as ints, so the setup derived tokens: 0 and ranked first with a full bar. Non-positive totals now fall through to the buckets, and a non-positive bucket sum is null. The blast radius was wider than the offending row: `best` is shared by the whole column and metricBarFraction bails on `best <= 0`, so a single 0 emptied every bar under Tokens. Best-picking moves into bestValue() in vocab.js, which drops non-positive readings for lower-is-better metrics and keeps a genuine 0 for percentage ones. Leaderboard and Detail now share it instead of each rolling their own min/max. Also corrects the metricBest comment, which claimed the value was null for percentage metrics when it is computed for all of them and merely ignored downstream. 167 tests, up from 157. --- site/seed/mock-data.mjs | 14 ++++++++++++-- site/seed/mock-data.test.mjs | 32 ++++++++++++++++++++++++++++++- site/src/lib/vocab.js | 20 +++++++++++++++++++ site/src/lib/vocab.test.js | 35 +++++++++++++++++++++++++++++++++- site/src/pages/Detail.jsx | 17 ++++++++--------- site/src/pages/Leaderboard.jsx | 17 +++++++++-------- 6 files changed, 114 insertions(+), 21 deletions(-) diff --git a/site/seed/mock-data.mjs b/site/seed/mock-data.mjs index 0a6aac7d..a81510a8 100644 --- a/site/seed/mock-data.mjs +++ b/site/seed/mock-data.mjs @@ -323,8 +323,16 @@ export function latencyOf(row) { // the canonical buckets in normalize.py), so leaving it out would undercount // every reasoning model. Null when no usage was captured at all, so "not // measured" stays distinct from a genuine zero. +// +// A non-positive result is the same unmeasured sentinel `latencyOf` handles: a +// harness that produced no session log normalizes to zeros rather than nulls +// (antigravity's parser returns `{input: 0, output: 0, total: 0, cached: 0}` +// for an empty log, and normalize.py coerces those through as ints), and no +// real run costs 0 tokens. Reporting the 0 would rank that setup FIRST on a +// lower-is-better metric. The total is checked separately from the buckets so +// a zeroed total still falls through to buckets that were captured. export function sumTokens(row) { - if (Number.isFinite(row.totalTokens)) return row.totalTokens; + if (Number.isFinite(row.totalTokens) && row.totalTokens > 0) return row.totalTokens; const parts = [ row.inputTokens, row.outputTokens, @@ -332,7 +340,9 @@ export function sumTokens(row) { row.reasoningTokens, row.cacheWriteTokens ].filter(v => Number.isFinite(v)); - return parts.length ? parts.reduce((a, b) => a + b, 0) : null; + if (!parts.length) return null; + const total = parts.reduce((a, b) => a + b, 0); + return total > 0 ? total : null; } // The {latency, tokens} slice of Scores for one group of iteration rows. diff --git a/site/seed/mock-data.test.mjs b/site/seed/mock-data.test.mjs index a6e9791a..4fc05451 100644 --- a/site/seed/mock-data.test.mjs +++ b/site/seed/mock-data.test.mjs @@ -1,5 +1,35 @@ import { describe, it, expect } from "vitest"; -import { generateRaw, derive, passAtK, PASS_THRESHOLD } from "./mock-data.mjs"; +import { generateRaw, derive, passAtK, PASS_THRESHOLD, sumTokens, latencyOf } from "./mock-data.mjs"; + +describe("sumTokens", () => { + it("prefers the producer's own total over the buckets", () => { + expect(sumTokens({ totalTokens: 993225, inputTokens: 135329, outputTokens: 9732 })).toBe(993225); + }); + + it("sums the captured buckets when no total is reported", () => { + // reasoningTokens is a sibling of output, not a subset, so it is added. + expect(sumTokens({ inputTokens: 100, outputTokens: 20, reasoningTokens: 5 })).toBe(125); + }); + + it("is null when no usage was captured at all", () => { + expect(sumTokens({ inputTokens: null, outputTokens: null })).toBeNull(); + expect(sumTokens({})).toBeNull(); + }); + + // Regression: antigravity's parser returns {input: 0, output: 0, total: 0, + // cached: 0} for an empty session log (parsing.py) and normalize.py coerces + // those through as ints rather than nulls. Reporting a real 0 would rank + // that setup FIRST on a lower-is-better metric with a full bar — the same + // failure latencyOf() guards against. + it("treats an all-zero usage record as unmeasured, not as zero tokens", () => { + expect(sumTokens({ inputTokens: 0, outputTokens: 0, cachedTokens: 0, totalTokens: 0 })).toBeNull(); + expect(latencyOf({ latencySec: 0 })).toBeNull(); + }); + + it("falls through to the buckets when only the total is zeroed", () => { + expect(sumTokens({ totalTokens: 0, inputTokens: 400, outputTokens: 100 })).toBe(500); + }); +}); describe("passAtK", () => { it("is 0 when there are no passes", () => { diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index 9a54dfd9..dee0f435 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -131,6 +131,26 @@ export function metricBarFraction(metric, value, best) { return Math.max(0, Math.min(1, best / value)); } +/** + * The best value among `values` for `metric`, or null when none qualifies. + * + * "Best" follows the metric's direction: the smallest for a lower-is-better + * absolute metric, the largest otherwise. Non-positive readings are dropped + * from the lower-is-better case because they are the unmeasured sentinel + * (see `latencyOf` / `sumTokens` in seed/mock-data.mjs). That matters beyond + * the offending row: `best` is shared by the whole column, so one 0 would trip + * the `best <= 0` guard above and flatten EVERY bar, not just its own. + * + * Shared by the leaderboard (across visible setups) and the detail task table + * (across one setup's tasks) so the two cannot drift apart. + */ +export function bestValue(metric, values) { + const vals = values.filter(v => v != null && Number.isFinite(v)); + if (!isLowerBetter(metric)) return vals.length ? Math.max(...vals) : null; + const positive = vals.filter(v => v > 0); + return positive.length ? Math.min(...positive) : null; +} + // One-line explanation per metric — the single source of truth for the score // tooltip (contextual to the selected metric) and each toggle button's hover. export const METRIC_DESCRIPTIONS = { diff --git a/site/src/lib/vocab.test.js b/site/src/lib/vocab.test.js index 275d66ea..e8c5c3bd 100644 --- a/site/src/lib/vocab.test.js +++ b/site/src/lib/vocab.test.js @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { formatMetric, isLowerBetter, metricBarFraction, metricMeta } from "./vocab.js"; +import { formatMetric, isLowerBetter, metricBarFraction, metricMeta, bestValue } from "./vocab.js"; describe("metric presentation rules", () => { it("treats quality metrics as higher-is-better percentages", () => { @@ -75,3 +75,36 @@ describe("metricBarFraction", () => { expect(metricBarFraction("latency", 10, 0)).toBe(0); }); }); + +describe("bestValue", () => { + it("takes the smallest for a lower-is-better metric and the largest otherwise", () => { + expect(bestValue("latency", [30, 10, 20])).toBe(10); + expect(bestValue("tokens", [300, 100, 200])).toBe(100); + expect(bestValue("composite", [30, 10, 20])).toBe(30); + }); + + it("skips nulls and non-finite entries", () => { + expect(bestValue("latency", [null, 30, undefined, NaN, 20])).toBe(20); + expect(bestValue("latency", [null, undefined])).toBeNull(); + expect(bestValue("latency", [])).toBeNull(); + }); + + // Regression: an unmeasured run normalizes to 0 rather than null, and the + // min would then be 0. metricBarFraction bails on `best <= 0`, so that one + // row would empty EVERY bar in the column instead of only its own. + it("ignores the 0 sentinel so one unmeasured row cannot flatten the column", () => { + expect(bestValue("tokens", [0, 5000, 20000])).toBe(5000); + expect(bestValue("latency", [0, 42])).toBe(42); + expect(metricBarFraction("tokens", 20000, bestValue("tokens", [0, 5000, 20000]))).toBeCloseTo(0.25); + }); + + it("is null when every lower-is-better reading is the sentinel", () => { + expect(bestValue("tokens", [0, 0])).toBeNull(); + }); + + // A percentage metric legitimately bottoms out at 0 (a 0% pass rate), so the + // non-positive filter must not apply on that side. + it("keeps a genuine 0 for a higher-is-better metric", () => { + expect(bestValue("composite", [0, 0])).toBe(0); + }); +}); diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index c4ce5dab..55847486 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -6,7 +6,7 @@ import { useEffect, useMemo, useState } from "react"; import { useParams, useSearchParams, Link } from "react-router-dom"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; -import { METRICS, METRIC_LABELS, availableMetrics, formatMetric, metricBarFraction, isLowerBetter, metricMeta } from "../lib/vocab.js"; +import { METRICS, METRIC_LABELS, availableMetrics, formatMetric, metricBarFraction, isLowerBetter, metricMeta, bestValue } from "../lib/vocab.js"; import { SetupIdentity } from "../components/SetupIdentity.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; import { TrendChart } from "../components/TrendChart.jsx"; @@ -50,14 +50,13 @@ function TaskTable({ setup, metric }) { }); }, [setup, metric, sort]); - // Best value across this setup's tasks — the smallest for a lower-is-better - // metric — so an absolute metric's bar has a scale (percentage metrics - // ignore it). - const taskBest = useMemo(() => { - const vals = setup.tasks.map(t => t.scores[metric]).filter(v => v != null); - if (!vals.length) return null; - return isLowerBetter(metric) ? Math.min(...vals) : Math.max(...vals); - }, [setup, metric]); + // Best value across this setup's tasks, so an absolute metric's bar has a + // scale (percentage metrics ignore it). Same helper the leaderboard uses, + // so the two cannot disagree about what "best" means. + const taskBest = useMemo( + () => bestValue(metric, setup.tasks.map(t => t.scores[metric])), + [setup, metric] + ); function sortBy(key) { setSort(prev => prev.key === key diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 24f51a20..da23e480 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -6,7 +6,7 @@ import { useMemo, useState } from "react"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { buildFilterGroups, getFilteredSetups, emptyFilterState } from "../lib/filters.js"; import { setupScore } from "../lib/accessors.js"; -import { METRIC_LABELS, availableMetrics, metricDescription, isLowerBetter } from "../lib/vocab.js"; +import { METRIC_LABELS, availableMetrics, metricDescription, isLowerBetter, bestValue } from "../lib/vocab.js"; import { FilterBar } from "../components/FilterBar.jsx"; import { LeaderboardRow } from "../components/LeaderboardRow.jsx"; import { MetricToggle } from "../components/MetricToggle.jsx"; @@ -42,13 +42,14 @@ export function Leaderboard() { }); }, [filtered, metric]); - // Best value on screen — the smallest, for the lower-is-better absolute - // metrics — so their bars have a scale. Null for percentage metrics, which - // need none. - const metricBest = useMemo(() => { - const vals = sorted.map(s => setupScore(s, metric)).filter(v => v != null); - return vals.length ? Math.min(...vals) : null; - }, [sorted, metric]); + // Best value on screen, so an absolute metric's bars have a scale. Computed + // for every metric; a percentage metric ignores it downstream in + // metricBarFraction(). See bestValue() for why non-positive readings are + // excluded rather than min'd over. + const metricBest = useMemo( + () => bestValue(metric, sorted.map(s => setupScore(s, metric))), + [sorted, metric] + ); function toggleFilter(groupKey, value) { setFilterState(prev => { From eca3f1850d42cd44a6da1cede33b8325ad9a4982 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 31 Aug 2026 13:28:08 -0700 Subject: [PATCH 14/18] feat(site): split Tokens into input, output and cached axes A single Tokens column summed buckets that providers bill at very different rates, so it reported whichever bucket happened to be largest rather than anything a reader could act on. On the fleet's week-2 drop, output is 0.7% of the summed buckets (2.4% and 5.5% on our two drops), so the combined figure was the input count wearing a different label and the most expensive axis was invisible. It was also not comparable across harnesses: cache reads are reported by 134 of 260 fleet rows and by none of ours, and folding them in ranked a harness worse for being more forthcoming about its telemetry. The same model on two harnesses differed 4.8x on the combined metric but only 2.1x on output; across the whole board the old column spread 101x where output spreads 3.2x. The buckets are now grouped by billed rate, so each axis is a quantity of one kind of thing: input = inputTokens + cacheWriteTokens (prompt content, ~input rate) output = outputTokens + reasoningTokens (generated, ~output rate) cached = cachedTokens (cache reads, steep discount) Reasoning stays with output because it is a sibling bucket and not a subset, which is what keeps reasoning models from being undercounted. None of the three consults totalTokens any more: a single reported total cannot be attributed to an axis. No row in any dataset we hold carries one without the buckets, so this costs no coverage; a producer that ever reports only a total now renders blank rather than being mis-attributed. The unmeasured-zero sentinel is preserved per axis, so a zeroed output bucket no longer blanks input alongside it. Also fills in MetricKey, which never listed the efficiency metrics. --- site/ingest/derive.mjs | 4 +- site/ingest/derive.test.mjs | 52 +++++++++++--- site/seed/mock-data.mjs | 81 +++++++++++++++------ site/seed/mock-data.test.mjs | 88 ++++++++++++++++++----- site/src/components/LeaderboardRow.jsx | 2 +- site/src/components/MetricToggle.jsx | 6 +- site/src/components/MetricToggle.test.jsx | 6 +- site/src/lib/accessors.test.js | 4 +- site/src/lib/schema.d.ts | 20 ++++-- site/src/lib/vocab.js | 40 ++++++++--- site/src/lib/vocab.test.js | 42 ++++++++--- site/src/pages/Detail.jsx | 4 +- site/src/pages/Detail.test.jsx | 22 +++--- site/src/pages/Leaderboard.jsx | 2 +- 14 files changed, 278 insertions(+), 95 deletions(-) diff --git a/site/ingest/derive.mjs b/site/ingest/derive.mjs index ec12bef1..d6290b36 100644 --- a/site/ingest/derive.mjs +++ b/site/ingest/derive.mjs @@ -107,7 +107,9 @@ function meanScores(scoreList) { correctness: avg("correctness"), recoverableSafety: avg("recoverableSafety"), latency: avg("latency"), - tokens: avg("tokens") + inputTokens: avg("inputTokens"), + outputTokens: avg("outputTokens"), + cachedTokens: avg("cachedTokens") }; } diff --git a/site/ingest/derive.test.mjs b/site/ingest/derive.test.mjs index 215766f1..896e7899 100644 --- a/site/ingest/derive.test.mjs +++ b/site/ingest/derive.test.mjs @@ -84,9 +84,12 @@ describe("derive — data-driven", () => { recoverableSafety: null, // Efficiency is telemetry, not a score: an iteration that never // scored still consumed wall-clock, so latency survives while every - // score is null. Tokens stay null because the fixture captured none. + // score is null. The token axes stay null because the fixture + // captured no usage. latency: 1, - tokens: null + inputTokens: null, + outputTokens: null, + cachedTokens: null }); }); @@ -112,23 +115,50 @@ describe("derive — data-driven", () => { expect(mixed[0].tasks[0].scores.latency).toBe(10); }); - it("counts reasoning tokens, which are a sibling bucket of output", () => { - // Regression: omitting reasoningTokens undercounted every reasoning model. + it("projects the token buckets onto three axes rather than one total", () => { + // Regression: a single summed figure was ~the input count wearing a + // different label. Output was 0.7% of the fleet's summed buckets, so the + // most expensive axis was invisible in the number that ranked setups. const base = { setupId: "s", model: "m", harness: "h", augmentation: [], runId: "run_20260101_000000", t: "2026-01-01T00:00:00Z", taskFolder: "task-a", taskName: "Task A", status: "success", toolScore: null, latencySec: 5, outcomeScore: 0.9, iteration: 0 }; - const summed = derive([ - { ...base, inputTokens: 100, outputTokens: 200, reasoningTokens: 700 } + const split = derive([ + { + ...base, + inputTokens: 100, + cacheWriteTokens: 50, + outputTokens: 200, + reasoningTokens: 700, + cachedTokens: 4000, + // A provider total no longer overrides the buckets: it cannot be + // attributed to an axis. + totalTokens: 950 + } ]); - expect(summed[0].tasks[0].scores.tokens).toBe(1000); + expect(split[0].tasks[0].scores).toMatchObject({ + inputTokens: 150, + outputTokens: 900, + cachedTokens: 4000 + }); + }); - // A provider-reported total still wins over the bucket sum. - const totalled = derive([ - { ...base, inputTokens: 100, outputTokens: 200, reasoningTokens: 700, totalTokens: 950 } + it("leaves the cached axis null for a harness that reports no cache reads", () => { + // Only some harnesses report cache reads. A blank cell has to stay + // distinct from a 0, or a silent harness would rank best on a + // lower-is-better axis purely for being less talkative. + const setups = derive([ + { + setupId: "s", model: "m", harness: "h", augmentation: [], + runId: "run_20260101_000000", t: "2026-01-01T00:00:00Z", + taskFolder: "task-a", taskName: "Task A", status: "success", + toolScore: null, latencySec: 5, outcomeScore: 0.9, iteration: 0, + inputTokens: 100, outputTokens: 200 + } ]); - expect(totalled[0].tasks[0].scores.tokens).toBe(950); + expect(setups[0].tasks[0].scores.cachedTokens).toBeNull(); + expect(setups[0].tasks[0].scores.inputTokens).toBe(100); }); }); diff --git a/site/seed/mock-data.mjs b/site/seed/mock-data.mjs index a81510a8..42266f4a 100644 --- a/site/seed/mock-data.mjs +++ b/site/seed/mock-data.mjs @@ -214,6 +214,14 @@ export function generateRaw() { latencySec: round(20 + rng() * 60, 2), inputTokens: Math.round(8000 + rng() * 30000), outputTokens: Math.round(300 + rng() * 1500), + // Cache reads are reported by SOME harnesses only (in the + // fleet's week-2 drop, 134 of 260 rows; in ours, none), so + // the mock leaves them absent on the API runner. That keeps + // a blank Cached cell on screen next to populated ones — + // the case the separate axis exists to make visible. + ...(def.harness === "api-loop" + ? {} + : { cachedTokens: Math.round(12000 + rng() * 60000) }), // Mock rows are all vetted so the seeded demo renders; // real rows carry per-task validated from the harness. validated: true @@ -317,37 +325,66 @@ export function latencyOf(row) { return Number.isFinite(row.latencySec) && row.latencySec > 0 ? row.latencySec : null; } -// Total tokens for one row. Prefers the producer's own total when present (it -// may count buckets the row does not break out); otherwise sums the captured -// buckets. `reasoningTokens` is a SIBLING of output, not a subset of it (see -// the canonical buckets in normalize.py), so leaving it out would undercount -// every reasoning model. Null when no usage was captured at all, so "not -// measured" stays distinct from a genuine zero. -// +// Sum of the named buckets for one row, or null when none of them was captured. // A non-positive result is the same unmeasured sentinel `latencyOf` handles: a // harness that produced no session log normalizes to zeros rather than nulls // (antigravity's parser returns `{input: 0, output: 0, total: 0, cached: 0}` // for an empty log, and normalize.py coerces those through as ints), and no // real run costs 0 tokens. Reporting the 0 would rank that setup FIRST on a -// lower-is-better metric. The total is checked separately from the buckets so -// a zeroed total still falls through to buckets that were captured. -export function sumTokens(row) { - if (Number.isFinite(row.totalTokens) && row.totalTokens > 0) return row.totalTokens; - const parts = [ - row.inputTokens, - row.outputTokens, - row.cachedTokens, - row.reasoningTokens, - row.cacheWriteTokens - ].filter(v => Number.isFinite(v)); +// lower-is-better metric. +function bucketSum(row, keys) { + const parts = keys.map(k => row[k]).filter(v => Number.isFinite(v)); if (!parts.length) return null; const total = parts.reduce((a, b) => a + b, 0); return total > 0 ? total : null; } -// The {latency, tokens} slice of Scores for one group of iteration rows. +// Token usage is reported as three axes, not one number, because the buckets are +// not interchangeable: a provider bills generated tokens at several times the +// prompt rate and a cache read at a fraction of it. Summing them reports +// whichever bucket happens to be largest — in our own drops output is 0.7% +// (fleet week-2), 2.4% and 5.5% of the summed buckets, so a combined figure is +// the input count wearing a different label, and the axis a reader cares about +// is the one it hides. +// +// Grouping is by BILLED RATE, so each axis is a quantity of one kind of thing: +// +// input = inputTokens + cacheWriteTokens — cache creation is prompt content +// sent for this run, billed at roughly the input rate. +// output = outputTokens + reasoningTokens — reasoning is a SIBLING bucket of +// output, not a subset (see the canonical buckets in normalize.py), +// and both are generated at the output rate. Folding it in here is +// what keeps reasoning models from being undercounted. +// cached = cachedTokens — the cheapest bucket, and reported by only some +// harnesses (134 of 260 fleet rows; none of ours). Folded into input +// it would make a harness that reports cache reads look more +// expensive than one that stays silent, which is a ranking artifact +// of telemetry verbosity rather than a real cost difference. +// +// None of these consults `totalTokens`: a single provider-reported total cannot +// be attributed to an axis, and no row in any dataset we hold carries one +// without also carrying the buckets. A producer that ever reports only a total +// renders blank here rather than being silently mis-attributed. +export function inputTokensOf(row) { + return bucketSum(row, ["inputTokens", "cacheWriteTokens"]); +} + +export function outputTokensOf(row) { + return bucketSum(row, ["outputTokens", "reasoningTokens"]); +} + +export function cachedTokensOf(row) { + return bucketSum(row, ["cachedTokens"]); +} + +// The efficiency slice of Scores for one group of iteration rows. export function efficiencyFor(rows) { - return { latency: rawMean(rows, latencyOf), tokens: rawMean(rows, sumTokens) }; + return { + latency: rawMean(rows, latencyOf), + inputTokens: rawMean(rows, inputTokensOf), + outputTokens: rawMean(rows, outputTokensOf), + cachedTokens: rawMean(rows, cachedTokensOf) + }; } // Mean over a list of score objects, per metric. Skips non-numeric entries so a @@ -365,7 +402,9 @@ function meanScores(scoreList) { correctness: avg("correctness"), recoverableSafety: avg("recoverableSafety"), latency: avg("latency"), - tokens: avg("tokens") + inputTokens: avg("inputTokens"), + outputTokens: avg("outputTokens"), + cachedTokens: avg("cachedTokens") }; } diff --git a/site/seed/mock-data.test.mjs b/site/seed/mock-data.test.mjs index 4fc05451..f154f837 100644 --- a/site/seed/mock-data.test.mjs +++ b/site/seed/mock-data.test.mjs @@ -1,19 +1,63 @@ import { describe, it, expect } from "vitest"; -import { generateRaw, derive, passAtK, PASS_THRESHOLD, sumTokens, latencyOf } from "./mock-data.mjs"; - -describe("sumTokens", () => { - it("prefers the producer's own total over the buckets", () => { - expect(sumTokens({ totalTokens: 993225, inputTokens: 135329, outputTokens: 9732 })).toBe(993225); - }); - - it("sums the captured buckets when no total is reported", () => { - // reasoningTokens is a sibling of output, not a subset, so it is added. - expect(sumTokens({ inputTokens: 100, outputTokens: 20, reasoningTokens: 5 })).toBe(125); - }); - - it("is null when no usage was captured at all", () => { - expect(sumTokens({ inputTokens: null, outputTokens: null })).toBeNull(); - expect(sumTokens({})).toBeNull(); +import { + generateRaw, + derive, + passAtK, + PASS_THRESHOLD, + inputTokensOf, + outputTokensOf, + cachedTokensOf, + latencyOf +} from "./mock-data.mjs"; + +describe("token axes", () => { + // The whole point of the split: these three must never be summed into one + // figure, so each has to pick up exactly its own billed-rate family. + const row = { + inputTokens: 100, + cacheWriteTokens: 7, + outputTokens: 20, + reasoningTokens: 5, + cachedTokens: 400 + }; + + it("groups the buckets by billed rate, keeping the axes disjoint", () => { + expect(inputTokensOf(row)).toBe(107); // input + cache write + expect(outputTokensOf(row)).toBe(25); // output + reasoning + expect(cachedTokensOf(row)).toBe(400); // cache reads alone + }); + + it("counts reasoning as output, since it is a sibling bucket and not a subset", () => { + // Dropping it would undercount every reasoning model on the one axis + // that is billed at the highest rate. + expect(outputTokensOf({ outputTokens: 20, reasoningTokens: 5 })).toBe(25); + expect(outputTokensOf({ outputTokens: 20 })).toBe(20); + }); + + it("ignores a provider total, which cannot be attributed to an axis", () => { + // sumTokens used to prefer totalTokens. A total says nothing about the + // input/output split, so a row carrying only one reads as unmeasured + // rather than being silently attributed to a single axis. + const totalOnly = { totalTokens: 993225 }; + expect(inputTokensOf(totalOnly)).toBeNull(); + expect(outputTokensOf(totalOnly)).toBeNull(); + expect(cachedTokensOf(totalOnly)).toBeNull(); + // ...and a total alongside buckets never overrides them. + expect(inputTokensOf({ totalTokens: 993225, inputTokens: 135329 })).toBe(135329); + }); + + it("keeps cache reads off the input axis", () => { + // Only some harnesses report cache reads (134 of 260 fleet rows, none of + // ours). Folding them into input would make a harness that reports them + // look more expensive than one that stays silent. + expect(inputTokensOf({ inputTokens: 100, cachedTokens: 400 })).toBe(100); + expect(cachedTokensOf({ inputTokens: 100 })).toBeNull(); + }); + + it("is null when the axis was never captured at all", () => { + expect(inputTokensOf({ inputTokens: null })).toBeNull(); + expect(outputTokensOf({})).toBeNull(); + expect(cachedTokensOf({})).toBeNull(); }); // Regression: antigravity's parser returns {input: 0, output: 0, total: 0, @@ -22,12 +66,17 @@ describe("sumTokens", () => { // that setup FIRST on a lower-is-better metric with a full bar — the same // failure latencyOf() guards against. it("treats an all-zero usage record as unmeasured, not as zero tokens", () => { - expect(sumTokens({ inputTokens: 0, outputTokens: 0, cachedTokens: 0, totalTokens: 0 })).toBeNull(); + const zeroed = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, totalTokens: 0 }; + expect(inputTokensOf(zeroed)).toBeNull(); + expect(outputTokensOf(zeroed)).toBeNull(); + expect(cachedTokensOf(zeroed)).toBeNull(); expect(latencyOf({ latencySec: 0 })).toBeNull(); }); - it("falls through to the buckets when only the total is zeroed", () => { - expect(sumTokens({ totalTokens: 0, inputTokens: 400, outputTokens: 100 })).toBe(500); + it("keeps an axis that was captured when a sibling axis is zeroed", () => { + // A zeroed bucket is per-axis: output going quiet must not blank input. + expect(inputTokensOf({ inputTokens: 400, outputTokens: 0 })).toBe(400); + expect(outputTokensOf({ inputTokens: 400, outputTokens: 0 })).toBeNull(); }); }); @@ -136,6 +185,7 @@ describe("derive", () => { // Efficiency is telemetry, not a score: the blanked cell still consumed // wall-clock and tokens, so those survive while every score is null. expect(task.scores.latency).toBeGreaterThan(0); - expect(task.scores.tokens).toBeGreaterThan(0); + expect(task.scores.inputTokens).toBeGreaterThan(0); + expect(task.scores.outputTokens).toBeGreaterThan(0); }); }); diff --git a/site/src/components/LeaderboardRow.jsx b/site/src/components/LeaderboardRow.jsx index 86d183f1..1089099d 100644 --- a/site/src/components/LeaderboardRow.jsx +++ b/site/src/components/LeaderboardRow.jsx @@ -7,7 +7,7 @@ import { setupScore, setupLabel } from "../lib/accessors.js"; import { formatMetric, metricBarFraction } from "../lib/vocab.js"; // `metricBest` is the best value for this metric across the visible rows — for -// absolute metrics (latency, tokens) that is the SMALLEST, and the bar shows +// absolute metrics (latency, the token axes) that is the SMALLEST, and the bar shows // each row's ratio to it, since those metrics have no natural ceiling. Unused by // percentage metrics. export function LeaderboardRow({ setup, models, harnesses, metric, metricBest }) { diff --git a/site/src/components/MetricToggle.jsx b/site/src/components/MetricToggle.jsx index 68d12eb4..f7fd2ea2 100644 --- a/site/src/components/MetricToggle.jsx +++ b/site/src/components/MetricToggle.jsx @@ -2,11 +2,11 @@ // hero. `available` (optional) marks which metrics have data — the others render // DISABLED rather than hidden, so the UI advertises the axis exists and says why // it's empty. The reason is per-metric (see metricUnavailableReason): pass@k is -// waiting on multi-iteration runs, latency/tokens on harness telemetry. When -// `available` is omitted (or empty), every metric is enabled (back-compat). +// waiting on multi-iteration runs, the efficiency axes on harness telemetry. +// When `available` is omitted (or empty), every metric is enabled (back-compat). // // The metrics are split into two pills, quality and efficiency, rather than one -// long strip: eight buttons overflow the leaderboard's score column, and the +// long strip: ten buttons overflow the leaderboard's score column, and the // break should fall BETWEEN the two families rather than mid-strip. Wrapping the // pills puts it there, and the seam doubles as the visual cue that the efficiency // axes read the other way — lower is better, absolute units. diff --git a/site/src/components/MetricToggle.test.jsx b/site/src/components/MetricToggle.test.jsx index 30334750..97c62ed9 100644 --- a/site/src/components/MetricToggle.test.jsx +++ b/site/src/components/MetricToggle.test.jsx @@ -56,8 +56,8 @@ describe("MetricToggle", () => { expect(buttonFor("latency")).toHaveAttribute("aria-pressed", "true"); expect(buttonFor("composite")).toHaveAttribute("aria-pressed", "false"); - fireEvent.click(buttonFor("tokens")); - expect(onChange).toHaveBeenCalledWith("tokens"); + fireEvent.click(buttonFor("outputTokens")); + expect(onChange).toHaveBeenCalledWith("outputTokens"); }); it("disables metrics missing from `available` rather than hiding them", () => { @@ -82,7 +82,7 @@ describe("MetricToggle", () => { render( {}} available={["composite"]} />); expect(buttonFor("pass5")).toHaveAttribute("title", "Available once multi-iteration runs land"); expect(buttonFor("latency")).toHaveAttribute("title", "Not reported by these runs"); - expect(buttonFor("tokens")).toHaveAttribute("title", "Not reported by these runs"); + expect(buttonFor("outputTokens")).toHaveAttribute("title", "Not reported by these runs"); }); it("describes an enabled metric instead of explaining its absence", () => { diff --git a/site/src/lib/accessors.test.js b/site/src/lib/accessors.test.js index 2b3330be..b5e82fd1 100644 --- a/site/src/lib/accessors.test.js +++ b/site/src/lib/accessors.test.js @@ -140,7 +140,7 @@ describe("yAxisBounds", () => { it("follows the data instead of clamping to 100", () => { // Token means run to five figures; a [0, 100] clamp would push every // series off the top of the chart. - const b = yAxisBounds([withHistory("tokens", [23200, 24100, 25200])], "tokens"); + const b = yAxisBounds([withHistory("outputTokens", [23200, 24100, 25200])], "outputTokens"); expect(b.min).toBeGreaterThan(100); expect(b.min).toBeLessThan(23200); expect(b.max).toBeGreaterThan(25200); @@ -150,7 +150,7 @@ describe("yAxisBounds", () => { // Raw padding gave [45.3, 54.7], printing "54.8s" against "54.0s". expect(yAxisBounds([withHistory("latency", [47.9, 50.1, 52.2])], "latency")) .toEqual({ min: 44, max: 56 }); - expect(yAxisBounds([withHistory("tokens", [23200, 24100, 25200])], "tokens")) + expect(yAxisBounds([withHistory("outputTokens", [23200, 24100, 25200])], "outputTokens")) .toEqual({ min: 21000, max: 27000 }); }); diff --git a/site/src/lib/schema.d.ts b/site/src/lib/schema.d.ts index 7d11338d..e4118d36 100644 --- a/site/src/lib/schema.d.ts +++ b/site/src/lib/schema.d.ts @@ -26,6 +26,12 @@ * The scoring metrics, in display order. `composite` is the scoring-framework v1 * headline (cat_v · √(c · rec_v)); `correctness` / `recoverableSafety` are its * sub-scores. All are 0..100 means. `pass1/5/Max` are pass rates. + * + * The efficiency keys are NOT percentages: they are absolute per-task means in + * seconds and token counts, where lower is better (see METRIC_META in vocab.js). + * The three token axes are separate because their buckets are billed at + * different rates and cannot be meaningfully added — see the grouping rules on + * `inputTokensOf` in seed/mock-data.mjs. */ export type MetricKey = | "composite" @@ -33,13 +39,19 @@ export type MetricKey = | "recoverableSafety" | "pass1" | "pass5" - | "passMax"; + | "passMax" + | "latency" + | "inputTokens" + | "outputTokens" + | "cachedTokens"; /** - * Per-metric scores as percentages (0..100). `null` where a metric has no - * scored data for the task/run. `pass5` and `passMax` are null today — they + * Per-metric values, keyed by MetricKey. Quality metrics are percentages + * (0..100); efficiency metrics are absolute magnitudes. `null` where a metric + * has no data for the task/run. `pass5` and `passMax` are null today — they * stay null until the harness produces multi-iteration runs (then `derive()` - * recomputes them from the same raw rows). + * recomputes them from the same raw rows). `cachedTokens` is null for any + * harness that does not report cache reads. */ export type Scores = Record; diff --git a/site/src/lib/vocab.js b/site/src/lib/vocab.js index dee0f435..be2560de 100644 --- a/site/src/lib/vocab.js +++ b/site/src/lib/vocab.js @@ -35,16 +35,26 @@ export const METRIC_LABELS = { pass5: "Pass@5", passMax: "Pass^5", latency: "Latency", - tokens: "Tokens" + inputTokens: "Input Tokens", + outputTokens: "Output Tokens", + cachedTokens: "Cached Tokens" }; -// Abbreviated labels for the metric toggle only, where eight buttons compete for +// Abbreviated labels for the metric toggle only, where ten buttons compete for // the width of one table column. "Recoverable Safety" is ~2.5x the width of any // other button, so it alone decides whether the group fits on one line. Headings, // tooltips and the accessible name of the button all keep the full METRIC_LABELS // text — this shortens the visible glyphs, not the vocabulary. +// +// The token axes drop the shared "Tokens" noun because the efficiency pill +// already groups them next to each other: three buttons reading "Input Tokens / +// Output Tokens / Cached Tokens" repeat the word twice for no discrimination, +// and the full name survives in the tooltip and the accessible name. const METRIC_SHORT_LABELS = { - recoverableSafety: "Rec. Safety" + recoverableSafety: "Rec. Safety", + inputTokens: "Input", + outputTokens: "Output", + cachedTokens: "Cached" }; /** Toggle-button text for a metric, falling back to the full label. */ @@ -62,7 +72,9 @@ export const METRICS = [ "pass5", "passMax", "latency", - "tokens" + "inputTokens", + "outputTokens", + "cachedTokens" ]; // Per-metric presentation rules. Quality metrics are 0..100 percentages where @@ -71,6 +83,7 @@ export const METRICS = [ // be scaled against the visible range rather than read as a percentage, and the // sort has to invert. Anything not listed defaults to the percentage rules. const PERCENT = { unit: "%", lowerIsBetter: false, percentage: true }; +const TOKENS = { unit: "", lowerIsBetter: true, percentage: false }; export const METRIC_META = { composite: PERCENT, correctness: PERCENT, @@ -79,7 +92,9 @@ export const METRIC_META = { pass5: PERCENT, passMax: PERCENT, latency: { unit: "s", lowerIsBetter: true, percentage: false }, - tokens: { unit: "", lowerIsBetter: true, percentage: false } + inputTokens: TOKENS, + outputTokens: TOKENS, + cachedTokens: TOKENS }; /** Presentation rules for a metric, defaulting to the percentage rules. */ @@ -137,7 +152,7 @@ export function metricBarFraction(metric, value, best) { * "Best" follows the metric's direction: the smallest for a lower-is-better * absolute metric, the largest otherwise. Non-positive readings are dropped * from the lower-is-better case because they are the unmeasured sentinel - * (see `latencyOf` / `sumTokens` in seed/mock-data.mjs). That matters beyond + * (see `latencyOf` / `bucketSum` in seed/mock-data.mjs). That matters beyond * the offending row: `best` is shared by the whole column, so one 0 would trip * the `best <= 0` guard above and flatten EVERY bar, not just its own. * @@ -165,7 +180,12 @@ export const METRIC_DESCRIPTIONS = { pass5: "Pass@5: needs multi-iteration runs (not produced yet).", passMax: "Pass^5: needs multi-iteration runs (not produced yet).", latency: "Latency: mean agent wall-clock seconds per task. Lower is better, so the bar is scaled against the fastest setup on screen — a full bar is the fastest, half a bar is twice as slow.", - tokens: "Tokens: mean total tokens per task (the provider total when reported, else the sum of the captured buckets). Lower is better." + inputTokens: + "Input tokens: mean prompt tokens sent per task, including cache writes. Kept separate from output because providers bill it at a fraction of the generated rate. Lower is better.", + outputTokens: + "Output tokens: mean generated tokens per task, including reasoning tokens (a sibling of output, not a subset). The most expensive axis, and typically a few percent of the volume. Lower is better.", + cachedTokens: + "Cached tokens: mean cache-read tokens per task — prompt content billed at a steep discount. Reported by some harnesses only, so a blank cell means not reported, not zero. Lower is better." }; // Description for a metric key, falling back to its label. @@ -182,7 +202,11 @@ const METRIC_UNAVAILABLE_REASONS = { pass5: "Available once multi-iteration runs land", passMax: "Available once multi-iteration runs land", latency: "Not reported by these runs", - tokens: "Not reported by these runs" + inputTokens: "Not reported by these runs", + outputTokens: "Not reported by these runs", + // Cache reads are the one axis a harness can legitimately omit while + // reporting everything else, so the reason names the harness, not the run. + cachedTokens: "Not reported by these harnesses" }; /** Tooltip for a metric with no data in the current dataset. */ diff --git a/site/src/lib/vocab.test.js b/site/src/lib/vocab.test.js index e8c5c3bd..bc05082e 100644 --- a/site/src/lib/vocab.test.js +++ b/site/src/lib/vocab.test.js @@ -1,5 +1,14 @@ import { describe, it, expect } from "vitest"; -import { formatMetric, isLowerBetter, metricBarFraction, metricMeta, bestValue } from "./vocab.js"; +import { + formatMetric, + isLowerBetter, + metricBarFraction, + metricMeta, + bestValue, + METRICS, + METRIC_LABELS, + metricDescription +} from "./vocab.js"; describe("metric presentation rules", () => { it("treats quality metrics as higher-is-better percentages", () => { @@ -10,12 +19,25 @@ describe("metric presentation rules", () => { }); it("treats efficiency metrics as lower-is-better magnitudes", () => { - for (const m of ["latency", "tokens"]) { + for (const m of ["latency", "inputTokens", "outputTokens", "cachedTokens"]) { expect(metricMeta(m).percentage).toBe(false); expect(isLowerBetter(m)).toBe(true); } }); + it("keeps the token buckets on separate axes, never summed into one", () => { + // The buckets are billed at different rates, so a single combined + // metric reports whichever happens to be largest. Guard the vocabulary + // against a combined key creeping back in. + expect(METRICS).toEqual(expect.arrayContaining(["inputTokens", "outputTokens", "cachedTokens"])); + expect(METRICS).not.toContain("tokens"); + // Each carries its own label and its own explanation. + const labels = ["inputTokens", "outputTokens", "cachedTokens"].map(m => METRIC_LABELS[m]); + expect(new Set(labels).size).toBe(3); + const notes = ["inputTokens", "outputTokens", "cachedTokens"].map(metricDescription); + expect(new Set(notes).size).toBe(3); + }); + it("defaults an unknown metric to the percentage rules", () => { expect(metricMeta("nope").percentage).toBe(true); }); @@ -30,14 +52,14 @@ describe("formatMetric", () => { it("renders latency in seconds and compacts large token counts", () => { expect(formatMetric("latency", 42.66)).toBe("42.7s"); expect(formatMetric("latency", 8)).toBe("8.0s"); - expect(formatMetric("tokens", 38412)).toBe("38.4k"); - expect(formatMetric("tokens", 850)).toBe("850"); + expect(formatMetric("outputTokens", 38412)).toBe("38.4k"); + expect(formatMetric("outputTokens", 850)).toBe("850"); }); it("renders a missing value as an em dash, never as zero", () => { expect(formatMetric("latency", null)).toBe("—"); expect(formatMetric("composite", undefined)).toBe("—"); - expect(formatMetric("tokens", NaN)).toBe("—"); + expect(formatMetric("outputTokens", NaN)).toBe("—"); }); }); @@ -57,7 +79,7 @@ describe("metricBarFraction", () => { // Regression: filtering down to one row made value === the scale, which // previously floored the bar at 2% for the fastest setup on screen. expect(metricBarFraction("latency", 42, 42)).toBeCloseTo(1); - expect(metricBarFraction("tokens", 38412, 38412)).toBeCloseTo(1); + expect(metricBarFraction("outputTokens", 38412, 38412)).toBeCloseTo(1); }); it("keeps near-equal values near-equal instead of full vs empty", () => { @@ -79,7 +101,7 @@ describe("metricBarFraction", () => { describe("bestValue", () => { it("takes the smallest for a lower-is-better metric and the largest otherwise", () => { expect(bestValue("latency", [30, 10, 20])).toBe(10); - expect(bestValue("tokens", [300, 100, 200])).toBe(100); + expect(bestValue("outputTokens", [300, 100, 200])).toBe(100); expect(bestValue("composite", [30, 10, 20])).toBe(30); }); @@ -93,13 +115,13 @@ describe("bestValue", () => { // min would then be 0. metricBarFraction bails on `best <= 0`, so that one // row would empty EVERY bar in the column instead of only its own. it("ignores the 0 sentinel so one unmeasured row cannot flatten the column", () => { - expect(bestValue("tokens", [0, 5000, 20000])).toBe(5000); + expect(bestValue("outputTokens", [0, 5000, 20000])).toBe(5000); expect(bestValue("latency", [0, 42])).toBe(42); - expect(metricBarFraction("tokens", 20000, bestValue("tokens", [0, 5000, 20000]))).toBeCloseTo(0.25); + expect(metricBarFraction("outputTokens", 20000, bestValue("outputTokens", [0, 5000, 20000]))).toBeCloseTo(0.25); }); it("is null when every lower-is-better reading is the sentinel", () => { - expect(bestValue("tokens", [0, 0])).toBeNull(); + expect(bestValue("outputTokens", [0, 0])).toBeNull(); }); // A percentage metric legitimately bottoms out at 0 (a 0% pass rate), so the diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 55847486..8d18be2e 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -179,8 +179,8 @@ export function Detail() { // "Avg Speed", which was independent of the metric back when latency wasn't // selectable; now that it is, selecting Latency makes "Average" the mean // latency and the two cards print the same figure side by side. Taking the - // first efficiency metric other than the selected one gives Tokens under - // Latency and Latency everywhere else, without naming either key here. + // first efficiency metric other than the selected one gives Input Tokens + // under Latency and Latency everywhere else, without naming either key here. const companion = METRICS.find(m => !metricMeta(m).percentage && m !== metric); const companionVals = companion ? setup.tasks.map(t => t.scores[companion]).filter(v => v != null) diff --git a/site/src/pages/Detail.test.jsx b/site/src/pages/Detail.test.jsx index c9e187f7..db4ccfc4 100644 --- a/site/src/pages/Detail.test.jsx +++ b/site/src/pages/Detail.test.jsx @@ -26,13 +26,15 @@ function makeBenchmark(overrides = {}) { { id: SETUP_ID, order: 0, model: "alpha-pro", harness: "gemini-cli", augmentation: [], color: "#3b82f6", - // Latency/tokens are chosen so their means are round (50.0s, - // 20.0k) and their best is the SMALLEST, which is the opposite - // end from the percentage metrics above. + // The efficiency figures are chosen so their means are round + // (50.0s, 20.0k input) and their best is the SMALLEST, which is + // the opposite end from the percentage metrics above. No + // cachedTokens: this harness reports no cache reads, so that + // axis stays null the way a real gemini-cli row does. tasks: [ - { folder: "a", name: "Apple", scores: { composite: 60, pass1: 60, pass5: 65, passMax: 70, latency: 40, tokens: 10000 } }, - { folder: "b", name: "Banana", scores: { composite: 90, pass1: 90, pass5: 95, passMax: 100, latency: 50, tokens: 20000 } }, - { folder: "c", name: "Cherry", scores: { composite: 80, pass1: 80, pass5: 85, passMax: 90, latency: 60, tokens: 30000 } } + { folder: "a", name: "Apple", scores: { composite: 60, pass1: 60, pass5: 65, passMax: 70, latency: 40, inputTokens: 10000, outputTokens: 400 } }, + { folder: "b", name: "Banana", scores: { composite: 90, pass1: 90, pass5: 95, passMax: 100, latency: 50, inputTokens: 20000, outputTokens: 500 } }, + { folder: "c", name: "Cherry", scores: { composite: 80, pass1: 80, pass5: 85, passMax: 90, latency: 60, inputTokens: 30000, outputTokens: 600 } } ], history: [ { t: "2026-01-15T00:00:00Z", scores: { composite: 70, pass1: 70, pass5: 75, passMax: 80 } }, @@ -171,7 +173,9 @@ describe("Detail", () => { renderAt(`/setup/${SETUP_ID}?metric=latency`); expect(within(card("Average")).getByText("50.0s")).toBeInTheDocument(); expect(screen.queryByText("Avg Latency")).not.toBeInTheDocument(); - expect(within(card("Avg Tokens")).getByText("20.0k")).toBeInTheDocument(); + // Falls to the next efficiency axis in METRICS order, which is now the + // input-token axis rather than a combined token count. + expect(within(card("Avg Input Tokens")).getByText("20.0k")).toBeInTheDocument(); }); it("orients the stat cards by the metric's direction", () => { @@ -186,7 +190,7 @@ describe("Detail", () => { it("heads the task column with the metric rather than calling it a score", () => { // A token count is telemetry, not a score; the old "Score (Tokens)" // header said otherwise. - renderAt(`/setup/${SETUP_ID}?metric=tokens`); + renderAt(`/setup/${SETUP_ID}?metric=inputTokens`); expect(screen.getByRole("columnheader", { name: /Tokens/ })).toBeInTheDocument(); expect(screen.queryByRole("columnheader", { name: /Score/ })).not.toBeInTheDocument(); }); @@ -231,7 +235,7 @@ describe("Detail", () => { expect(header(/Outcome/)).toHaveTextContent("▼"); // 90 → 60, descending cleanup(); - renderAt(`/setup/${SETUP_ID}?metric=tokens`); + renderAt(`/setup/${SETUP_ID}?metric=inputTokens`); expect(header(/Tokens/)).toHaveTextContent("▲"); // 10.0k → 30.0k, ascending fireEvent.click(header(/Tokens/)); expect(header(/Tokens/)).toHaveTextContent("▼"); diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index da23e480..2b3ecc9c 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -104,7 +104,7 @@ export function Leaderboard() {
{/* "METRIC", not "SCORE": the toggle below can select - latency or tokens, and neither is a score. Naming the + latency or a token axis, none of which is a score. Naming the selected metric here instead would just echo the highlighted button an inch beneath it. */} METRIC From 32f1a923c82cea66f8add25a9b3c7e7830bc3cf7 Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 31 Aug 2026 14:27:42 -0700 Subject: [PATCH 15/18] fix(site): drop the catastrophic badge from the efficiency columns A catastrophic safety violation zeroes the OUTCOME score. The seconds and tokens a run consumed are unaffected by it and still valid readings, but the badge renders immediately left of the figure, so on a latency or token column it reads as annotating a number it has no bearing on. Gate it on the metric family. Quality columns keep the fixed-width slot reserved on every row so figures and bars stay aligned; efficiency columns drop the slot entirely rather than reserving empty space, since no row can badge there and the bar can have the width back. --- site/src/components/LeaderboardRow.jsx | 39 +++++++---- site/src/components/LeaderboardRow.test.jsx | 73 +++++++++++++++++++++ 2 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 site/src/components/LeaderboardRow.test.jsx diff --git a/site/src/components/LeaderboardRow.jsx b/site/src/components/LeaderboardRow.jsx index 1089099d..62343e59 100644 --- a/site/src/components/LeaderboardRow.jsx +++ b/site/src/components/LeaderboardRow.jsx @@ -4,7 +4,7 @@ import { Link } from "react-router-dom"; import { SetupIdentity } from "./SetupIdentity.jsx"; import { setupScore, setupLabel } from "../lib/accessors.js"; -import { formatMetric, metricBarFraction } from "../lib/vocab.js"; +import { formatMetric, metricBarFraction, metricMeta } from "../lib/vocab.js"; // `metricBest` is the best value for this metric across the visible rows — for // absolute metrics (latency, the token axes) that is the SMALLEST, and the bar shows @@ -16,6 +16,13 @@ export function LeaderboardRow({ setup, models, harnesses, metric, metricBest }) const score = setupScore(setup, metric); const barPct = metricBarFraction(metric, score, metricBest) * 100; const to = `/setup/${encodeURIComponent(setup.id)}?metric=${encodeURIComponent(metric)}`; + // The badge sits immediately left of the figure, so it reads as annotating + // it — and what a catastrophic violation zeroes is the OUTCOME score, not + // the seconds or tokens a run consumed. Those readings are unaffected and + // still valid, so under an efficiency metric the badge would flag a number + // it has no bearing on. It stays on the quality columns, where the zeroing + // is what the reader is looking at. + const badgeable = metricMeta(metric).percentage; return (
- {/* Score progression meter — fixed-width badge slot (reserved on every row) - keeps the %, bar, and column start identical whether or not a badge shows. */} + {/* Score progression meter. On a quality metric the badge slot is a + fixed width reserved on EVERY row, so the figure, bar and column + start line up whether or not a given row carries a badge. An + efficiency metric drops the slot entirely rather than reserving + empty space — no row can badge there, so the columns still agree + with each other and the bar gets the width back. */}
- - {setup.catastrophicCount > 0 && ( - - ⚠ {setup.catastrophicCount} - - )} - + {badgeable && ( + + {setup.catastrophicCount > 0 && ( + + ⚠ {setup.catastrophicCount} + + )} + + )} {formatMetric(metric, score)} diff --git a/site/src/components/LeaderboardRow.test.jsx b/site/src/components/LeaderboardRow.test.jsx new file mode 100644 index 00000000..70cd315b --- /dev/null +++ b/site/src/components/LeaderboardRow.test.jsx @@ -0,0 +1,73 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { render, screen, cleanup } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; +import { LeaderboardRow } from "./LeaderboardRow.jsx"; + +const MODELS = { "alpha-pro": { name: "Alpha Pro", provider: "Acme", logo: "alpha" } }; +const HARNESSES = { + "gemini-cli": { name: "Gemini CLI", type: "cli", accent: "#0ea5e9", logo: "terminal" } +}; + +// Two catastrophic tasks, and readings on both metric families so the row has +// something to print whichever one is selected. +const SETUP = { + id: "alpha-pro-gemini-cli", + order: 0, + model: "alpha-pro", + harness: "gemini-cli", + augmentation: [], + color: "#3b82f6", + catastrophicCount: 2, + tasks: [{ folder: "a", name: "A", scores: { composite: 84, latency: 40, outputTokens: 900 } }], + history: [{ t: "2026-01-15T00:00:00Z", scores: { composite: 84, latency: 40, outputTokens: 900 } }] +}; + +function renderRow(metric, setup = SETUP) { + return render( + + + + ); +} + +describe("LeaderboardRow catastrophic badge", () => { + afterEach(cleanup); + + it("badges a quality metric, where the violation is what zeroed the figure", () => { + renderRow("composite"); + expect(screen.getByText("⚠ 2")).toBeInTheDocument(); + }); + + it("drops the badge on the efficiency metrics, which a violation does not zero", () => { + // A catastrophic violation zeroes the OUTCOME score. The seconds and + // tokens a run consumed are unaffected and still valid, so a badge + // beside them would flag a reading it has no bearing on. + for (const metric of ["latency", "inputTokens", "outputTokens", "cachedTokens"]) { + renderRow(metric); + expect(screen.queryByText("⚠ 2")).not.toBeInTheDocument(); + cleanup(); + } + }); + + it("still prints the efficiency figure it was hiding the badge next to", () => { + // Guard against "fixing" the badge by dropping the whole slot including + // the value: the number is the point of the column. + renderRow("latency"); + expect(screen.getByText("40.0s")).toBeInTheDocument(); + }); + + it("shows no badge for a clean setup on any metric", () => { + const clean = { ...SETUP, catastrophicCount: 0 }; + for (const metric of ["composite", "latency"]) { + renderRow(metric, clean); + expect(screen.queryByText(/⚠/)).not.toBeInTheDocument(); + cleanup(); + } + }); +}); From 780540db0d26ac3717eed50cb32c62088c556f8a Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 31 Aug 2026 15:20:15 -0700 Subject: [PATCH 16/18] feat(site): sortable leaderboard headers, so row order can hold across metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ranking by the selected metric answers "who won this category", but it reshuffles on every tab, so there was no way to follow one setup across Outcome → Latency → Input → Output and read it as a profile. Make MODEL/HARNESS and METRIC clickable headers, the same pattern the detail page's task table already uses (its comment even says it mirrors the leaderboard's ordering; the leaderboard just never got the other half). Rank-first stays the default, and the sort key deliberately survives a metric change, which is the entire point of choosing it. Direction follows the same convention as the task table: "desc" is best-first, which on latency and the token axes is the smallest value, and the arrow reports which way the VALUES run rather than the internal flag, so a ▼ never sits above a column reading 20.0s → 50.0s. These are buttons in a CSS grid rather than real columnheaders, so the direction rides in visually-hidden text instead of an invalid aria-sort. Also gate the detail page's Catastrophic stat card on the metric family, matching the row badge: in a row of cards that otherwise all describe the selected metric, it read as qualifying a latency or token figure that a violation does not touch. --- site/src/lib/accessors.js | 19 ++++++++ site/src/pages/Detail.jsx | 36 ++++++++------ site/src/pages/Detail.test.jsx | 15 ++++++ site/src/pages/Leaderboard.jsx | 69 +++++++++++++++++++++++---- site/src/pages/Leaderboard.test.jsx | 74 +++++++++++++++++++++++++++-- 5 files changed, 185 insertions(+), 28 deletions(-) diff --git a/site/src/lib/accessors.js b/site/src/lib/accessors.js index 94217dd5..e99b8d6f 100644 --- a/site/src/lib/accessors.js +++ b/site/src/lib/accessors.js @@ -42,6 +42,25 @@ export function setupLabel(setup, models, harnesses) { return parts.join(" · "); } +// Alphabetical ordering over the identity a row actually displays — model, then +// harness, then the augmentation chips — rather than over setup.id, which is a +// slug and would sort "api-loop" above "Gemini CLI" on punctuation the reader +// cannot see. `numeric` keeps gpt-5-2 ahead of gpt-5-10 instead of ordering the +// version digits as text. +/** + * @param {Setup} a + * @param {Setup} b + * @param {ModelMap} models + * @param {HarnessMap} harnesses + * @returns {number} + */ +export function compareByName(a, b, models, harnesses) { + const cmp = (x, y) => x.localeCompare(y, undefined, { numeric: true, sensitivity: "base" }); + return cmp(models[a.model].name, models[b.model].name) + || cmp(harnesses[a.harness].name, harnesses[b.harness].name) + || cmp(a.augmentation.join(","), b.augmentation.join(",")); +} + // Secondary modifier chips — one per augmentation token, or a single neutral // "Baseline" chip when the augmentation array is empty. The harness type chip // is built separately at the call site (it needs the per-harness accent color). diff --git a/site/src/pages/Detail.jsx b/site/src/pages/Detail.jsx index 8d18be2e..035bb6ed 100644 --- a/site/src/pages/Detail.jsx +++ b/site/src/pages/Detail.jsx @@ -224,20 +224,28 @@ export function Detail() { - + {/* Only on the quality metrics. What a catastrophic + violation zeroes is the Outcome score — the seconds and + tokens the run consumed are untouched and still valid, so + in a row of cards that otherwise all describe the selected + metric, this one would read as qualifying a figure it has + no bearing on. Same rule as the leaderboard's ⚠ badge. */} + {metricMeta(metric).percentage && ( + + )} {companion ? ( { expect(within(card("Catastrophic")).getByText("outcomes zeroed")).toBeInTheDocument(); }); + it("drops the catastrophic card on the efficiency metrics", () => { + // A catastrophic violation zeroes the OUTCOME score. The seconds and + // tokens the run consumed are untouched, so beside a latency or token + // headline the card qualifies a figure it has no bearing on. + benchmark.setups[0].catastrophicCount = 3; + for (const m of ["latency", "inputTokens", "outputTokens"]) { + renderAt(`/setup/${SETUP_ID}?metric=${m}`); + expect(screen.queryByText("Catastrophic")).not.toBeInTheDocument(); + cleanup(); + } + // ...and is still there on the quality side, where it explains the score. + renderAt(`/setup/${SETUP_ID}?metric=composite`); + expect(screen.getByText("Catastrophic")).toBeInTheDocument(); + }); + it("reports the efficiency axis the toggle is not showing", () => { const card = label => screen.getByText(label).closest("div"); diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index 2b3ecc9c..75188696 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -5,7 +5,7 @@ import { useMemo, useState } from "react"; import { useBenchmark } from "../context/BenchmarkContext.jsx"; import { buildFilterGroups, getFilteredSetups, emptyFilterState } from "../lib/filters.js"; -import { setupScore } from "../lib/accessors.js"; +import { setupScore, compareByName } from "../lib/accessors.js"; import { METRIC_LABELS, availableMetrics, metricDescription, isLowerBetter, bestValue } from "../lib/vocab.js"; import { FilterBar } from "../components/FilterBar.jsx"; import { LeaderboardRow } from "../components/LeaderboardRow.jsx"; @@ -17,6 +17,11 @@ export function Leaderboard() { const { models, harnesses, setups, loading, error } = useBenchmark(); const [metric, setMetric] = useState("composite"); const [filterState, setFilterState] = useState(emptyFilterState); + // Rank-first is the default, so the page still opens as a leaderboard. The + // key deliberately SURVIVES a metric change: the reason to sort by name is + // to hold every row still while you click across the metric tabs and read + // one setup down the columns, which re-ranking on each tab makes impossible. + const [sort, setSort] = useState({ key: "metric", dir: "desc" }); const groups = useMemo(() => buildFilterGroups(models, harnesses, setups), [models, harnesses, setups]); const available = useMemo(() => availableMetrics(setups), [setups]); @@ -26,21 +31,25 @@ export function Leaderboard() { [setups, groups, filterState] ); - // Sort the filtered setups by aggregated score under the selected metric. - // Efficiency metrics rank ascending (lower latency / fewer tokens is better). - // A setup with no value for the metric sorts last either way rather than - // being treated as a 0, which would make it look like the best latency. + // Sort the filtered setups, either alphabetically or by aggregated score + // under the selected metric. "desc" means BEST first, which for latency and + // the token axes is the SMALLEST value — same convention as the detail + // page's task table, so the two cannot disagree about which way is up. + // A setup with no value for the metric sorts last in either direction rather + // than being treated as a 0, which would make it look like the best latency. const sorted = useMemo(() => { + const dir = sort.dir === "asc" ? 1 : -1; const lower = isLowerBetter(metric); return [...filtered].sort((a, b) => { + if (sort.key === "name") return dir * compareByName(a, b, models, harnesses); const av = setupScore(a, metric); const bv = setupScore(b, metric); if (av == null && bv == null) return 0; if (av == null) return 1; if (bv == null) return -1; - return lower ? av - bv : bv - av; + return dir * (lower ? bv - av : av - bv); }); - }, [filtered, metric]); + }, [filtered, metric, sort, models, harnesses]); // Best value on screen, so an absolute metric's bars have a scale. Computed // for every metric; a percentage metric ignores it downstream in @@ -64,6 +73,37 @@ export function Leaderboard() { setFilterState(emptyFilterState()); } + // Clicking the header you are already sorted by flips direction; switching + // headers picks that column's natural default — A→Z for names, best-first + // for the metric. + function sortBy(key) { + setSort(prev => prev.key === key + ? { key, dir: prev.dir === "asc" ? "desc" : "asc" } + : { key, dir: key === "name" ? "asc" : "desc" }); + } + + // The arrow reports which way the VALUES run, not the internal sort flag. + // "desc" means best-first, and best-first under latency or a token axis is + // ascending numbers — so the glyph has to invert or the column reads + // 22.3k → 28.0k under a ▼. Same rule as the detail page's table. + // These headers are buttons in a CSS grid, not real columnheaders, so + // aria-sort would be invalid here — the direction rides along as + // visually-hidden text in the button's accessible name instead. + const Arrow = ({ k }) => { + if (sort.key !== k) return ; + const ascending = k === "name" + ? sort.dir === "asc" + : (sort.dir === "asc") !== isLowerBetter(metric); + return ( + <> + + , sorted {ascending ? "ascending" : "descending"} + + ); + }; + + const headerBtn = "flex items-center gap-1 hover:text-slate-700 dark:hover:text-slate-200 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-indigo-500 rounded"; + return (
@@ -93,13 +133,20 @@ export function Leaderboard() { {/* Controls & column headers */}
- MODEL + {/* Both halves of the pairing sort the same way — the + comparator runs model then harness — so either one is + a live target rather than only the leading word. */} + - HARNESS & config +
@@ -107,7 +154,9 @@ export function Leaderboard() { latency or a token axis, none of which is a score. Naming the selected metric here instead would just echo the highlighted button an inch beneath it. */} - METRIC +
; - const ascending = k === "name" - ? sort.dir === "asc" - : (sort.dir === "asc") !== isLowerBetter(metric); + const down = sort.dir === "desc"; + const said = k === "name" + ? (down ? "Z to A" : "A to Z") + : (down ? "best first" : "worst first"); return ( <> - - , sorted {ascending ? "ascending" : "descending"} + + , sorted {said} ); }; @@ -154,7 +162,7 @@ export function Leaderboard() { latency or a token axis, none of which is a score. Naming the selected metric here instead would just echo the highlighted button an inch beneath it. */} -
diff --git a/site/src/pages/Leaderboard.test.jsx b/site/src/pages/Leaderboard.test.jsx index 639cc28c..cd3deff2 100644 --- a/site/src/pages/Leaderboard.test.jsx +++ b/site/src/pages/Leaderboard.test.jsx @@ -27,8 +27,8 @@ const FIXTURE = { { id: "gamma-coder-openclaw-mcp-skills", order: 1, model: "gamma-coder", harness: "openclaw", augmentation: ["mcp", "skills"], color: "#ec4899", - tasks: [{ folder: "a", name: "A", scores: { pass1: 70, pass5: 75, passMax: 80, composite: 90, latency: 50 } }], - history: [{ t: "2026-01-15T00:00:00Z", scores: { pass1: 70, pass5: 75, passMax: 80, composite: 90, latency: 50 } }] + tasks: [{ folder: "a", name: "A", scores: { pass1: 70, pass5: 75, passMax: 80, composite: 90, latency: 50, cachedTokens: 5000 } }], + history: [{ t: "2026-01-15T00:00:00Z", scores: { pass1: 70, pass5: 75, passMax: 80, composite: 90, latency: 50, cachedTokens: 5000 } }] } ], loading: false, @@ -136,13 +136,48 @@ describe("Leaderboard sorting", () => { expect(order()[0]).toMatch(/Alpha Pro/); }); - it("points the arrow at the values, not the sort flag, on a lower-is-better metric", () => { - // Best-first on latency is ASCENDING numbers. A ▼ there would sit above - // a column reading 20.0s → 50.0s. + it("never changes sort direction on its own when the metric changes", () => { + // Regression: the arrow used to describe the raw digits, and best-first + // is DESCENDING numbers on composite but ASCENDING numbers on latency. + // Crossing between the two families inverted it with no click, which + // reads as the sort mode switching by itself. renderPage(); - fireEvent.click(screen.getByRole("button", { name: "Latency" })); - expect(metricHeader()).toHaveAccessibleName(/sorted ascending/); + expect(metricHeader()).toHaveAccessibleName(/best first/); + // Latency/Cached are lower-is-better and Pass@1/Outcome higher — the + // crossing that used to invert the glyph. + for (const m of ["Latency", "Pass@1", "Cached Tokens", "Outcome"]) { + fireEvent.click(screen.getByRole("button", { name: m })); + expect(metricHeader()).toHaveAccessibleName(/best first/); + } + // Only an explicit click moves it, and then it stays moved. fireEvent.click(metricHeader()); - expect(metricHeader()).toHaveAccessibleName(/sorted descending/); + expect(metricHeader()).toHaveAccessibleName(/worst first/); + fireEvent.click(screen.getByRole("button", { name: "Latency" })); + expect(metricHeader()).toHaveAccessibleName(/worst first/); + }); + + it("keeps the winner on top across every metric, quality or efficiency", () => { + // The arrow no longer tracks the digits, but best-first is still what + // the default MEANS: highest composite, lowest latency. + renderPage(); + expect(order()[0]).toMatch(/Gamma Coder/); // composite 90 > 70 + fireEvent.click(screen.getByRole("button", { name: "Latency" })); + expect(order()[0]).toMatch(/Alpha Pro/); // latency 20s < 50s + }); + + it("gives an identical alphabetical order on every metric tab", () => { + // The name comparator reads model/harness/augmentation only, so the + // ordering cannot vary by metric — including on tabs where a setup has + // no reading at all and the rank sort would shuffle it to the bottom. + renderPage(); + fireEvent.click(modelHeader()); + const baseline = order(); + expect(baseline[0]).toMatch(/Alpha Pro/); + for (const m of ["Latency", "Input Tokens", "Output Tokens", "Cached Tokens", "Pass@1", "Outcome"]) { + const pill = screen.queryByRole("button", { name: m }); + if (!pill || pill.disabled) continue; + fireEvent.click(pill); + expect(order()).toEqual(baseline); + } }); }); From bd0fc26c910f31b0967db970d1ba6a6265810aba Mon Sep 17 00:00:00 2001 From: Jessie Liu Date: Mon, 31 Aug 2026 16:54:39 -0700 Subject: [PATCH 18/18] refactor(site): name the arrow's direction label for what it is --- site/src/pages/Leaderboard.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/src/pages/Leaderboard.jsx b/site/src/pages/Leaderboard.jsx index c1d2c55c..c41da230 100644 --- a/site/src/pages/Leaderboard.jsx +++ b/site/src/pages/Leaderboard.jsx @@ -99,13 +99,13 @@ export function Leaderboard() { const Arrow = ({ k }) => { if (sort.key !== k) return ; const down = sort.dir === "desc"; - const said = k === "name" + const directionLabel = k === "name" ? (down ? "Z to A" : "A to Z") : (down ? "best first" : "worst first"); return ( <> - , sorted {said} + , sorted {directionLabel} ); };