From fb45e29fee20e23bc3a883572cd8b6b5dd29f683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A1n=20Coronel?= Date: Fri, 25 Sep 2026 12:39:10 -0300 Subject: [PATCH] feat(engines): add Claude Code CLI as a ringer engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the claude/claude-sonnet engine identity, model registry entries, and a claude-code-json result parser for the `claude -p --output-format json` envelope. The parser distinguishes transport/API failures (rate limits, timeouts — excluded from the scoreboard as non-scoreable "infrastructure", no verify/retry) from agent-level failures (max turns, tool/permission rejection — still scoreable, so a real quality loss isn't hidden from the scoreboard). It reads the JSON result structurally (whole-payload decode, one JSON object per line) instead of scanning for a bare `{`, so text an agent quotes inside its own result can't be mistaken for the envelope. Model attribution from `modelUsage` only returns a family match when every candidate of that family agrees on the same canonical model, rather than guessing the first one when a director and a same-family subagent both appear. Adds a schema v3->v4 migration (scoreable, failure_kind columns) with a backward-compatible ALTER TABLE. Co-Authored-By: Claude Sonnet 5 --- README.md | 1 + config.sample.toml | 13 ++ registry/model-identity.toml | 62 ++++++++ ringer.py | 256 +++++++++++++++++++++++++++++--- tests/test_claude_engine.py | 225 ++++++++++++++++++++++++++++ tests/test_identity_evidence.py | 21 +-- tests/test_model_db.py | 4 +- tests/test_model_log.py | 2 + tests/test_taxonomy.py | 4 +- 9 files changed, 552 insertions(+), 36 deletions(-) create mode 100644 tests/test_claude_engine.py diff --git a/README.md b/README.md index 68dc7ab19..c7c5212d4 100644 --- a/README.md +++ b/README.md @@ -397,6 +397,7 @@ Every community PR that lands in main is credited here — that's a project rule - [@davekopecek](https://github.com/davekopecek) (Dave Kopecek) — committed the design-reference fixture so the design-token guard runs on every machine (#30) - [@snapsynapse](https://github.com/snapsynapse) (Sam Rogers) — graceful shutdown on SIGINT/SIGTERM with worker-tree cleanup and finished state, plus the 14-test end-to-end CLI regression suite (#4) - [@mlava](https://github.com/mlava) (Mark Lavercombe) — named setup failures across every diagnostic surface (#37), `run --baseline`, the no-workers check preflight (#38), guidance on check-writing failure modes (#57), early warnings for missing worker commands (#59), and preserving fix-swarm patches across retries (#56) +- [@hcoronel1](https://github.com/hcoronel1) (Hernán Coronel) — added Claude Code CLI as a ringer engine, with a `claude-code-json` result parser that keeps API/infra failures out of the scoreboard without hiding real agent failures Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the philosophy and what gets a PR merged fast. The short version: small and scoped, rebased on current main, every claim backed by an executed test. Authorship is always preserved — where a maintainer pushes a mechanical fix to your branch, you remain the commit author. diff --git a/config.sample.toml b/config.sample.toml index 1527dead3..c97a4611a 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -91,6 +91,19 @@ token_regex = "tokens\\s+used\\s*:?\\s*([0-9][0-9,]*)" # default reads the `model:` header; override it here only if that format changes. model_report_regex = "(?m)^model:[ \\t]*([^ \\t\\r\\n]+)[ \\t]*\\r?$" +# Claude Code CLI (opt in). Claude Code permissions are not an OS filesystem +# sandbox; use an OS-level wrapper when filesystem containment is required. +# [engines.claude] +# bin = "claude" +# model_default = "claude-sonnet-5" +# args_template = [ +# "-p", "--model", "{model}", "--output-format", "json", +# "{access_args}", "{engine_args}", "{spec}", +# ] +# sandbox_args = ["--permission-mode", "acceptEdits"] +# full_access_args = ["--dangerously-skip-permissions"] +# result_parser = "claude-code-json" + # Grok Build CLI (xAI). Install: curl -fsSL https://x.ai/cli/install.sh | bash # (or: npm install -g @xai-official/grok), then `grok login` — OAuth on a # SuperGrok or X Premium Plus plan. Headless mode (-p) runs a full agentic diff --git a/registry/model-identity.toml b/registry/model-identity.toml index 2807de915..4e5e40d73 100644 --- a/registry/model-identity.toml +++ b/registry/model-identity.toml @@ -71,6 +71,68 @@ access = "OpenRouter API" # names. Unlisted slugs are marked unregistered and derive a display name; # their complete raw value appears only in scoreboard diagnostics until verified. +[engines.claude] +harness = "Claude Code CLI" +access = "Claude subscription" +default_model_key = "claude-sonnet-5" + +[engines.claude.models."claude-sonnet-5"] +display = "Claude Sonnet 5" +lab = "Anthropic" +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude.models."sonnet"] +display = "Claude Sonnet 5 (alias)" +lab = "Anthropic" +alias = true +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude.models."claude-opus-5"] +display = "Claude Opus 5" +lab = "Anthropic" +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude.models."claude-fable-5-1"] +display = "Claude Fable 5.1" +lab = "Anthropic" +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude.models."claude-haiku-4-5"] +display = "Claude Haiku 4.5" +lab = "Anthropic" +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude-sonnet] +# Legacy engine name retained for historical rows; the harness is unchanged. +harness = "Claude Code CLI" +access = "Claude subscription" +default_model_key = "claude-sonnet-5" + +[engines.claude-sonnet.models."claude-sonnet-5"] +display = "Claude Sonnet 5" +lab = "Anthropic" +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + +[engines.claude-sonnet.models."sonnet"] +display = "Claude Sonnet 5 (alias)" +lab = "Anthropic" +alias = true +confidence = "verified" +source = "https://docs.anthropic.com/en/docs/about-claude/models/overview" +last_verified = 2026-09-16 + [engines.opencode.models."openrouter/z-ai/glm-5.2"] display = "GLM 5.2" lab = "Z.ai (Zhipu AI)" diff --git a/ringer.py b/ringer.py index 68663ac50..ae98c28c2 100755 --- a/ringer.py +++ b/ringer.py @@ -737,6 +737,7 @@ class EngineConfig: sandbox_args: tuple[str, ...] token_regex: str | None = DEFAULT_TOKEN_REGEX model_report_regex: str | None = None + result_parser: str | None = None # Fills the {model} placeholder in args_template when a task does not set # its own "model" — this is what makes a harness engine (OpenCode) model # agnostic instead of hard-coding one model into the command line. @@ -1698,6 +1699,11 @@ def load_engines(raw: Any) -> dict[str, EngineConfig]: model_default = str( section.get("model_default", base.model_default if base else "") ).strip() + result_parser = optional_string(section.get("result_parser")) + if result_parser not in {None, "claude-code-json"}: + raise ValueError( + f"engines.{clean_name}.result_parser must be 'claude-code-json'" + ) engines[clean_name] = EngineConfig( name=clean_name, bin=bin_path, @@ -1706,6 +1712,7 @@ def load_engines(raw: Any) -> dict[str, EngineConfig]: sandbox_args=sandbox_args, token_regex=token_regex, model_report_regex=model_report_regex, + result_parser=result_parser, model_default=model_default, ) return engines @@ -2205,6 +2212,8 @@ class WorkerResult: tokens: int | None error: str | None = None reported_model: str | None = None + scoreable: bool = True + failure_kind: str | None = None @dataclass(frozen=True) @@ -5726,7 +5735,7 @@ def log_message(self, _format: str, *_args: Any) -> None: url = f"http://127.0.0.1:{self.port}" if self.open_viewer: with contextlib.suppress(Exception): - webbrowser.open(url) + open_in_browser(url) print(f"Ringside: {url}", flush=True) return self.port @@ -6112,6 +6121,7 @@ def aggregate_model_log_rows( task_type: str | None = None, model: str | None = None, ) -> list[dict[str, Any]]: + rows = [row for row in rows if row.get("scoreable") is not False] groups: dict[tuple[str, str, str, str, bool], dict[str, Any]] = {} effort_keys = model_reasoning_effort_keys(rows) for task_rows in group_model_log_tasks(rows): @@ -6673,7 +6683,7 @@ def create_read_model_schema(conn: Any) -> None: row = conn.execute("SELECT version FROM schema_version LIMIT 1").fetchone() if row is not None: schema_version = int(row[0]) - needs_stamp = user_version != 3 or schema_version != 3 + needs_stamp = user_version != 4 or schema_version != 4 conn.executescript( """ CREATE TABLE IF NOT EXISTS schema_version ( @@ -6692,6 +6702,8 @@ def create_read_model_schema(conn: Any) -> None: reasoning_effort TEXT, task_type TEXT, retry INTEGER, + scoreable INTEGER, + failure_kind TEXT, verdict TEXT, duration_ms INTEGER, worker_tokens INTEGER, @@ -6752,6 +6764,10 @@ def create_read_model_schema(conn: Any) -> None: conn.execute("ALTER TABLE attempts ADD COLUMN reported_model TEXT") if not read_model_column_exists(conn, "attempts", "expected_model"): conn.execute("ALTER TABLE attempts ADD COLUMN expected_model TEXT") + if not read_model_column_exists(conn, "attempts", "scoreable"): + conn.execute("ALTER TABLE attempts ADD COLUMN scoreable INTEGER") + if not read_model_column_exists(conn, "attempts", "failure_kind"): + conn.execute("ALTER TABLE attempts ADD COLUMN failure_kind TEXT") if not read_model_column_exists(conn, "identity", "lab"): conn.execute("ALTER TABLE identity ADD COLUMN lab TEXT") if not read_model_column_exists(conn, "identity", "alias"): @@ -6762,8 +6778,8 @@ def create_read_model_schema(conn: Any) -> None: conn.executescript( """ DELETE FROM schema_version; - INSERT INTO schema_version(version) VALUES (3); - PRAGMA user_version = 3; + INSERT INTO schema_version(version) VALUES (4); + PRAGMA user_version = 4; """ ) @@ -6854,6 +6870,8 @@ def insert_attempt_rows(conn: Any, rows: list[dict[str, Any]]) -> int: model_log_row_reasoning_effort(row), model_log_text(row.get("task_type")), 1 if model_log_row_is_retry(row) else 0, + 0 if row.get("scoreable") is False else 1, + model_log_text(row.get("failure_kind")) or None, model_log_text(row.get("verdict")), model_log_int(row.get("duration_ms")), model_log_int(row.get("worker_tokens")), @@ -6866,9 +6884,9 @@ def insert_attempt_rows(conn: Any, rows: list[dict[str, Any]]) -> int: INSERT INTO attempts ( run_id, task_key, logged_at, engine, model, reported_model, expected_model, reasoning_effort, task_type, retry, - verdict, duration_ms, worker_tokens, orchestrator + scoreable, failure_kind, verdict, duration_ms, worker_tokens, orchestrator ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, payloads, ) @@ -7181,7 +7199,7 @@ def db_attempt_rows( with contextlib.closing(connect_read_model_db_readonly(db_path)) as conn: query = """ SELECT run_id, task_key, logged_at, engine, model, reported_model, expected_model, - reasoning_effort, task_type, retry, + reasoning_effort, task_type, retry, scoreable, failure_kind, verdict, duration_ms, worker_tokens, orchestrator FROM attempts """ @@ -7202,6 +7220,8 @@ def db_attempt_rows( "reasoning_effort": row["reasoning_effort"], "task_type": row["task_type"], "retry": bool(row["retry"]), + "scoreable": row["scoreable"] != 0, + "failure_kind": row["failure_kind"], "verdict": row["verdict"], "duration_ms": row["duration_ms"], "worker_tokens": row["worker_tokens"], @@ -7541,6 +7561,7 @@ def aggregate_model_scoreboard_rows( task_type: str | None = None, model: str | None = None, ) -> list[dict[str, Any]]: + rows = [row for row in rows if row.get("scoreable") is not False] models: dict[tuple[str, str, str, bool], dict[str, Any]] = {} effort_keys = model_reasoning_effort_keys(rows) for task_rows in group_model_log_tasks(rows): @@ -8879,10 +8900,18 @@ async def _run_task(self, runtime: TaskRuntime) -> None: worker = await self._run_worker(runtime, current_spec, attempt) with self.lock: runtime.worker_pid = None - runtime.status = "verifying" + runtime.status = "verifying" if worker.scoreable else "fail" if worker.tokens is not None: runtime.tokens = (runtime.tokens or 0) + worker.tokens - verify = await self.verifier.verify(runtime.task, runtime.taskdir) + if worker.scoreable: + verify = await self.verifier.verify(runtime.task, runtime.taskdir) + else: + verify = VerifyResult( + ok=False, + check_returncode=None, + check_timed_out=False, + raw_output_excerpt=worker.error or "non-scoreable worker result", + ) verdict = verdict_for(worker, verify) with self.lock: runtime.last_check_returncode = verify.check_returncode @@ -9089,16 +9118,23 @@ async def _record_prepare_error(self, runtime: TaskRuntime, error: str) -> None: async def _run_worker(self, runtime: TaskRuntime, spec: str, attempt: int) -> WorkerResult: log_path = runtime.log_path + + def finish(result: WorkerResult) -> WorkerResult: + if result.timed_out: + append_text(log_path, f"\n[ringer.py] worker timed out after {runtime.task.timeout_s}s\n") + append_text(log_path, f"[ringer.py] attempt {attempt} exited rc={result.returncode}\n") + return result + engine = self.config.engines.get(runtime.task.engine) if engine is None: - return WorkerResult( + return finish(WorkerResult( returncode=None, timed_out=False, tokens=None, error=f"unknown worker engine: {runtime.task.engine}", - ) + )) if runtime.task.full_access and not self.config.allow_full_access: - return WorkerResult( + return finish(WorkerResult( returncode=None, timed_out=False, tokens=None, @@ -9106,7 +9142,7 @@ async def _run_worker(self, runtime: TaskRuntime, spec: str, attempt: int) -> Wo f"task requested full_access with engine {runtime.task.engine}, " "but config allow_full_access is false" ), - ) + )) cmd = build_worker_command( engine, taskdir=runtime.taskdir, @@ -9183,7 +9219,7 @@ async def _run_worker(self, runtime: TaskRuntime, spec: str, attempt: int) -> Wo try: log_fh = log_path.open("ab") except OSError as exc: - return WorkerResult(returncode=None, timed_out=False, tokens=None, error=str(exc)) + return finish(WorkerResult(returncode=None, timed_out=False, tokens=None, error=str(exc))) async with AsyncFileCloser(log_fh): try: proc = await asyncio.create_subprocess_exec( @@ -9198,7 +9234,7 @@ async def _run_worker(self, runtime: TaskRuntime, spec: str, attempt: int) -> Wo message = f"[ringer.py] worker spawn failed: {exc}\n" log_fh.write(message.encode("utf-8", errors="replace")) log_fh.flush() - return WorkerResult(returncode=None, timed_out=False, tokens=None, error=str(exc)) + return finish(WorkerResult(returncode=None, timed_out=False, tokens=None, error=str(exc))) with self.lock: runtime.worker_pid = proc.pid self.active_processes[proc.pid] = proc @@ -9222,17 +9258,49 @@ async def _run_worker(self, runtime: TaskRuntime, spec: str, attempt: int) -> Wo await reader self.active_processes.pop(proc.pid, None) output_tail = capture.text() - tokens = parse_token_count(output_tail, engine.token_regex) - reported_model = parse_reported_model(output_tail, engine.model_report_regex) - if timed_out: - append_text(log_path, f"\n[ringer.py] worker timed out after {runtime.task.timeout_s}s\n") - append_text(log_path, f"[ringer.py] attempt {attempt} exited rc={proc.returncode}\n") - return WorkerResult( + if engine.result_parser == "claude-code-json": + structured = parse_claude_code_json_result( + output_tail, + runtime.task.model or engine.model_default, + ) + if structured is not None: + tokens = structured.tokens + reported_model = structured.reported_model + if structured.error: + append_text( + log_path, + f"[ringer.py] structured result: {structured.error}\n", + ) + if structured.failure_kind: + return finish(WorkerResult( + returncode=proc.returncode, + timed_out=timed_out, + tokens=tokens, + error=structured.error, + reported_model=reported_model, + scoreable=structured.scoreable, + failure_kind=structured.failure_kind, + )) + else: + error = "Claude Code structured output contained no result envelope" + append_text(log_path, f"[ringer.py] structured result: {error}\n") + return finish(WorkerResult( + returncode=proc.returncode, + timed_out=timed_out, + tokens=None, + error=error, + scoreable=False, + failure_kind="harness", + )) + else: + tokens = parse_token_count(output_tail, engine.token_regex) + reported_model = parse_reported_model(output_tail, engine.model_report_regex) + return finish(WorkerResult( returncode=proc.returncode, timed_out=timed_out, tokens=tokens, reported_model=reported_model, - ) + )) async def _tee_stream( self, @@ -9293,6 +9361,9 @@ def _log_attempt( ] if worker.error: notes_parts.append(f"worker_error={worker.error}") + notes_parts.append(f"scoreable={'true' if worker.scoreable else 'false'}") + if worker.failure_kind: + notes_parts.append(f"failure_kind={worker.failure_kind}") if verify.missing_files: notes_parts.append(f"missing_expect_files={json.dumps(list(verify.missing_files))}") notes_parts.append("raw_check_output_first_2000_chars:") @@ -9331,6 +9402,8 @@ def _log_attempt( "reasoning_effort": reasoning_effort, "task_type": runtime.task.task_type, "retry": retrying, + "scoreable": worker.scoreable, + "failure_kind": worker.failure_kind, } ) @@ -9525,6 +9598,145 @@ def parse_env_file(path: Path) -> dict[str, str]: return values +@dataclass(frozen=True) +class StructuredResult: + tokens: int | None + reported_model: str | None + error: str | None = None + scoreable: bool = True + failure_kind: str | None = None + + +def _last_json_result(text: str) -> dict[str, Any] | None: + candidates: list[dict[str, Any]] = [] + for line in reversed(text.splitlines()): + stripped = line.strip() + if not stripped: + continue + try: + value = json.loads(stripped) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and value.get("type") == "result": + candidates.append(value) + if candidates: + return candidates[0] + # Non-streaming `--output-format json` can pretty-print the single result + # object across multiple lines. Decode the whole trimmed payload rather + # than scanning for a `{` anywhere in the text: a scan can match braces + # inside a string field (e.g. the agent quoting JSON in its answer) and + # misattribute tokens/model/error to that embedded fragment. + stripped_text = text.strip() + if stripped_text: + try: + value = json.loads(stripped_text) + except json.JSONDecodeError: + return None + if isinstance(value, dict) and value.get("type") == "result": + return value + return None + + +def _claude_model_family(value: str) -> str | None: + lowered = value.strip().lower() + for family in ("sonnet", "opus", "fable", "haiku"): + if family in lowered: + return family + return None + + +def _claude_reported_model(model_usage: Any, requested_model: str) -> str | None: + if not isinstance(model_usage, dict): + return None + requested = requested_model.strip().lower() + requested_family = _claude_model_family(requested) + family_matches: list[str] = [] + for key, value in model_usage.items(): + key_text = str(key) + canonical = value.get("canonicalModel") if isinstance(value, dict) else None + candidates = [key_text, str(canonical or "")] + if requested in {candidate.lower() for candidate in candidates if candidate}: + return str(canonical or key_text) + if requested_family and any( + _claude_model_family(candidate) == requested_family for candidate in candidates + ): + family_matches.append(str(canonical or key_text)) + # No exact match: fall back to family only when every family match agrees + # on the same canonical model. modelUsage can list more than one entry of + # the same family (e.g. a director and a subagent both on Sonnet, but + # different versions) — guessing the first one risks misattributing a + # run's identity, which this project treats as an unforgivable bug class. + unique_family_matches = set(family_matches) + if len(unique_family_matches) == 1: + return next(iter(unique_family_matches)) + return None + + +def parse_claude_code_json_result( + text: str, requested_model: str = "" +) -> StructuredResult: + result = _last_json_result(text) + if result is None: + return StructuredResult( + tokens=None, + reported_model=None, + error="Claude Code structured output contained no result envelope", + scoreable=False, + failure_kind="harness", + ) + usage = result.get("usage") + tokens = None + if isinstance(usage, dict): + counters = ( + "input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "output_tokens", + ) + values: list[int] = [] + valid = True + for name in counters: + if name not in usage: + continue + value = usage[name] + if not isinstance(value, int) or isinstance(value, bool): + valid = False + break + values.append(value) + if valid and values: + tokens = sum(values) + reported_model = _claude_reported_model(result.get("modelUsage"), requested_model) + api_error_status = result.get("api_error_status") + # Only a transport/API-level failure is "infrastructure" (not the + # model's fault, excluded from the scoreboard, no verify/retry). An + # agent-level failure (max turns, tool/permission rejection, giving up) + # is still scoreable — it's a real quality outcome for that model on + # that task, and excluding it would hide it from the scoreboard instead + # of just recording a loss. + is_api_error = result.get("terminal_reason") == "api_error" or api_error_status is not None + if is_api_error: + status = api_error_status or result.get("status") or "unknown status" + message = result.get("message") or result.get("error") or result.get("result") or "unknown error" + return StructuredResult( + tokens=tokens, + reported_model=reported_model, + error=f"Claude Code API error ({status}): {message}", + scoreable=False, + failure_kind="infrastructure", + ) + if result.get("is_error") is True: + subtype = result.get("subtype") or result.get("terminal_reason") or "unknown" + message = result.get("message") or result.get("error") or result.get("result") or "unknown error" + return StructuredResult( + tokens=tokens, + reported_model=reported_model, + error=f"Claude Code agent error ({subtype}): {message}", + scoreable=True, + failure_kind="agent", + ) + return StructuredResult(tokens=tokens, reported_model=reported_model) + + def parse_token_count(text: str, token_regex: str | None = DEFAULT_TOKEN_REGEX) -> int | None: if token_regex: matches = list(re.finditer(token_regex, text, flags=re.IGNORECASE)) diff --git a/tests/test_claude_engine.py b/tests/test_claude_engine.py new file mode 100644 index 000000000..e7b84b2c4 --- /dev/null +++ b/tests/test_claude_engine.py @@ -0,0 +1,225 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from ringer import ( + AppConfig, + ArtifactConfig, + EngineConfig, + EvalConfig, + Manifest, + RingerRunner, + aggregate_model_log_rows, + parse_claude_code_json_result, + VerifyResult, + WorkerResult, +) + + +class ClaudeCodeResultTests(unittest.TestCase): + def envelope(self, requested: str) -> str: + return json.dumps( + { + "type": "result", + "usage": { + "input_tokens": 10, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + "output_tokens": 40, + }, + "modelUsage": { + "claude-haiku-4-5-20251001": {"canonicalModel": "claude-haiku-4-5"}, + requested: {"canonicalModel": requested}, + }, + "is_error": False, + } + ) + + def test_tokens_exclude_internal_title_model_and_match_requested_model(self) -> None: + result = parse_claude_code_json_result(self.envelope("claude-sonnet-5"), "sonnet") + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(100, result.tokens) + self.assertEqual("claude-sonnet-5", result.reported_model) + + def test_all_alias_families_match_the_requested_entry(self) -> None: + for alias, key in ( + ("sonnet", "claude-sonnet-5"), + ("opus", "claude-opus-5"), + ("fable", "claude-fable-5-1"), + ("haiku", "claude-haiku-4-5"), + ): + result = parse_claude_code_json_result(self.envelope(key), alias) + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(key, result.reported_model) + + def test_api_error_is_infrastructure_and_non_scoreable(self) -> None: + result = parse_claude_code_json_result( + json.dumps( + { + "type": "result", + "is_error": True, + "terminal_reason": "api_error", + "api_error_status": 429, + "message": "rate limit exceeded", + } + ), + "sonnet", + ) + self.assertIsNotNone(result) + assert result is not None + self.assertFalse(result.scoreable) + self.assertEqual("infrastructure", result.failure_kind) + self.assertIn("429", result.error or "") + self.assertIn("rate limit", result.error or "") + + def test_agent_error_without_api_markers_stays_scoreable(self) -> None: + result = parse_claude_code_json_result( + json.dumps( + { + "type": "result", + "is_error": True, + "subtype": "error_max_turns", + "message": "agent gave up after max turns", + } + ), + "sonnet", + ) + self.assertTrue(result.scoreable) + self.assertEqual("agent", result.failure_kind) + self.assertIn("error_max_turns", result.error or "") + + def test_embedded_json_in_result_string_is_not_mistaken_for_the_envelope(self) -> None: + # A single pretty-printed envelope whose own `result` text happens to + # quote something that looks like a JSON result object. The bracket + # scan this replaces would have grabbed the embedded fragment. + text = json.dumps( + { + "type": "result", + "usage": {"input_tokens": 1, "output_tokens": 1}, + "result": 'example: {"type": "result", "usage": {"output_tokens": 999}}', + "is_error": False, + }, + indent=2, + ) + result = parse_claude_code_json_result(text) + self.assertEqual(2, result.tokens) + + def test_ambiguous_same_family_model_usage_returns_none(self) -> None: + result = parse_claude_code_json_result( + json.dumps( + { + "type": "result", + "usage": {"input_tokens": 1, "output_tokens": 1}, + "modelUsage": { + "claude-sonnet-5": {"canonicalModel": "claude-sonnet-5"}, + "claude-sonnet-4-6": {"canonicalModel": "claude-sonnet-4-6"}, + }, + } + ), + "sonnet", + ) + self.assertIsNone(result.reported_model) + + def test_last_result_envelope_wins_over_diagnostic_object(self) -> None: + text = "\n".join([ + json.dumps({"type": "result", "usage": {"input_tokens": 2, "output_tokens": 3}}), + json.dumps({"type": "diagnostic", "message": "not a result"}), + json.dumps({"type": "result", "usage": {"input_tokens": 7, "output_tokens": 11}}), + ]) + result = parse_claude_code_json_result(text) + self.assertEqual(18, result.tokens) + + def test_missing_result_envelope_is_non_scoreable_harness_failure(self) -> None: + result = parse_claude_code_json_result(json.dumps({"type": "diagnostic"})) + self.assertFalse(result.scoreable) + self.assertEqual("harness", result.failure_kind) + + def test_usage_sums_present_counters_and_rejects_bool(self) -> None: + result = parse_claude_code_json_result(json.dumps({ + "type": "result", + "usage": {"input_tokens": 4, "output_tokens": 6}, + "modelUsage": {"sonnet": {"inputTokens": 9000}}, + })) + self.assertEqual(10, result.tokens) + rejected = parse_claude_code_json_result(json.dumps({ + "type": "result", "usage": {"input_tokens": True, "output_tokens": 6} + })) + self.assertIsNone(rejected.tokens) + + +class RunnerNonScoreableTests(unittest.IsolatedAsyncioTestCase): + async def test_non_scoreable_infrastructure_skips_verifier_and_retry(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + manifest = Manifest.from_obj({ + "run_name": "non-scoreable", + "workdir": str(root / "work"), + "tasks": [{ + "key": "task", + "spec": "unused", + "check": "exit 0", + "max_attempts": 3, + "engine": "fake", + }], + }) + engine = EngineConfig( + name="fake", bin="unused", args_template=("{spec}",), + full_access_args=(), sandbox_args=(), model_default="fake-model", + ) + config = AppConfig( + path=None, identity_default=None, state_dir=root / "state", + dashboard_port_base=8787, hud_port=8700, hud_app_path=None, + allow_full_access=False, + eval=EvalConfig(backend="jsonl", jsonl_path=root / "eval.jsonl"), + engines={"fake": engine}, + artifact=ArtifactConfig( + enabled=False, out_template=str(root / "live.html"), + report_template=str(root / "report.html"), index_out=root / "index.html", + ), + ) + runner = RingerRunner(manifest, config=config, identity="tester", dashboard_enabled=False) + calls = {"worker": 0, "verify": 0} + + async def fake_worker(runtime, spec, attempt): + calls["worker"] += 1 + return WorkerResult(1, False, None, error="network unavailable", scoreable=False, failure_kind="infrastructure") + + async def verify(*args, **kwargs): + calls["verify"] += 1 + return VerifyResult(True, 0, False, "should not run") + + runner._run_worker = fake_worker + runner.verifier.verify = verify + await runner._run_task(runner.runtimes[0]) + row = json.loads((root / "eval.jsonl").read_text(encoding="utf-8")) + self.assertEqual(1, calls["worker"]) + self.assertEqual(0, calls["verify"]) + self.assertEqual("ERROR", row["verdict"]) + self.assertFalse(row["scoreable"]) + self.assertEqual("infrastructure", row["failure_kind"]) + + def test_aggregation_excludes_explicitly_non_scoreable_rows_but_keeps_old_rows(self) -> None: + base = { + "run_id": "run", + "worker_engine": "claude", + "model": "claude-sonnet-5", + "task_type": "code-feature", + "retry": False, + "duration_ms": 1, + "worker_tokens": 100, + } + rows = [ + {**base, "task_key": "infra-only", "verdict": "ERROR", "scoreable": False}, + {**base, "task_key": "historical", "verdict": "PASS"}, + ] + groups = aggregate_model_log_rows(rows) + self.assertEqual(1, len(groups)) + self.assertEqual(1, groups[0]["tasks"]) + self.assertEqual(1, groups[0]["attempts"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_identity_evidence.py b/tests/test_identity_evidence.py index 0d8ed3ce0..b49d2cd6c 100644 --- a/tests/test_identity_evidence.py +++ b/tests/test_identity_evidence.py @@ -161,35 +161,36 @@ def log_attempts(self, *workers: WorkerResult) -> list[dict[str, object]]: runner._log_attempt(runtime, runtime.task.spec, False, worker, verify, "PASS", 10) return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines()] - def test_schema_v3_migration_preserves_v2_attempt(self) -> None: + def test_schema_v4_migration_preserves_v3_attempt(self) -> None: db = self.root / "ringer.db" with sqlite3.connect(db) as conn: conn.executescript( """ - PRAGMA user_version = 2; + PRAGMA user_version = 3; CREATE TABLE schema_version(version INTEGER NOT NULL); - INSERT INTO schema_version VALUES (2); + INSERT INTO schema_version VALUES (3); CREATE TABLE attempts ( id INTEGER PRIMARY KEY, run_id TEXT, task_key TEXT, logged_at TEXT, - engine TEXT, model TEXT, reasoning_effort TEXT, task_type TEXT, + engine TEXT, model TEXT, reported_model TEXT, expected_model TEXT, + reasoning_effort TEXT, task_type TEXT, retry INTEGER, verdict TEXT, duration_ms INTEGER, worker_tokens INTEGER, orchestrator TEXT ); - INSERT INTO attempts(model, verdict, reasoning_effort) - VALUES ('gpt-5.6-sol', 'PASS', 'high'); + INSERT INTO attempts(model, reported_model, expected_model, verdict, reasoning_effort) + VALUES ('gpt-5.6-sol', 'gpt-5.6-sol', 'gpt-5.6-sol', 'PASS', 'high'); """ ) create_read_model_schema(conn) columns = {row[1] for row in conn.execute("PRAGMA table_info(attempts)")} - self.assertTrue({"reported_model", "expected_model"}.issubset(columns)) + self.assertTrue({"reported_model", "expected_model", "scoreable", "failure_kind"}.issubset(columns)) self.assertEqual( - ("gpt-5.6-sol", "PASS", "high", None, None), + ("gpt-5.6-sol", "PASS", "high", "gpt-5.6-sol", "gpt-5.6-sol"), conn.execute( "SELECT model, verdict, reasoning_effort, reported_model, expected_model FROM attempts" ).fetchone(), ) - self.assertEqual(3, conn.execute("PRAGMA user_version").fetchone()[0]) - self.assertEqual(3, conn.execute("SELECT version FROM schema_version").fetchone()[0]) + self.assertEqual(4, conn.execute("PRAGMA user_version").fetchone()[0]) + self.assertEqual(4, conn.execute("SELECT version FROM schema_version").fetchone()[0]) conn.execute( """ INSERT INTO attempts(model, reported_model, expected_model, verdict) diff --git a/tests/test_model_db.py b/tests/test_model_db.py index 70f2ea71c..1b7a71988 100644 --- a/tests/test_model_db.py +++ b/tests/test_model_db.py @@ -180,8 +180,8 @@ def test_schema_creation_uses_wal_mode(self) -> None: conn.commit() self.assertEqual("wal", str(journal_mode).lower()) - self.assertEqual(3, user_version) - self.assertEqual(3, version) + self.assertEqual(4, user_version) + self.assertEqual(4, version) def test_rebuild_ingests_rows_and_counts_skipped_lines(self) -> None: write_jsonl( diff --git a/tests/test_model_log.py b/tests/test_model_log.py index 0a79ef9d9..6256e9f6f 100644 --- a/tests/test_model_log.py +++ b/tests/test_model_log.py @@ -118,6 +118,8 @@ def test_eval_row_carries_model_task_type_and_retry(self) -> None: self.assertEqual("openrouter/z-ai/glm-5.2", payload["model"]) self.assertEqual("code-feature", payload["task_type"]) self.assertIs(payload["retry"], True) + self.assertIs(payload["scoreable"], True) + self.assertIsNone(payload["failure_kind"]) self.assertIn("model=openrouter/z-ai/glm-5.2", payload["notes"]) self.assertIn("task_type=code-feature", payload["notes"]) self.assertIn("retry=true", payload["notes"]) diff --git a/tests/test_taxonomy.py b/tests/test_taxonomy.py index 27c60e633..7c0bfee83 100644 --- a/tests/test_taxonomy.py +++ b/tests/test_taxonomy.py @@ -296,8 +296,8 @@ def test_existing_database_migrates_reasoning_effort_without_data_loss(self) -> self.assertEqual(("gpt-5.5", "PASS", None), conn.execute( "SELECT model, verdict, reasoning_effort FROM attempts" ).fetchone()) - self.assertEqual(3, conn.execute("PRAGMA user_version").fetchone()[0]) - self.assertEqual(3, conn.execute("SELECT version FROM schema_version").fetchone()[0]) + self.assertEqual(4, conn.execute("PRAGMA user_version").fetchone()[0]) + self.assertEqual(4, conn.execute("SELECT version FROM schema_version").fetchone()[0]) if __name__ == "__main__":