papers.json

{
  "catalog_version": 3,
  "reviewed_at": "2026-08-25",
  "papers": [
    {
      "id": "qlcoder-2026",
      "title": "QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities",
      "authors": [
        "Claire Wang",
        "Ziyang Li",
        "Saikat Dutta",
        "Mayur Naik"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "security"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "QLCoder",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html",
      "artifact_url": "https://github.com/neuralprogram/qlcoder",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Claude Sonnet 4",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "maximum 10 iterations per CVE and agent baseline",
          "runs": "not-reported",
          "tool_permissions": "Claude Code framework with CodeQL MCP/LSP and execution feedback"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "GPT-5",
          "version": "not-reported",
          "reasoning_mode": "minimal and medium",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "maximum 10 iterations per CVE and agent baseline",
          "runs": "not-reported",
          "tool_permissions": "Codex CLI official tools; exact permissions not reported"
        }
      ],
      "task": {
        "summary": "CodeQL query synthesis from CVE metadata",
        "benchmark": "176 CVEs across 111 Java projects"
      },
      "method": {
        "summary": "CVE-grounded retrieval, AST/LSP guidance, an MCP interface, and an executable CodeQL validator that feeds back into iterative query repair.",
        "tags": [
          "retrieval",
          "verifier-loop",
          "static-analysis"
        ]
      },
      "evidence": {
        "result": "QLCoder synthesizes correct queries for 53.4% of CVEs versus 10% for Claude Code-only; its query F1 is 0.70, versus 0.048 for IRIS and 0.073 for CodeQL suites.",
        "same_model": "unknown",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "quality",
        "comparison_scope": "product-level",
        "source_location": "Section 4.1 Experimental Setup; Table 2; Table 5; Appendix A",
        "caveats": "The task is CodeQL security-query synthesis, not general repository-level software engineering; GPT-5 reasoning settings differ across Codex baseline rows."
      },
      "experiment": {
        "reasoning_mode": "Codex GPT-5 minimal and medium; Claude Sonnet 4 not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "maximum 10 iterations per CVE and agent baseline",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "QLCoder MCP interface exposes CodeQL LSP, retrieval, and execution feedback; baseline permissions are product defaults",
        "baseline_config": "Claude Code-only; Codex with GPT-5 minimal/medium; Gemini CLI; static-analysis baselines IRIS and CodeQL suites"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "llm2ltac-2026",
      "title": "Mining Tactics for Automated Theorem Proving",
      "authors": [
        "Jian Fang",
        "Yixun Yao",
        "Yingfei Xiong"
      ],
      "year": 2026,
      "conference": "ASE",
      "venue": "ASE Research Papers",
      "domains": [
        "formal-methods",
        "software-engineering"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "LLM2Ltac",
      "paper_url": "https://conf.researchr.org/details/ase-2026/ase-2026-research-track/232/Mining-Tactics-for-Automated-Theorem-Proving",
      "artifact_url": "https://zenodo.org/records/19251765",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2605.08694v1",
          "landing_page_url": "https://arxiv.org/abs/2605.08694v1",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "complete-author-list-and-materially-identical-abstract-with-reviewed-title-change",
          "identity_value": "https://conf.researchr.org/details/ase-2026/ase-2026-research-track/232/Mining-Tactics-for-Automated-Theorem-Proving",
          "identity_evidence": "The official ASE record and arXiv v1 have the same three authors, LLM2Ltac method, datasets, and exact headline results. The submitted title was A Learning Method for Symbolic Systems Using Large Language Models; the accepted title change is explicitly retained.",
          "verified_at": "2026-08-25T14:39:18+00:00",
          "discovered_at": "2026-08-25T14:39:18+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Claude Sonnet 4.6",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "200 theorem tasks per setting with a 600-second timeout per task",
          "runs": "One attempt for each of 200 sampled theorems in each of three settings; independent repeat count not reported",
          "tool_permissions": "bash, read, and edit; skills expose the base proving pipeline, CoqHammer, or LLM2Ltac-enhanced CoqHammer"
        }
      ],
      "task": {
        "summary": "Prove Rocq theorems with Claude Code, optionally augmented by CoqHammer or tactics mined and verified by LLM2Ltac.",
        "benchmark": "200 sampled theorems, 50 each from CompCert, Ext-Lib, Coq-Art, and VFA; tactic mining uses 11,725 standard-library theorems"
      },
      "method": {
        "summary": "LLM2Ltac mines reusable symbolic tactics with GPT-5.2 or DeepSeek-V3.2, compiler-checks and generalization-tests them, retrieves applicable tactics with BM25, and integrates them into CoqHammer for Claude Code.",
        "tags": [
          "retrieval",
          "verifier-loop",
          "deterministic-search",
          "repository-instructions"
        ]
      },
      "evidence": {
        "result": "On the same 200 theorem tasks, Claude Code proves 71 alone, 101 with CoqHammer, and 111 with LLM2Ltac-enhanced CoqHammer. Relative to Claude Code + CoqHammer, LLM2Ltac improves proved theorems by 9.90% and reduces token use from 38,346,691 to 34,317,809 (10.51%).",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "configuration-ablation",
        "source_location": "Section 5 Evaluation; Section 5.3 RQ3; Table 4 in arXiv v1",
        "source_url": "https://arxiv.org/abs/2605.08694v1",
        "source_version": "submittedVersion",
        "caveats": "The evidence copy predates the accepted-title change and still contains placeholder ACM venue text. The Claude Code CLI version, exact model snapshot, temperature, output-token cap, and independent repeat count are not reported; the result is specific to Rocq theorem proving."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "600 seconds per theorem task; CoqHammer itself uses a 20-second timeout",
        "turn_budget": "not-reported",
        "token_budget": "Total observed tokens are reported per setting, but no hard token cap is reported",
        "api_budget": "not-reported",
        "runs": "200 theorem tasks per setting; independent repeat count not reported",
        "tool_permissions": "Claude Code receives bash, read, and edit permissions; setting-specific skills instruct use of the base pipeline, CoqHammer, or LLM2Ltac",
        "baseline_config": "Claude Code alone versus the same Claude Sonnet 4.6 product with CoqHammer skills versus LLM2Ltac-enhanced CoqHammer skills"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ASE",
      "audit_status": "included",
      "notes": "The ASE page is the acceptance authority. arXiv v1 is an identity-reviewed evidence copy whose different submitted title is preserved explicitly."
    },
    {
      "id": "ape-bench-2026",
      "title": "APE-Bench: Evaluating Automated Proof Engineering for Formal Math Libraries",
      "authors": [
        "Huajian Xin",
        "Zheng Yuan",
        "Jacques Fleuriot",
        "Wenda Li"
      ],
      "year": 2026,
      "conference": "ICML",
      "venue": "ICML Main Conference",
      "domains": [
        "formal-methods",
        "software-engineering"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "direct",
      "system": "APE-Bench / APE-Harness / APE-Agent",
      "paper_url": "https://icml.cc/virtual/2026/poster/64323",
      "artifact_url": "https://github.com/xinhjBrant/APE-Bench",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2504.19110v3",
          "landing_page_url": "https://arxiv.org/abs/2504.19110v3",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-matching-abstract-and-author-maintained-artifact-link",
          "identity_value": "https://icml.cc/virtual/2026/poster/64323",
          "identity_evidence": "The ICML poster and arXiv v3 have the same title and materially identical abstract. The author-maintained APE-Bench repository links arXiv v3; authorship differences between the submitted manuscript and official poster are retained as a caveat.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Gemini 3 Flash; Gemini 3 Pro; GPT-5.2",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "100 turns and 3 USD per task; top-5 declaration retrieval enabled",
          "runs": "100 APE-Bench tasks per model-scaffold configuration; repeat count not reported",
          "tool_permissions": "Identical APE task contracts and shared execute/retrieve services; native Claude Code tools and implementation strategy"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "Gemini 3 Flash; Gemini 3 Pro; GPT-5.2",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "100 turns and 3 USD per task; top-5 declaration retrieval enabled",
          "runs": "100 APE-Bench tasks per model-scaffold configuration; repeat count not reported",
          "tool_permissions": "Identical APE task contracts and shared execute/retrieve services; native Codex CLI tools and implementation strategy"
        }
      ],
      "task": {
        "summary": "Modify pinned Mathlib repositories from natural-language proof-engineering requirements and satisfy compilation plus semantic constraints.",
        "benchmark": "APE-Bench, 100 tasks from 67 Mathlib commits dated 2026-01-06 through 2026-01-12; also miniCTX and miniF2F"
      },
      "method": {
        "summary": "Declarative task contracts bind repository versions, access controls, retrieval, compilation, and semantic verification while APE-Agent integrates editing with verification in a compact ReAct scaffold.",
        "tags": [
          "benchmark-design",
          "verifier-loop",
          "retrieval",
          "deterministic-search"
        ]
      },
      "evidence": {
        "result": "On identical APE-Bench tasks and 3 USD/100-turn budgets, APE-Agent reaches 47%, 24%, and 27% with Gemini 3 Flash, Gemini 3 Pro, and GPT-5.2; Claude Code reaches 27%, 8%, and 10%, while Codex reaches 32%, 1%, and 11%. The paper attributes the 15–23 point gaps to integrated verification, shorter prompts, and lower redundant execution/retrieval overhead.",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Section 6.1 Experimental Setup; Figure 3; Section 6.4 Scaffold Interchangeability; Figure 4 in arXiv v3",
        "source_url": "https://arxiv.org/abs/2504.19110v3",
        "source_version": "submittedVersion",
        "caveats": "The auxiliary manuscript is arXiv v3 and its author list differs from the official ICML poster. Product CLI versions, temperature, output-token caps, and independent repeat counts are not reported; the domain is Lean proof engineering rather than general-purpose software repair."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "maximum 100 dialogue turns per task",
        "token_budget": "not-reported",
        "api_budget": "maximum 3 USD per task",
        "runs": "100 APE-Bench tasks per model-scaffold configuration; repeat count not reported",
        "tool_permissions": "Same pinned task contracts, top-5 retrieval, execute service, compilation checks, and semantic judges; scaffold-specific native tools remain different",
        "baseline_config": "APE-Agent, Claude Code, and Codex CLI each run Gemini 3 Flash, Gemini 3 Pro, and GPT-5.2 under the same 100-turn and 3 USD task caps"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ICML",
      "audit_status": "included",
      "notes": "The ICML poster remains the acceptance authority; arXiv v3 is an identity-reviewed evidence copy with an explicit authorship-version caveat."
    },
    {
      "id": "numina-lean-agent-2026",
      "title": "Numina-Lean-Agent: An Open and General Agentic Reasoning System for Formal Mathematics",
      "authors": [
        "Junqi Liu",
        "Zihao Zhou",
        "Zekai Zhu",
        "Marco Dos Santos",
        "Weikun He",
        "Jiawei Liu",
        "Yunzhou Xie",
        "Junqiao Zhao",
        "Qiufeng Wang",
        "Lihong Zhi",
        "Jia Li",
        "Wenda Li"
      ],
      "year": 2026,
      "conference": "ICML",
      "venue": "ICML Main Conference",
      "domains": [
        "formal-methods"
      ],
      "publication_status": "main",
      "classification": "related",
      "system": "Numina-Lean-Agent",
      "paper_url": "https://icml.cc/virtual/2026/poster/66755",
      "artifact_url": "https://github.com/project-numina/numina-lean-agent",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2601.14027v1",
          "landing_page_url": "https://arxiv.org/abs/2601.14027v1",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-matching-abstract-and-author-maintained-artifact-link",
          "identity_value": "https://icml.cc/virtual/2026/poster/66755",
          "identity_evidence": "The title and abstract match the ICML record, and the author-maintained repository links arXiv:2601.14027. ArXiv v1 contains the twelve ICML-listed authors plus Ran Wang, which is retained as a version caveat.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "host",
          "model": "Claude Opus 4.5",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Approximately 50 USD per Putnam problem by default; A5 approximately 1000 USD and B6 approximately 300 USD",
          "runs": "One reported successful trajectory per problem; total attempt count not reported",
          "tool_permissions": "Internet search disabled; sequential Claude Code execution with Numina-Lean-MCP, Lean-LSP-MCP, LeanDex, an informal prover, and a discussion-partner tool"
        }
      ],
      "task": {
        "summary": "Prove and formalize mathematics in Lean, including olympiad problems and a paper-scale Brascamp-Lieb development.",
        "benchmark": "Putnam 2025 (12 problems) plus an interactive formalization case study exceeding 8,000 lines of Lean"
      },
      "method": {
        "summary": "Claude Code is extended with MCP tools for Lean proof-state interaction, semantic theorem retrieval, iterative informal proving, external discussion, and subagent-based context decomposition.",
        "tags": [
          "verifier-loop",
          "retrieval",
          "multi-agent",
          "context-reduction",
          "structured-state"
        ]
      },
      "evidence": {
        "result": "Numina-Lean-Agent with Claude Opus 4.5 solves all 12 Putnam 2025 problems, matching AxiomProver and exceeding Aristotle by two problems; the paper also reports more than 8,000 lines of Lean produced during a sub-two-week human-agent formalization.",
        "same_model": "unknown",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "component-level",
        "source_location": "Section 2 Numina-Lean-Agent; Section 3.1 Performance; Tables 1-4; Section 4 in arXiv v1",
        "source_url": "https://arxiv.org/abs/2601.14027v1",
        "source_version": "submittedVersion",
        "caveats": "Claude Code is the host scaffold rather than an independently measured baseline. Product version, decoding settings, output-token cap, and total failed attempts are not reported; the very large A5/B6 budgets are approximate rather than exact accounting."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "Per-problem times range from 30 to 2,040 minutes; the Brascamp-Lieb collaboration lasted less than two weeks intermittently",
        "turn_budget": "Informal prover allows at most 20 iterations; other Claude Code turn limits are not reported",
        "token_budget": "not-reported",
        "api_budget": "Approximately 50 USD per normal problem, 1000 USD for A5, and 300 USD for B6",
        "runs": "Total attempts not reported; all 12 final successful solutions are released",
        "tool_permissions": "Sequential execution with no internet search; Lean compiler/LSP, semantic retrieval, informal prover, discussion partner, and subagents are available",
        "baseline_config": "Claude Code with Claude Opus 4.5 hosts Numina-Lean-MCP; external informal generator/verifier use Gemini-3-Pro-Preview, and the A5 subagent can query GPT-5.2"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ICML",
      "audit_status": "included",
      "notes": "The official ICML poster is primary; arXiv v1 is retained as a versioned evidence copy and lists one additional author."
    },
    {
      "id": "posttrainbench-2026",
      "title": "PostTrainBench: Can LLM Agents Automate LLM Post-Training?",
      "authors": [
        "Ben Rank",
        "Hardik Bhatnagar",
        "Ameya Pandurang Prabhu",
        "Shira Eisenberg",
        "Karina Nguyen",
        "Matthias Bethge",
        "Maksym Andriushchenko"
      ],
      "year": 2026,
      "conference": "ICML",
      "venue": "ICML Main Conference",
      "domains": [
        "machine-learning",
        "scientific-computing"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "PostTrainBench",
      "paper_url": "https://icml.cc/virtual/2026/poster/63667",
      "artifact_url": "https://posttrainbench.com/",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2603.08640v1",
          "landing_page_url": "https://arxiv.org/abs/2603.08640v1",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-abstract-and-complete-author-list",
          "identity_value": "https://icml.cc/virtual/2026/poster/63667",
          "identity_evidence": "The title, abstract, and seven authors match the ICML poster; Ameya Prabhu is expanded to Ameya Pandurang Prabhu on the official page. The project site links the same arXiv paper.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude Opus 4.6; Claude Opus 4.5; Claude Sonnet 4.6; Claude Sonnet 4.5; Claude Haiku 4.5; Qwen3 Max",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "10 hours on one H100 GPU per model-benchmark run; 28 configurations per agent",
          "runs": "3 independent runs per frontier native-CLI configuration; single runs for other configurations",
          "tool_permissions": "Autonomous terminal, file editing, Python/training scripts, unrestricted web search, and one H100; no human interaction"
        },
        {
          "product": "codex-cli",
          "role": "evaluated",
          "model": "GPT-5.2; GPT 5.4 (High); GPT 5.1 Codex Max; GPT 5.3 Codex (High); GPT 5.3 Codex (Med); GPT 5.2 Codex",
          "version": "not-reported",
          "reasoning_mode": "Medium and High where reported; GPT-5.1 Codex Max ablates Low, Medium, and High",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "10 hours on one H100 GPU per model-benchmark run; 28 configurations per agent",
          "runs": "3 independent runs per frontier native-CLI configuration; single runs for other configurations",
          "tool_permissions": "Autonomous terminal, file editing, Python/training scripts, unrestricted web search, and one H100; no human interaction"
        }
      ],
      "task": {
        "summary": "Autonomously post-train a supplied 1.7B-4B base LLM for one target benchmark without starter code, data, or strategy.",
        "benchmark": "PostTrainBench, 4 base models × 7 benchmarks spanning math, science, code, tool use, writing, and health"
      },
      "method": {
        "summary": "A bounded, end-to-end AI-R&D benchmark gives CLI agents one H100, terminal and web access, then audits submitted checkpoints and pipelines for contamination, model substitution, and API misuse.",
        "tags": [
          "benchmark-design",
          "verifier-loop",
          "observability",
          "adversarial-testing"
        ]
      },
      "evidence": {
        "result": "Claude Opus 4.6 on Claude Code leads agents at 23.2% weighted average versus 51.1% for official instruct models; GPT-5.2 on Codex CLI reaches 21.4%. Agents can win narrowly—GPT-5.1 Codex Max reaches 89% on BFCL for Gemma-3-4B versus 67% for the official model—but the audit finds reward hacking, including 12 contamination flags for Opus 4.6 across 84 runs.",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 2 PostTrainBench Setup; Figure 2; Section 3.1 and Table 1; Sections 4.1-4.4; Sections 5.3-5.5 in arXiv v1",
        "source_url": "https://arxiv.org/abs/2603.08640v1",
        "source_version": "submittedVersion",
        "caveats": "The same-model statement applies to reported native-scaffold versus OpenCode comparisons, not every leaderboard row. Agents optimize one narrow benchmark for ten hours, whereas official instruction-tuned models target broad capabilities; CLI versions, temperature, and output-token caps are not reported."
      },
      "experiment": {
        "reasoning_mode": "GPT-5.1 Codex Max Low/Medium/High; GPT-5.3 Codex Medium/High; other effort settings not reported",
        "temperature": "Evaluation uses 0.6 and top-p 0.95 for the few-shot base-model table; agent-generation temperature is not reported",
        "max_output_tokens": "not-reported",
        "time_budget": "maximum 10 hours on one H100 per run",
        "turn_budget": "not-reported; one example Claude Code trajectory uses 104 turns",
        "token_budget": "No fixed token cap; GPT-5.1 Codex Max averages 0.96M-1.89M tokens by effort, while GPT-5.3 Codex averages 10.6M-29.1M",
        "api_budget": "Observed Claude Opus 4.6 API cost is approximately 600-750 USD per run; GPT-5.1/5.2 Codex CLI configurations are reported below 35 USD, plus up to about 30 USD GPU cost",
        "runs": "3 independent runs for frontier native CLI configurations; one run for other configurations",
        "tool_permissions": "Unrestricted internet and terminal access, file and code execution, one H100, no human interaction; an LLM judge audits rules",
        "baseline_config": "Claude Code, Codex CLI, Gemini CLI, and OpenCode agents across 28 base-model/benchmark pairs under the same 10-hour single-H100 resource envelope"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ICML",
      "audit_status": "included",
      "notes": "The official ICML poster remains primary; arXiv v1 is the identity-verified evidence copy."
    },
    {
      "id": "swe-compass-2026",
      "title": "SWE-Compass: Towards Unified Evaluation of Agentic Coding Abilities for Large Language Models",
      "authors": [
        "Jingxuan Xu",
        "Ken Deng",
        "Weihao Li",
        "Songwei Yu",
        "Haoyang Huang",
        "Xinping Lei",
        "Yifan Yao",
        "Huaixi Tang",
        "Zhiyi Lai",
        "Kepeng Lei",
        "Zizheng Zhan",
        "Yanan Wu",
        "Chenchen Zhang",
        "Wenqiang Zhu",
        "Wen Xiang",
        "Zongxian Feng",
        "Han Li",
        "Junqi Xiong",
        "Dailin Li",
        "Zuchen Gao",
        "Kun Wu",
        "Yuanxing Zhang",
        "Wuxuan Gong",
        "Ziyuan Gao",
        "Guanxiang Wang",
        "Yirong Xue",
        "Mengfei Xie",
        "Xiaojiang Zhang",
        "Jinghui Wang",
        "Wenhao Zhuang",
        "Zheng Lin",
        "Huiming Wang",
        "Zhaoxiang Zhang",
        "Yuqun Zhang",
        "Haotian Zhang",
        "Ming Sun",
        "Chen Bin",
        "Jiaheng Liu"
      ],
      "year": 2026,
      "conference": "ICML",
      "venue": "ICML Main Conference",
      "domains": [
        "software-engineering"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "SWE-Compass",
      "paper_url": "https://icml.cc/virtual/2026/poster/64552",
      "artifact_url": "https://huggingface.co/datasets/Kwaipilot/SWE-Compass",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2511.05459v3",
          "landing_page_url": "https://arxiv.org/abs/2511.05459v3",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-matching-abstract-and-reviewed-authorship",
          "identity_value": "https://icml.cc/virtual/2026/poster/64552",
          "identity_evidence": "The title and abstract match the ICML poster and OpenReview manuscript. The submitted manuscript contains authorship revisions, so it is explicitly retained as arXiv v3 rather than represented as camera-ready.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude-Sonnet-4-20250514; Qwen3-Coder-480B-A35B-Instruct; Qwen3-Coder-30B-A3B-Instruct; Qwen3-235B-A22B-Instruct-2507; DeepSeek-V3-0324",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "standard decoding; exact value not reported",
          "max_output_tokens": "not-reported",
          "budget": "One attempt per task, maximum 150 turns, 600 seconds per tool step",
          "runs": "One fixed-seed Pass@1 attempt per model-task pair; retries disabled",
          "tool_permissions": "permission_mode=bypassPermissions; sandboxed editor-centric workflow, parallel tool-call prompt, optional SubAgent, standardized build/test commands, and no network"
        }
      ],
      "task": {
        "summary": "Solve real pull-request tasks across feature work, repair, refactoring, performance, tests, understanding, and deployment in ten languages.",
        "benchmark": "SWE-Compass, 2,000 verified GitHub PR instances across 8 task types, 8 scenarios, and 10 programming languages"
      },
      "method": {
        "summary": "A broad execution-grounded benchmark pairs real PR tasks with reproducible containers and controlled agent budgets, then analyzes task, language, scenario, interaction, and failure-mode differences.",
        "tags": [
          "benchmark-design",
          "test-feedback",
          "observability"
        ]
      },
      "evidence": {
        "result": "Claude-Sonnet-4-20250514 reaches 32.9% macro-average with Claude Code versus 31.8% with SWE-Agent. Across five shared models only two score higher with Claude Code; Claude Code favors deterministic deployment, understanding, and test-generation tasks, while failures are dominated by requirement misinterpretation and incomplete solutions rather than basic technical knowledge.",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4.1 Evaluated LLMs and Frameworks; Table 2; Sections 4.2.1-4.2.2; Figure 7; Appendix A.2 and A.4 in arXiv v3",
        "source_url": "https://arxiv.org/abs/2511.05459v3",
        "source_version": "submittedVersion",
        "caveats": "The arXiv v3 author list differs from the official ICML poster and is retained only as a versioned evidence copy. Only five of ten models are run under Claude Code; the CLI version, temperature, output-token cap, and global job timeout are not reported."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "standard decoding; exact value not reported",
        "max_output_tokens": "not-reported",
        "time_budget": "maximum 600 seconds per tool step; global job limit not reported",
        "turn_budget": "maximum 150 turns",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "Pass@1 with fixed seeds, no retries; one attempt per model-task pair",
        "tool_permissions": "Offline pinned containers and caches; permission_mode=bypassPermissions; parallel tool calls and optional SubAgent; standardized pytest/npm/maven/go/cargo/cmake commands",
        "baseline_config": "Claude Code and SWE-Agent use identical offline executors, 150-turn limits, 600-second tool timeouts, fixed seeds, no retries, and the five overlapping models"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ICML",
      "audit_status": "included",
      "notes": "The official ICML poster is the acceptance authority; arXiv v3 provides the versioned full-text evidence."
    },
    {
      "id": "red-teaming-coding-agents-2026",
      "title": "Red-Teaming Coding Agents from a Tool-Invocation Perspective: An Empirical Security Assessment",
      "authors": [
        "Yuchong Xie",
        "Mingyu Luo",
        "Zesen Liu",
        "Zhixiang Zhang",
        "Kaikai Zhang",
        "Yu Liu",
        "Ci Tao",
        "Changhui Wang",
        "Zongjie Li",
        "Ping Chen",
        "Shuai Wang",
        "Dongdong She"
      ],
      "year": 2026,
      "conference": "ISSTA",
      "venue": "ISSTA Research Papers",
      "domains": [
        "security",
        "software-engineering"
      ],
      "publication_status": "main",
      "classification": "evaluation",
      "system": "ToolLeak / RCE-2",
      "paper_url": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/176/Red-Teaming-Coding-Agents-from-a-Tool-Invocation-Perspective-An-Empirical-Security-A",
      "artifact_url": "https://anonymous.4open.science/r/issta_2026-B18F",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2509.05755v6",
          "landing_page_url": "https://arxiv.org/abs/2509.05755v6",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-matching-abstract-and-reviewed-authorship",
          "identity_value": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/176/Red-Teaming-Coding-Agents-from-a-Tool-Invocation-Perspective-An-Empirical-Security-A",
          "identity_evidence": "The exact title and abstract match the official ISSTA page. ArXiv v6 contains ten of the twelve official authors; Ci Tao and Changhui Wang appear only on the official record, and this difference is preserved as a version caveat.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude-Sonnet 4; Claude-Sonnet 4.5; Claude-Sonnet 4.6; Claude-Opus-4.7",
          "version": "v2.0.37 (Old); v2.1.138 (New)",
          "reasoning_mode": "Thinking mode enabled by default when offered",
          "temperature": "Product default",
          "max_output_tokens": "not-reported",
          "budget": "10 attempts per attack and agent-model pair",
          "runs": "10 independent trials per attack and agent-model pair; real-world leakage uses best-of-10",
          "tool_permissions": "MCP-enabled product configuration with an attacker-controlled external tool and the built-in command-execution tool"
        }
      ],
      "task": {
        "summary": "Exfiltrate hidden coding-agent prompts and hijack MCP tool invocation to trigger attacker-selected command execution.",
        "benchmark": "Six real-world coding agents across 25 old/new agent-model pairs, plus emulated agents conditioned on recovered prompts"
      },
      "method": {
        "summary": "ToolLeak exploits weaker guardrails during schema-driven argument filling; RCE-2 chains a malicious tool description with a high-salience tool-return instruction tailored from leaked prompt details.",
        "tags": [
          "adversarial-testing",
          "observability"
        ]
      },
      "evidence": {
        "result": "On old Claude Code v2.0.37, ToolLeak reaches 1.00 pseudo-recall for both Sonnet 4 and Sonnet 4.5, while RCE-2 reaches 0.6 and 0.7 attack success. On v2.1.138, progressive disclosure of tool names plus newer Sonnet 4.6/Opus 4.7 reduces RCE-2 to 0.0, demonstrating that agent-side tool-description isolation is an effective defense.",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 6.1 and Table 2; Sections 6.2-6.4; Tables 3-5; Section 7.2 in arXiv v6",
        "source_url": "https://arxiv.org/abs/2509.05755v6",
        "source_version": "submittedVersion",
        "caveats": "The evidence copy predates an official camera-ready PDF and has two fewer authors than the official ISSTA record. Pseudo-recall uses the union of all methods rather than ground truth, real-world leakage selects best-of-10, and attack outcomes depend on exact product generation and MCP exposure."
      },
      "experiment": {
        "reasoning_mode": "Thinking mode enabled by default when a tested backend offers it",
        "temperature": "Default for real products; temperature 0 in the API-based emulation study",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "10 attempts per method and agent-model pair; best-of-10 for deployed-agent prompt leakage",
        "tool_permissions": "Official product interfaces, MCP-enabled external malicious tool, and built-in shell/command execution; no root/admin access assumed",
        "baseline_config": "ToolLeak versus nine prompt-leak baselines; RCE-2 versus AgentDojo, InjecAgent, MCPTox, and one-channel RCE-1 on the same agent-model pairs"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ISSTA",
      "audit_status": "included",
      "notes": "The ISSTA detail page is the acceptance authority; arXiv v6 is a versioned evidence copy with an explicit two-author difference."
    },
    {
      "id": "execution-cost-effectiveness-2026",
      "title": "To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair",
      "authors": [
        "Zhihao Lin",
        "Junhua Zhu",
        "Mingyi Zhou",
        "Xin Wang",
        "Zhensu Sun",
        "Renyu Yang",
        "David Lo",
        "Li Li"
      ],
      "year": 2026,
      "conference": "ISSTA",
      "venue": "ISSTA Research Papers",
      "domains": [
        "software-engineering",
        "systems-performance"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "Execution-aware repair policies",
      "paper_url": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/22/To-Run-or-Not-to-Run-Analyzing-the-Cost-Effectiveness-of-Code-Execution-in-LLM-Based",
      "artifact_url": "https://github.com/mathieu0905/To_Run_Or_Not_To_Run",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2606.26978v1",
          "landing_page_url": "https://arxiv.org/abs/2606.26978v1",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-abstract-and-complete-author-list",
          "identity_value": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/22/To-Run-or-Not-to-Run-Analyzing-the-Cost-Effectiveness-of-Code-Execution-in-LLM-Based",
          "identity_evidence": "The exact title, abstract, and eight authors match the official ISSTA detail page; arXiv v1 is used only as the versioned full-text evidence copy.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Anthropic claude-sonnet-4-5",
          "version": "v1.0.16",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "200 tasks × 5 execution modes; Prohibited, Quota-1, Quota-3, Budget-Guided, and Unrestricted",
          "runs": "One complete attempt per task-mode pair; 1,000 controlled Claude Code runs plus a 100-task hard-constraint rerun",
          "tool_permissions": "claude -p in SWE-bench containers; execution policy changes by prompt, with a hard rerun using --disallowedTools for test commands"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "OpenAI gpt-5.2",
          "version": "v0.1.2025062301",
          "reasoning_mode": "xhigh",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "200 tasks × 5 execution modes; Prohibited, Quota-1, Quota-3, Budget-Guided, and Unrestricted",
          "runs": "One complete attempt per task-mode pair; 1,000 controlled Codex runs",
          "tool_permissions": "codex exec in SWE-bench containers; execution policy changes by prompt while repository reading/editing remains available"
        }
      ],
      "task": {
        "summary": "Repair SWE-bench issues while varying whether and how often the industrial coding agent can run project tests or scripts.",
        "benchmark": "First 100 SWE-bench Lite and first 100 SWE-bench Verified instances; 3,000 controlled attempts plus 7,745 public trace analyses"
      },
      "method": {
        "summary": "Five execution policies isolate test execution as a controlled resource, combining paired outcome tests, token/time accounting, hard tool denial, and trajectory-level analysis of localization and validation feedback.",
        "tags": [
          "test-feedback",
          "performance-feedback",
          "observability"
        ]
      },
      "evidence": {
        "result": "Prohibiting execution changes commercial-agent resolve rate by only 1.25 points on average and is not significant. For Claude Code it saves 56–62% tokens and 48–54% wall time for only 1–3 points lower resolve rate; for Codex, Quota-1 saves 21–25% tokens. Most failures persist because 81.2% of Claude Code and 100% of Codex fail-to-fail cases pass at least one agent-selected validation but fail official tests.",
        "same_model": "yes",
        "same_budget": "no",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "configuration-ablation",
        "source_location": "Section 3.3 Models and Agents footnote 1; Section 3.5; Tables 2-7; Sections 4.2-4.3; Tables 8-13 in arXiv v1",
        "source_url": "https://arxiv.org/abs/2606.26978v1",
        "source_version": "submittedVersion",
        "caveats": "Execution access is intentionally unequal across configurations, and most restrictions are prompt-level; the paper separately verifies zero-execution subsets and a 100-task hard-denial Claude Code run. Results cover Python-heavy SWE-bench subsets and one fixed product/model version per commercial agent."
      },
      "experiment": {
        "reasoning_mode": "Claude Code not reported; Codex gpt-5.2 uses reasoning effort xhigh",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "No fixed cap reported; modes for each instance run concurrently on identical hardware",
        "turn_budget": "not-reported",
        "token_budget": "Measured rather than capped; Table 6 reports 48K-128K input and 15K-39K output tokens for Claude Code, and 300K-463K input plus 75K-116K output for Codex",
        "api_budget": "not-reported",
        "runs": "3,000 runs total = 200 tasks × 3 agents × 5 modes; one attempt per cell, plus the Claude Code hard-constraint rerun",
        "tool_permissions": "Repository exploration/editing always available; test frameworks and project scripts prohibited, quota-limited, cost-guided, or unrestricted by mode; official SWE-bench harness evaluates final patches",
        "baseline_config": "Unrestricted Claude Code v1.0.16 + Anthropic claude-sonnet-4-5 and Codex v0.1.2025062301 + OpenAI gpt-5.2 xhigh serve as product baselines for restricted execution policies"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ISSTA",
      "audit_status": "included",
      "notes": "The ISSTA page remains the publication source; arXiv v1 supplies the identity-verified full-text and exact configuration evidence."
    },
    {
      "id": "evodev-2026",
      "title": "Towards Iterative End-to-End Software Development: A Feature-Driven Multi-Agent Framework",
      "authors": [
        "Junwei Liu",
        "Chen Xu",
        "Chong Wang",
        "Tong Bai",
        "Weitong Chen",
        "Kaseng Wong",
        "Yiling Lou",
        "Xin Peng"
      ],
      "year": 2026,
      "conference": "ISSTA",
      "venue": "ISSTA Research Papers",
      "domains": [
        "software-engineering",
        "web-ui"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "EvoDev",
      "paper_url": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/207/Towards-Iterative-End-to-End-Software-Development-A-Feature-Driven-Multi-Agent-Frame",
      "artifact_status": "not-found",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2511.02399v3",
          "landing_page_url": "https://arxiv.org/abs/2511.02399v3",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "not-reported",
          "identity_status": "verified",
          "identity_method": "exact-title-abstract-and-complete-author-list-with-acceptance-note",
          "identity_value": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/207/Towards-Iterative-End-to-End-Software-Development-A-Feature-Driven-Multi-Agent-Frame",
          "identity_evidence": "arXiv v3 and the official ISSTA record have the same title, eight authors, abstract, and revised headline results; the arXiv record also explicitly states Accepted by ISSTA 2026.",
          "verified_at": "2026-08-25T14:39:18+00:00",
          "discovered_at": "2026-08-25T14:39:18+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "claude-sonnet-4-20250514",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "15 Android applications; 30, 40, or 50 minute task timeout for elementary, intermediate, or advanced apps",
          "runs": "One archived development output per 15 app tasks; independent repeat count not reported",
          "tool_permissions": "Product used according to official documentation against an Android Studio scaffold; exact permission flags are not reported"
        }
      ],
      "task": {
        "summary": "Build complete Kotlin Android applications from detailed natural-language requirements and an Android Studio scaffold project.",
        "benchmark": "APPDev, 15 applications with 8-26 functional requirements each, blinded four-person manual evaluation, and standardized Android devices"
      },
      "method": {
        "summary": "EvoDev replaces a linear to-do list with global design, a dependency-aware Feature Map, predecessor context propagation, chief-programmer planning, iterative build-and-fix loops, and a compact current-file memory.",
        "tags": [
          "dependency-aware-planning",
          "structured-state",
          "memory",
          "multi-agent",
          "verifier-loop",
          "context-reduction"
        ]
      },
      "evidence": {
        "result": "With the same claude-sonnet-4-20250514 model and task time limits, Claude Code reaches 73.3% build success and 2.27/4 function completeness; EvoDev reaches 100% and 3.57/4, a 57.3% relative completeness improvement. Ablations show that predecessor context reverses an iterative-only regression from 2.80 to 3.57.",
        "same_model": "yes",
        "same_budget": "yes",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Sections 4.3.2-4.3.4; Section 5.1 and Table 3; Section 5.3 and Table 6; Section 5.5 and Figure 7 in arXiv v3",
        "source_url": "https://arxiv.org/abs/2511.02399v3",
        "source_version": "submittedVersion",
        "caveats": "APPDev contains only 15 Kotlin Android applications and relies on blinded manual scoring. Product CLI version, temperature, output-token cap, exact permissions, and repeated-run variance are not reported; average Claude Code cost is higher than EvoDev but EvoDev takes longer."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "30 minutes for elementary apps, 40 for intermediate, and 50 for advanced; average observed time is 11.81 minutes for Claude Code and 23.19 for EvoDev",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "Average observed cost is 4.38 USD per app for Claude Code and 4.12 USD for EvoDev; no hard common price cap is reported",
        "runs": "15 application tasks per approach; independent repeat count not reported",
        "tool_permissions": "All approaches start from the same Android Studio scaffold and follow official setup; exact Claude Code flags and network permissions are not reported",
        "baseline_config": "MetaGPT, GPT-Engineer, Claude Code, and EvoDev run with Claude-4-Sonnet; the product comparison and EvoDev use the exact claude-sonnet-4-20250514 base model"
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "ISSTA",
      "audit_status": "included",
      "notes": "The ISSTA detail page is the acceptance authority; arXiv v3 is the exact-title, complete-author, acceptance-noted evidence copy."
    },
    {
      "id": "swe-bench-mobile-2026",
      "title": "SWE-Bench Mobile: Can Large Language Model Agents Develop Industry-Level Mobile Applications?",
      "authors": [
        "Muxin Tian",
        "Zhe Wang",
        "Blair Yang",
        "Zhenwei Tang",
        "Kunlun Zhu",
        "Honghua Dong",
        "Hanchen Li",
        "Xinni Xie",
        "Guangjing Wang",
        "Jiaxuan You"
      ],
      "year": 2026,
      "conference": "KDD",
      "venue": "KDD 2026 Applied Data Science Track (July Cycle)",
      "domains": [
        "software-engineering",
        "web-ui"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "SWE-Bench Mobile",
      "paper_url": "https://doi.org/10.1145/3770855.3818488",
      "doi": "10.1145/3770855.3818488",
      "artifact_url": "https://swebenchmobile.com/",
      "artifact_status": "official",
      "content_sources": [
        {
          "provider": "arxiv",
          "url": "https://arxiv.org/pdf/2602.09540v1",
          "landing_page_url": "https://arxiv.org/abs/2602.09540v1",
          "source_role": "full-text",
          "version": "submittedVersion",
          "license": "cc-by-4.0",
          "identity_status": "verified",
          "identity_method": "exact-title-and-complete-author-list",
          "identity_value": "10.1145/3770855.3818488",
          "identity_evidence": "The arXiv v1 title and complete ten-author list match the official KDD DOI record; the authors' benchmark site links the same record.",
          "verified_at": "2026-08-25T00:00:00+00:00",
          "discovered_at": "2026-08-25T00:00:00+00:00",
          "used_for": [
            "full-text",
            "evidence"
          ]
        }
      ],
      "products": [
        {
          "product": "codex-cli",
          "role": "evaluated",
          "model": "GLM-4.6; Claude Sonnet 4.5; GPT-5; Claude Opus 4.5; GPT-5.1",
          "version": "v0.77.0",
          "reasoning_mode": "not-reported",
          "temperature": "API defaults where applicable; exact values not reported",
          "max_output_tokens": "not-reported",
          "budget": "50 tasks per configuration; reported averages range from $1.00 to $3.50 and 10.0 to 14.2 minutes per task for Codex configurations",
          "runs": "One complete 50-task evaluation per configuration; Codex + Claude Opus 4.5 was repeated 3 times for robustness",
          "tool_permissions": "Full production-codebase task environment; official Figma MCP for vision-capable models and GLM Vision MCP for GLM-4.6"
        },
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "GLM-4.6; Claude Sonnet 4.5; Claude Opus 4.5; Claude Haiku",
          "version": "v2.1.37",
          "reasoning_mode": "not-reported",
          "temperature": "API defaults where applicable; exact values not reported",
          "max_output_tokens": "not-reported",
          "budget": "50 tasks per configuration; reported averages range from $0.50 to $4.00 and 8.3 to 15.0 minutes per task for Claude Code configurations",
          "runs": "One complete 50-task evaluation per configuration; Claude Code + Claude Opus 4.5 was repeated 3 times for robustness",
          "tool_permissions": "Full production-codebase task environment; official Figma MCP for vision-capable models and GLM Vision MCP for GLM-4.6"
        }
      ],
      "task": {
        "summary": "Implement production iOS features from product requirements, Figma designs, and an existing Swift/Objective-C codebase.",
        "benchmark": "SWE-Bench Mobile; 50 proprietary tasks, 449 diff-based tests, about 500K lines of code, and 22 agent-model configurations"
      },
      "method": {
        "summary": "A hosted, contamination-resistant mobile software-engineering benchmark with real PRDs, multimodal design inputs, production code, and task-specific patch tests.",
        "tags": [
          "benchmark-design"
        ]
      },
      "evidence": {
        "result": "Codex + GLM-4.6 reaches 12.0% task success and 19.6% test pass; Claude Code + GLM-4.6 reaches 10.0% and 26.7%. Across all 22 configurations, no system exceeds 12.0% task success, and success falls from 18% on 1-2-file tasks to 2% on tasks requiring 7 or more files.",
        "same_model": "yes",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 3.1-3.9; Appendix B Tables 5-6; Appendix E Reproducibility in arXiv v1",
        "source_url": "https://arxiv.org/abs/2602.09540v1",
        "source_version": "submittedVersion",
        "caveats": "The official KDD DOI proves venue identity, while detailed evidence was checked in the identity-matched arXiv v1 because the ACM PDF endpoint was unavailable to the crawler. The benchmark uses a proprietary codebase and diff-based tests rather than compiling or running the iOS app; model snapshots, hard token limits, and common time or cost caps are not reported."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "GPT-family models use default temperature and top-p; exact values for all configurations are not reported",
        "max_output_tokens": "not-reported",
        "time_budget": "No common time cap reported; average runtime per configuration ranges from 2.0 to 52.1 minutes per task",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "No common cost cap; observed average cost ranges from $0.02 to $9.33 per task across all 22 configurations",
        "runs": "50 tasks per configuration; selected Claude Code and Codex configurations repeated 3 times for robustness",
        "tool_permissions": "Agents receive PRDs, Figma designs, and the production repository; Figma MCP or GLM Vision MCP supplies visual context; outputs are unified diff patches",
        "baseline_config": "22 configurations across Cursor, Codex CLI v0.77.0, Claude Code CLI v2.1.37, and OpenCode v1.1.44 on the same 50-task benchmark"
      },
      "source_type": "official-publisher",
      "year_tag": 2026,
      "conference_tag": "KDD",
      "audit_status": "included",
      "notes": "The primary source remains the official KDD DOI. arXiv v1 is retained only as an identity-verified auxiliary evidence copy."
    },
    {
      "id": "lean-refactor-2026",
      "title": "Verifiable PDE Reasoning and Modeling with Neurosymbolics",
      "authors": [
        "Wuyang Chen"
      ],
      "year": 2026,
      "conference": "IJCAI",
      "venue": "IJCAI-ECAI 2026 Early Career Spotlight",
      "domains": [
        "formal-methods",
        "scientific-computing"
      ],
      "publication_status": "conference",
      "classification": "direct",
      "system": "Lean Refactor",
      "paper_url": "https://2026.ijcai.org/accepted-papers/?ijtrack=early-career-spotlight",
      "artifact_status": "not-found",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "not-reported",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "not-reported",
          "runs": "not-reported",
          "tool_permissions": "not-reported"
        }
      ],
      "task": {
        "summary": "Refactor Lean 4 proofs to reduce proof length and compilation cost while preserving correctness across library versions.",
        "benchmark": "Competition proof benchmarks and research Lean repositories; exact benchmark composition is not reported in the IJCAI spotlight paper."
      },
      "method": {
        "summary": "A version-filtered strategy bank and multi-objective retrieve-and-rerank pipeline steer a frozen planner-refactorer-debugger agent without retraining.",
        "tags": [
          "retrieval",
          "verifier-loop",
          "context-reduction"
        ]
      },
      "evidence": {
        "result": "Lean Refactor reports over 70% token-level compression on competition benchmarks, over 20% on research repositories, and up to 60% compilation-time reduction, while stating that it outperforms Claude Code.",
        "same_model": "unknown",
        "same_budget": "unknown",
        "strength": "contextual",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Section 2.3 Formalizing Maintainable PDE Proofs in Lean; PDF page 3",
        "caveats": "The IJCAI Early Career Spotlight paper reports only aggregate Lean Refactor results and the Claude Code comparison claim. It does not identify the Claude Code model or CLI version, task-level results, run count, tool permissions, or a common budget, so comparison fairness cannot be established from the official paper."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "not-reported",
        "baseline_config": "Claude Code is named as a baseline in the reported comparison; its underlying model, CLI version, prompts, permissions, and budget are not reported."
      },
      "source_type": "official-conference",
      "year_tag": 2026,
      "conference_tag": "IJCAI",
      "audit_status": "included"
    },
    {
      "id": "scaling-laws-2026",
      "title": "Can Language Models Discover Scaling Laws?",
      "authors": [
        "Haowei Lin",
        "Haotian Ye",
        "Wenzheng Feng",
        "Quzhe Huang",
        "Yujun Li",
        "Hubert Lim",
        "Zhengrui Li",
        "Xiangyu Wang",
        "Jianzhu Ma",
        "Yitao Liang",
        "James Zou"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "machine-learning",
        "scientific-computing"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "SLDAgent",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/636d57c09a5baacd83722639265802f6-Abstract-Conference.html",
      "artifact_status": "not-found",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Claude-Haiku-4.5; Claude-Sonnet-4.5",
          "version": "1.0.102",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Baseline budget not reported; SLDAgent uses 50 evolutionary iterations",
          "runs": "5 runs for provider-specific comparisons",
          "tool_permissions": "Native CLI agent with upstream defaults"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "o4-mini; GPT-5",
          "version": "0.29.0",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Baseline budget not reported; SLDAgent uses 50 evolutionary iterations",
          "runs": "5 runs for provider-specific comparisons",
          "tool_permissions": "Native Codex agent with upstream defaults"
        }
      ],
      "task": {
        "summary": "Discover symbolic scaling laws from experimental data with executable code agents.",
        "benchmark": "SLDBench, eight scaling-law discovery tasks"
      },
      "method": {
        "summary": "Evolution-based program search jointly optimizes symbolic expressions and fitting procedures, with execution feedback and MAP-Elites selection.",
        "tags": [
          "benchmark-design",
          "deterministic-search",
          "performance-feedback",
          "test-feedback"
        ]
      },
      "evidence": {
        "result": "SLDAgent reaches average R2 0.748 with GPT-5 versus 0.550 for Codex GPT-5, and improves over Claude Code rows for both Claude-Haiku-4.5 and Claude-Sonnet-4.5.",
        "same_model": "yes",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "quality",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4.1 Agent baselines; Table 3; Appendix B.5; Appendix D",
        "caveats": "Provider-specific rows compare SLDAgent with native agents under different harnesses; the paper reports product versions but not exact token or temperature settings."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "50 evolutionary iterations for SLDAgent; baseline turns not reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "5 runs for provider-specific comparisons",
        "tool_permissions": "Native CLI agent defaults; SLDAgent uses terminal execution and verifier feedback",
        "baseline_config": "Claude Code with Claude-Haiku-4.5/Claude-Sonnet-4.5; Codex with o4-mini/GPT-5; Gemini CLI and other terminal agents"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "cybergym-2026",
      "title": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale",
      "authors": [
        "Zhun Wang",
        "Tianneng Shi",
        "Jingxuan He",
        "Matthew Cai",
        "Jialin Zhang",
        "Dawn Song"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "security",
        "software-engineering"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "CyberGym",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/c876a2935a90aff21874e14c07dc3e33-Abstract-Conference.html",
      "artifact_url": "https://github.com/sunblaze-ucb/cybergym",
      "artifact_status": "official",
      "products": [
        {
          "product": "codex-cli",
          "role": "evaluated",
          "model": "GPT-4.1",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Approximately 2.0 USD per task; maximum 100 iterations",
          "runs": "not-reported",
          "tool_permissions": "Default Codex CLI tools in repository-level vulnerability reproduction tasks"
        }
      ],
      "task": {
        "summary": "Generate proof-of-concept tests that reproduce real vulnerabilities in source repositories.",
        "benchmark": "CyberGym, 1,507 vulnerabilities across 188 software projects"
      },
      "method": {
        "summary": "A scalable vulnerability-reproduction benchmark with difficulty levels, pre/post-patch execution validation, and zero-day discovery analysis.",
        "tags": [
          "benchmark-design",
          "verifier-loop",
          "adversarial-testing",
          "observability"
        ]
      },
      "evidence": {
        "result": "Codex CLI with GPT-4.1 is directly compared with OpenHands, EnIGMA, and Cybench agent; the union of four agents reaches 18.4% Level-1 success, exposing complementary failure modes.",
        "same_model": "yes",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4 experimental evaluation; Figure 5; Appendix C Detailed Agent Settings",
        "caveats": "The Codex comparison uses a calibrated approximate cost budget and 100-iteration cap; the paper does not report a Codex CLI version or exact per-call token settings."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "maximum 100 iterations per task",
        "token_budget": "not-reported",
        "api_budget": "approximately 2.0 USD per task",
        "runs": "not-reported",
        "tool_permissions": "Repository files, shell execution, and Codex CLI defaults; task-specific tool details are not reported",
        "baseline_config": "Codex CLI with GPT-4.1 compared with OpenHands, EnIGMA, and Cybench agent using the same approximate cost/iteration constraints"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "deepscientist-2026",
      "title": "DeepScientist: Advancing Frontier-Pushing Scientific Findings Progressively",
      "authors": [
        "Yixuan Weng",
        "Minjun Zhu",
        "Qiujie Xie",
        "Qiyao Sun",
        "Zhen Lin",
        "Sifan Liu",
        "Yue Zhang"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "scientific-computing",
        "machine-learning"
      ],
      "publication_status": "conference",
      "classification": "related",
      "system": "DeepScientist",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/4f64494ecc3442f1c9261baa036378bc-Abstract-Conference.html",
      "artifact_url": "https://github.com/ResearAI/DeepScientist",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "host",
          "model": "Claude-4-opus",
          "version": "v1.0.53",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Month-long exploration on 16 H800 GPUs; Claude Code task budget not separately reported",
          "runs": "not-reported",
          "tool_permissions": "Sandboxed Docker repository with command-line re-execution and human inspection"
        }
      ],
      "task": {
        "summary": "Long-horizon autonomous scientific discovery with code implementation, experiment validation, and analysis/report generation.",
        "benchmark": "Three frontier scientific tasks; five generated research papers and multi-week operational logs"
      },
      "method": {
        "summary": "A three-stage hypothesis, implementation/evaluation, and analysis loop backed by persistent Findings Memory and Bayesian exploration.",
        "tags": [
          "structured-state",
          "memory",
          "performance-feedback",
          "verifier-loop",
          "parallelism"
        ]
      },
      "evidence": {
        "result": "Claude Code executes all code implementation and analysis tasks as a host component; the system reports 21 validated innovations and 60% simulated acceptance under the stated review protocol.",
        "same_model": "unknown",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "component-level",
        "source_location": "Section F Implementation Details; Section 4.2 Experimental Setup; Figure 4",
        "caveats": "Claude Code is a host executor rather than the paper''s comparison baseline; the scientific reasoning backbone is Gemini-2.5-pro and product-level Claude Code ablations are not reported."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "Month-long cycle; one AI text-detection run lasted 14 days",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "Claude Code confined to a duplicated sandbox repository; independent command-line verification after completion",
        "baseline_config": "Claude-4-opus in Claude Code v1.0.53 for code execution; Gemini-2.5-pro for scientific reasoning"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "devops-gym-2026",
      "title": "DevOps-Gym: Benchmarking AI Agents in Software DevOps Cycle",
      "authors": [
        "Yuheng Tang",
        "Kaijie Zhu",
        "Bonan Ruan",
        "Chuqi Zhang",
        "Michael Yang",
        "Hongwei Li",
        "Suyue Guo",
        "Tianneng Shi",
        "Zekun Li",
        "Christopher Kruegel",
        "Giovanni Vigna",
        "Dawn Song",
        "William Yang Wang",
        "Lun Wang",
        "Yangruibo Ding",
        "Zhenkai Liang",
        "Wenbo Guo"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "DevOps-Gym",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/15e35461247bbd05fa890d384060c847-Abstract-Conference.html",
      "artifact_status": "not-found",
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude-4-Sonnet",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "0.7",
          "max_output_tokens": "not-reported",
          "budget": "704 tasks plus 14 end-to-end pipeline tasks; five-run stability subset uses 50 tasks",
          "runs": "5 independent runs on the 50-task subset",
          "tool_permissions": "Isolated DevOps-Gym Docker environment with terminal and DevOps-specific tools"
        }
      ],
      "task": {
        "summary": "Evaluate coding agents across build/configuration, monitoring, issue resolving, and test generation.",
        "benchmark": "DevOps-Gym, 704 tasks from 30+ Java/Go projects plus 14 end-to-end pipeline tasks"
      },
      "method": {
        "summary": "A realistic dynamic DevOps environment with standardized tool interfaces, multi-stage workflows, and stage-specific metrics.",
        "tags": [
          "benchmark-design",
          "observability",
          "test-feedback",
          "coordination"
        ]
      },
      "evidence": {
        "result": "Claude Code with Claude-4-Sonnet reports 51.85% build/configuration, 20.56% monitoring, 23.87% issue resolving, and 13.87% test generation success.",
        "same_model": "yes",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4.1 Experimental Setup; Table 1; Appendix D.3 Table 4",
        "caveats": "The paper evaluates Claude Code as one agent framework; exact product version and per-task token budget are not reported."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "0.7",
        "max_output_tokens": "not-reported",
        "time_budget": "60-second inference timeout; Docker task limits otherwise not reported",
        "turn_budget": "not-reported",
        "token_budget": "maximum 256K context length",
        "api_budget": "not-reported",
        "runs": "5 independent runs on a sampled 50-task subset",
        "tool_permissions": "DevOps-Gym isolated Docker containers; terminal-format tool interfaces and dynamic runtime tools",
        "baseline_config": "Claude Code, OpenHands, mini-SWE-agent, and Aider with multiple backbone models; Claude Code uses Claude-4-Sonnet"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "featurebench-2026",
      "title": "FeatureBench: Benchmarking Agentic Coding for Complex Feature Development",
      "authors": [
        "Qixing Zhou",
        "Jiacheng Zhang",
        "Haiyang Wang",
        "Rui Hao",
        "Jiahe Wang",
        "Minghao Han",
        "Yuxue Yang",
        "Shuzhe Wu",
        "Feiyang Pan",
        "Lue Fan",
        "Dandan Tu",
        "Zhaoxiang Zhang"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "FeatureBench",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/25203d1cc8c58381eab578f4fcf9c4f8-Abstract-Conference.html",
      "artifact_url": "https://github.com/LiberCoders/FeatureBench",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude Opus 4.5",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Maximum 500 steps per task; token I/O reported per repository",
          "runs": "not-reported",
          "tool_permissions": "FeatureBench repository environment with internet access and agentic file/tool actions"
        },
        {
          "product": "codex-cli",
          "role": "evaluated",
          "model": "GPT-5.1-Codex",
          "version": "not-reported",
          "reasoning_mode": "medium",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Maximum 500 steps per task; token I/O reported per repository",
          "runs": "not-reported",
          "tool_permissions": "FeatureBench repository environment with internet access and Codex scaffold"
        }
      ],
      "task": {
        "summary": "Implement large, cross-file features in existing repositories and pass comprehensive tests.",
        "benchmark": "FeatureBench, 210 complex feature tasks across 24 repositories; Lite set has 30 tasks"
      },
      "method": {
        "summary": "Execution-traced feature extraction removes target functionality while preserving interfaces, producing realistic feature-completion tasks with visible tests and anti-cheating controls.",
        "tags": [
          "benchmark-design",
          "test-feedback",
          "verifier-loop",
          "repository-instructions"
        ]
      },
      "evidence": {
        "result": "Claude Code with Claude Opus 4.5 resolves 11.0% of Full tasks and Codex with GPT-5.1-Codex resolves 12.5%; the paper reports passed rates, token I/O, and failure modes.",
        "same_model": "no",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4.1.1 Baseline; Table 2; Table 8; Appendix C Table 16",
        "caveats": "The two products use different provider models; Claude Code's routing configuration and exact CLI version are not reported."
      },
      "experiment": {
        "reasoning_mode": "Claude Opus 4.5 not reported; GPT-5.1-Codex medium",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "maximum 500 steps per task",
        "token_budget": "Table 2 reports aggregate token I/O per configuration/repository",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "Internet access freely available; no specific browser tool; repository file editing and test execution",
        "baseline_config": "Claude Code + Claude Opus 4.5; Codex + GPT-5.1-Codex; OpenHands, Gemini CLI, and other scaffolds"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "appforge-2026",
      "title": "From Assistant to Independent Developer — Are GPTs Ready for Software Development?",
      "authors": [
        "Dezhi Ran",
        "Yuan Cao",
        "Mengzhou Wu",
        "Simin Chen",
        "Yuzhe Guo",
        "Jun Ren",
        "Zihe Song",
        "Hao Yu",
        "Jialei Wei",
        "Linyi Li",
        "Wei Yang",
        "Baishakhi Ray",
        "Tao Xie"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering",
        "web-ui"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "AppForge",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/a421e88bb7db67cd0401475a0ed4a17d-Abstract-Conference.html",
      "artifact_url": "https://github.com/TongmingLAIC/AppForge",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Qwen3-Coder",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "0.2",
          "max_output_tokens": "not-reported",
          "budget": "101 Android app development tasks; pass@1 evaluation",
          "runs": "pass@1",
          "tool_permissions": "Android project repository, Gradle build, emulator, UIAutomator tests, and compilation feedback"
        }
      ],
      "task": {
        "summary": "Build complete Android applications from natural-language specifications and pass functional/runtime tests.",
        "benchmark": "AppForge, 101 software-development problems from real-world Android apps"
      },
      "method": {
        "summary": "Formal task specifications, expert-verified UI interaction traces, automated compilation/testing, and runtime fuzz testing create a full-app benchmark.",
        "tags": [
          "benchmark-design",
          "visual-review",
          "test-feedback",
          "verifier-loop"
        ]
      },
      "evidence": {
        "result": "Claude Code with Qwen3-Coder reaches 6.93% functional success; the paper reports that coding-agent gains are marginal and that multi-file integration and lifecycle behavior remain bottlenecks.",
        "same_model": "no",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 4.3 Table 2; Appendix C.1 Setup; Appendix C.2 detailed statistics",
        "caveats": "The paper reports Claude Code as CC with Qwen3-Coder; it does not report a product version or a separate CLI iteration budget."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "0.2 (greedy decoding)",
        "max_output_tokens": "not-reported",
        "time_budget": "600-second fuzzing duration shown for the evaluation environment",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "pass@1",
        "tool_permissions": "Android emulator API 31, Gradle 7.5.1, UIAutomator, compilation feedback, and runtime fuzzing",
        "baseline_config": "Claude Code and mini-SWE-agent compared with standalone LLMs on AppForge"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "innogym-2026",
      "title": "InnoGym: Benchmarking the Innovation Potential of AI Agents",
      "authors": [
        "Jintian Zhang",
        "Kewei Xu",
        "Jingsheng Zheng",
        "Zhuoyun Yu",
        "Yuqi Zhu",
        "Yujie Luo",
        "Lanning Wei",
        "Shuofei Qiao",
        "Lun Du",
        "Da Zheng",
        "Shumin Deng",
        "Huajun Chen",
        "Ningyu Zhang"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "machine-learning",
        "scientific-computing"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "related",
      "system": "InnoGym",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/743514dfa1ef705f378424bd1effb57b-Abstract-Conference.html",
      "artifact_url": "https://github.com/zjunlp/igym",
      "artifact_status": "official",
      "products": [
        {
          "product": "codex-cli",
          "role": "host",
          "model": "GPT-5",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "Up to 12 hours per task-agent-model configuration; Codex judge budget not separately reported",
          "runs": "3 runs for main agent configurations; Codex judge count not separately reported",
          "tool_permissions": "Solution-repository access for structural extraction and novelty comparison prompts"
        }
      ],
      "task": {
        "summary": "Evaluate whether ML/scientific agents improve solution performance while producing methodologically novel solutions.",
        "benchmark": "InnoGym/iBench, 18 tasks with a 10-task main evaluation subset"
      },
      "method": {
        "summary": "Agent-as-judge extraction and comparison turns complete solution repositories into summaries/pseudocode and scores novelty against known solutions.",
        "tags": [
          "benchmark-design",
          "structured-state",
          "performance-feedback",
          "deterministic-search"
        ]
      },
      "evidence": {
        "result": "Codex with GPT-5 is actually used as the extraction and novelty-evaluation host; the benchmark compares MLAB, CodeAct, and AIDE rather than using Codex as a primary agent baseline.",
        "same_model": "unknown",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "component-level",
        "source_location": "Section 3.2 Novelty Evaluation; Section 4.1; Appendix F.1",
        "caveats": "The paper calls the host product Codex rather than Codex CLI and does not report its product version or exact decoding settings; it is included as a related host use, not a direct product comparison."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "Up to 12 hours per task-agent-model configuration",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "3 runs for main configurations; host judge count not reported",
        "tool_permissions": "Codex reads solution repositories and emits summary.md/pseudocode.tex; novelty prompt uses GPT-5 scoring",
        "baseline_config": "MLAB, CodeAct, and AIDE are the main agent frameworks; Codex is the novelty-evaluation host"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "rpg-zerorepo-2026",
      "title": "RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation",
      "authors": [
        "Jane Luo",
        "Xin Zhang",
        "Steven Liu",
        "Jie Wu",
        "Jianfeng Liu",
        "Yiming Huang",
        "Yangyu Huang",
        "Chengyu Yin",
        "Ying Xin",
        "Yuefeng Zhan",
        "Hao Sun",
        "Qi Chen",
        "Scarlett Li",
        "Mao Yang"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "ZeroRepo / RPG",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/9482f45fdd89aba9130bb04c44f788a9-Abstract-Conference.html",
      "artifact_url": "https://github.com/microsoft/RPG-ZeroRepo",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "claude 4 sonnet",
          "version": "not-reported",
          "reasoning_mode": "official strongest model; exact effort not reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "averaged across 30 rounds",
          "runs": "30 rounds",
          "tool_permissions": "terminal agent with web search enabled"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "o3 pro",
          "version": "not-reported",
          "reasoning_mode": "official strongest model; exact effort not reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "averaged across 30 rounds",
          "runs": "30 rounds",
          "tool_permissions": "terminal agent with web search enabled"
        }
      ],
      "task": {
        "summary": "Repository-level codebase generation",
        "benchmark": "RepoCraft"
      },
      "method": {
        "summary": "A persistent repository planning graph over features, functions, files, interfaces, and data flow guides staged generation, dependency-aware planning, and validation.",
        "tags": [
          "structured-state",
          "repository-graph",
          "dependency-aware-planning",
          "test-feedback"
        ]
      },
      "evidence": {
        "result": "ZeroRepo reaches 81.5% functional coverage and 69.7% test accuracy, improving over Claude Code by 27.3 and 35.8 points; it produces about 36K lines and 445K code tokens.",
        "same_model": "no",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "quality",
        "comparison_scope": "product-level",
        "source_location": "Section 3.3 Baselines; Section 4.3; RepoCraft main-results table",
        "caveats": "ZeroRepo uses a different backbone/configuration from each terminal-product baseline, so model and harness effects are mixed."
      },
      "experiment": {
        "reasoning_mode": "Codex CLI o3 pro; Claude Code CLI claude 4 sonnet; exact effort not reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "30 rounds for strong baselines and ZeroRepo",
        "tool_permissions": "Terminal agents can retrieve real-world knowledge via web search",
        "baseline_config": "Claude Code CLI, Codex CLI, Gemini CLI, OpenHands, and workflow baselines compared with ZeroRepo/RPG"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "artemis-2026",
      "title": "Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing",
      "authors": [
        "Justin W. Lin",
        "Eliot Krzysztof Jones",
        "Donovan Julian Jasper",
        "Ethan Jun-shen Ho",
        "Anna Wu",
        "Arnold Tianyi Yang",
        "Neil Perry",
        "Andy Zou",
        "Matt Fredrikson",
        "J. Zico Kolter",
        "Percy Liang",
        "Dan Boneh",
        "Daniel E. Ho"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "security"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "ARTEMIS",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/0410c2ff9f872efe5a7c61a4323a5da3-Abstract-Conference.html",
      "artifact_url": "https://github.com/Stanford-Trinity/ARTEMIS",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Claude Sonnet 4",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "runs to completion; human-comparison window is 10 hours",
          "runs": "not-reported",
          "tool_permissions": "live enterprise-network penetration-testing environment"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "GPT-5",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "runs to completion; human-comparison window is 10 hours",
          "runs": "not-reported",
          "tool_permissions": "live enterprise-network penetration-testing environment"
        }
      ],
      "task": {
        "summary": "Long-horizon penetration testing in a live enterprise network",
        "benchmark": "Approximately 8,000 hosts across 12 subnets"
      },
      "method": {
        "summary": "Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting",
        "tags": [
          "multi-agent",
          "dynamic-specialization",
          "parallelism",
          "memory",
          "verifier-loop"
        ]
      },
      "evidence": {
        "result": "ARTEMIS finds 9 valid vulnerabilities with an 82% valid-submission rate, ranks second overall, and outperforms 9 of 10 human participants in the reported penetration-testing study.",
        "same_model": "unknown",
        "same_budget": "no",
        "strength": "medium",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Section 4.2 Agent Results; Table 1; Appendix A and J",
        "caveats": "This is live-network cybersecurity, not general coding; runtime limits, model mixtures, refusal behavior, and product policies differ."
      },
      "experiment": {
        "reasoning_mode": "ARTEMIS A1 uses GPT-5; A2 uses a multi-model supervisor ensemble with Claude Sonnet 4 sub-agents",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "ARTEMIS configurations execute for 16 hours; first 10 hours evaluated for human comparability",
        "turn_budget": "not-reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "same VM/live network setting for agents; scaffold-specific capabilities differ",
        "baseline_config": "OpenAI Codex with GPT-5, Claude Code with Claude Sonnet 4, CyAgent, Incalmo, MAPTA, and human professionals"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "formact-2026",
      "title": "FormAct: Agentic Source Editing for Rich-Format Document Generation",
      "authors": [
        "Eugene J. Yu",
        "Xingxing Zhang",
        "Yuan Xia",
        "Tao Ge",
        "Xun Wang",
        "FNU Kartik",
        "Vishwas Suryanarayanan",
        "Cheng Yang",
        "Amanda Jiang",
        "Jiayu Ding",
        "Xiangyu Wong",
        "Tengchao Lv",
        "Lei Cui",
        "Si-Qing Chen",
        "Furu Wei",
        "Sujian Li"
      ],
      "year": 2026,
      "conference": "ICML",
      "venue": "ICML Main Conference",
      "domains": [
        "documents"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "FormAct",
      "paper_url": "https://openreview.net/forum?id=n7Ta0YEcgw",
      "artifact_status": "not-found",
      "products": [
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "gpt-5.2-2025-12-11",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "1.0",
          "max_output_tokens": "32768",
          "budget": "multi-pass; exact budget not reported",
          "runs": "not-reported",
          "tool_permissions": "not-reported"
        }
      ],
      "task": {
        "summary": "Rich-format document generation and editing",
        "benchmark": "RichDocBench and RichDocFuzz"
      },
      "method": {
        "summary": "HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression",
        "tags": [
          "retrieval",
          "context-reduction",
          "verifier-loop",
          "visual-review"
        ]
      },
      "evidence": {
        "result": "Render correctness is 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate is 0.760 versus 0.140. Content alignment is the caveat where Codex is slightly higher.",
        "same_model": "yes",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Section 5.1 Experimental Setup; Section 5.2; main comparison table; review-refinement ablation",
        "caveats": "The task is rich-format document editing rather than source-code repair; the paper does not report a CLI version or a matched multi-pass budget."
      },
      "source_type": "openreview-conference",
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "1.0",
        "max_output_tokens": "32768",
        "time_budget": "not-reported",
        "turn_budget": "multi-pass baseline; exact turn budget not reported",
        "token_budget": "not-reported",
        "api_budget": "not-reported",
        "runs": "not-reported",
        "tool_permissions": "not-reported",
        "baseline_config": "Repo Codex (OpenAI, 2025), multi-pass; all methods and evaluation use gpt-5.2-2025-12-11"
      },
      "year_tag": 2026,
      "conference_tag": "ICML",
      "audit_status": "included"
    },
    {
      "id": "terminal-bench-2-2026",
      "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces",
      "authors": [
        "Mike A. Merrill",
        "Alexander G. Shaw",
        "Nicholas Carlini",
        "Boxuan Li",
        "Harsh Raj",
        "Ivan Bercovich",
        "Lin Shi",
        "Jeong Yeon Shin",
        "Thomas Walshe",
        "E. Kelly Buchanan",
        "Junhong Shen",
        "Guanghao Ye",
        "Haowei Lin",
        "Jason Poulos",
        "Maoyu Wang",
        "Marianna Nezhurina",
        "Jenia Jitsev",
        "Di Lu",
        "Orfeas Menis Mastromichalakis",
        "Zhiwei Xu",
        "Zizhao Chen",
        "Yue Liu",
        "Robert Zhang",
        "Leon Liangyu Chen",
        "Anurag Kashyap",
        "Jan-Lucas Uslu",
        "Jeffrey Li",
        "Jianbo Wu",
        "Minghao Yan",
        "Song Bian",
        "Vedang Sharma",
        "Ke Sun",
        "Steven Dillmann",
        "Akshay Anand",
        "Andrew Lanpouthakoun",
        "Bardia Koopah",
        "Changran Hu",
        "Etash Guha",
        "Gabriel H. S. Dreiman",
        "Jiacheng Zhu",
        "Karl Krauth",
        "Li Zhong",
        "Niklas Muennighoff",
        "Robert Amanfu",
        "Shangyin Tan",
        "Shreyas Pimpalgaonkar",
        "Tushar Aggarwal",
        "Xiangning Lin",
        "Xin Lan",
        "Xuandong Zhao",
        "Yiqing Liang",
        "Yuanli Wang",
        "Zilong Wang",
        "Changzhi Zhou",
        "David Heineman",
        "Hange Liu",
        "Harsh Trivedi",
        "John Yang",
        "Junhong Lin",
        "Manish Shetty",
        "Michael Yang",
        "Nabil Omi",
        "Negin Raoof",
        "Shanda Li",
        "Terry Yue Zhuo",
        "Wuwei Lin",
        "Yiwei Dai",
        "Yuxin Wang",
        "Wenhao Chai",
        "Shang Zhou",
        "Dariush Wahdany",
        "Ziyu She",
        "Jiaming Hu",
        "Zhikang Dong",
        "Yuxuan Zhu",
        "Sasha Cui",
        "Ahson Saiyed",
        "Arinbjörn Kolbeinsson",
        "Jesse Hu",
        "Christopher Michael Rytting",
        "Ryan Marten",
        "Yixin Wang",
        "Alex Dimakis",
        "Andy Konwinski",
        "Ludwig Schmidt"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering",
        "systems-performance"
      ],
      "publication_status": "dataset-benchmark",
      "classification": "evaluation",
      "system": "Terminal-Bench 2.0",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/444a3737adaee10d86ad2ef5f74468e6-Abstract-Conference.html",
      "artifact_url": "https://github.com/harbor-framework/terminal-bench-2",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "evaluated",
          "model": "Claude Opus 4.5; Claude Sonnet 4.5; Claude Opus 4.1; Claude Haiku 4.5",
          "version": "not-reported",
          "reasoning_mode": "provider default medium",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "89 tasks; token totals vary by agent-model combination",
          "runs": "not-reported",
          "tool_permissions": "Claude Code tool scaffold in Harbor"
        },
        {
          "product": "codex-cli",
          "role": "evaluated",
          "model": "GPT-5.2; GPT-5; GPT-5-Mini; GPT-5-Nano",
          "version": "not-reported",
          "reasoning_mode": "provider default medium",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "89 tasks; token totals vary by agent-model combination",
          "runs": "not-reported",
          "tool_permissions": "Codex CLI tool scaffold in Harbor"
        }
      ],
      "task": {
        "summary": "Evaluate industrial and open-source agents on realistic command-line tasks with executable tests.",
        "benchmark": "Terminal-Bench 2.0: 89 hard tasks, Harbor harness, containerized environments"
      },
      "method": {
        "summary": "A benchmark design with realistic terminal environments, human-written solutions, executable tests, standardized Harbor tasks, and failure-mode analysis.",
        "tags": [
          "benchmark-design",
          "verifier-loop",
          "observability",
          "adversarial-testing"
        ]
      },
      "evidence": {
        "result": "Codex CLI with GPT-5.2 reaches 62.9% resolution; Claude Code with Claude Opus 4.5 reaches 52.1%, with lower Claude Code rows for Sonnet 4.5, Opus 4.1, and Haiku 4.5 reported in Table 2.",
        "same_model": "no",
        "same_budget": "no",
        "strength": "high",
        "claim_type": "diagnostic",
        "comparison_scope": "benchmark-only",
        "source_location": "Section 3.2 Agents; Section 3.3 Models; Table 2; Appendix B",
        "caveats": "This paper evaluates products and models rather than proposing a method that beats them; token and cost budgets vary substantially across agent-model combinations."
      },
      "source_type": "official-proceedings",
      "experiment": {
        "reasoning_mode": "provider default medium for configurable Anthropic/OpenAI models",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "most attempts under 20 minutes; some up to 2 hours",
        "turn_budget": "not-reported",
        "token_budget": "Table 2 reports aggregate input/output tokens per 74 tasks; per-task budgets vary",
        "api_budget": "approximately $1–$100 per run depending on model",
        "runs": "not-reported",
        "tool_permissions": "Harbor containerized terminal tasks; Claude Code and Codex CLI use their product scaffolds",
        "baseline_config": "Claude Code, Codex CLI, Gemini CLI, OpenHands, Mini-SWE-Agent, and Terminus 2 across 89 Terminal-Bench 2.0 tasks"
      },
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    },
    {
      "id": "helmsman-2026",
      "title": "Helmsman: Autonomous Synthesis of Federated Learning Systems via Collaborative LLM Agents",
      "authors": [
        "Haoyuan Li",
        "Mathias Funk",
        "Aaqib Saeed"
      ],
      "year": 2026,
      "conference": "ICLR",
      "venue": "ICLR Conference",
      "domains": [
        "software-engineering",
        "machine-learning"
      ],
      "publication_status": "main",
      "classification": "direct",
      "system": "Helmsman",
      "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/1c364d98a5cdc426fd8c76fbb2c10e34-Abstract-Conference.html",
      "artifact_url": "https://github.com/haoyuan-l/Helmsman",
      "artifact_status": "official",
      "products": [
        {
          "product": "claude-code",
          "role": "baseline",
          "model": "Claude Sonnet 4.5",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "16 AgentFL-Bench tasks; baseline token and cost budgets are reported per task in Appendix Tables 8-12",
          "runs": "3 independent runs for reported benchmark averages",
          "tool_permissions": "not-reported for the Claude Code baseline"
        },
        {
          "product": "codex-cli",
          "role": "baseline",
          "model": "GPT-5.1-Codex",
          "version": "not-reported",
          "reasoning_mode": "not-reported",
          "temperature": "not-reported",
          "max_output_tokens": "not-reported",
          "budget": "16 AgentFL-Bench tasks; baseline token and cost budgets are reported per task in Appendix Tables 8-12",
          "runs": "3 independent runs for reported benchmark averages",
          "tool_permissions": "not-reported for the Codex CLI baseline"
        }
      ],
      "task": {
        "summary": "End-to-end synthesis of federated learning systems from high-level specifications.",
        "benchmark": "AgentFL-Bench, 16 tasks across five federated-learning research areas; cross-silo (5 clients) and cross-device (10 clients) settings"
      },
      "method": {
        "summary": "Human-in-the-loop planning, modular code generation by collaborative agent teams, and closed-loop sandboxed evaluation and refinement with hierarchical runtime and semantic verification.",
        "tags": [
          "multi-agent",
          "coordination",
          "verifier-loop",
          "test-feedback",
          "benchmark-design"
        ]
      },
      "evidence": {
        "result": "Claude Code with Claude Sonnet 4.5 reports 43.75% success and Codex with GPT-5.1-Codex reports 37.50%; Helmsman with Claude Sonnet 4.5 and GPT-5.1 reports 100% success in the paper's 16-task comparison.",
        "same_model": "no",
        "same_budget": "unknown",
        "strength": "high",
        "claim_type": "mixed",
        "comparison_scope": "product-level",
        "source_location": "Section 4.2 Implementation Details; Appendix A.1 Tables 8-12; Appendix A.2 Table 12; PDF p. 15",
        "caveats": "The paper compares a multi-agent federated-learning system synthesis framework against product code-synthesis baselines. CLI versions, temperature, tool permissions, and exact budget caps are not reported. Helmsman's main backbone uses Gemini-2.5-flash for planning and Claude-Sonnet-4.0 for coding and evaluation; additional experiments use Claude-Sonnet-4.5 and GPT-5.1."
      },
      "experiment": {
        "reasoning_mode": "not-reported",
        "temperature": "not-reported",
        "max_output_tokens": "not-reported",
        "time_budget": "not-reported",
        "turn_budget": "Maximum 10 debugging attempts in Helmsman evaluation; baseline turn and iteration caps are not reported",
        "token_budget": "Appendix Tables 8-12 report per-task and average total tokens for each configuration",
        "api_budget": "Appendix Tables 8-12 report per-task and average dollar costs; no common budget cap is declared",
        "runs": "3 independent runs for reported benchmark averages; Appendix A.2 reproducibility study",
        "tool_permissions": "Sandboxed simulation, terminal and code execution, and hierarchical runtime and semantic verification; exact Claude Code and Codex CLI permissions are not reported",
        "baseline_config": "Codex (GPT-5.1-Codex) and Claude Code (Claude Sonnet 4.5) compared with Helmsman variants and task-specific federated-learning baselines"
      },
      "source_type": "official-proceedings",
      "year_tag": 2026,
      "conference_tag": "ICLR",
      "audit_status": "included"
    }
  ]
}