From 5847d353040978b107a474a0b42f2c2fa659f90d Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 12:45:07 -0700 Subject: [PATCH 001/166] =?UTF-8?q?=F0=9F=90=9B=20agents:=20Copilot=20CLI?= =?UTF-8?q?=20ET=5FEXEC=20binary=20fails=20with=20ENOEXEC=20on=20kernel=20?= =?UTF-8?q?6.12=20=E2=80=94=20exec=20via=20ld.so=20+=20startup=20preflight?= =?UTF-8?q?=20(#17737)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #17729. Refs #17642. Signed-off-by: Andrew Anderson --- bin/install-copilot-exec-shim.sh | 78 ++++++++ .../fixed-17729-copilot-enoexec-ldso-shim.md | 1 + src/Dockerfile | 8 + src/Dockerfile.contributor | 4 + src/crux/agent/exec_format.go | 178 ++++++++++++++++++ src/crux/agent/exec_format_test.go | 87 +++++++++ src/crux/agent/manager_credentials.go | 20 ++ src/crux/runtime/runtime.go | 24 +++ 8 files changed, 400 insertions(+) create mode 100755 bin/install-copilot-exec-shim.sh create mode 100644 changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md create mode 100644 src/crux/agent/exec_format.go create mode 100644 src/crux/agent/exec_format_test.go diff --git a/bin/install-copilot-exec-shim.sh b/bin/install-copilot-exec-shim.sh new file mode 100755 index 0000000000..1db16d94c7 --- /dev/null +++ b/bin/install-copilot-exec-shim.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# install-copilot-exec-shim.sh — wrap the Copilot CLI platform binary so it is +# started through the glibc dynamic loader instead of execve'd directly. +# +# Why (hivecommons/hive#17729): the @github/copilot platform package ships a +# Bun single-file executable — an ELF64 ET_EXEC with a 0x1000-aligned appended +# JS segment after 0x10000-aligned code/data LOADs. Linux 6.12 (seen on OKE +# UEK8 6.12.0-207, aarch64) rejects that layout in binfmt_elf with ENOEXEC +# ("Exec format error"); the npm-loader shim's sh fallback then "runs" the +# ELF as a script and every copilot agent crash-loops. glibc's ld.so maps the +# very same file without complaint, so `ld.so ` is a drop-in. +# +# Build-time only. Idempotent. Tolerant of a missing copilot install (the +# Dockerfile layer that installs it is itself tolerant). Usage: +# install-copilot-exec-shim.sh [npm-global-root] +set -eu + +ROOT="${1:-$(npm root -g 2>/dev/null || echo /usr/local/lib/node_modules)}" +PKG_DIR="$ROOT/@github/copilot/node_modules/@github" + +# Arch → loader, resolved at BUILD time for the target arch. The wrapper falls +# back to a direct exec when the loader is absent (musl, unusual distros). +case "$(uname -m)" in + aarch64|arm64) LDSO=/lib/ld-linux-aarch64.so.1 ;; + x86_64|amd64) LDSO=/lib64/ld-linux-x86-64.so.2 ;; + *) LDSO="" ;; +esac + +if [ ! -d "$PKG_DIR" ]; then + echo "install-copilot-exec-shim: no @github/copilot platform package under $PKG_DIR — skipping" + exit 0 +fi + +installed=0 +for dir in "$PKG_DIR"/copilot-linux-*; do + [ -d "$dir" ] || continue + bin="$dir/copilot" + real="$dir/copilot.real" + if [ -f "$real" ] && head -c 2 "$bin" 2>/dev/null | grep -q '#!'; then + echo "install-copilot-exec-shim: $bin already wrapped" + installed=$((installed + 1)) + continue + fi + if [ ! -f "$bin" ]; then + echo "install-copilot-exec-shim: $bin missing — skipping" + continue + fi + mv "$bin" "$real" + cat > "$bin" </dev/null || true + chmod 0755 "$bin" "$real" + echo "install-copilot-exec-shim: wrapped $bin via ${LDSO:-direct exec}" + installed=$((installed + 1)) +done + +if [ "$installed" -eq 0 ]; then + echo "install-copilot-exec-shim: no copilot-linux-* platform binary found — skipping" + exit 0 +fi + +# Prove the wrapped CLI starts, through the npm shim, the way an agent runs it. +if ! copilot --version >/dev/null 2>&1; then + echo "install-copilot-exec-shim: WARNING: 'copilot --version' failed after wrapping" >&2 + exit 0 +fi +echo "install-copilot-exec-shim: copilot --version OK ($(copilot --version 2>/dev/null | head -1))" diff --git a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md new file mode 100644 index 0000000000..80a3ef1606 --- /dev/null +++ b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md @@ -0,0 +1 @@ +- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). diff --git a/src/Dockerfile b/src/Dockerfile index ac69b5335e..65c9b3a47d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -291,6 +291,14 @@ ARG COPILOT_VERSION=1.0.88 RUN npm install -g --ignore-scripts --no-fund --no-audit \ "@github/copilot@${COPILOT_VERSION}" && npm cache clean --force || \ echo "WARN: GitHub Copilot CLI install failed — skipping" +# Wrap the platform binary so it starts through glibc's ld.so (#17729): the +# Bun single-file ET_EXEC the package ships is rejected with ENOEXEC by Linux +# 6.12 binfmt_elf (OKE UEK8 upgrade took every copilot agent down), while the +# dynamic loader maps the same file fine. The installer is idempotent and a +# no-op when the copilot layer above was skipped. +COPY bin/install-copilot-exec-shim.sh /usr/local/bin/install-copilot-exec-shim.sh +RUN chmod +x /usr/local/bin/install-copilot-exec-shim.sh && \ + /usr/local/bin/install-copilot-exec-shim.sh /usr/local/lib/node_modules # --- Layer 3b: GitHub Copilot SDK (pinned; powers SDK-based model discovery) --- # The dashboard's copilot model discovery prefers a probe through the official diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 17fa574e92..be037588fc 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -271,6 +271,10 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit \ && npm cache clean --force \ && node "$(npm root -g)/@anthropic-ai/claude-code/install.cjs" \ && claude --version +# ld.so wrapper for the copilot platform binary (#17729) — see src/Dockerfile. +COPY bin/install-copilot-exec-shim.sh /usr/local/bin/install-copilot-exec-shim.sh +RUN chmod +x /usr/local/bin/install-copilot-exec-shim.sh && \ + /usr/local/bin/install-copilot-exec-shim.sh "$(npm root -g)" # Install Muse Code (backend: muse). Mirrors Layer 8.3 of src/Dockerfile so a # containerized contributor relay can actually run the muse backend, instead of diff --git a/src/crux/agent/exec_format.go b/src/crux/agent/exec_format.go new file mode 100644 index 0000000000..90a010a6a1 --- /dev/null +++ b/src/crux/agent/exec_format.go @@ -0,0 +1,178 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// ENOEXEC detection (hivecommons/hive#17729). +// +// After the OKE nodes moved to Linux 6.12 (UEK8), the kernel's ELF loader +// rejected the Copilot CLI platform binary — a Bun single-file ET_EXEC — with +// ENOEXEC. Nothing in the launch path said so: the npm shim's sh fell back to +// interpreting the ELF as a script ("copilot: 1: : not found"), the hang +// diagnostic saw no copilot output and timed out, and the restart-storm +// breaker backed every agent off for up to 30 minutes. Fifteen hives, zero +// agents, and the only log line was "copilot hung with no output". +// +// Two things make that loud now. paneShowsExecFormatError recognises the +// kernel's message and the sh-fallback signature in a diagnostic pane so the +// diagnostic fails the agent with a remedy instead of relaunching. And +// PreflightBackendBinaries execs every installed agent CLI once at boot so the +// same breakage is one ERROR line at startup, before any agent is launched. + +// execFormatErrorIndicators are the pane signatures of an ELF the kernel +// refused: the libc strerror text, and what dash/bash print when they fall +// back to running the binary as a script (the ELF's PT_INTERP string lands on +// the first "line", so the "command" sh reports is the loader path). +var execFormatErrorIndicators = []string{ + "Exec format error", + "cannot execute binary file", + "ld-linux-aarch64.so.1GNU", + "ld-linux-x86-64.so.2GNU", +} + +// paneShowsExecFormatError reports whether a diagnostic pane shows the CLI +// binary being rejected by the kernel rather than failing on its own. +func paneShowsExecFormatError(lines []string) bool { + for _, line := range lines { + for _, ind := range execFormatErrorIndicators { + if strings.Contains(line, ind) { + return true + } + } + // sh fell back to "running" the ELF as a script and reported the + // PT_INTERP string (the first printable run in the file) as a + // missing command. + if strings.Contains(line, "ld-linux-") && strings.Contains(line, ": not found") { + return true + } + } + return false +} + +// execFormatRemedy is the operator-facing explanation attached to every +// ENOEXEC finding. It names the one thing that fixes it (run the binary +// through ld.so, which the hive image now does via +// bin/install-copilot-exec-shim.sh) so nobody has to rediscover it from a +// pane full of "not found". +const execFormatRemedy = "the kernel refused to exec the CLI binary (ENOEXEC / Exec format error): " + + "usually a Bun/Node single-file ET_EXEC on a kernel whose ELF loader rejects its segment layout (seen with Linux 6.12 on aarch64). " + + "Relaunching cannot help. Start it through the dynamic loader instead (`/lib/ld-linux-aarch64.so.1 ` / `/lib64/ld-linux-x86-64.so.2 `) " + + "or roll a hive image built with bin/install-copilot-exec-shim.sh — see hivecommons/hive#17729" + +// execFormatLastError is what the agent's LastError is set to when the +// diagnostic sees ENOEXEC. +func execFormatLastError(binary string) string { + return fmt.Sprintf("%s cannot be executed on this kernel: %s", binary, execFormatRemedy) +} + +// Preflight pacing. Vars so tests can shrink them. +var ( + binaryPreflightTimeout = 20 * time.Second +) + +// BinaryPreflightFinding is one CLI binary the preflight could not start. +type BinaryPreflightFinding struct { + Backend string + Binary string + Path string + // ExecFormat is true when the failure is the kernel refusing the ELF + // (ENOEXEC) rather than the CLI exiting non-zero on its own. + ExecFormat bool + Err error + Output string +} + +func (f BinaryPreflightFinding) String() string { + if f.ExecFormat { + return fmt.Sprintf("backend %s: %s", f.Backend, execFormatLastError(f.Path)) + } + return fmt.Sprintf("backend %s: %s --version failed: %v", f.Backend, f.Path, f.Err) +} + +// classifyPreflightError decides whether a `--version` failure is the kernel +// refusing the binary. exec.Command surfaces ENOEXEC directly from execve +// (it does not go through a shell), and some CLIs are themselves shims that +// spawn the real binary and print the kernel's message. +func classifyPreflightError(err error, output string) bool { + if errors.Is(err, syscall.ENOEXEC) { + return true + } + return paneShowsExecFormatError(strings.Split(output, "\n")) +} + +// PreflightBackendBinaries runs ` --version` once for every CLI backend +// in use (or every installed CLI backend when backends is empty), with a +// timeout per binary, and returns the ones that could not start. Binaries +// not on PATH are skipped — an absent CLI is a configuration matter the +// launch path already reports; this is about CLIs that are present and +// broken. It never blocks boot: findings are returned for the caller to log. +func PreflightBackendBinaries(ctx context.Context, backends []string) []BinaryPreflightFinding { + if len(backends) == 0 { + backends = config.CLIBackends + } + seen := map[string]bool{} + var findings []BinaryPreflightFinding + for _, backend := range backends { + binary, err := backendBinaryName(backend) + if err != nil || seen[binary] { + continue + } + seen[binary] = true + path, err := exec.LookPath(binary) + if err != nil { + continue + } + if f, bad := preflightOneBinary(ctx, backend, binary, path); bad { + findings = append(findings, f) + } + } + return findings +} + +func preflightOneBinary(ctx context.Context, backend, binary, path string) (BinaryPreflightFinding, bool) { + ctx, cancel := context.WithTimeout(ctx, binaryPreflightTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, path, "--version") + cmd.Env = append(cmd.Environ(), "NO_COLOR=1", "CI=1") + out, err := cmd.CombinedOutput() + output := string(out) + if err == nil { + return BinaryPreflightFinding{}, false + } + execFormat := classifyPreflightError(err, output) + // A CLI that runs but dislikes `--version` (exit 1, usage text) is not + // what this preflight is for; only the kernel-level refusal is a finding. + // A timeout is not either — a slow cold start is the hang diagnostic's job. + if !execFormat { + return BinaryPreflightFinding{}, false + } + return BinaryPreflightFinding{ + Backend: backend, + Binary: binary, + Path: path, + ExecFormat: true, + Err: err, + Output: truncateStr(strings.TrimSpace(output), 300), + }, true +} + +// LogBackendBinaryPreflight runs PreflightBackendBinaries and logs each +// finding at ERROR with the remedy. Called once at boot by cmd/hive. +func LogBackendBinaryPreflight(ctx context.Context, logger *slog.Logger, backends []string) []BinaryPreflightFinding { + findings := PreflightBackendBinaries(ctx, backends) + for _, f := range findings { + logger.Error("agent CLI preflight: binary cannot be executed on this kernel", + "backend", f.Backend, "binary", f.Path, "error", f.Err, "output", f.Output, "remedy", execFormatRemedy) + } + return findings +} diff --git a/src/crux/agent/exec_format_test.go b/src/crux/agent/exec_format_test.go new file mode 100644 index 0000000000..7ddd159478 --- /dev/null +++ b/src/crux/agent/exec_format_test.go @@ -0,0 +1,87 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// Pane signatures from the OKE 6.12 outage (hivecommons/hive#17729): the npm +// shim's sh fell back to running the ELF as a script, so the pane had no +// copilot output at all — only sh "not found" lines naming the PT_INTERP. +func TestPaneShowsExecFormatError(t *testing.T) { + cases := []struct { + name string + pane string + want bool + }{ + {"kernel strerror", "sh: 1: /usr/local/lib/node_modules/@github/copilot/node_modules/@github/copilot-linux-arm64/copilot: Exec format error", true}, + {"bash wording", "bash: ./copilot: cannot execute binary file: Exec format error", true}, + {"sh fallback interp line", "copilot: 1: : not found\ncopilot: 2: /lib/ld-linux-aarch64.so.1GNU: not found", true}, + {"sh fallback x86 interp", "/lib64/ld-linux-x86-64.so.2GNU: not found", true}, + {"ordinary not found is not enoexec", "sh: 1: foo: not found", false}, + {"auth failure is not enoexec", "Bad credentials\n❯", false}, + {"ready prompt", "GitHub Copilot CLI 1.0.88\n❯", false}, + {"empty", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := paneShowsExecFormatError(strings.Split(tc.pane, "\n")); got != tc.want { + t.Fatalf("paneShowsExecFormatError(%q) = %v, want %v", tc.pane, got, tc.want) + } + }) + } +} + +func TestExecFormatLastErrorNamesBinaryAndRemedy(t *testing.T) { + got := execFormatLastError("/usr/local/bin/copilot") + for _, want := range []string{"/usr/local/bin/copilot", "ENOEXEC", "ld-linux-aarch64.so.1", "#17729"} { + if !strings.Contains(got, want) { + t.Fatalf("LastError %q lacks %q", got, want) + } + } +} + +// A real execve of a non-ELF, non-shebang file is the kernel's ENOEXEC; a +// binary that merely exits non-zero is not a finding. +func TestPreflightOneBinaryClassifiesENOEXEC(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("needs a POSIX execve") + } + dir := t.TempDir() + garbage := filepath.Join(dir, "garbage-cli") + // 0x7f 'E' 'L' 'F' with an impossible class byte: the kernel refuses it + // before any loader runs, and there is no shebang for sh to fall back to. + if err := os.WriteFile(garbage, append([]byte{0x7f, 'E', 'L', 'F', 0x09}, make([]byte, 64)...), 0o755); err != nil { + t.Fatal(err) + } + exits := filepath.Join(dir, "exits-cli") + if err := os.WriteFile(exits, []byte("#!/bin/sh\necho usage >&2\nexit 2\n"), 0o755); err != nil { + t.Fatal(err) + } + old := binaryPreflightTimeout + binaryPreflightTimeout = 10 * time.Second + t.Cleanup(func() { binaryPreflightTimeout = old }) + + f, bad := preflightOneBinary(context.Background(), "copilot", "copilot", garbage) + if !bad || !f.ExecFormat { + t.Fatalf("garbage ELF: want ENOEXEC finding, got bad=%v finding=%+v", bad, f) + } + if !strings.Contains(f.String(), "ENOEXEC") { + t.Fatalf("finding string lacks remedy: %q", f.String()) + } + if _, bad := preflightOneBinary(context.Background(), "codex", "codex", exits); bad { + t.Fatalf("a CLI that runs and exits non-zero must not be a finding") + } +} + +func TestPreflightBackendBinariesSkipsMissingCLIs(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if got := PreflightBackendBinaries(context.Background(), []string{"copilot", "codex", "not-a-backend"}); len(got) != 0 { + t.Fatalf("nothing on PATH must yield no findings, got %+v", got) + } +} diff --git a/src/crux/agent/manager_credentials.go b/src/crux/agent/manager_credentials.go index 5fb1a2d898..b681e0a9c2 100644 --- a/src/crux/agent/manager_credentials.go +++ b/src/crux/agent/manager_credentials.go @@ -204,6 +204,26 @@ func (m *Manager) runCopilotDiagnostic(ctx context.Context, agent *AgentProcess) if output == "" { continue } + if paneShowsExecFormatError(strings.Split(output, "\n")) { + // The kernel refused the binary (#17729). No relaunch can fix + // that, and the restart-storm breaker would only turn it into + // 30-minute silences; park the agent with the remedy instead. + agent.LastError = execFormatLastError(binary) + m.logger.Error("diagnostic: copilot binary cannot be executed on this kernel (ENOEXEC); not relaunching", + "agent", agent.Name, "binary", binary, "output_tail", truncateStr(compactPaneTail(output, 8), 500), "remedy", execFormatRemedy) + if agent.UID > 0 { + killAgentProcesses(agent.UID, m.logger) + } + _ = m.tmuxCmd(agent, "kill-session", "-t", agent.tmuxSession).Run() + agent.State = StateFailed + m.audit(AuditAgentStartFailed, agent.Name, auditFields( + "outcome", "failure", + "backend", agent.effectiveBackend(), + "model", agent.effectiveModel(), + "error", agent.LastError, + )) + return + } if matchesAuthError(output) { agent.LastError = "auth token expired or invalid" // Prefer to RESTORE the stored token over merely clearing it: an diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go index 5f3070e61e..d27a2c224a 100644 --- a/src/crux/runtime/runtime.go +++ b/src/crux/runtime/runtime.go @@ -26,6 +26,7 @@ import ( "fmt" "io" "log/slog" + "sort" "strings" "time" @@ -148,9 +149,32 @@ func newInProcess(o options) crux.Runtime { if o.sharedPool != nil { mopts = append(mopts, agent.WithSharedSessionPool(o.sharedPool)) } + // The in-process runtime is the one that execs agent CLIs on this host, + // so this is where a kernel that refuses one of them (ENOEXEC, #17729) + // gets said out loud once at boot — in the background, so a slow + // `--version` never delays the manager. + go agent.LogBackendBinaryPreflight(context.Background(), o.logger, backendsInUse(o.defs)) return agent.AsRuntime(agent.NewManagerWithOptions(o.defs, o.logger, o.project, mopts...)) } +// backendsInUse lists the distinct backends the agent definitions name, so +// the boot preflight probes the CLIs this hive will actually launch (and +// every installed CLI when no definitions were handed in). +func backendsInUse(defs map[string]config.AgentConfig) []string { + seen := map[string]bool{} + var out []string + for _, d := range defs { + b := strings.TrimSpace(d.Backend) + if b == "" || seen[b] { + continue + } + seen[b] = true + out = append(out, b) + } + sort.Strings(out) + return out +} + // HostOf returns rt's crux.Host when rt is the in-process implementation. // A sidecar runtime has no Host: its capabilities arrive over the API // (stage 2 and 3), so callers treat ok == false as "nothing to wire". From 2764dbca81915b2bd77dae55d1725d885d48f8e2 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 20:01:54 +0000 Subject: [PATCH 002/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.221.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.221.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/added-workflowmodel-marketplace.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-workflowmodel-marketplace.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 647f4ebfbb..f1a05a1eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.221.0) + +### Added + +- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) + ## 2026-11-11 (v5.220.0) ### Added diff --git a/changelog.d/added-workflowmodel-marketplace.md b/changelog.d/added-workflowmodel-marketplace.md deleted file mode 100644 index bae3f3071c..0000000000 --- a/changelog.d/added-workflowmodel-marketplace.md +++ /dev/null @@ -1 +0,0 @@ -- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) From 75515eac1e9bec0eb57af11826ef0c7af22e157e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:16:43 +0000 Subject: [PATCH 003/166] fix: warn once per agent about user identity in agent gitconfig (#17742) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-gitconfig-identity-warning-once.md | 1 + src/crux/agent/git_identity_warn_dedupe.go | 37 +++++++++++++++++++ src/crux/agent/interactive_home.go | 3 ++ .../agent/interactive_home_gitconfig_test.go | 34 +++++++++++++++++ src/crux/agent/manager.go | 3 +- 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-gitconfig-identity-warning-once.md create mode 100644 src/crux/agent/git_identity_warn_dedupe.go diff --git a/changelog.d/fixed-gitconfig-identity-warning-once.md b/changelog.d/fixed-gitconfig-identity-warning-once.md new file mode 100644 index 0000000000..55515cc892 --- /dev/null +++ b/changelog.d/fixed-gitconfig-identity-warning-once.md @@ -0,0 +1 @@ +- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). diff --git a/src/crux/agent/git_identity_warn_dedupe.go b/src/crux/agent/git_identity_warn_dedupe.go new file mode 100644 index 0000000000..b3d04f9913 --- /dev/null +++ b/src/crux/agent/git_identity_warn_dedupe.go @@ -0,0 +1,37 @@ +package agent + +import ( + "crypto/sha256" + "os" + "sync" +) + +// strayGitIdentityWarned remembers, per (agent, config path), the content hash +// of the git config the stray-identity WARN was last emitted for. The launch +// check runs every cycle; without this the identical informational WARN would +// repeat and trip the repeated-error selfreport (hivecommons/hive#17703). +// State is in-memory, so a hive restart warns once more. +type strayGitIdentityWarned struct { + mu sync.Mutex + seen map[string][sha256.Size]byte +} + +// shouldWarn reports whether the finding at path is new for agentName: true on +// first sight and whenever the file content changed since the last warning. +func (s *strayGitIdentityWarned) shouldWarn(agentName, path string) bool { + var sum [sha256.Size]byte + if data, err := os.ReadFile(path); err == nil { + sum = sha256.Sum256(data) + } + key := agentName + "\x00" + path + s.mu.Lock() + defer s.mu.Unlock() + if prev, ok := s.seen[key]; ok && prev == sum { + return false + } + if s.seen == nil { + s.seen = make(map[string][sha256.Size]byte) + } + s.seen[key] = sum + return true +} diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 02e5974e1e..114e404d1e 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -521,6 +521,9 @@ func (m *Manager) warnOnStrayGitIdentity(agentName, home string) { if !gitConfigDeclaresUserIdentity(path) { continue } + if !m.strayGitIdentity.shouldWarn(agentName, path) { + continue + } m.logger.Warn("agent git config declares a user identity outside the system /etc/gitconfig; commits still use the pinned per-lane GIT_AUTHOR_*/GIT_COMMITTER_* identity, but `git config user.email` reads this shared value instead (hivecommons/hive#9478)", "agent", agentName, "config", path) } diff --git a/src/crux/agent/interactive_home_gitconfig_test.go b/src/crux/agent/interactive_home_gitconfig_test.go index 6140b0c4cd..c2014e8cab 100644 --- a/src/crux/agent/interactive_home_gitconfig_test.go +++ b/src/crux/agent/interactive_home_gitconfig_test.go @@ -132,6 +132,40 @@ func TestWarnOnStrayGitIdentity(t *testing.T) { } } +// TestWarnOnStrayGitIdentity_OncePerAgentAndContent: the check runs every +// cycle, so an unchanged file must warn once, a changed file or another agent +// must warn again (hivecommons/hive#17703). +func TestWarnOnStrayGitIdentity_OncePerAgentAndContent(t *testing.T) { + const marker = "declares a user identity" + home := t.TempDir() + path := filepath.Join(home, ".gitconfig") + write := func(body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("[user]\n\temail = a@example.com\n") + + m, logBuf := gitConfigHomeManager(t) + m.warnOnStrayGitIdentity("scanner", home) + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 1 { + t.Fatalf("repeat with unchanged config warned %d times, want 1: %s", got, logBuf.String()) + } + + write("[user]\n\temail = b@example.com\n") + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 2 { + t.Fatalf("changed config warned %d times total, want 2", got) + } + + m.warnOnStrayGitIdentity("reviewer", home) + if got := strings.Count(logBuf.String(), marker); got != 3 { + t.Fatalf("different agent warned %d times total, want 3", got) + } +} + func TestGitConfigDeclaresUserIdentity(t *testing.T) { dir := t.TempDir() cases := []struct { diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 9a03a1dcc0..64b76f38c6 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -552,7 +552,8 @@ type Manager struct { // own mutex: the pane pollers that consult it must not take m.mu. rateLimitRelaunchMu sync.Mutex lastRateLimitRelaunch time.Time - cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + strayGitIdentity strayGitIdentityWarned // warn-once state, see git_identity_warn_dedupe.go // thrashMu guards thrash — its own mutex, NEVER m.mu: the breaker runs on // the output-capture goroutines, and taking m.mu there risks startup From c3d36628f15f520768438bcb0a7ecf46ef242553 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:26:56 +0000 Subject: [PATCH 004/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20name=20ENOEXEC=20?= =?UTF-8?q?instead=20of=20dumping=20binary=20stderr=20in=20model=20discove?= =?UTF-8?q?ry=20error=20(#17756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: name ENOEXEC instead of dumping binary stderr in model discovery error Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix: import unicode/utf8 and keep ANSI-coloured stderr out of the ENOEXEC check cli_models.go used utf8.ValidString without importing unicode/utf8, so the dashboard package failed to build. isBinaryStderr also treated ESC (0x1b) as a binary control byte, so coloured stderr would have been mislabelled as an ENOEXEC dump; ESC is now allowed alongside the whitespace controls, with a regression test. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/fixed-discovery-binary-stderr.md | 1 + src/pkg/dashboard/cli_models.go | 19 +++++++++++++++++ .../dashboard/cli_models_last_good_test.go | 21 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 changelog.d/fixed-discovery-binary-stderr.md diff --git a/changelog.d/fixed-discovery-binary-stderr.md b/changelog.d/fixed-discovery-binary-stderr.md new file mode 100644 index 0000000000..18be9f2d91 --- /dev/null +++ b/changelog.d/fixed-discovery-binary-stderr.md @@ -0,0 +1 @@ +- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). diff --git a/src/pkg/dashboard/cli_models.go b/src/pkg/dashboard/cli_models.go index c8342e6eae..81000fd55a 100644 --- a/src/pkg/dashboard/cli_models.go +++ b/src/pkg/dashboard/cli_models.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/claude" @@ -1613,12 +1614,30 @@ func copilotSDKHelperExecErr(ctx context.Context, err error, stderr string) erro errCopilotSDKHelperTimeout, copilotSDKProbeTimeout, filepath.Base(copilotSDKHelperPath)) } msg := strings.TrimSpace(stderr) + if isBinaryStderr(msg) { + return fmt.Errorf("sdk helper: %w (stderr was binary: the kernel refused to exec the Copilot CLI ELF and sh tried to run it as a script — ENOEXEC, see the copilot exec shim; not a model-catalog fault)", err) + } if len(msg) > copilotSDKStderrLogLimit { msg = msg[:copilotSDKStderrLogLimit] } return fmt.Errorf("sdk helper: %w (stderr: %s)", err, msg) } +// isBinaryStderr reports stderr that is not text: invalid UTF-8 or control +// bytes other than whitespace and ESC (ANSI colour), as when sh dumps an ELF +// it could not exec. +func isBinaryStderr(s string) bool { + if !utf8.ValidString(s) { + return true + } + for _, r := range s { + if r < 0x20 && r != '\n' && r != '\r' && r != '\t' && r != 0x1b { + return true + } + } + return false +} + // copilotSDKModel mirrors one entry of the helper's stdout JSON. The helper // also emits name/efforts/defaultEffort for future use; discovery only needs // the id and the policy gate today. diff --git a/src/pkg/dashboard/cli_models_last_good_test.go b/src/pkg/dashboard/cli_models_last_good_test.go index 97c8434bba..79edcf6b26 100644 --- a/src/pkg/dashboard/cli_models_last_good_test.go +++ b/src/pkg/dashboard/cli_models_last_good_test.go @@ -11,6 +11,7 @@ import ( "sync" "testing" "time" + "unicode/utf8" ) // newLastGoodTestCache builds a cache with a pinned clock and an isolated @@ -63,6 +64,26 @@ func TestCopilotSDKHelperExecErr_OtherKeepsStderr(t *testing.T) { } } +// TestCopilotSDKHelperExecErr_BinaryStderrNamed: an ELF dumped as stderr (#17735) +// must not reach the self-report as garbage; it is named as an ENOEXEC. +func TestCopilotSDKHelperExecErr_BinaryStderrNamed(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "copilot: 1: \x7fELF\x02\x01\x00\xff\xfe\x06") + got := err.Error() + if !strings.Contains(got, "ENOEXEC") || strings.Contains(got, "ELF\x02") || !utf8.ValidString(got) { + t.Fatalf("binary stderr not summarised: %q", got) + } +} + +// TestCopilotSDKHelperExecErr_ANSIStderrKept: coloured stderr is text, not an +// ELF dump — it must be passed through rather than mislabelled as ENOEXEC. +func TestCopilotSDKHelperExecErr_ANSIStderrKept(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "\x1b[31merror:\x1b[0m auth required\n") + got := err.Error() + if strings.Contains(got, "ENOEXEC") || !strings.Contains(got, "auth required") { + t.Fatalf("ANSI stderr mislabelled: %q", got) + } +} + // TestQueryCLIModels_SDKFailureServesLastGoodCatalog is the #11865 flap: a // clean SDK probe, then one that dies. The served list must stay the CLI's // catalog — authoritative, labelled stale with the helper failure as the From b005987d969509b192bb9780ce01453500467749 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 14:45:31 -0700 Subject: [PATCH 005/166] =?UTF-8?q?=E2=9C=A8=20crux:=20Substrate=20as=20th?= =?UTF-8?q?e=204th=20sandbox=20runtime=20(opt-in,=20preflight-gated)=20(#1?= =?UTF-8?q?7741)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: Substrate as the 4th sandbox runtime (opt-in, preflight-gated) Add Agent Substrate (CNCF Sandbox) as sandbox.runtime: substrate behind the existing sandbox.Launcher seam, mirroring the OpenShell runtime (ADR-0026 phase 2). - crux/sandbox/substrate*.go: SubstrateLauncher (ActorTemplate + Actor per kick, atenet-egress policy derived from the proxy host set + allow_hosts, workspace push/run/pull through an in-actor bridge over the atenet-router mTLS CONNECT tunnel, snapshot_on_idle suspends instead of deleting), real gRPC client constructed lazily, Control API preflight registered into crux/config. - crux/sandbox/substrateapi: trimmed, wire-compatible proto subset of ate.control.v1 generated in-repo (no go.ate.dev/substrate import: it requires Go 1.27 and is very large). Zero new Go modules. - crux/sandbox/substratebridge: the HTTP bridge served inside the actor. - crux/config/agent_config_substrate.go: sandbox.substrate.* leaves with per-agent merge; pkg/config gate warnings (digest-pinned image). - Settings parity allowlist entries for the new leaves (#17699). Fixes #17697 Part of #17695 Signed-off-by: Andrew Anderson * changelog: fragment for #17741 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🌱 changelog: make the substrate fragment an entry bullet Signed-off-by: Andrew Anderson * 🌱 changelog: drop the auto-generated duplicate fragment Signed-off-by: Andrew Anderson * 🌱 substrateapi: drop a never-true nil comparison flagged by staticcheck Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../added-substrate-sandbox-runtime.md | 1 + src/crux/agent/manager.go | 3 + src/crux/agent/manager_sandbox.go | 14 +- src/crux/agent/manager_sandbox_substrate.go | 78 ++ .../agent/sandbox_substrate_runtime_test.go | 141 ++++ src/crux/config/agent_config.go | 8 +- src/crux/config/agent_config_substrate.go | 179 +++++ .../config/agent_config_substrate_test.go | 121 ++++ src/crux/go.mod | 4 +- src/crux/sandbox/substrate.go | 387 ++++++++++ src/crux/sandbox/substrate_bridge.go | 107 +++ src/crux/sandbox/substrate_client.go | 292 ++++++++ src/crux/sandbox/substrate_client_test.go | 681 ++++++++++++++++++ src/crux/sandbox/substrate_preflight.go | 125 ++++ src/crux/sandbox/substrate_spec.go | 234 ++++++ src/crux/sandbox/substrate_test.go | 652 +++++++++++++++++ src/crux/sandbox/substrateapi/actor.pb.go | 586 +++++++++++++++ src/crux/sandbox/substrateapi/actor.proto | 79 ++ src/crux/sandbox/substrateapi/actor_rpc.pb.go | 572 +++++++++++++++ src/crux/sandbox/substrateapi/actor_rpc.proto | 72 ++ src/crux/sandbox/substrateapi/buf.gen.yaml | 8 + src/crux/sandbox/substrateapi/buf.yaml | 3 + src/crux/sandbox/substrateapi/common.pb.go | 428 +++++++++++ src/crux/sandbox/substrateapi/common.proto | 59 ++ src/crux/sandbox/substrateapi/container.pb.go | 401 +++++++++++ src/crux/sandbox/substrateapi/container.proto | 54 ++ src/crux/sandbox/substrateapi/control.pb.go | 135 ++++ src/crux/sandbox/substrateapi/control.proto | 48 ++ .../sandbox/substrateapi/control_grpc.pb.go | 490 +++++++++++++ src/crux/sandbox/substrateapi/egress.pb.go | 553 ++++++++++++++ src/crux/sandbox/substrateapi/egress.proto | 68 ++ src/crux/sandbox/substrateapi/generate.sh | 6 + .../sandbox/substrateapi/substrateapi_test.go | 364 ++++++++++ src/crux/sandbox/substrateapi/template.pb.go | 616 ++++++++++++++++ src/crux/sandbox/substrateapi/template.proto | 76 ++ src/crux/sandbox/substrateapi/worker.pb.go | 499 +++++++++++++ src/crux/sandbox/substrateapi/worker.proto | 67 ++ src/crux/sandbox/substratebridge/bridge.go | 338 +++++++++ .../sandbox/substratebridge/bridge_test.go | 394 ++++++++++ .../config/agent_sandbox_substrate_test.go | 115 +++ src/pkg/config/config.go | 19 +- src/pkg/config/crux_aliases.go | 2 + .../dashboard/settings_parity_allowlist.txt | 38 + 43 files changed, 9112 insertions(+), 5 deletions(-) create mode 100644 changelog.d/added-substrate-sandbox-runtime.md create mode 100644 src/crux/agent/manager_sandbox_substrate.go create mode 100644 src/crux/agent/sandbox_substrate_runtime_test.go create mode 100644 src/crux/config/agent_config_substrate.go create mode 100644 src/crux/config/agent_config_substrate_test.go create mode 100644 src/crux/sandbox/substrate.go create mode 100644 src/crux/sandbox/substrate_bridge.go create mode 100644 src/crux/sandbox/substrate_client.go create mode 100644 src/crux/sandbox/substrate_client_test.go create mode 100644 src/crux/sandbox/substrate_preflight.go create mode 100644 src/crux/sandbox/substrate_spec.go create mode 100644 src/crux/sandbox/substrate_test.go create mode 100644 src/crux/sandbox/substrateapi/actor.pb.go create mode 100644 src/crux/sandbox/substrateapi/actor.proto create mode 100644 src/crux/sandbox/substrateapi/actor_rpc.pb.go create mode 100644 src/crux/sandbox/substrateapi/actor_rpc.proto create mode 100644 src/crux/sandbox/substrateapi/buf.gen.yaml create mode 100644 src/crux/sandbox/substrateapi/buf.yaml create mode 100644 src/crux/sandbox/substrateapi/common.pb.go create mode 100644 src/crux/sandbox/substrateapi/common.proto create mode 100644 src/crux/sandbox/substrateapi/container.pb.go create mode 100644 src/crux/sandbox/substrateapi/container.proto create mode 100644 src/crux/sandbox/substrateapi/control.pb.go create mode 100644 src/crux/sandbox/substrateapi/control.proto create mode 100644 src/crux/sandbox/substrateapi/control_grpc.pb.go create mode 100644 src/crux/sandbox/substrateapi/egress.pb.go create mode 100644 src/crux/sandbox/substrateapi/egress.proto create mode 100755 src/crux/sandbox/substrateapi/generate.sh create mode 100644 src/crux/sandbox/substrateapi/substrateapi_test.go create mode 100644 src/crux/sandbox/substrateapi/template.pb.go create mode 100644 src/crux/sandbox/substrateapi/template.proto create mode 100644 src/crux/sandbox/substrateapi/worker.pb.go create mode 100644 src/crux/sandbox/substrateapi/worker.proto create mode 100644 src/crux/sandbox/substratebridge/bridge.go create mode 100644 src/crux/sandbox/substratebridge/bridge_test.go create mode 100644 src/pkg/config/agent_sandbox_substrate_test.go diff --git a/changelog.d/added-substrate-sandbox-runtime.md b/changelog.d/added-substrate-sandbox-runtime.md new file mode 100644 index 0000000000..f9c190943b --- /dev/null +++ b/changelog.d/added-substrate-sandbox-runtime.md @@ -0,0 +1 @@ +- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 64b76f38c6..92102c3b9d 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -767,6 +767,9 @@ type Manager struct { // sandboxOpenShellLauncherFactory is the same seam for // sandbox.runtime: openshell (ADR-0026). sandboxOpenShellLauncherFactory func(config.SandboxOpenShellConfig) sandbox.Launcher + // sandboxSubstrateLauncherFactory is the same seam for + // sandbox.runtime: substrate (ADR-0026 phase 2). + sandboxSubstrateLauncherFactory func(config.SandboxSubstrateConfig) sandbox.Launcher sandboxRunner sandboxCommandRunner // sandboxRunPersister overrides persistSandboxRun in tests. sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index d0d4d31235..93906a0079 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -18,6 +18,7 @@ import ( func init() { config.SetOpenShellPreflight(openShellPreflightWarnings) + config.SetSubstratePreflight(substratePreflightWarnings) } // openShellPreflightWarnings checks each distinct gateway the openshell @@ -112,7 +113,7 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) // does, instead of failing it after the state machine has moved. runtime := agent.Config.SandboxRuntime(m.sandboxConfig) if !config.ValidSandboxRuntimes[runtime] { - return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell) + return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell, config.SandboxRuntimeSubstrate) } switch runtime { case config.SandboxRuntimeJob: @@ -129,6 +130,15 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) if effectiveBackend(agent) == "cline" { return "", fmt.Errorf("agent %s: the Cline backend is not supported on the openshell sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) } + case config.SandboxRuntimeSubstrate: + // ADR-0026 phase 2: same contract as openshell — inert unless + // selected, refused before the state machine moves. + if err := SubstrateOptionsFromConfig(agent.Config.SandboxSubstrate(m.sandboxConfig)).Validate(); err != nil { + return "", fmt.Errorf("agent %s sandbox substrate runtime: %w", agent.Name, err) + } + if effectiveBackend(agent) == "cline" { + return "", fmt.Errorf("agent %s: the Cline backend is not supported on the substrate sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) + } } if effectiveBackend(agent) == "cline" { if err := validateClineSandbox(agent, runtime, agent.Config.SandboxJob(m.sandboxConfig)); err != nil { @@ -149,6 +159,8 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message string) er launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig)) case config.SandboxRuntimeOpenShell: launcher = m.openshellLauncherLocked(agent.Config.SandboxOpenShell(m.sandboxConfig)) + case config.SandboxRuntimeSubstrate: + launcher = m.substrateLauncherLocked(agent.Config.SandboxSubstrate(m.sandboxConfig)) } repo := m.project.PrimaryRepo() now := time.Now() diff --git a/src/crux/agent/manager_sandbox_substrate.go b/src/crux/agent/manager_sandbox_substrate.go new file mode 100644 index 0000000000..4d535bf238 --- /dev/null +++ b/src/crux/agent/manager_sandbox_substrate.go @@ -0,0 +1,78 @@ +package agent + +import ( + "context" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// sandbox.runtime: substrate wiring (ADR-0026 phase 2); the sibling of the +// openshell helpers in manager_sandbox.go. + +// substratePreflightWarnings checks each distinct control plane the +// substrate agents resolve to, once. +func substratePreflightWarnings(blocks []config.SandboxSubstrateConfig) []string { + var out []string + seen := map[string]bool{} + for _, b := range blocks { + opts := SubstrateOptionsFromConfig(b) + key := opts.Endpoint + "\x00" + opts.Atespace + "\x00" + opts.Token + if seen[key] { + continue + } + seen[key] = true + out = append(out, sandbox.DefaultSubstratePreflighter.Check(context.Background(), opts)...) + } + return out +} + +func (m *Manager) setSandboxSubstrateLauncherFactoryForTest(f func(config.SandboxSubstrateConfig) sandbox.Launcher) { + m.mu.Lock() + defer m.mu.Unlock() + m.sandboxSubstrateLauncherFactory = f +} + +// substrateLauncherLocked returns the launcher for a sandbox.runtime: +// substrate kick: the injected factory's, or a real Control-API-backed +// sandbox.SubstrateLauncher. +func (m *Manager) substrateLauncherLocked(cfg config.SandboxSubstrateConfig) sandbox.Launcher { + if m.sandboxSubstrateLauncherFactory != nil { + return m.sandboxSubstrateLauncherFactory(cfg) + } + return &sandbox.SubstrateLauncher{Options: SubstrateOptionsFromConfig(cfg)} +} + +// SubstrateOptionsFromConfig converts the operator's substrate block into +// the launcher's options. A token named by token_env is read from the +// environment so it never sits in hive.yaml; otherwise the launcher reads +// the projected token file per request. +func SubstrateOptionsFromConfig(cfg config.SandboxSubstrateConfig) sandbox.SubstrateOptions { + opts := sandbox.SubstrateOptions{ + Endpoint: cfg.Endpoint, + Atespace: cfg.Atespace, + TokenFile: cfg.TokenFile, + CAFile: cfg.CAFile, + InsecureTLS: cfg.InsecureTLS, + Timeout: time.Duration(cfg.TimeoutS) * time.Second, + RouterURL: cfg.RouterURL, + RouterClientBundle: cfg.RouterClientBundle, + RouterCAFile: cfg.RouterCAFile, + BridgePort: cfg.BridgePort, + Image: cfg.Image, + Command: append([]string(nil), cfg.BridgeCommand...), + CPUMillicores: cfg.CPUMillicores, + MemoryBytes: cfg.MemoryBytes, + WorkerSelector: cfg.WorkerSelector, + SnapshotOnIdle: cfg.SnapshotOnIdle, + AllowHosts: append([]string(nil), cfg.AllowHosts...), + PolicyProfile: cfg.PolicyProfile, + } + if env := strings.TrimSpace(cfg.TokenEnv); env != "" { + opts.Token = os.Getenv(env) + } + return opts +} diff --git a/src/crux/agent/sandbox_substrate_runtime_test.go b/src/crux/agent/sandbox_substrate_runtime_test.go new file mode 100644 index 0000000000..dfd75d04fa --- /dev/null +++ b/src/crux/agent/sandbox_substrate_runtime_test.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// The Substrate runtime (ADR-0026 phase 2) routes a sandboxed kick through +// the launcher built from the agent's substrate block, refuses a kick whose +// options are invalid before the state machine moves, and stays inert for +// every agent that did not select it. + +const substrateTestImage = "ghcr.io/hivecommons/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func substrateRuntimeManager(t *testing.T, override *config.AgentSandboxOverride, global config.AgentSandboxConfig) (*Manager, *int32, *int32) { + t.Helper() + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: override}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + if global.WorkspaceDir == "" { + global.WorkspaceDir = t.TempDir() + } + m.SetSandboxConfig(global) + var podmanRuns, substrateRuns int32 + m.SetSandboxLauncher(countingLauncher{runs: &podmanRuns}) + m.setSandboxSubstrateLauncherFactoryForTest(func(sb config.SandboxSubstrateConfig) sandbox.Launcher { + if sb.Atespace != "hive-test" { + t.Errorf("factory got atespace=%q", sb.Atespace) + } + return countingLauncher{runs: &substrateRuns} + }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, &podmanRuns, &substrateRuns +} + +func TestSubstrateRuntimeKickUsesTheSubstrateLauncher(t *testing.T) { + yes := true + m, podman, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: substrateTestImage}}) + if err := m.SendKick("deps", "bump the pinned dependency"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "deps") + if got := atomic.LoadInt32(sbRuns); got != 1 { + t.Errorf("substrate launcher runs = %d, want 1", got) + } + if got := atomic.LoadInt32(podman); got != 0 { + t.Errorf("podman launcher ran %d time(s) for a substrate-runtime agent", got) + } +} + +func TestSubstrateRuntimeRefusesKickWithTagOnlyImage(t *testing.T) { + yes := true + m, _, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: "ghcr.io/x/y:latest"}}) + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("SendKick error = %v, want a refusal naming the digest rule", err) + } + if got := atomic.LoadInt32(sbRuns); got != 0 { + t.Errorf("launcher ran %d time(s) despite the refusal", got) + } + if st := m.AllStatuses()["deps"].State; st == StateRunning { + t.Errorf("agent marked running after a refused kick") + } +} + +func TestSubstrateRuntimeRefusesCline(t *testing.T) { + yes := true + cfg := map[string]config.AgentConfig{"deps": {Backend: "cline", Sandbox: &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "dev-image", WorkspaceDir: t.TempDir(), + Substrate: &config.SandboxSubstrateConfig{Image: substrateTestImage}}) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "Cline") { + t.Fatalf("SendKick error = %v, want the Cline refusal", err) + } +} + +func TestSubstrateOptionsFromConfig(t *testing.T) { + t.Setenv("SUBSTRATE_TEST_TOKEN", "sekrit") + opts := SubstrateOptionsFromConfig(config.SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + PolicyProfile: "strict", + TimeoutS: 90, + TokenEnv: "SUBSTRATE_TEST_TOKEN", + Image: substrateTestImage, + BridgeCommand: []string{"/bin/bridge"}, + AllowHosts: []string{"litellm.example.com"}, + WorkerSelector: map[string]string{"class": "gvisor"}, + SnapshotOnIdle: true, + }) + if opts.Token != "sekrit" || opts.Atespace != "hive" || opts.PolicyProfile != "strict" || opts.Timeout != 90*time.Second || !opts.SnapshotOnIdle { + t.Errorf("opts = %+v", opts) + } + if len(opts.Command) != 1 || opts.Command[0] != "/bin/bridge" || opts.WorkerSelector["class"] != "gvisor" { + t.Errorf("command/selector = %+v", opts) + } + if err := opts.Validate(); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestSubstrateLauncherDefaultsToControlAPI(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) + l := m.substrateLauncherLocked(config.SandboxSubstrateConfig{Atespace: "hive"}) + if _, ok := l.(*sandbox.SubstrateLauncher); !ok { + t.Fatalf("launcher = %T, want *sandbox.SubstrateLauncher", l) + } +} + +func TestSubstratePreflightWarningsDedupesControlPlanes(t *testing.T) { + // Two agents on the same control plane probe once. A CA file that does + // not exist fails the client constructor before any network I/O, which + // is reported as the control plane being unreachable. + blocks := []config.SandboxSubstrateConfig{ + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + } + got := substratePreflightWarnings(blocks) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Fatalf("warnings = %q, want exactly one unreachable warning for the shared control plane", got) + } +} diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index de810c3a0b..c3b9d573e8 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -10,10 +10,11 @@ const ( SandboxRuntimePodman = "podman" SandboxRuntimeJob = "job" SandboxRuntimeOpenShell = "openshell" + SandboxRuntimeSubstrate = "substrate" ) // ValidSandboxRuntimes is the closed set a config may name. -var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true} +var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true, SandboxRuntimeSubstrate: true} // SandboxJobConfig is everything the Job runtime needs beyond the image // (#6311). The pod-template fields pass through to Kubernetes verbatim. @@ -499,6 +500,8 @@ type AgentSandboxConfig struct { Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` // OpenShell holds the settings for sandbox.runtime: openshell (ADR-0026). OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate holds the settings for sandbox.runtime: substrate (ADR-0026 phase 2). + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } @@ -540,6 +543,9 @@ type AgentSandboxOverride struct { // OpenShell, when set, overrides the global openshell block field by // field for this agent; see AgentConfig.SandboxOpenShell. OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate, when set, overrides the global substrate block field by + // field for this agent; see AgentConfig.SandboxSubstrate. + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish, when set, replaces the global publish preflight for this // agent; an empty verify list here turns the preflight off for it. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` diff --git a/src/crux/config/agent_config_substrate.go b/src/crux/config/agent_config_substrate.go new file mode 100644 index 0000000000..327eb6959b --- /dev/null +++ b/src/crux/config/agent_config_substrate.go @@ -0,0 +1,179 @@ +package config + +import ( + "strings" + "sync" +) + +var ( + substratePreflightMu sync.RWMutex + substratePreflight func([]SandboxSubstrateConfig) []string +) + +// SetSubstratePreflight registers the cluster-prerequisite check +// AgentSandboxGateWarnings runs for substrate-runtime agents. This package +// (and pkg/config above it) cannot import the Substrate client or +// crux/sandbox, so the code that owns the Control API client plugs itself in +// here. Passing nil unregisters it. +func SetSubstratePreflight(f func([]SandboxSubstrateConfig) []string) { + substratePreflightMu.Lock() + defer substratePreflightMu.Unlock() + substratePreflight = f +} + +// SubstratePreflightWarnings runs the registered preflight against the given +// substrate blocks; with none registered it reports nothing. +func SubstratePreflightWarnings(cfgs []SandboxSubstrateConfig) []string { + substratePreflightMu.RLock() + f := substratePreflight + substratePreflightMu.RUnlock() + if f == nil || len(cfgs) == 0 { + return nil + } + return f(cfgs) +} + +// SandboxSubstrateConfig is everything the Agent Substrate runtime needs +// beyond the agent's kick (ADR-0026 phase 2). Hive talks to Substrate's +// Control API (ate-api-server) over gRPC; Substrate owns the gVisor/microVM +// sandbox, the snapshot store and the atenet-egress gateway, so most of what +// the Job runtime spells out as a pod template is an ActorTemplate hive +// writes per kick from these settings. +// +// The workspace is NOT shared the way the Job and OpenShell runtimes share +// it: Substrate cannot mount an existing PersistentVolumeClaim, only a +// per-actor durableDir or a CSI volume it provisions itself. Hive therefore +// copies the per-kick workspace into the actor and back out through the +// bridge the sandbox image runs (crux substrate-bridge), reached over the +// atenet-router mTLS CONNECT tunnel. The local workspace stays the source of +// truth after the kick, so the push broker and PR flow are unchanged. +// +// AllowHosts is the egress allowlist handed to Substrate as the actor's +// EgressPolicy: the hosts hive's own MITM proxy (ADR-0002) would let a Podman +// sandbox reach — github.com, api.github.com — plus whatever the operator +// adds. Nothing else is reachable from the actor; Substrate default-denies. +// Credentials follow the Job rule: none of hive's enter it. Substrate's own +// egress credential injection is complementary and not used by hive. +type SandboxSubstrateConfig struct { + // Endpoint is the Control API address (host:port, or an http(s):// URL + // whose scheme picks TLS). Default api.ate-system.svc:443. + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // Atespace is the Substrate atespace (its tenant namespace, not a + // Kubernetes namespace) actors are created in. Default "hive". + Atespace string `yaml:"atespace,omitempty" json:"atespace,omitempty"` + // TokenEnv names the environment variable holding a Control API bearer + // token. Empty reads TokenFile instead. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` + // TokenFile is the projected ServiceAccount token (audience + // api.ate-system.svc) hive authenticates with. Default + // /var/run/secrets/ate/token. + TokenFile string `yaml:"token_file,omitempty" json:"token_file,omitempty"` + // CAFile is the PEM bundle the Control API serving certificate chains + // to (the servicedns ClusterTrustBundle projection). Empty uses + // /var/run/secrets/ate/ca.crt when present, else the system roots. + CAFile string `yaml:"ca_file,omitempty" json:"ca_file,omitempty"` + // InsecureTLS skips Control API and router certificate verification. + // Dev only. + InsecureTLS bool `yaml:"insecure_tls,omitempty" json:"insecure_tls,omitempty"` + // TimeoutS bounds one actor from create to delete. Non-positive uses + // the sandbox timeout (agent_sandbox.timeout_s) hive already applies to + // the kick. + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + // RouterURL is the atenet-router CONNECT listener hive reaches the + // actor's bridge through. Default https://atenet-router.ate-system.svc:8444. + RouterURL string `yaml:"router_url,omitempty" json:"router_url,omitempty"` + // RouterClientBundle and RouterCAFile are the podCertificate bundle the + // router authenticates hive with and the CA its serving certificate + // chains to. Defaults /run/router-client/credential-bundle.pem and + // /run/router-client/router-ca.pem. + RouterClientBundle string `yaml:"router_client_bundle,omitempty" json:"router_client_bundle,omitempty"` + RouterCAFile string `yaml:"router_ca_file,omitempty" json:"router_ca_file,omitempty"` + // BridgePort is the port the in-actor bridge listens on. Default 8477. + BridgePort int `yaml:"bridge_port,omitempty" json:"bridge_port,omitempty"` + // Image is the digest-pinned sandbox image (…@sha256:…) actors run. + // Substrate refuses tag-only references, so this overrides the agent + // sandbox image, which is usually a tag. + Image string `yaml:"image,omitempty" json:"image,omitempty"` + // BridgeCommand is the actor entrypoint that serves the bridge. Empty + // uses the crux binary's substrate-bridge subcommand. + BridgeCommand []string `yaml:"bridge_command,omitempty" json:"bridge_command,omitempty"` + // CPUMillicores and MemoryBytes are the ActorTemplate resource limits; + // Substrate requires both. Defaults 2000 and 4 GiB. + CPUMillicores int64 `yaml:"cpu_millicores,omitempty" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `yaml:"memory_bytes,omitempty" json:"memory_bytes,omitempty"` + // WorkerSelector pins actors to WorkerPools carrying these labels; the + // pool decides the sandbox class (gvisor or microvm). Empty means any + // pool. + WorkerSelector map[string]string `yaml:"worker_selector,omitempty" json:"worker_selector,omitempty"` + // SnapshotOnIdle suspends the actor (asynchronous snapshot) after the + // kick instead of deleting it, so Substrate keeps a resumable image. + // Default false: a kick is one-shot and snapshots cost object storage. + SnapshotOnIdle bool `yaml:"snapshot_on_idle,omitempty" json:"snapshot_on_idle,omitempty"` + // AllowHosts are additional egress hosts (host or host:port, leading + // *. wildcard allowed) the actor may reach, on top of the GitHub hosts + // hive always allows. + AllowHosts []string `yaml:"allow_hosts,omitempty" json:"allow_hosts,omitempty"` + // PolicyProfile names an operator-side profile; hive records it on the + // template so pools and in-actor tooling can act on it. + PolicyProfile string `yaml:"policy_profile,omitempty" json:"policy_profile,omitempty"` +} + +// SandboxSubstrate returns the effective Substrate runtime settings. The +// endpoint, atespace, credentials, TLS, router and bridge fields are cluster +// facts, so a per-agent block falls back to the global one for each it +// leaves empty; the image, resources, pool selector, snapshot choice and +// extra egress hosts are per-agent choices and replace the global ones +// wholesale when the per-agent block sets them. +func (a *AgentConfig) SandboxSubstrate(global AgentSandboxConfig) SandboxSubstrateConfig { + var g SandboxSubstrateConfig + if global.Substrate != nil { + g = *global.Substrate + } + if a == nil || a.Sandbox == nil || a.Sandbox.Substrate == nil { + return g + } + out := *a.Sandbox.Substrate + fallback := func(dst *string, src string) { + if strings.TrimSpace(*dst) == "" { + *dst = src + } + } + fallback(&out.Endpoint, g.Endpoint) + fallback(&out.Atespace, g.Atespace) + fallback(&out.TokenEnv, g.TokenEnv) + fallback(&out.TokenFile, g.TokenFile) + fallback(&out.CAFile, g.CAFile) + fallback(&out.RouterURL, g.RouterURL) + fallback(&out.RouterClientBundle, g.RouterClientBundle) + fallback(&out.RouterCAFile, g.RouterCAFile) + fallback(&out.Image, g.Image) + fallback(&out.PolicyProfile, g.PolicyProfile) + if out.TimeoutS == 0 { + out.TimeoutS = g.TimeoutS + } + if out.BridgePort == 0 { + out.BridgePort = g.BridgePort + } + if out.CPUMillicores == 0 { + out.CPUMillicores = g.CPUMillicores + } + if out.MemoryBytes == 0 { + out.MemoryBytes = g.MemoryBytes + } + if !out.InsecureTLS { + out.InsecureTLS = g.InsecureTLS + } + if !out.SnapshotOnIdle { + out.SnapshotOnIdle = g.SnapshotOnIdle + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = g.BridgeCommand + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = g.AllowHosts + } + if len(out.WorkerSelector) == 0 { + out.WorkerSelector = g.WorkerSelector + } + return out +} diff --git a/src/crux/config/agent_config_substrate_test.go b/src/crux/config/agent_config_substrate_test.go new file mode 100644 index 0000000000..5b55bc2998 --- /dev/null +++ b/src/crux/config/agent_config_substrate_test.go @@ -0,0 +1,121 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxSubstrateFallsBackToGlobal(t *testing.T) { + var nilA *AgentConfig + if got := nilA.SandboxSubstrate(AgentSandboxConfig{}); !reflect.DeepEqual(got, SandboxSubstrateConfig{}) { + t.Fatalf("nil agent, nil global = %+v, want zero value", got) + } + + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", Atespace: "hive", TokenEnv: "ATE_TOKEN", TokenFile: "/tok", + CAFile: "/ca", InsecureTLS: true, TimeoutS: 900, RouterURL: "https://router:8444", + RouterClientBundle: "/bundle", RouterCAFile: "/router-ca", BridgePort: 9000, + Image: "img@sha256:abc", BridgeCommand: []string{"/bridge"}, CPUMillicores: 1000, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"class": "gvisor"}, SnapshotOnIdle: true, + AllowHosts: []string{"hub.example"}, PolicyProfile: "strict", + }} + if got := nilA.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("nil agent = %+v, want global block", got) + } + noBlock := &AgentConfig{Sandbox: &AgentSandboxOverride{Image: "a-img"}} + if got := noBlock.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("agent without substrate block = %+v, want global block", got) + } + + // A per-agent block that names nothing but whitespace inherits every + // field from the global one. + empty := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{Endpoint: " "}}} + if got := empty.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("blank per-agent block = %+v, want global block", got) + } +} + +func TestSandboxSubstratePerAgentFieldsWin(t *testing.T) { + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-global:443", Atespace: "global", TokenEnv: "GLOBAL_TOKEN", TokenFile: "/global-tok", + CAFile: "/global-ca", TimeoutS: 900, RouterURL: "https://router-global:8444", + RouterClientBundle: "/global-bundle", RouterCAFile: "/global-router-ca", BridgePort: 8477, + Image: "global@sha256:aaa", BridgeCommand: []string{"/global-bridge"}, CPUMillicores: 2000, MemoryBytes: 4 << 30, + WorkerSelector: map[string]string{"pool": "global"}, + AllowHosts: []string{"global.example", "hub.example"}, PolicyProfile: "global-policy", + }} + a := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-agent:443", TimeoutS: 60, InsecureTLS: true, BridgePort: 9001, + Image: "agent@sha256:bbb", BridgeCommand: []string{"/agent-bridge"}, CPUMillicores: 500, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"gpu": "a100"}, SnapshotOnIdle: true, + AllowHosts: []string{"models.example"}, + }}} + got := a.SandboxSubstrate(global) + + if got.Endpoint != "api-agent:443" || got.TimeoutS != 60 || !got.InsecureTLS || got.BridgePort != 9001 || got.Image != "agent@sha256:bbb" { + t.Fatalf("per-agent fields not kept: %+v", got) + } + if got.Atespace != "global" || got.TokenEnv != "GLOBAL_TOKEN" || got.TokenFile != "/global-tok" || got.CAFile != "/global-ca" || + got.RouterURL != "https://router-global:8444" || got.RouterClientBundle != "/global-bundle" || got.RouterCAFile != "/global-router-ca" || got.PolicyProfile != "global-policy" { + t.Fatalf("empty per-agent cluster facts not inherited: %+v", got) + } + if got.CPUMillicores != 500 || got.MemoryBytes != 1<<30 || !got.SnapshotOnIdle { + t.Fatalf("per-agent resources/snapshot not kept: %+v", got) + } + if !reflect.DeepEqual(got.AllowHosts, []string{"models.example"}) { + t.Fatalf("allow_hosts = %v, want per-agent list only", got.AllowHosts) + } + if !reflect.DeepEqual(got.BridgeCommand, []string{"/agent-bridge"}) { + t.Fatalf("bridge_command = %v, want per-agent list only", got.BridgeCommand) + } + if !reflect.DeepEqual(got.WorkerSelector, map[string]string{"gpu": "a100"}) { + t.Fatalf("worker_selector = %v, want per-agent map only", got.WorkerSelector) + } + + // Booleans: a global true cannot be switched off per-agent. + global.Substrate.InsecureTLS = true + global.Substrate.SnapshotOnIdle = true + a.Sandbox.Substrate.InsecureTLS = false + a.Sandbox.Substrate.SnapshotOnIdle = false + if merged := a.SandboxSubstrate(global); !merged.InsecureTLS || !merged.SnapshotOnIdle { + t.Fatal("per-agent false inherits global true") + } + + // The merge must not write the inherited values back into its inputs. + if global.Substrate.AllowHosts[0] != "global.example" || a.Sandbox.Substrate.Atespace != "" { + t.Fatal("merge mutated its inputs") + } +} + +func TestSubstratePreflightRegistry(t *testing.T) { + t.Cleanup(func() { SetSubstratePreflight(nil) }) + cfgs := []SandboxSubstrateConfig{{Endpoint: "api:443"}} + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("unregistered preflight = %v, want nil", got) + } + + var seen []SandboxSubstrateConfig + SetSubstratePreflight(func(c []SandboxSubstrateConfig) []string { + seen = c + return []string{"control api unreachable"} + }) + if got := SubstratePreflightWarnings(nil); got != nil || seen != nil { + t.Fatalf("empty cfgs = %v (called=%v), want nil and no call", got, seen != nil) + } + if got := SubstratePreflightWarnings(cfgs); !reflect.DeepEqual(got, []string{"control api unreachable"}) || !reflect.DeepEqual(seen, cfgs) { + t.Fatalf("registered preflight = %v (seen %v), want its warnings for the given cfgs", got, seen) + } + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("after unregister = %v, want nil", got) + } +} + +func TestValidSandboxRuntimesIncludesSubstrate(t *testing.T) { + if !ValidSandboxRuntimes[SandboxRuntimeSubstrate] { + t.Fatal("substrate is not a valid sandbox runtime") + } +} diff --git a/src/crux/go.mod b/src/crux/go.mod index ec82f74fff..f9e6c9f12a 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -16,6 +16,8 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/sys v0.49.0 + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) @@ -50,6 +52,4 @@ require ( golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/grpc v1.85.0 // indirect - google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go new file mode 100644 index 0000000000..1ed56c0ee9 --- /dev/null +++ b/src/crux/sandbox/substrate.go @@ -0,0 +1,387 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Agent Substrate runtime (ADR-0026 phase 2; sibling of the OpenShell runtime +// in openshell.go). +// +// SubstrateLauncher implements Launcher on top of a Substrate control plane +// (ate-api-server): it registers a per-kick ActorTemplate from the kick spec, +// creates an Actor from it, attaches the egress policy, resumes the actor, +// drives the kick through the in-actor bridge (substratebridge) over the +// atenet-router CONNECT tunnel, and deletes — or, with snapshot_on_idle, +// suspends — the actor afterwards. Substrate owns the gVisor/microVM sandbox, +// the snapshot store and the egress gateway; hive's part is the spec, the +// policy and the same cleanup and timeout semantics the Job launcher has. +// +// Two points differ from the OpenShell and Job runtimes and are deliberate: +// +// - Substrate has no exec/attach RPC and cannot mount an existing PVC, so +// the workspace is a per-actor durableDir and hive copies it in and out +// through the bridge (push, run, pull). The local workspace hive opened +// the kick in is therefore still the source of truth after the kick. +// - Substrate refuses tag-only images; the sandbox image must be pinned by +// digest (…@sha256:…). Validate enforces it up front so the failure is a +// config error, not a CreateActorTemplate rejection mid-kick. + +const ( + // SubstrateDefaultEndpoint is the in-cluster Control API address the + // default ate-setup install exposes. + SubstrateDefaultEndpoint = "api.ate-system.svc:443" + // SubstrateDefaultRouterURL is the in-cluster atenet-router CONNECT + // listener hive reaches actors through. + SubstrateDefaultRouterURL = "https://atenet-router.ate-system.svc:8444" + // SubstrateDefaultAtespace is the atespace actors land in when none is + // configured. + SubstrateDefaultAtespace = "hive" + // SubstrateDefaultTokenFile is the projected ServiceAccount token + // (audience api.ate-system.svc) the Control API authenticates. + SubstrateDefaultTokenFile = "/var/run/secrets/ate/token" + // SubstrateDefaultCAFile is the servicedns ClusterTrustBundle projection + // the Control API serving certificate chains to. + SubstrateDefaultCAFile = "/var/run/secrets/ate/ca.crt" + // SubstrateDefaultRouterClientBundle is the projected podCertificate + // bundle atenet-router authenticates hive with. + SubstrateDefaultRouterClientBundle = "/run/router-client/credential-bundle.pem" + // SubstrateDefaultRouterCAFile is the servicedns ClusterTrustBundle the + // router's serving certificate chains to. + SubstrateDefaultRouterCAFile = "/run/router-client/router-ca.pem" + // SubstrateWorkspaceMount is where the actor sees the per-kick + // workspace, matching the other launchers. + SubstrateWorkspaceMount = substratebridge.DefaultWorkspace + // SubstrateDefaultCPUMillicores and SubstrateDefaultMemoryBytes are the + // template resource limits when the operator sets none; Substrate + // requires both. + SubstrateDefaultCPUMillicores = 2000 + SubstrateDefaultMemoryBytes = 4 << 30 + // SubstrateExitCodeUnknown mirrors OpenShellExitCodeUnknown. + SubstrateExitCodeUnknown = -1 + + // SubstratePolicyProfileLabel carries sandbox.substrate.policy_profile on + // the per-kick template's worker selector key space; Substrate has no + // annotations, so the profile travels as a template label hive and + // operators can filter on. + SubstratePolicyProfileLabel = "hive.dev/policy-profile" + + substrateNamePrefix = "hive-kick-" + substrateNameMax = 63 + substrateRequestWait = 30 * time.Second + substrateReadyPoll = 2 * time.Second + substrateEgressPolicy = "default" + substrateContainer = "agent" + substrateVolume = "workspace" + substrateHTTPSPort = 443 +) + +// SubstrateOptions is the launcher's configuration; crux/agent builds it +// from config.SandboxSubstrateConfig. +type SubstrateOptions struct { + Endpoint string + Atespace string + Token string + TokenFile string + CAFile string + InsecureTLS bool + Timeout time.Duration + RouterURL string + RouterClientBundle string + RouterCAFile string + BridgePort int + Image string + Command []string + CPUMillicores int64 + MemoryBytes int64 + WorkerSelector map[string]string + SnapshotOnIdle bool + AllowHosts []string + PolicyProfile string +} + +// Validate reports the first configuration error a kick would hit. +func (o SubstrateOptions) Validate() error { + if _, _, err := substrateAddress(o.endpoint()); err != nil { + return err + } + if _, err := substrateRouterAddress(o.routerURL()); err != nil { + return err + } + if o.Image != "" && !substrateImageDigest.MatchString(o.Image) { + return fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); Substrate refuses tag-only images", o.Image) + } + if o.CPUMillicores < 0 || o.MemoryBytes < 0 { + return errors.New("substrate: cpu_millicores and memory_bytes must not be negative") + } + if o.BridgePort < 0 || o.BridgePort > 65535 { + return fmt.Errorf("substrate: bridge_port %d is out of range", o.BridgePort) + } + _, err := SubstrateEgressPolicy(o.AllowHosts) + return err +} + +func (o SubstrateOptions) endpoint() string { + if e := strings.TrimSpace(o.Endpoint); e != "" { + return e + } + return SubstrateDefaultEndpoint +} + +func (o SubstrateOptions) atespace() string { + if a := strings.TrimSpace(o.Atespace); a != "" { + return a + } + return SubstrateDefaultAtespace +} + +func (o SubstrateOptions) routerURL() string { + if r := strings.TrimSpace(o.RouterURL); r != "" { + return r + } + return SubstrateDefaultRouterURL +} + +func (o SubstrateOptions) bridgePort() int { + if o.BridgePort > 0 { + return o.BridgePort + } + return substratebridge.DefaultPort +} + +// substrateControl is the slice of the Control API the launcher uses; it +// exists so tests can drive the launcher with a fake and so an upstream +// RPC rename shows up here, not in every call site. +type substrateControl interface { + CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) + GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) + ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error + SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error + DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error + ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) +} + +// substrateBridge is the launcher's view of the in-actor bridge. +type substrateBridge interface { + WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error + Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error + Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) + Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error +} + +type substrateClient interface { + Control() substrateControl + Bridge() substrateBridge + Close() error +} + +// SubstrateLauncher runs sandboxed kicks as Substrate actors. +type SubstrateLauncher struct { + Options SubstrateOptions + + // newClient overrides the client constructor in tests. + newClient func(SubstrateOptions) (substrateClient, error) + // now overrides the clock in tests. + now func() time.Time +} + +func (l *SubstrateLauncher) client() (substrateClient, error) { + if l.newClient != nil { + return l.newClient(l.Options) + } + return newSubstrateGRPCClient(l.Options) +} + +// Run implements Launcher. +func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, error) { + res := Result{ExitCode: SubstrateExitCodeUnknown} + if spec.ClineProvider != nil { + _ = spec.ClineProvider.Close() + return res, errors.New("substrate: scoped Cline provider files are Podman-only; the Substrate runtime takes no credential files") + } + if err := l.Options.Validate(); err != nil { + return res, err + } + // Validate just proved the hosts parse, so this cannot fail; the policy + // is built before anything is created so a kick never has a half-built + // actor to clean up on a config error. + policy, err := SubstrateEgressPolicy(l.Options.AllowHosts) + if err != nil { + return res, err + } + image := strings.TrimSpace(l.Options.Image) + if image == "" { + image = strings.TrimSpace(spec.Image) + } + if image == "" { + return res, errors.New("substrate: image is required") + } + if !substrateImageDigest.MatchString(image) { + return res, fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); set sandbox.substrate.image to the digest-pinned sandbox image", image) + } + if strings.TrimSpace(spec.Workspace) == "" { + return res, errors.New("substrate: workspace is required") + } + if len(spec.Command) == 0 { + return res, errors.New("substrate: command is required") + } + if l.Options.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, l.Options.Timeout) + defer cancel() + } + + client, err := l.client() + if err != nil { + return res, err + } + defer func() { _ = client.Close() }() + control, bridge := client.Control(), client.Bridge() + + now := time.Now + if l.now != nil { + now = l.now + } + name := substrateActorName(spec.Name, now()) + ref := &substrateapi.ObjectRef{Atespace: l.Options.atespace(), Name: name} + + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests; cleanup falls back to a fresh, request-bounded + // context so a finished actor is still removed. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), substrateRequestWait) + cancels = append(cancels, cancel) + return fresh + } + + if _, err := control.CreateActorTemplate(ctx, l.actorTemplate(ref, image)); err != nil { + return res, fmt.Errorf("substrate: creating actor template %s: %w", name, err) + } + cleanupTemplate := func() { _ = control.DeleteActorTemplate(live(), ref) } + + if _, err := control.CreateActor(ctx, &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: name}, + ActorTemplate: ref, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + }); err != nil { + cleanupTemplate() + return res, fmt.Errorf("substrate: creating actor %s: %w", name, err) + } + // Delete is the default: a kick is a one-shot and keeping a snapshot + // around costs object storage. snapshot_on_idle keeps a resumable + // snapshot instead, so the next kick's warm state (caches, builds) is + // a resume rather than a cold boot. + cleanupActor := func() { + if l.Options.SnapshotOnIdle && ctx.Err() == nil { + if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { + cleanupTemplate() + return + } + } + _ = control.DeleteActor(live(), ref) + cleanupTemplate() + } + + if spec.NetworkNone { + policy.Rules = nil + } + if err := control.CreateActorEgressPolicy(ctx, ref, policy); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: setting egress policy on actor %s: %w", name, err) + } + if err := control.ResumeActor(ctx, ref); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: resuming actor %s: %w", name, err) + } + if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: actor %s never became running: %w", name, err) + } + if err := bridge.WaitReady(ctx, ref, substrateReadyPoll); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: bridge in actor %s never answered: %w", name, err) + } + if err := bridge.Push(ctx, ref, spec.Workspace); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: pushing workspace into actor %s: %w", name, err) + } + workDir := spec.WorkDir + if workDir == "" { + workDir = SubstrateWorkspaceMount + } + out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}) + if runErr == nil { + res.Stdout, res.Stderr, res.ExitCode = out.Stdout, out.Stderr, out.ExitCode + if out.Error != "" { + runErr = errors.New(out.Error) + } + } + var pullErr error + if ctx.Err() == nil { + pullErr = bridge.Pull(ctx, ref, spec.Workspace) + } + cleanupActor() + if ctx.Err() != nil { + // Kick timeout or cancel: the actor has been removed above so the + // workload does not keep running on the cluster. + return res, ctx.Err() + } + if runErr != nil { + return res, fmt.Errorf("substrate: running kick in actor %s: %w", name, runErr) + } + if pullErr != nil { + return res, fmt.Errorf("substrate: pulling workspace out of actor %s: %w", name, pullErr) + } + return res, nil +} + +// substrateWaitRunning polls GetActor until the actor is RUNNING, or fails +// when it crashes. A deleted or suspended actor during the wait is a failure +// too: nothing hive did asked for it. +func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + actor, err := control.GetActor(ctx, ref) + if err != nil { + return err + } + switch state := actor.GetStatus().GetState(); state { + case substrateapi.ActorState_ACTOR_STATE_RUNNING: + return nil + case substrateapi.ActorState_ACTOR_STATE_CRASHED: + return fmt.Errorf("actor crashed: %s", actor.GetStatus().GetCrash().GetMessage()) + case substrateapi.ActorState_ACTOR_STATE_DELETING, substrateapi.ActorState_ACTOR_STATE_SUSPENDING: + return fmt.Errorf("actor entered %s while being resumed", state) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} diff --git a/src/crux/sandbox/substrate_bridge.go b/src/crux/sandbox/substrate_bridge.go new file mode 100644 index 0000000000..77f3ca4a16 --- /dev/null +++ b/src/crux/sandbox/substrate_bridge.go @@ -0,0 +1,107 @@ +package sandbox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// routerBridge's substrateBridge implementation: the four verbs the launcher +// drives a kick with, each one HTTP exchange over its own CONNECT tunnel. + +// WaitReady polls the bridge's health endpoint until it answers. The actor +// is RUNNING by now, but the bridge process inside still has to come up. +func (b *routerBridge) WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + var last error + for { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil) + if err == nil { + resp.Body.Close() + return nil + } + last = err + select { + case <-ctx.Done(): + return fmt.Errorf("%w (last: %v)", ctx.Err(), last) + case <-ticker.C: + } + } +} + +// Push replaces the actor's workspace with the local one. +func (b *routerBridge) Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + var buf bytes.Buffer + if err := substratebridge.Pack(workspace, &buf); err != nil { + return fmt.Errorf("packing %s: %w", workspace, err) + } + resp, err := b.do(ctx, ref, http.MethodPut, substratebridge.PathWorkspace, &buf) + if err != nil { + return err + } + return resp.Body.Close() +} + +// Run executes the kick command in the actor and waits for it to exit. +func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + body, err := json.Marshal(req) + if err != nil { + return out, err + } + resp, err := b.do(ctx, ref, http.MethodPost, substratebridge.PathRun, bytes.NewReader(body)) + if err != nil { + return out, err + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return out, fmt.Errorf("decoding run response: %w", err) + } + return out, nil +} + +// Pull replaces the local workspace with the actor's, so the commits the +// agent made are what hive's post-kick push broker sees. The new tree is +// unpacked beside the old one and swapped in, so a failed transfer leaves +// the original workspace untouched. +func (b *routerBridge) Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathWorkspace, nil) + if err != nil { + return err + } + defer resp.Body.Close() + return substrateReplaceDir(resp.Body, workspace) +} + +func substrateReplaceDir(stream io.Reader, workspace string) error { + parent := filepath.Dir(filepath.Clean(workspace)) + staging, err := os.MkdirTemp(parent, ".substrate-pull-*") + if err != nil { + return err + } + if err := substratebridge.Unpack(stream, staging); err != nil { + _ = os.RemoveAll(staging) + return fmt.Errorf("unpacking into %s: %w", staging, err) + } + old := staging + ".old" + if err := os.Rename(workspace, old); err != nil { + _ = os.RemoveAll(staging) + return err + } + if err := os.Rename(staging, workspace); err != nil { + _ = os.Rename(old, workspace) + _ = os.RemoveAll(staging) + return err + } + return os.RemoveAll(old) +} diff --git a/src/crux/sandbox/substrate_client.go b/src/crux/sandbox/substrate_client.go new file mode 100644 index 0000000000..2e72b7e4a5 --- /dev/null +++ b/src/crux/sandbox/substrate_client.go @@ -0,0 +1,292 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Real Substrate client: the Control API over gRPC plus the in-actor bridge +// over an atenet-router CONNECT tunnel. Constructed lazily by +// SubstrateLauncher.client() so unit tests never dial anything. + +// substrateRouterHeader selects the target actor on the router's CONNECT +// request (api-guide "Workload Connectivity"). +const substrateRouterHeader = "ate-target-actor" + +type grpcSubstrateClient struct { + conn *grpc.ClientConn + control *grpcControl + bridge *routerBridge +} + +func (c *grpcSubstrateClient) Control() substrateControl { return c.control } +func (c *grpcSubstrateClient) Bridge() substrateBridge { return c.bridge } +func (c *grpcSubstrateClient) Close() error { return c.conn.Close() } + +func newSubstrateGRPCClient(o SubstrateOptions) (substrateClient, error) { + addr, useTLS, err := substrateAddress(o.endpoint()) + if err != nil { + return nil, err + } + creds := insecure.NewCredentials() + if useTLS { + cfg, err := substrateControlTLS(o) + if err != nil { + return nil, err + } + creds = credentials.NewTLS(cfg) + } + opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + if o.Token != "" || o.TokenFile != "" || useTLS { + opts = append(opts, grpc.WithPerRPCCredentials(substrateBearer{token: o.Token, file: o.tokenFile(), secure: useTLS})) + } + conn, err := grpc.NewClient(addr, opts...) + if err != nil { + return nil, fmt.Errorf("substrate: connecting to control api %s: %w", addr, err) + } + bridge, err := newRouterBridge(o) + if err != nil { + _ = conn.Close() + return nil, err + } + return &grpcSubstrateClient{conn: conn, control: &grpcControl{c: substrateapi.NewControlClient(conn)}, bridge: bridge}, nil +} + +func (o SubstrateOptions) tokenFile() string { + if f := strings.TrimSpace(o.TokenFile); f != "" { + return f + } + return SubstrateDefaultTokenFile +} + +// substrateControlTLS verifies the Control API against ca_file (the +// servicedns ClusterTrustBundle projection) when present, the system roots +// otherwise, or nothing with insecure_tls. +func substrateControlTLS(o SubstrateOptions) (*tls.Config, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + if o.InsecureTLS { + return cfg, nil + } + caFile := strings.TrimSpace(o.CAFile) + if caFile == "" { + if _, err := os.Stat(SubstrateDefaultCAFile); err == nil { + caFile = SubstrateDefaultCAFile + } + } + if caFile != "" { + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: ca_file: %w", err) + } + cfg.RootCAs = pool + } + return cfg, nil +} + +func substrateCertPool(path string) (*x509.CertPool, error) { + pem, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("%s holds no PEM certificates", path) + } + return pool, nil +} + +// substrateBearer sends the projected ServiceAccount token as a bearer +// token, re-reading the file per RPC so kubelet rotation is picked up. +type substrateBearer struct { + token string + file string + secure bool +} + +func (b substrateBearer) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + tok := b.token + if tok == "" { + raw, err := os.ReadFile(b.file) + if err != nil { + return nil, fmt.Errorf("substrate: reading token file %s: %w", b.file, err) + } + tok = strings.TrimSpace(string(raw)) + } + if tok == "" { + return nil, errors.New("substrate: control api token is empty") + } + return map[string]string{"authorization": "Bearer " + tok}, nil +} + +func (b substrateBearer) RequireTransportSecurity() bool { return b.secure } + +// grpcControl adapts the generated ControlClient to substrateControl. +type grpcControl struct{ c substrateapi.ControlClient } + +func (g *grpcControl) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + return g.c.CreateActorTemplate(ctx, &substrateapi.CreateActorTemplateRequest{ActorTemplate: tpl}) +} + +func (g *grpcControl) DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActorTemplate(ctx, &substrateapi.DeleteActorTemplateRequest{ActorTemplate: ref}) + return err +} + +func (g *grpcControl) CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) { + return g.c.CreateActor(ctx, &substrateapi.CreateActorRequest{Actor: actor}) +} + +func (g *grpcControl) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + return g.c.GetActor(ctx, &substrateapi.GetActorRequest{Actor: ref}) +} + +func (g *grpcControl) ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.ResumeActor(ctx, &substrateapi.ResumeActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + _, err := g.c.SuspendActor(ctx, &substrateapi.SuspendActorRequest{Actor: ref, Options: &substrateapi.SuspendOptions{Mode: mode}}) + return err +} + +func (g *grpcControl) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActor(ctx, &substrateapi.DeleteActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error { + _, err := g.c.CreateActorEgressPolicy(ctx, &substrateapi.CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: policy}) + return err +} + +func (g *grpcControl) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + var out []*substrateapi.Worker + req := &substrateapi.ListWorkersRequest{Options: &substrateapi.ListOptions{}} + for { + resp, err := g.c.ListWorkers(ctx, req) + if err != nil { + return nil, err + } + out = append(out, resp.GetWorkers()...) + if resp.GetNextPageToken() == "" { + return out, nil + } + req.Options.PageToken = resp.GetNextPageToken() + } +} + +// routerBridge reaches the in-actor bridge through atenet-router: an mTLS +// CONNECT to the router with the actor named in ate-target-actor, then plain +// HTTP to the bridge over the tunnel. +type routerBridge struct { + routerAddr string + tlsConfig *tls.Config + port int + dial func(ctx context.Context, network, addr string) (net.Conn, error) +} + +func newRouterBridge(o SubstrateOptions) (*routerBridge, error) { + addr, err := substrateRouterAddress(o.routerURL()) + if err != nil { + return nil, err + } + host, _, _ := net.SplitHostPort(addr) + cfg := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: host, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + bundle := o.RouterClientBundle + if bundle == "" { + bundle = SubstrateDefaultRouterClientBundle + } + cert, err := tls.LoadX509KeyPair(bundle, bundle) + if err != nil { + return nil, fmt.Errorf("substrate: router_client_bundle %s: %w — atenet-router needs a podCertificate projection for hive (see deploy/substrate.md)", bundle, err) + } + cfg.Certificates = []tls.Certificate{cert} + if !o.InsecureTLS { + caFile := o.RouterCAFile + if caFile == "" { + caFile = SubstrateDefaultRouterCAFile + } + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: router_ca_file: %w", err) + } + cfg.RootCAs = pool + } + d := &net.Dialer{Timeout: substrateRequestWait} + return &routerBridge{routerAddr: addr, tlsConfig: cfg, port: o.bridgePort(), dial: d.DialContext}, nil +} + +// tunnel opens one CONNECT tunnel to the actor's bridge port. +func (b *routerBridge) tunnel(ctx context.Context, ref *substrateapi.ObjectRef) (net.Conn, error) { + raw, err := b.dial(ctx, "tcp", b.routerAddr) + if err != nil { + return nil, err + } + conn := tls.Client(raw, b.tlsConfig) + if err := conn.HandshakeContext(ctx); err != nil { + raw.Close() + return nil, fmt.Errorf("router tls handshake: %w", err) + } + target := net.JoinHostPort("actor", fmt.Sprint(b.port)) + req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: http.Header{}} + req.Header.Set(substrateRouterHeader, ref.GetAtespace()+"/"+ref.GetName()) + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + if err := req.Write(conn); err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + conn.Close() + return nil, fmt.Errorf("router refused connect: %s", resp.Status) + } + _ = conn.SetDeadline(time.Time{}) + return conn, nil +} + +func (b *routerBridge) do(ctx context.Context, ref *substrateapi.ObjectRef, method, path string, body io.Reader) (*http.Response, error) { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return b.tunnel(ctx, ref) }, + DisableKeepAlives: true, + } + defer transport.CloseIdleConnections() + req, err := http.NewRequestWithContext(ctx, method, "http://actor"+path, body) + if err != nil { + return nil, err + } + resp, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + return nil, fmt.Errorf("bridge %s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(msg))) + } + return resp, nil +} diff --git a/src/crux/sandbox/substrate_client_test.go b/src/crux/sandbox/substrate_client_test.go new file mode 100644 index 0000000000..fa9f0c8ea4 --- /dev/null +++ b/src/crux/sandbox/substrate_client_test.go @@ -0,0 +1,681 @@ +package sandbox + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "io" + "log" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Real-client tests: the gRPC adapter against an in-memory Control server, +// TLS/credential plumbing against files in a temp dir, and the router bridge +// against a TLS server that speaks CONNECT and then the bridge protocol. + +// selfSignedBundle writes a PEM file holding a self-signed cert and its key +// (the shape of a podCertificate credential bundle) plus a CA-only file. +func selfSignedBundle(t *testing.T, dir string) (bundle, caOnly string, cert tls.Certificate) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "hive-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + bundle = filepath.Join(dir, "credential-bundle.pem") + caOnly = filepath.Join(dir, "router-ca.pem") + if err := os.WriteFile(bundle, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(caOnly, certPEM, 0o600); err != nil { + t.Fatal(err) + } + cert, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + return bundle, caOnly, cert +} + +func TestNewSubstrateGRPCClientConstructs(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + + // TLS endpoint verified against ca_file, router verified against its CA. + c, err := newSubstrateGRPCClient(SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", CAFile: ca, TokenFile: filepath.Join(dir, "token"), + RouterClientBundle: bundle, RouterCAFile: ca, + }) + if err != nil { + t.Fatalf("tls client: %v", err) + } + if c.Control() == nil || c.Bridge() == nil { + t.Error("nil control or bridge") + } + if err := c.Close(); err != nil { + t.Errorf("Close: %v", err) + } + + // Plaintext endpoint with an env token and insecure router. + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", Token: "t", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext client: %v", err) + } + _ = c.Close() + + // Plaintext without any token still constructs (no per-RPC creds). + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext no-token client: %v", err) + } + _ = c.Close() + + // Insecure TLS skips the CA file entirely. + c, err = newSubstrateGRPCClient(SubstrateOptions{InsecureTLS: true, CAFile: "/nonexistent", RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("insecure client: %v", err) + } + _ = c.Close() +} + +func TestNewSubstrateGRPCClientErrors(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + noPEM := filepath.Join(dir, "empty.pem") + if err := os.WriteFile(noPEM, []byte("not a cert"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + o SubstrateOptions + want string + }{ + {"endpoint", SubstrateOptions{Endpoint: "nope"}, "endpoint"}, + {"ca missing", SubstrateOptions{CAFile: "/nonexistent/ca.crt"}, "ca_file"}, + {"ca not pem", SubstrateOptions{CAFile: noPEM}, "no PEM certificates"}, + {"router url", SubstrateOptions{CAFile: ca, RouterURL: "http://r"}, "router_url"}, + {"router bundle", SubstrateOptions{CAFile: ca, RouterClientBundle: "/nonexistent/b.pem"}, "router_client_bundle"}, + {"router ca", SubstrateOptions{CAFile: ca, RouterClientBundle: bundle, RouterCAFile: "/nonexistent/ca.pem"}, "router_ca_file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := newSubstrateGRPCClient(tc.o) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestSubstrateControlTLSDefaults(t *testing.T) { + cfg, err := substrateControlTLS(SubstrateOptions{}) + if err != nil || cfg.RootCAs != nil || cfg.InsecureSkipVerify { + t.Errorf("default tls = %+v, %v (system roots expected when the projected CA is absent)", cfg, err) + } + _, ca, _ := selfSignedBundle(t, t.TempDir()) + cfg, err = substrateControlTLS(SubstrateOptions{CAFile: ca}) + if err != nil || cfg.RootCAs == nil { + t.Errorf("ca_file tls = %+v, %v", cfg, err) + } +} + +func TestSubstrateBearer(t *testing.T) { + ctx := context.Background() + md, err := substrateBearer{token: "abc", secure: true}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer abc" { + t.Errorf("static token: %v %v", md, err) + } + dir := t.TempDir() + f := filepath.Join(dir, "token") + if err := os.WriteFile(f, []byte(" from-file\n"), 0o600); err != nil { + t.Fatal(err) + } + md, err = substrateBearer{file: f}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer from-file" { + t.Errorf("file token: %v %v", md, err) + } + if _, err := (substrateBearer{file: filepath.Join(dir, "missing")}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "reading token file") { + t.Errorf("missing file: %v", err) + } + if err := os.WriteFile(f, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (substrateBearer{file: f}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "token is empty") { + t.Errorf("empty file: %v", err) + } + if !(substrateBearer{secure: true}).RequireTransportSecurity() || (substrateBearer{}).RequireTransportSecurity() { + t.Error("RequireTransportSecurity must follow the TLS choice") + } +} + +// controlServer is the in-memory Control API the adapter is tested against. +type controlServer struct { + substrateapi.UnimplementedControlServer + fail bool + pages int +} + +func (s *controlServer) err() error { + if s.fail { + return status.Error(codes.Internal, "injected") + } + return nil +} + +func (s *controlServer) CreateActorTemplate(_ context.Context, r *substrateapi.CreateActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return r.GetActorTemplate(), s.err() +} +func (s *controlServer) DeleteActorTemplate(context.Context, *substrateapi.DeleteActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return &substrateapi.ActorTemplate{}, s.err() +} +func (s *controlServer) CreateActor(_ context.Context, r *substrateapi.CreateActorRequest) (*substrateapi.Actor, error) { + return r.GetActor(), s.err() +} +func (s *controlServer) GetActor(_ context.Context, r *substrateapi.GetActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: r.GetActor().GetName()}}, s.err() +} +func (s *controlServer) ResumeActor(context.Context, *substrateapi.ResumeActorRequest) (*substrateapi.ResumeActorResponse, error) { + return &substrateapi.ResumeActorResponse{}, s.err() +} +func (s *controlServer) SuspendActor(_ context.Context, r *substrateapi.SuspendActorRequest) (*substrateapi.SuspendActorResponse, error) { + if r.GetOptions().GetMode() != substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + return nil, status.Error(codes.InvalidArgument, "mode not forwarded") + } + return &substrateapi.SuspendActorResponse{}, s.err() +} +func (s *controlServer) DeleteActor(context.Context, *substrateapi.DeleteActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{}, s.err() +} +func (s *controlServer) CreateActorEgressPolicy(_ context.Context, r *substrateapi.CreateActorEgressPolicyRequest) (*substrateapi.EgressPolicy, error) { + return r.GetEgressPolicy(), s.err() +} +func (s *controlServer) ListWorkers(_ context.Context, r *substrateapi.ListWorkersRequest) (*substrateapi.ListWorkersResponse, error) { + if err := s.err(); err != nil { + return nil, err + } + s.pages++ + if r.GetOptions().GetPageToken() == "" { + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w1"}}}, NextPageToken: "next"}, nil + } + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w2"}}}}, nil +} + +func newBufconnControl(t *testing.T, srv substrateapi.ControlServer) *grpcControl { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + substrateapi.RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &grpcControl{c: substrateapi.NewControlClient(conn)} +} + +func TestGRPCControlAdapter(t *testing.T) { + srv := &controlServer{} + g := newBufconnControl(t, srv) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a1"} + if tpl, err := g.CreateActorTemplate(ctx, &substrateapi.ActorTemplate{Metadata: &substrateapi.ResourceMetadata{Name: "t"}}); err != nil || tpl.GetMetadata().GetName() != "t" { + t.Errorf("CreateActorTemplate: %v %v", tpl, err) + } + if err := g.DeleteActorTemplate(ctx, ref); err != nil { + t.Errorf("DeleteActorTemplate: %v", err) + } + if a, err := g.CreateActor(ctx, &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: "a1"}}); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("CreateActor: %v %v", a, err) + } + if a, err := g.GetActor(ctx, ref); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("GetActor: %v %v", a, err) + } + if err := g.ResumeActor(ctx, ref); err != nil { + t.Errorf("ResumeActor: %v", err) + } + if err := g.SuspendActor(ctx, ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err != nil { + t.Errorf("SuspendActor: %v", err) + } + if err := g.DeleteActor(ctx, ref); err != nil { + t.Errorf("DeleteActor: %v", err) + } + if err := g.CreateActorEgressPolicy(ctx, ref, &substrateapi.EgressPolicy{}); err != nil { + t.Errorf("CreateActorEgressPolicy: %v", err) + } + workers, err := g.ListWorkers(ctx) + if err != nil || len(workers) != 2 || srv.pages != 2 { + t.Errorf("ListWorkers: %d workers, %d pages, %v", len(workers), srv.pages, err) + } + + srv.fail = true + if _, err := g.ListWorkers(ctx); status.Code(err) != codes.Internal { + t.Errorf("ListWorkers error not propagated: %v", err) + } + if _, err := g.CreateActorTemplate(ctx, nil); status.Code(err) != codes.Internal { + t.Errorf("CreateActorTemplate error not propagated: %v", err) + } +} + +// connectRouter is a TLS server that accepts CONNECT like atenet-router and +// then answers one bridge request over the tunnel. +type connectRouter struct { + t *testing.T + handler http.Handler + refuse bool + lastHdr string + lastHost string +} + +func (r *connectRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodConnect { + http.Error(w, "not connect", http.StatusMethodNotAllowed) + return + } + r.lastHdr = req.Header.Get(substrateRouterHeader) + r.lastHost = req.Host + if r.refuse { + http.Error(w, "spiffe id not allowed", http.StatusForbidden) + return + } + hj, ok := w.(http.Hijacker) + if !ok { + r.t.Fatal("no hijacker") + } + conn, rw, err := hj.Hijack() + if err != nil { + r.t.Fatal(err) + } + defer conn.Close() + _, _ = rw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n") + _ = rw.Flush() + inner, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return + } + rec := httptest.NewRecorder() + r.handler.ServeHTTP(rec, inner) + resp := rec.Result() + resp.ContentLength = int64(rec.Body.Len()) + _ = resp.Write(conn) +} + +func newConnectRouter(t *testing.T, h http.Handler) (*connectRouter, *routerBridge) { + t.Helper() + cr := &connectRouter{t: t, handler: h} + srv := httptest.NewUnstartedServer(cr) + // The untrusted-certificate case below makes the server log a handshake + // error; that is the expected outcome, not noise worth printing. + srv.Config.ErrorLog = log.New(io.Discard, "", 0) + srv.StartTLS() + t.Cleanup(srv.Close) + addr := strings.TrimPrefix(srv.URL, "https://") + b := &routerBridge{ + routerAddr: addr, + tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, //nolint:gosec // test server + port: substratebridge.DefaultPort, + dial: (&net.Dialer{}).DialContext, + } + return cr, b +} + +func TestRouterBridgeDrivesTheBridgeProtocol(t *testing.T) { + actorWS := filepath.Join(t.TempDir(), "actor-ws") + cr, b := newConnectRouter(t, &substratebridge.Handler{Workspace: actorWS}) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "hive-kick-deps-1"} + + if err := b.WaitReady(ctx, ref, time.Millisecond); err != nil { + t.Fatalf("WaitReady: %v", err) + } + if cr.lastHdr != "hive/hive-kick-deps-1" || cr.lastHost != "actor:8477" { + t.Errorf("CONNECT header=%q host=%q", cr.lastHdr, cr.lastHost) + } + + local := filepath.Join(t.TempDir(), "ws") + if err := os.MkdirAll(filepath.Join(local, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(local, "sub", "file.txt"), []byte("before"), 0o644); err != nil { + t.Fatal(err) + } + if err := b.Push(ctx, ref, local); err != nil { + t.Fatalf("Push: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(actorWS, "sub", "file.txt")); string(got) != "before" { + t.Fatalf("actor workspace = %q", got) + } + out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}) + if err != nil || out.ExitCode != 4 || !strings.Contains(out.Stdout, "ran") { + t.Fatalf("Run: %+v %v", out, err) + } + if err := b.Pull(ctx, ref, local); err != nil { + t.Fatalf("Pull: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(local, "sub", "file.txt")); strings.TrimSpace(string(got)) != "after" { + t.Errorf("local workspace after pull = %q", got) + } + if entries, _ := os.ReadDir(filepath.Dir(local)); len(entries) != 1 { + t.Errorf("staging dirs left behind: %v", entries) + } +} + +func TestRouterBridgeErrors(t *testing.T) { + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + + // Router refuses the CONNECT. + cr, b := newConnectRouter(t, http.NotFoundHandler()) + cr.refuse = true + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "router refused connect") { + t.Errorf("refused connect: %v", err) + } + + // Bridge answers non-2xx. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unpack failed", http.StatusBadRequest) + })) + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "unpack failed") { + t.Errorf("non-2xx: %v", err) + } + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil { + t.Error("Run swallowed a non-2xx") + } + if err := b.Pull(ctx, ref, t.TempDir()); err == nil { + t.Error("Pull swallowed a non-2xx") + } + + // Bridge answers 200 with garbage JSON / garbage tar. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("{not json")) + })) + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Errorf("bad json: %v", err) + } + ws := filepath.Join(t.TempDir(), "ws") + if err := os.Mkdir(ws, 0o755); err != nil { + t.Fatal(err) + } + if err := b.Pull(ctx, ref, ws); err == nil || !strings.Contains(err.Error(), "unpacking") { + t.Errorf("bad tar: %v", err) + } + + // Push of a workspace that does not exist fails before any I/O. + if err := b.Push(ctx, ref, filepath.Join(t.TempDir(), "missing")); err == nil || !strings.Contains(err.Error(), "packing") { + t.Errorf("missing workspace: %v", err) + } + + // TLS to a server whose certificate is not trusted. + _, strict := newConnectRouter(t, http.NotFoundHandler()) + strict.tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: x509.NewCertPool(), ServerName: "localhost"} + if _, err := strict.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "tls handshake") { + t.Errorf("untrusted server: %v", err) + } + + // Dial failure. + b.dial = func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("no route") } + if _, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil); err == nil || !strings.Contains(err.Error(), "no route") { + t.Errorf("dial failure: %v", err) + } + + // Server closes the tunnel without a CONNECT response. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + _, _, serverCert := selfSignedBundle(t, t.TempDir()) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + tc := tls.Server(c, &tls.Config{Certificates: []tls.Certificate{serverCert}, MinVersion: tls.VersionTLS12}) + _ = tc.Handshake() + _ = tc.Close() + } + }() + closing := &routerBridge{routerAddr: ln.Addr().String(), tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, port: 1, dial: (&net.Dialer{}).DialContext} //nolint:gosec // test server + if _, err := closing.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "router connect") { + t.Errorf("closed tunnel: %v", err) + } +} + +func TestRouterBridgeWaitReadyGivesUpOnContext(t *testing.T) { + // The bridge answers 503 and the kick is cancelled right after that + // first probe, so WaitReady must stop without waiting for the ticker. + ctx, cancel := context.WithCancel(context.Background()) + _, b := newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + cancel() + http.Error(w, "starting", http.StatusServiceUnavailable) + })) + err := b.WaitReady(ctx, &substrateapi.ObjectRef{Atespace: "hive", Name: "a"}, time.Hour) + if !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "last:") { + t.Fatalf("err = %v, want context.Canceled wrapping the last probe error", err) + } +} + +func TestRouterBridgeRequestContextCancelled(t *testing.T) { + _, b := newConnectRouter(t, http.NotFoundHandler()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := b.do(ctx, &substrateapi.ObjectRef{}, http.MethodGet, substratebridge.PathHealth, nil); err == nil { + t.Error("do with a cancelled context succeeded") + } + if _, err := b.do(context.Background(), &substrateapi.ObjectRef{}, "bad method", "/x", nil); err == nil { + t.Error("do with an invalid method succeeded") + } +} + +func TestNewRouterBridgeLoadsCredentials(t *testing.T) { + bundle, ca, _ := selfSignedBundle(t, t.TempDir()) + b, err := newRouterBridge(SubstrateOptions{RouterClientBundle: bundle, RouterCAFile: ca, BridgePort: 9001, RouterURL: "https://router.example:8444"}) + if err != nil { + t.Fatalf("newRouterBridge: %v", err) + } + if b.routerAddr != "router.example:8444" || b.port != 9001 || len(b.tlsConfig.Certificates) != 1 || b.tlsConfig.RootCAs == nil || b.tlsConfig.ServerName != "router.example" { + t.Errorf("bridge = %+v", b) + } + // Dialer is wired to the real network. + if _, err := b.dial(context.Background(), "tcp", "127.0.0.1:1"); err == nil { + t.Error("dial to a closed port succeeded") + } +} + +func TestSubstrateReplaceDirErrors(t *testing.T) { + var good bytes.Buffer + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := substratebridge.Pack(src, &good); err != nil { + t.Fatal(err) + } + // Parent of the workspace is not writable for the staging dir. + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), "/nonexistent-parent/ws"); err == nil { + t.Error("staging in a missing parent succeeded") + } + // Workspace itself missing: rename of the old tree fails and staging is removed. + parent := t.TempDir() + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), filepath.Join(parent, "missing")); err == nil { + t.Error("replacing a missing workspace succeeded") + } + if entries, _ := os.ReadDir(parent); len(entries) != 0 { + t.Errorf("staging left behind: %v", entries) + } +} + +// Preflight. + +type fakeSubstrateFactory struct { + fake *fakeSubstrate + err error +} + +func (f fakeSubstrateFactory) new(SubstrateOptions) (substrateClient, error) { + if f.err != nil { + return nil, f.err + } + return f.fake, nil +} + +func worker(name, state string, labels map[string]string, classes ...string) *substrateapi.Worker { + w := &substrateapi.Worker{Metadata: &substrateapi.ResourceMetadata{Name: name}, Labels: labels, Status: &substrateapi.WorkerStatus{}} + switch state { + case "active": + w.Status.State = substrateapi.WorkerState_WORKER_STATE_ACTIVE + default: + w.Status.State = substrateapi.WorkerState_WORKER_STATE_UNSPECIFIED + } + for _, c := range classes { + w.Status.SandboxClasses = append(w.Status.SandboxClasses, &substrateapi.WorkerSandboxClass{SandboxClass: c}) + } + return w +} + +func TestSubstratePreflighterWarnings(t *testing.T) { + ctx := context.Background() + opts := SubstrateOptions{Endpoint: "api:443", Token: "t"} + + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{err: errors.New("no bundle")}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") || !strings.Contains(got[0], "no bundle") { + t.Errorf("constructor failure: %q", got) + } + + fake := &fakeSubstrate{listErr: errors.New("connection refused")} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("list failure: %q", got) + } + if !fake.closed { + t.Error("client not closed after probe") + } + + fake = &fakeSubstrate{} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "no active workers") { + t.Errorf("no workers: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", map[string]string{"class": "microvm"}, "microvm")}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + sel := opts + sel.WorkerSelector = map[string]string{"class": "gvisor"} + if got := p.Check(ctx, sel); len(got) != 1 || !strings.Contains(got[0], "worker_selector") { + t.Errorf("selector mismatch: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil)}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "none reports a sandbox class") { + t.Errorf("no class: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{ + worker("down", "unspecified", nil, "gvisor"), + worker("up", "active", map[string]string{"class": "gvisor"}, "GVISOR", ""), + }} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 0 { + t.Errorf("healthy control plane warned: %q", got) + } +} + +func TestSubstratePreflighterCachesPerControlPlane(t *testing.T) { + ctx := context.Background() + fake := &fakeSubstrate{listErr: errors.New("down")} + now := time.Unix(1_700_000_000, 0) + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new, now: func() time.Time { return now }} + a := SubstrateOptions{Endpoint: "a:443", Token: "t"} + b := SubstrateOptions{Endpoint: "b:443", Token: "t"} + p.Check(ctx, a) + p.Check(ctx, a) + p.Check(ctx, b) + if n := len(fake.calls); n != 2 { + t.Errorf("ListWorkers calls = %d, want 2 (one per control plane, second a cache hit)", n) + } + now = now.Add(substratePreflightTTL + time.Second) + p.Check(ctx, a) + if n := len(fake.calls); n != 3 { + t.Errorf("ListWorkers calls after TTL = %d, want 3", n) + } + // Returned slices are copies: mutating one must not poison the cache. + got := p.Check(ctx, a) + got[0] = "mutated" + if again := p.Check(ctx, a); again[0] == "mutated" { + t.Error("cache returned its own backing slice") + } +} + +func TestSubstrateActiveWorkersAndLabels(t *testing.T) { + n, classes := substrateActiveWorkers([]*substrateapi.Worker{ + worker("a", "active", map[string]string{"k": "v"}, "gvisor"), + worker("b", "active", map[string]string{"k": "v"}, "microvm", "gvisor"), + worker("c", "active", map[string]string{"k": "other"}, "firecracker"), + nil, + }, map[string]string{"k": "v"}) + if n != 2 || strings.Join(classes, ",") != "gvisor,microvm" { + t.Errorf("active=%d classes=%v", n, classes) + } + if !substrateLabelsMatch(nil, nil) || substrateLabelsMatch(nil, map[string]string{"a": "b"}) { + t.Error("label matching") + } +} + +func TestDefaultSubstratePreflighterUsesRealClient(t *testing.T) { + p := &SubstratePreflighter{} + got := p.Check(context.Background(), SubstrateOptions{Endpoint: "127.0.0.1:1", CAFile: "/nonexistent/ca.crt"}) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("warnings = %q", got) + } +} diff --git a/src/crux/sandbox/substrate_preflight.go b/src/crux/sandbox/substrate_preflight.go new file mode 100644 index 0000000000..94964cc5e9 --- /dev/null +++ b/src/crux/sandbox/substrate_preflight.go @@ -0,0 +1,125 @@ +package sandbox + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Substrate cluster-prerequisite preflight (ADR-0026 phase 2; the sibling +// of the OpenShell preflight in openshell_preflight.go). Hive cannot see the +// node kernel or the WorkerPool CRDs from its pod without cluster RBAC, but +// the Control API answers whether it is up and which workers — and so which +// sandbox classes — exist; that is what decides whether a kick can land. + +const substratePreflightTTL = 5 * time.Minute + +type substratePreflightEntry struct { + at time.Time + warnings []string +} + +// SubstratePreflighter checks a control plane and remembers the answer for a +// TTL, so the dashboard polling the same warnings does not hit it per call. +type SubstratePreflighter struct { + mu sync.Mutex + cache map[string]substratePreflightEntry + + newClient func(SubstrateOptions) (substrateClient, error) + now func() time.Time +} + +// DefaultSubstratePreflighter backs the warnings AgentSandboxGateWarnings emits. +var DefaultSubstratePreflighter = &SubstratePreflighter{} + +// Check returns the warnings for one control plane. Never an error: an +// unreachable control plane is itself a warning. +func (p *SubstratePreflighter) Check(ctx context.Context, opts SubstrateOptions) []string { + now := time.Now + if p.now != nil { + now = p.now + } + key := opts.endpoint() + "\x00" + opts.atespace() + "\x00" + opts.Token + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.cache[key]; ok && now().Sub(e.at) < substratePreflightTTL { + return append([]string(nil), e.warnings...) + } + warnings := p.probe(ctx, opts) + if p.cache == nil { + p.cache = map[string]substratePreflightEntry{} + } + p.cache[key] = substratePreflightEntry{at: now(), warnings: warnings} + return append([]string(nil), warnings...) +} + +func (p *SubstratePreflighter) probe(ctx context.Context, opts SubstrateOptions) []string { + ep := opts.endpoint() + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + + newClient := newSubstrateGRPCClient + if p.newClient != nil { + newClient = p.newClient + } + client, err := newClient(opts) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + defer func() { _ = client.Close() }() + + workers, err := client.Control().ListWorkers(ctx) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + active, classes := substrateActiveWorkers(workers, opts.WorkerSelector) + if active == 0 { + if len(opts.WorkerSelector) > 0 { + return []string{fmt.Sprintf("substrate control api %s has no active worker matching worker_selector %v; actors will never be scheduled — check the WorkerPool labels and that its pods are running", ep, opts.WorkerSelector)} + } + return []string{fmt.Sprintf("substrate control api %s reports no active workers; actors will never be scheduled — install a WorkerPool (see deploy/substrate.md) and check its pods are running", ep)} + } + var out []string + if len(classes) == 0 { + out = append(out, fmt.Sprintf("substrate control api %s has %d active worker(s) but none reports a sandbox class; the WorkerPool's SandboxConfig may not have resolved (kubectl get workerpool -A)", ep, active)) + } + return out +} + +// substrateActiveWorkers counts the ACTIVE workers matching the selector and +// returns the sorted set of sandbox classes they advertise. +func substrateActiveWorkers(workers []*substrateapi.Worker, selector map[string]string) (int, []string) { + active := 0 + classSet := map[string]bool{} + for _, w := range workers { + if w.GetStatus().GetState() != substrateapi.WorkerState_WORKER_STATE_ACTIVE || !substrateLabelsMatch(w.GetLabels(), selector) { + continue + } + active++ + for _, c := range w.GetStatus().GetSandboxClasses() { + if c.GetSandboxClass() != "" { + classSet[strings.ToLower(c.GetSandboxClass())] = true + } + } + } + classes := make([]string, 0, len(classSet)) + for c := range classSet { + classes = append(classes, c) + } + sort.Strings(classes) + return active, classes +} + +func substrateLabelsMatch(labels, selector map[string]string) bool { + for k, v := range selector { + if labels[k] != v { + return false + } + } + return true +} diff --git a/src/crux/sandbox/substrate_spec.go b/src/crux/sandbox/substrate_spec.go new file mode 100644 index 0000000000..d4df1e8a16 --- /dev/null +++ b/src/crux/sandbox/substrate_spec.go @@ -0,0 +1,234 @@ +package sandbox + +import ( + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// SubstrateDefaultBridgeCommand is the actor entrypoint when the operator +// names none: the bridge subcommand of the crux binary every hive sandbox +// image ships (ADR-0026 phase 2). +var SubstrateDefaultBridgeCommand = []string{"/usr/local/bin/crux", "substrate-bridge"} + +// SubstrateAlwaysAllowedHosts are the egress hosts every hive actor gets: +// the same GitHub hosts hive's MITM proxy (ADR-0002) brokers for a Podman +// sandbox and the OpenShell policy allows. Substrate's atenet-egress has no +// per-method read-only preset, so api.github.com is allowed as TLS +// passthrough like github.com; the actor still holds no token, and hive does +// every write itself after the kick. +var SubstrateAlwaysAllowedHosts = []string{"github.com", "api.github.com"} + +// substrateImageDigest accepts only digest-pinned references. +var substrateImageDigest = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +// actorTemplate maps the hive LaunchSpec onto a per-kick ActorTemplate: one +// container running the bridge, the workspace as a durableDir, golden +// snapshots off (a template used once has nothing to warm), and the +// operator's resource limits and pool selector. +func (l *SubstrateLauncher) actorTemplate(ref *substrateapi.ObjectRef, image string) *substrateapi.ActorTemplate { + command := l.Options.Command + if len(command) == 0 { + command = SubstrateDefaultBridgeCommand + } + cpu, mem := l.Options.CPUMillicores, l.Options.MemoryBytes + if cpu == 0 { + cpu = SubstrateDefaultCPUMillicores + } + if mem == 0 { + mem = SubstrateDefaultMemoryBytes + } + env := []*substrateapi.EnvVar{{Name: "HIVE_SUBSTRATE_BRIDGE_PORT", Value: strconv.Itoa(l.Options.bridgePort())}} + if p := strings.TrimSpace(l.Options.PolicyProfile); p != "" { + // Substrate has no annotations; the profile travels as an env var the + // bridge and in-actor tooling can read, and as a selector label an + // operator can pin a pool on. + env = append(env, &substrateapi.EnvVar{Name: "HIVE_POLICY_PROFILE", Value: p}) + } + return &substrateapi.ActorTemplate{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + Containers: []*substrateapi.Container{{ + Name: substrateContainer, + Image: image, + Command: command, + Env: env, + VolumeMounts: []*substrateapi.VolumeMount{{Name: substrateVolume, MountPath: SubstrateWorkspaceMount}}, + }}, + Volumes: []*substrateapi.Volume{{Name: substrateVolume, DurableDir: &substrateapi.DurableDirVolumeSource{}}}, + SnapshotConfig: &substrateapi.SnapshotConfig{GoldenSnapshotConfig: &substrateapi.GoldenSnapshotConfig{Mode: substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}}, + Resources: &substrateapi.ResourceRequirements{Limits: &substrateapi.Resources{CpuMillicores: cpu, MemoryBytes: mem}}, + } +} + +func substrateSelector(labels map[string]string) *substrateapi.Selector { + if len(labels) == 0 { + return nil + } + out := make(map[string]string, len(labels)) + for k, v := range labels { + out[k] = v + } + return &substrateapi.Selector{MatchLabels: out} +} + +// substrateEnv is the environment the kick runs with: the allowlisted, +// credential-filtered subset of hive's own, plus the spec's explicit values — +// the same rule the Podman, Job and OpenShell launchers apply. +func substrateEnv(spec LaunchSpec) map[string]string { + env := map[string]string{} + for _, entry := range SanitizedEnv(os.Environ(), spec.EnvAllowlist, spec.Env) { + if k, v, ok := strings.Cut(entry, "="); ok { + env[k] = v + } + } + return env +} + +// SubstrateEgressPolicy derives the actor's egress policy from hive's +// always-allowed GitHub hosts plus the operator's extra hosts, all as TLS +// passthrough on their port: the actor talks TLS end to end to the same hosts +// hive's MITM proxy would broker for a Podman sandbox, and atenet-egress +// default-denies everything else (including all UDP). Hosts are sorted so the +// policy hive sends is stable across kicks. +func SubstrateEgressPolicy(allowHosts []string) (*substrateapi.EgressPolicy, error) { + byPort := map[int32][]string{} + seen := map[string]bool{} + add := func(host string, port int32) { + key := host + ":" + strconv.Itoa(int(port)) + if seen[key] { + return + } + seen[key] = true + byPort[port] = append(byPort[port], host) + } + for _, h := range SubstrateAlwaysAllowedHosts { + add(h, substrateHTTPSPort) + } + for _, h := range allowHosts { + host, port, err := substrateHostPort(h) + if err != nil { + return nil, err + } + add(host, int32(port)) + } + ports := make([]int32, 0, len(byPort)) + for p := range byPort { + ports = append(ports, p) + } + sort.Slice(ports, func(i, j int) bool { return ports[i] < ports[j] }) + rules := make([]*substrateapi.EgressRule, 0, len(ports)) + for _, p := range ports { + hosts := byPort[p] + sort.Strings(hosts) + rules = append(rules, &substrateapi.EgressRule{TlsPassthrough: &substrateapi.TlsPassthroughRule{ + Hostnames: hosts, + Ports: &substrateapi.Ports{Numbers: []int32{p}}, + }}) + } + return &substrateapi.EgressPolicy{ + Metadata: &substrateapi.ResourceMetadata{Name: substrateEgressPolicy}, + Rules: rules, + }, nil +} + +// substrateAddress turns endpoint into a gRPC host:port target and whether +// TLS is implied. Accepts `host:port`, `http://host:port`, `https://host[:port]` +// and the grpc(s) spellings; a bare host:port is TLS, which is what the +// Control API serves. +func substrateAddress(raw string) (addr string, tls bool, err error) { + raw = strings.TrimSpace(raw) + if !strings.Contains(raw, "://") { + if _, _, err := substrateHostPort(raw); err != nil || !strings.Contains(raw, ":") { + return "", false, fmt.Errorf("substrate: endpoint %q must be host:port or an http(s) URL", raw) + } + return raw, true, nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("substrate: endpoint %q is not a valid URL", raw) + } + switch u.Scheme { + case "https", "grpcs": + tls = true + case "http", "grpc": + default: + return "", false, fmt.Errorf("substrate: endpoint %q has unsupported scheme %q", raw, u.Scheme) + } + host := u.Host + if u.Port() == "" { + if tls { + host += ":443" + } else { + host += ":80" + } + } + return host, tls, nil +} + +// substrateRouterAddress validates router_url and returns its host:port; the +// router serves TLS only, so https is the one accepted scheme. +func substrateRouterAddress(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || u.Scheme != "https" { + return "", fmt.Errorf("substrate: router_url %q must be an https URL (atenet-router serves TLS only)", raw) + } + host := u.Host + if u.Port() == "" { + host += ":443" + } + return host, nil +} + +// substrateHostName is a literal DNS name or a leftmost-wildcard DNS name, +// the two shapes Substrate's egress hostnames accept. +var substrateHostName = regexp.MustCompile(`^(\*\.)?[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$`) + +// substrateHostPort parses an allow_hosts entry: `host` means 443. +func substrateHostPort(raw string) (string, uint32, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", 0, errors.New("substrate: allow_hosts entry is empty") + } + if strings.Contains(raw, "://") || strings.ContainsAny(raw, "/ ") { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be host or host:port, not a URL", raw) + } + host, portStr, found := strings.Cut(raw, ":") + if !substrateHostName.MatchString(host) { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be a DNS name with at most a leading *. wildcard — no userinfo, query or fragment", raw) + } + host = strings.ToLower(host) + if !found { + return host, substrateHTTPSPort, nil + } + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil || port == 0 { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q has an invalid port", raw) + } + return host, uint32(port), nil +} + +var substrateNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// substrateActorName builds a unique, DNS-label-safe actor (and template) +// name from the launch name and a timestamp: hive-kick--. +func substrateActorName(launchName string, now time.Time) string { + base := strings.Trim(substrateNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + if base == "" { + base = "agent" + } + suffix := fmt.Sprintf("-%d", now.Unix()) + room := substrateNameMax - len(substrateNamePrefix) - len(suffix) + if len(base) > room { + base = strings.TrimRight(base[:room], "-") + } + return substrateNamePrefix + base + suffix +} diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go new file mode 100644 index 0000000000..f306ab9d39 --- /dev/null +++ b/src/crux/sandbox/substrate_test.go @@ -0,0 +1,652 @@ +package sandbox + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// fakeSubstrate is an in-memory stand-in for the Control API slice and the +// in-actor bridge the launcher drives. It records the calls so tests can +// assert on the template, actor, policy and cleanup that reached the control +// plane, and lets each step fail or report a state on demand. +type fakeSubstrate struct { + mu sync.Mutex + + templates []*substrateapi.ActorTemplate + actors []*substrateapi.Actor + policies []*substrateapi.EgressPolicy + calls []string + templateErr error + actorErr error + policyErr error + resumeErr error + suspendErr error + deleteErr error + getErr error + states []substrateapi.ActorState + crashMsg string + getCancels context.CancelFunc + waitReadyErr error + pushErr error + runErr error + runResp substratebridge.RunResponse + runCancels context.CancelFunc + pullErr error + runReq substratebridge.RunRequest + pushedWorkspace string + pulledWorkspace string + workers []*substrateapi.Worker + listErr error + closed bool + cleanupCtxDead bool +} + +func (f *fakeSubstrate) note(c string, ctx context.Context) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, c) + if (c == "DeleteActor" || c == "DeleteActorTemplate" || c == "SuspendActor") && ctx.Err() != nil { + f.cleanupCtxDead = true + } +} + +func (f *fakeSubstrate) Control() substrateControl { return f } +func (f *fakeSubstrate) Bridge() substrateBridge { return f } +func (f *fakeSubstrate) Close() error { f.closed = true; return nil } + +func (f *fakeSubstrate) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + f.note("CreateActorTemplate", ctx) + if f.templateErr != nil { + return nil, f.templateErr + } + f.templates = append(f.templates, tpl) + return tpl, nil +} + +func (f *fakeSubstrate) DeleteActorTemplate(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("DeleteActorTemplate", ctx) + return nil +} + +func (f *fakeSubstrate) CreateActor(ctx context.Context, a *substrateapi.Actor) (*substrateapi.Actor, error) { + f.note("CreateActor", ctx) + if f.actorErr != nil { + return nil, f.actorErr + } + f.actors = append(f.actors, a) + return a, nil +} + +func (f *fakeSubstrate) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + f.note("GetActor", ctx) + if f.getErr != nil { + return nil, f.getErr + } + state := substrateapi.ActorState_ACTOR_STATE_RUNNING + if len(f.states) > 0 { + state, f.states = f.states[0], f.states[1:] + } + if f.getCancels != nil { + f.getCancels() + } + return &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Name: ref.GetName()}, + Status: &substrateapi.ActorStatus{State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}}, + }, nil +} + +func (f *fakeSubstrate) ResumeActor(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("ResumeActor", ctx) + return f.resumeErr +} + +func (f *fakeSubstrate) SuspendActor(ctx context.Context, _ *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + f.note("SuspendActor", ctx) + f.calls = append(f.calls, "mode="+mode.String()) + return f.suspendErr +} + +func (f *fakeSubstrate) DeleteActor(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("DeleteActor", ctx) + return f.deleteErr +} + +func (f *fakeSubstrate) CreateActorEgressPolicy(ctx context.Context, _ *substrateapi.ObjectRef, p *substrateapi.EgressPolicy) error { + f.note("CreateActorEgressPolicy", ctx) + if f.policyErr != nil { + return f.policyErr + } + f.policies = append(f.policies, p) + return nil +} + +func (f *fakeSubstrate) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + f.note("ListWorkers", ctx) + return f.workers, f.listErr +} + +func (f *fakeSubstrate) WaitReady(ctx context.Context, _ *substrateapi.ObjectRef, _ time.Duration) error { + f.note("WaitReady", ctx) + return f.waitReadyErr +} + +func (f *fakeSubstrate) Push(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Push", ctx) + f.pushedWorkspace = ws + return f.pushErr +} + +func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { + f.note("Run", ctx) + f.runReq = req + if f.runCancels != nil { + f.runCancels() + } + return f.runResp, f.runErr +} + +func (f *fakeSubstrate) Pull(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Pull", ctx) + f.pulledWorkspace = ws + return f.pullErr +} + +func (f *fakeSubstrate) saw(call string) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, c := range f.calls { + if c == call { + return true + } + } + return false +} + +const substrateTestImage = "registry.example/hive/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func newTestSubstrate(fake *fakeSubstrate) *SubstrateLauncher { + l := &SubstrateLauncher{Options: SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + Token: "tok", + Image: substrateTestImage, + AllowHosts: []string{"litellm.example.com", "*.pypi.org:8443"}, + }} + l.newClient = func(SubstrateOptions) (substrateClient, error) { return fake, nil } + l.now = func() time.Time { return time.Unix(1_700_000_000, 0) } + return l +} + +func substrateTestSpec() LaunchSpec { + return LaunchSpec{ + Image: "registry.example/hive/sandbox:dev", + Name: "deps-sandbox", + Workspace: "/data/agents/sandbox/deps", + Command: []string{"crux", "kick", "--agent", "deps"}, + Env: map[string]string{"HIVE_AGENT": "deps"}, + } +} + +func TestSubstrateRunHappyPath(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "done\n", Stderr: "warn\n", ExitCode: 0}} + l := newTestSubstrate(fake) + l.Options.PolicyProfile = "strict" + l.Options.WorkerSelector = map[string]string{"class": "gvisor"} + res, err := l.Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Stdout != "done\n" || res.Stderr != "warn\n" || res.ExitCode != 0 { + t.Errorf("result = %+v", res) + } + want := []string{"CreateActorTemplate", "CreateActor", "CreateActorEgressPolicy", "ResumeActor", "GetActor", "WaitReady", "Push", "Run", "Pull", "DeleteActor", "DeleteActorTemplate"} + if got := strings.Join(fake.calls, " "); got != strings.Join(want, " ") { + t.Errorf("calls = %s\nwant = %s", got, strings.Join(want, " ")) + } + if !fake.closed { + t.Error("client not closed") + } + if len(fake.templates) != 1 { + t.Fatalf("templates = %d", len(fake.templates)) + } + tpl := fake.templates[0] + if tpl.GetMetadata().GetName() != "hive-kick-deps-1700000000" || tpl.GetMetadata().GetAtespace() != "hive" { + t.Errorf("template metadata = %v", tpl.GetMetadata()) + } + c := tpl.GetContainers()[0] + if c.GetImage() != substrateTestImage || strings.Join(c.GetCommand(), " ") != strings.Join(SubstrateDefaultBridgeCommand, " ") { + t.Errorf("container = %v", c) + } + var envNames []string + for _, e := range c.GetEnv() { + envNames = append(envNames, e.GetName()+"="+e.GetValue()) + } + if joined := strings.Join(envNames, ","); !strings.Contains(joined, "HIVE_SUBSTRATE_BRIDGE_PORT=8477") || !strings.Contains(joined, "HIVE_POLICY_PROFILE=strict") { + t.Errorf("env = %s", joined) + } + if tpl.GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" || fake.actors[0].GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" { + t.Error("worker selector not propagated to template and actor") + } + if tpl.GetVolumes()[0].GetDurableDir() == nil || c.GetVolumeMounts()[0].GetMountPath() != SubstrateWorkspaceMount { + t.Errorf("workspace volume = %v / %v", tpl.GetVolumes(), c.GetVolumeMounts()) + } + if tpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() != substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { + t.Error("golden snapshot not disabled") + } + if lim := tpl.GetResources().GetLimits(); lim.GetCpuMillicores() != SubstrateDefaultCPUMillicores || lim.GetMemoryBytes() != SubstrateDefaultMemoryBytes { + t.Errorf("limits = %v", lim) + } + if fake.actors[0].GetActorTemplate().GetName() != tpl.GetMetadata().GetName() { + t.Error("actor does not reference the per-kick template") + } + if fake.pushedWorkspace != "/data/agents/sandbox/deps" || fake.pulledWorkspace != "/data/agents/sandbox/deps" { + t.Errorf("workspace push/pull = %q / %q", fake.pushedWorkspace, fake.pulledWorkspace) + } + if fake.runReq.WorkDir != SubstrateWorkspaceMount || strings.Join(fake.runReq.Command, " ") != "crux kick --agent deps" || fake.runReq.Env["HIVE_AGENT"] != "deps" { + t.Errorf("run request = %+v", fake.runReq) + } + // Egress: GitHub hosts plus the operator's, grouped by port, sorted. + pol := fake.policies[0] + if pol.GetMetadata().GetName() != "default" || len(pol.GetRules()) != 2 { + t.Fatalf("policy = %v", pol) + } + r443 := pol.GetRules()[0].GetTlsPassthrough() + if strings.Join(r443.GetHostnames(), ",") != "api.github.com,github.com,litellm.example.com" || r443.GetPorts().GetNumbers()[0] != 443 { + t.Errorf("443 rule = %v", r443) + } + r8443 := pol.GetRules()[1].GetTlsPassthrough() + if strings.Join(r8443.GetHostnames(), ",") != "*.pypi.org" || r8443.GetPorts().GetNumbers()[0] != 8443 { + t.Errorf("8443 rule = %v", r8443) + } + if fake.cleanupCtxDead { + t.Error("cleanup ran on a dead context although the caller's was live") + } +} + +func TestSubstrateRunHonoursSpecOverrides(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.Image = "" + l.Options.Command = []string{"/opt/bridge"} + l.Options.CPUMillicores = 500 + l.Options.MemoryBytes = 1 << 30 + l.Options.BridgePort = 9000 + l.Options.Timeout = time.Minute + spec := substrateTestSpec() + spec.Image = substrateTestImage + spec.WorkDir = "/workspace/sub" + spec.NetworkNone = true + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + c := fake.templates[0].GetContainers()[0] + if c.GetImage() != substrateTestImage || c.GetCommand()[0] != "/opt/bridge" || c.GetEnv()[0].GetValue() != "9000" { + t.Errorf("container = %v", c) + } + if lim := fake.templates[0].GetResources().GetLimits(); lim.GetCpuMillicores() != 500 || lim.GetMemoryBytes() != 1<<30 { + t.Errorf("limits = %v", lim) + } + if fake.templates[0].GetWorkerSelector() != nil { + t.Error("empty worker selector must be nil") + } + if len(fake.policies[0].GetRules()) != 0 { + t.Errorf("network_none must send an empty policy, got %v", fake.policies[0].GetRules()) + } + if fake.runReq.WorkDir != "/workspace/sub" { + t.Errorf("workdir = %q", fake.runReq.WorkDir) + } +} + +func TestSubstrateRunRefusesClineProvider(t *testing.T) { + spec, _ := clineLaunchFixture(t) + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec.Workspace = "/data/agents/sandbox/deps" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "Podman-only") { + t.Fatalf("err = %v", err) + } + if len(fake.calls) != 0 { + t.Error("control plane touched despite the Cline provider refusal") + } +} + +func TestSubstrateRunRejectsBadOptionsAndSpec(t *testing.T) { + cases := []struct { + name string + mut func(l *SubstrateLauncher, spec *LaunchSpec) + want string + }{ + {"bad endpoint", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Endpoint = "ftp://x" }, "unsupported scheme"}, + {"tag image in options", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Image = "img:latest" }, "pinned by digest"}, + {"no image anywhere", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "" }, "image is required"}, + {"tag image in spec", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "img:latest" }, "pinned by digest"}, + {"no workspace", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Workspace = " " }, "workspace is required"}, + {"no command", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Command = nil }, "command is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec := substrateTestSpec() + tc.mut(l, &spec) + _, err := l.Run(context.Background(), spec) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + if len(fake.calls) != 0 { + t.Errorf("control plane touched: %v", fake.calls) + } + }) + } +} + +func TestSubstrateRunClientConstructorError(t *testing.T) { + l := newTestSubstrate(&fakeSubstrate{}) + l.newClient = func(SubstrateOptions) (substrateClient, error) { return nil, errors.New("no router bundle") } + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil || !strings.Contains(err.Error(), "no router bundle") { + t.Fatalf("err = %v", err) + } +} + +func TestSubstrateRunControlPlaneFailuresCleanUp(t *testing.T) { + cases := []struct { + name string + mut func(f *fakeSubstrate) + want string + cleanup []string + }{ + {"template", func(f *fakeSubstrate) { f.templateErr = errors.New("boom") }, "creating actor template", nil}, + {"actor", func(f *fakeSubstrate) { f.actorErr = errors.New("boom") }, "creating actor hive-kick", []string{"DeleteActorTemplate"}}, + {"policy", func(f *fakeSubstrate) { f.policyErr = errors.New("boom") }, "setting egress policy", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"resume", func(f *fakeSubstrate) { f.resumeErr = errors.New("boom") }, "resuming actor", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"get actor", func(f *fakeSubstrate) { f.getErr = errors.New("boom") }, "never became running", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"crashed", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_CRASHED} + f.crashMsg = "OOM" + }, "actor crashed: OOM", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"deleting", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_DELETING} + }, "entered ACTOR_STATE_DELETING", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"bridge never ready", func(f *fakeSubstrate) { f.waitReadyErr = errors.New("boom") }, "never answered", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"push", func(f *fakeSubstrate) { f.pushErr = errors.New("boom") }, "pushing workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"run transport", func(f *fakeSubstrate) { f.runErr = errors.New("boom") }, "running kick", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"run reported", func(f *fakeSubstrate) { + f.runResp = substratebridge.RunResponse{ExitCode: -1, Error: "exec: not found"} + }, "exec: not found", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"pull", func(f *fakeSubstrate) { f.pullErr = errors.New("boom") }, "pulling workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + tc.mut(fake) + l := newTestSubstrate(fake) + _, err := l.Run(context.Background(), substrateTestSpec()) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + for _, c := range tc.cleanup { + if !fake.saw(c) { + t.Errorf("cleanup %s not called: %v", c, fake.calls) + } + } + if tc.name == "template" && fake.saw("DeleteActorTemplate") { + t.Error("template deleted although it was never created") + } + if !fake.closed { + t.Error("client not closed") + } + }) + } +} + +func TestSubstrateRunReportsNonZeroExit(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "x", ExitCode: 3}} + res, err := newTestSubstrate(fake).Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v (a non-zero exit is a result, not an error)", err) + } + if res.ExitCode != 3 || res.Stdout != "x" { + t.Errorf("result = %+v", res) + } +} + +func TestSubstrateRunSnapshotOnIdle(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("mode=DURABILITY_MODE_ASYNCHRONOUS") || fake.saw("DeleteActor") { + t.Errorf("snapshot_on_idle must suspend asynchronously instead of deleting: %v", fake.calls) + } + if !fake.saw("DeleteActorTemplate") { + t.Error("per-kick template must still be removed") + } + + // A failed suspend falls back to delete so no actor is left running. + fake = &fakeSubstrate{suspendErr: errors.New("no snapshot store")} + l = newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("DeleteActor") { + t.Errorf("suspend failure must fall back to delete: %v", fake.calls) + } +} + +func TestSubstrateRunContextCancelledMidKick(t *testing.T) { + // The kick's context dies while the command runs: Pull is skipped, the + // actor is deleted on a fresh context, and the caller gets ctx.Err(). + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{runCancels: cancel} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true // must not snapshot on a dead context + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Pull") || fake.saw("SuspendActor") { + t.Errorf("pull or suspend ran after cancel: %v", fake.calls) + } + if !fake.saw("DeleteActor") || !fake.saw("DeleteActorTemplate") { + t.Errorf("cleanup skipped: %v", fake.calls) + } + if fake.cleanupCtxDead { + t.Error("cleanup used the dead caller context instead of a fresh one") + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForRunning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{getCancels: cancel, states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_RESUMING}} + _, err := newTestSubstrate(fake).Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if !fake.saw("DeleteActor") || fake.saw("WaitReady") { + t.Errorf("calls = %v", fake.calls) + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForBridge(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{waitReadyErr: context.Canceled} + l := newTestSubstrate(fake) + // Cancel just before WaitReady by hooking GetActor (the step before). + fake.getCancels = cancel + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Push") { + t.Errorf("push ran after cancel: %v", fake.calls) + } +} + +func TestSubstrateWaitRunningPollsUntilRunning(t *testing.T) { + fake := &fakeSubstrate{states: []substrateapi.ActorState{ + substrateapi.ActorState_ACTOR_STATE_SUSPENDED, + substrateapi.ActorState_ACTOR_STATE_RESUMING, + substrateapi.ActorState_ACTOR_STATE_RUNNING, + }} + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err != nil { + t.Fatalf("err = %v", err) + } + if n := len(fake.calls); n != 3 { + t.Errorf("GetActor calls = %d, want 3", n) + } + fake = &fakeSubstrate{states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_SUSPENDING}} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { + t.Errorf("err = %v", err) + } +} + +func TestSubstrateOptionsValidate(t *testing.T) { + good := SubstrateOptions{Image: substrateTestImage, AllowHosts: []string{"a.example.com", "*.b.example.com:8443"}} + if err := good.Validate(); err != nil { + t.Fatalf("Validate(good) = %v", err) + } + if err := (SubstrateOptions{}).Validate(); err != nil { + t.Fatalf("Validate(defaults) = %v", err) + } + cases := map[string]SubstrateOptions{ + "endpoint no port": {Endpoint: "api.ate-system.svc"}, + "endpoint bad url": {Endpoint: "https://"}, + "endpoint scheme": {Endpoint: "ftp://x:1"}, + "router http": {RouterURL: "http://router:8444"}, + "router garbage": {RouterURL: "::not a url"}, + "image tag": {Image: "img:latest"}, + "negative cpu": {CPUMillicores: -1}, + "negative mem": {MemoryBytes: -1}, + "bridge port range": {BridgePort: 70000}, + "allow host url": {AllowHosts: []string{"https://x.example.com"}}, + "allow host empty": {AllowHosts: []string{" "}}, + "allow host userinfo": {AllowHosts: []string{"u@x.example.com"}}, + "allow host bad port": {AllowHosts: []string{"x.example.com:0"}}, + "allow host port alpha": {AllowHosts: []string{"x.example.com:abc"}}, + "allow host wildcard": {AllowHosts: []string{"*x.example.com"}}, + } + for name, o := range cases { + if err := o.Validate(); err == nil { + t.Errorf("%s: Validate accepted %+v", name, o) + } + } +} + +func TestSubstrateAddressForms(t *testing.T) { + cases := []struct { + in string + addr string + tls bool + }{ + {"api.ate-system.svc:443", "api.ate-system.svc:443", true}, + {"https://api.ate-system.svc", "api.ate-system.svc:443", true}, + {"grpcs://api:9443", "api:9443", true}, + {"http://localhost:8080", "localhost:8080", false}, + {"http://localhost", "localhost:80", false}, + {"grpc://api:50051", "api:50051", false}, + } + for _, tc := range cases { + addr, tls, err := substrateAddress(tc.in) + if err != nil || addr != tc.addr || tls != tc.tls { + t.Errorf("substrateAddress(%q) = %q %v %v", tc.in, addr, tls, err) + } + } + if got, err := substrateRouterAddress("https://atenet-router.ate-system.svc"); err != nil || got != "atenet-router.ate-system.svc:443" { + t.Errorf("router default port: %q %v", got, err) + } + if got, err := substrateRouterAddress("https://r:8444"); err != nil || got != "r:8444" { + t.Errorf("router explicit port: %q %v", got, err) + } +} + +func TestSubstrateHostPort(t *testing.T) { + host, port, err := substrateHostPort("API.Example.COM") + if err != nil || host != "api.example.com" || port != 443 { + t.Errorf("got %q %d %v", host, port, err) + } + host, port, err = substrateHostPort("*.pypi.org:8443") + if err != nil || host != "*.pypi.org" || port != 8443 { + t.Errorf("got %q %d %v", host, port, err) + } + for _, bad := range []string{"", "a b", "x/y", "x.example.com:99999", "-bad.example.com", "x.example.com:"} { + if _, _, err := substrateHostPort(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} + +func TestSubstrateEgressPolicyDedupesAndErrors(t *testing.T) { + pol, err := SubstrateEgressPolicy([]string{"github.com", "GitHub.com:443", "x.example.com:8443"}) + if err != nil { + t.Fatal(err) + } + if len(pol.GetRules()) != 2 || len(pol.GetRules()[0].GetTlsPassthrough().GetHostnames()) != 2 { + t.Errorf("policy = %v", pol) + } + if _, err := SubstrateEgressPolicy([]string{"bad host"}); err == nil { + t.Error("bad host accepted") + } +} + +func TestSubstrateActorName(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + if got := substrateActorName("Deps_Review-sandbox", at); got != "hive-kick-deps-review-1700000000" { + t.Errorf("got %q", got) + } + if got := substrateActorName("---", at); got != "hive-kick-agent-1700000000" { + t.Errorf("empty base: %q", got) + } + long := substrateActorName(strings.Repeat("a", 80)+"-b", at) + if len(long) > substrateNameMax || !strings.HasPrefix(long, "hive-kick-aaaa") || !strings.HasSuffix(long, "-1700000000") { + t.Errorf("long name = %q (%d)", long, len(long)) + } + trimmed := substrateActorName(strings.Repeat("a", 41)+"-"+strings.Repeat("b", 20), at) + if strings.Contains(trimmed, "--") { + t.Errorf("truncation left a dangling dash: %q", trimmed) + } +} + +func TestSubstrateOptionDefaults(t *testing.T) { + o := SubstrateOptions{} + if o.endpoint() != SubstrateDefaultEndpoint || o.atespace() != SubstrateDefaultAtespace || o.routerURL() != SubstrateDefaultRouterURL || o.bridgePort() != substratebridge.DefaultPort || o.tokenFile() != SubstrateDefaultTokenFile { + t.Errorf("defaults = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } + o = SubstrateOptions{Endpoint: " e:1 ", Atespace: " a ", RouterURL: " https://r ", BridgePort: 1, TokenFile: " /t "} + if o.endpoint() != "e:1" || o.atespace() != "a" || o.routerURL() != "https://r" || o.bridgePort() != 1 || o.tokenFile() != "/t" { + t.Errorf("trimmed = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } +} + +func TestSubstrateEnvFiltersCredentials(t *testing.T) { + t.Setenv("GITHUB_TOKEN", "ghp_secret") + t.Setenv("HIVE_SUBSTRATE_TEST_KEEP", "yes") + spec := substrateTestSpec() + spec.EnvAllowlist = []string{"HIVE_SUBSTRATE_TEST_KEEP"} + env := substrateEnv(spec) + if env["HIVE_SUBSTRATE_TEST_KEEP"] != "yes" || env["HIVE_AGENT"] != "deps" { + t.Errorf("allowlisted values missing: %v", env) + } + if _, ok := env["GITHUB_TOKEN"]; ok { + t.Error("GITHUB_TOKEN leaked into the actor env") + } +} + +func TestSubstrateLauncherDefaultClientIsGRPC(t *testing.T) { + // Without a test hook the launcher builds the real client; with no + // router bundle on disk that fails before any network I/O. + l := &SubstrateLauncher{Options: SubstrateOptions{InsecureTLS: true, RouterClientBundle: "/nonexistent/bundle.pem"}} + if _, err := l.client(); err == nil || !strings.Contains(err.Error(), "router_client_bundle") { + t.Fatalf("client() = %v", err) + } +} diff --git a/src/crux/sandbox/substrateapi/actor.pb.go b/src/crux/sandbox/substrateapi/actor.pb.go new file mode 100644 index 0000000000..d70548216f --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.pb.go @@ -0,0 +1,586 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ActorState int32 + +const ( + ActorState_ACTOR_STATE_UNSPECIFIED ActorState = 0 + ActorState_ACTOR_STATE_RESUMING ActorState = 1 + ActorState_ACTOR_STATE_RUNNING ActorState = 2 + ActorState_ACTOR_STATE_SUSPENDING ActorState = 3 + ActorState_ACTOR_STATE_SUSPENDED ActorState = 4 + ActorState_ACTOR_STATE_CRASHED ActorState = 5 + ActorState_ACTOR_STATE_DELETING ActorState = 6 + ActorState_ACTOR_STATE_REVERTING ActorState = 7 +) + +// Enum value maps for ActorState. +var ( + ActorState_name = map[int32]string{ + 0: "ACTOR_STATE_UNSPECIFIED", + 1: "ACTOR_STATE_RESUMING", + 2: "ACTOR_STATE_RUNNING", + 3: "ACTOR_STATE_SUSPENDING", + 4: "ACTOR_STATE_SUSPENDED", + 5: "ACTOR_STATE_CRASHED", + 6: "ACTOR_STATE_DELETING", + 7: "ACTOR_STATE_REVERTING", + } + ActorState_value = map[string]int32{ + "ACTOR_STATE_UNSPECIFIED": 0, + "ACTOR_STATE_RESUMING": 1, + "ACTOR_STATE_RUNNING": 2, + "ACTOR_STATE_SUSPENDING": 3, + "ACTOR_STATE_SUSPENDED": 4, + "ACTOR_STATE_CRASHED": 5, + "ACTOR_STATE_DELETING": 6, + "ACTOR_STATE_REVERTING": 7, + } +) + +func (x ActorState) Enum() *ActorState { + p := new(ActorState) + *p = x + return p +} + +func (x ActorState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ActorState) Descriptor() protoreflect.EnumDescriptor { + return file_actor_proto_enumTypes[0].Descriptor() +} + +func (ActorState) Type() protoreflect.EnumType { + return &file_actor_proto_enumTypes[0] +} + +func (x ActorState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ActorState.Descriptor instead. +func (ActorState) EnumDescriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +type Actor struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + ActorTemplate *ObjectRef `protobuf:"bytes,2,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,3,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + SourceTag *ObjectRef `protobuf:"bytes,4,opt,name=source_tag,json=sourceTag,proto3" json:"source_tag,omitempty"` + Status *ActorStatus `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Actor) Reset() { + *x = Actor{} + mi := &file_actor_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Actor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Actor) ProtoMessage() {} + +func (x *Actor) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Actor.ProtoReflect.Descriptor instead. +func (*Actor) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +func (x *Actor) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Actor) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *Actor) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *Actor) GetSourceTag() *ObjectRef { + if x != nil { + return x.SourceTag + } + return nil +} + +func (x *Actor) GetStatus() *ActorStatus { + if x != nil { + return x.Status + } + return nil +} + +type ActorStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State ActorState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.ActorState" json:"state,omitempty"` + WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` + Crash *ActorCrash `protobuf:"bytes,4,opt,name=crash,proto3" json:"crash,omitempty"` + AssignedNode string `protobuf:"bytes,5,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorStatus) Reset() { + *x = ActorStatus{} + mi := &file_actor_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorStatus) ProtoMessage() {} + +func (x *ActorStatus) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorStatus.ProtoReflect.Descriptor instead. +func (*ActorStatus) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{1} +} + +func (x *ActorStatus) GetState() ActorState { + if x != nil { + return x.State + } + return ActorState_ACTOR_STATE_UNSPECIFIED +} + +func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { + if x != nil { + return x.WorkerAssignment + } + return nil +} + +func (x *ActorStatus) GetCrash() *ActorCrash { + if x != nil { + return x.Crash + } + return nil +} + +func (x *ActorStatus) GetAssignedNode() string { + if x != nil { + return x.AssignedNode + } + return "" +} + +type ActorCrash struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + CrashTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=crash_time,json=crashTime,proto3" json:"crash_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorCrash) Reset() { + *x = ActorCrash{} + mi := &file_actor_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorCrash) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorCrash) ProtoMessage() {} + +func (x *ActorCrash) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorCrash.ProtoReflect.Descriptor instead. +func (*ActorCrash) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{2} +} + +func (x *ActorCrash) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *ActorCrash) GetCrashTime() *timestamppb.Timestamp { + if x != nil { + return x.CrashTime + } + return nil +} + +type WorkerAssignment struct { + state protoimpl.MessageState `protogen:"open.v1"` + Worker *ObjectRef `protobuf:"bytes,6,opt,name=worker,proto3" json:"worker,omitempty"` + WorkerNamespace string `protobuf:"bytes,1,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,2,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,3,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,4,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + WorkerPodIps []string `protobuf:"bytes,5,rep,name=worker_pod_ips,json=workerPodIps,proto3" json:"worker_pod_ips,omitempty"` + NodeName string `protobuf:"bytes,7,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerEpoch int64 `protobuf:"varint,8,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` + SandboxRuntime *ActorSandboxRuntime `protobuf:"bytes,9,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,10,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerAssignment) Reset() { + *x = WorkerAssignment{} + mi := &file_actor_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerAssignment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerAssignment) ProtoMessage() {} + +func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerAssignment.ProtoReflect.Descriptor instead. +func (*WorkerAssignment) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{3} +} + +func (x *WorkerAssignment) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *WorkerAssignment) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodIps() []string { + if x != nil { + return x.WorkerPodIps + } + return nil +} + +func (x *WorkerAssignment) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *WorkerAssignment) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 +} + +func (x *WorkerAssignment) GetSandboxRuntime() *ActorSandboxRuntime { + if x != nil { + return x.SandboxRuntime + } + return nil +} + +func (x *WorkerAssignment) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +type ActorSandboxRuntime struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorSandboxRuntime) Reset() { + *x = ActorSandboxRuntime{} + mi := &file_actor_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorSandboxRuntime) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorSandboxRuntime) ProtoMessage() {} + +func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. +func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{4} +} + +func (x *ActorSandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +var File_actor_proto protoreflect.FileDescriptor + +const file_actor_proto_rawDesc = "" + + "\n" + + "\vactor.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd2\x02\n" + + "\x05Actor\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12E\n" + + "\x0eactor_template\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12F\n" + + "\x0fworker_selector\x18\x03 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12=\n" + + "\n" + + "source_tag\x18\x04 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\tsourceTag\x128\n" + + "\x06status\x18\x05 \x01(\v2 .substrate.ateapi.v1.ActorStatusR\x06status\"\xf4\x01\n" + + "\vActorStatus\x125\n" + + "\x05state\x18\x01 \x01(\x0e2\x1f.substrate.ateapi.v1.ActorStateR\x05state\x12R\n" + + "\x11worker_assignment\x18\x02 \x01(\v2%.substrate.ateapi.v1.WorkerAssignmentR\x10workerAssignment\x125\n" + + "\x05crash\x18\x04 \x01(\v2\x1f.substrate.ateapi.v1.ActorCrashR\x05crash\x12#\n" + + "\rassigned_node\x18\x05 \x01(\tR\fassignedNode\"a\n" + + "\n" + + "ActorCrash\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + + "\n" + + "crash_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcrashTime\"\xbe\x03\n" + + "\x10WorkerAssignment\x126\n" + + "\x06worker\x18\x06 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x06worker\x12)\n" + + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x02 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x03 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x04 \x01(\tR\fworkerPodUid\x12$\n" + + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\x12Q\n" + + "\x0fsandbox_runtime\x18\t \x01(\v2(.substrate.ateapi.v1.ActorSandboxRuntimeR\x0esandboxRuntime\x12(\n" + + "\x10worker_spiffe_id\x18\n" + + " \x01(\tR\x0eworkerSpiffeId\":\n" + + "\x13ActorSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass*\xe1\x01\n" + + "\n" + + "ActorState\x12\x1b\n" + + "\x17ACTOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14ACTOR_STATE_RESUMING\x10\x01\x12\x17\n" + + "\x13ACTOR_STATE_RUNNING\x10\x02\x12\x1a\n" + + "\x16ACTOR_STATE_SUSPENDING\x10\x03\x12\x19\n" + + "\x15ACTOR_STATE_SUSPENDED\x10\x04\x12\x17\n" + + "\x13ACTOR_STATE_CRASHED\x10\x05\x12\x18\n" + + "\x14ACTOR_STATE_DELETING\x10\x06\x12\x19\n" + + "\x15ACTOR_STATE_REVERTING\x10\aBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_proto_rawDescOnce sync.Once + file_actor_proto_rawDescData []byte +) + +func file_actor_proto_rawDescGZIP() []byte { + file_actor_proto_rawDescOnce.Do(func() { + file_actor_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc))) + }) + return file_actor_proto_rawDescData +} + +var file_actor_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_actor_proto_goTypes = []any{ + (ActorState)(0), // 0: substrate.ateapi.v1.ActorState + (*Actor)(nil), // 1: substrate.ateapi.v1.Actor + (*ActorStatus)(nil), // 2: substrate.ateapi.v1.ActorStatus + (*ActorCrash)(nil), // 3: substrate.ateapi.v1.ActorCrash + (*WorkerAssignment)(nil), // 4: substrate.ateapi.v1.WorkerAssignment + (*ActorSandboxRuntime)(nil), // 5: substrate.ateapi.v1.ActorSandboxRuntime + (*ResourceMetadata)(nil), // 6: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 7: substrate.ateapi.v1.ObjectRef + (*Selector)(nil), // 8: substrate.ateapi.v1.Selector + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp +} +var file_actor_proto_depIdxs = []int32{ + 6, // 0: substrate.ateapi.v1.Actor.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 7, // 1: substrate.ateapi.v1.Actor.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 8, // 2: substrate.ateapi.v1.Actor.worker_selector:type_name -> substrate.ateapi.v1.Selector + 7, // 3: substrate.ateapi.v1.Actor.source_tag:type_name -> substrate.ateapi.v1.ObjectRef + 2, // 4: substrate.ateapi.v1.Actor.status:type_name -> substrate.ateapi.v1.ActorStatus + 0, // 5: substrate.ateapi.v1.ActorStatus.state:type_name -> substrate.ateapi.v1.ActorState + 4, // 6: substrate.ateapi.v1.ActorStatus.worker_assignment:type_name -> substrate.ateapi.v1.WorkerAssignment + 3, // 7: substrate.ateapi.v1.ActorStatus.crash:type_name -> substrate.ateapi.v1.ActorCrash + 9, // 8: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 7, // 9: substrate.ateapi.v1.WorkerAssignment.worker:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 10: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_actor_proto_init() } +func file_actor_proto_init() { + if File_actor_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_proto_goTypes, + DependencyIndexes: file_actor_proto_depIdxs, + EnumInfos: file_actor_proto_enumTypes, + MessageInfos: file_actor_proto_msgTypes, + }.Build() + File_actor_proto = out.File + file_actor_proto_goTypes = nil + file_actor_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor.proto b/src/crux/sandbox/substrateapi/actor.proto new file mode 100644 index 0000000000..d5df52af03 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.proto @@ -0,0 +1,79 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Actor { + ResourceMetadata metadata = 1; + ObjectRef actor_template = 2; + Selector worker_selector = 3; + ObjectRef source_tag = 4; + ActorStatus status = 5; +} + +enum ActorState { + ACTOR_STATE_UNSPECIFIED = 0; + ACTOR_STATE_RESUMING = 1; + ACTOR_STATE_RUNNING = 2; + ACTOR_STATE_SUSPENDING = 3; + ACTOR_STATE_SUSPENDED = 4; + ACTOR_STATE_CRASHED = 5; + ACTOR_STATE_DELETING = 6; + ACTOR_STATE_REVERTING = 7; +} + +message ActorStatus { + ActorState state = 1; + WorkerAssignment worker_assignment = 2; + ActorCrash crash = 4; + string assigned_node = 5; +} + +message ActorCrash { + string message = 1; + google.protobuf.Timestamp crash_time = 2; +} + +message WorkerAssignment { + ObjectRef worker = 6; + string worker_namespace = 1; + string worker_pool = 2; + string worker_pod = 3; + string worker_pod_uid = 4; + repeated string worker_pod_ips = 5; + string node_name = 7; + int64 worker_epoch = 8; + ActorSandboxRuntime sandbox_runtime = 9; + string worker_spiffe_id = 10; +} + +message ActorSandboxRuntime { + string sandbox_class = 1; +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.pb.go b/src/crux/sandbox/substrateapi/actor_rpc.pb.go new file mode 100644 index 0000000000..17d81ba51d --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.pb.go @@ -0,0 +1,572 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor_rpc.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DurabilityMode int32 + +const ( + DurabilityMode_DURABILITY_MODE_UNSPECIFIED DurabilityMode = 0 + DurabilityMode_DURABILITY_MODE_SYNCHRONOUS DurabilityMode = 1 + DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS DurabilityMode = 2 +) + +// Enum value maps for DurabilityMode. +var ( + DurabilityMode_name = map[int32]string{ + 0: "DURABILITY_MODE_UNSPECIFIED", + 1: "DURABILITY_MODE_SYNCHRONOUS", + 2: "DURABILITY_MODE_ASYNCHRONOUS", + } + DurabilityMode_value = map[string]int32{ + "DURABILITY_MODE_UNSPECIFIED": 0, + "DURABILITY_MODE_SYNCHRONOUS": 1, + "DURABILITY_MODE_ASYNCHRONOUS": 2, + } +) + +func (x DurabilityMode) Enum() *DurabilityMode { + p := new(DurabilityMode) + *p = x + return p +} + +func (x DurabilityMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DurabilityMode) Descriptor() protoreflect.EnumDescriptor { + return file_actor_rpc_proto_enumTypes[0].Descriptor() +} + +func (DurabilityMode) Type() protoreflect.EnumType { + return &file_actor_rpc_proto_enumTypes[0] +} + +func (x DurabilityMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DurabilityMode.Descriptor instead. +func (DurabilityMode) EnumDescriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +type GetActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorRequest) Reset() { + *x = GetActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorRequest) ProtoMessage() {} + +func (x *GetActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. +func (*GetActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +func (x *GetActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type CreateActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorRequest) Reset() { + *x = CreateActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorRequest) ProtoMessage() {} + +func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. +func (*CreateActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateActorRequest) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *SuspendOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorRequest) Reset() { + *x = SuspendActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorRequest) ProtoMessage() {} + +func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. +func (*SuspendActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{2} +} + +func (x *SuspendActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *SuspendActorRequest) GetOptions() *SuspendOptions { + if x != nil { + return x.Options + } + return nil +} + +type SuspendActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorResponse) Reset() { + *x = SuspendActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorResponse) ProtoMessage() {} + +func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. +func (*SuspendActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{3} +} + +func (x *SuspendActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode DurabilityMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.DurabilityMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendOptions) Reset() { + *x = SuspendOptions{} + mi := &file_actor_rpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendOptions) ProtoMessage() {} + +func (x *SuspendOptions) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendOptions.ProtoReflect.Descriptor instead. +func (*SuspendOptions) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{4} +} + +func (x *SuspendOptions) GetMode() DurabilityMode { + if x != nil { + return x.Mode + } + return DurabilityMode_DURABILITY_MODE_UNSPECIFIED +} + +type ResumeActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorRequest) Reset() { + *x = ResumeActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorRequest) ProtoMessage() {} + +func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. +func (*ResumeActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{5} +} + +func (x *ResumeActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type ResumeActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Resumed bool `protobuf:"varint,2,opt,name=resumed,proto3" json:"resumed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorResponse) Reset() { + *x = ResumeActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorResponse) ProtoMessage() {} + +func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. +func (*ResumeActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{6} +} + +func (x *ResumeActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +func (x *ResumeActorResponse) GetResumed() bool { + if x != nil { + return x.Resumed + } + return false +} + +type DeleteActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorRequest) Reset() { + *x = DeleteActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorRequest) ProtoMessage() {} + +func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *DeleteActorRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_actor_rpc_proto protoreflect.FileDescriptor + +const file_actor_rpc_proto_rawDesc = "" + + "\n" + + "\x0factor_rpc.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\fcommon.proto\"G\n" + + "\x0fGetActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"F\n" + + "\x12CreateActorRequest\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"\x8a\x01\n" + + "\x13SuspendActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12=\n" + + "\aoptions\x18\x02 \x01(\v2#.substrate.ateapi.v1.SuspendOptionsR\aoptions\"H\n" + + "\x14SuspendActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"I\n" + + "\x0eSuspendOptions\x127\n" + + "\x04mode\x18\x01 \x01(\x0e2#.substrate.ateapi.v1.DurabilityModeR\x04mode\"J\n" + + "\x12ResumeActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"a\n" + + "\x13ResumeActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\x12\x18\n" + + "\aresumed\x18\x02 \x01(\bR\aresumed\"\x88\x01\n" + + "\x12DeleteActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*t\n" + + "\x0eDurabilityMode\x12\x1f\n" + + "\x1bDURABILITY_MODE_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bDURABILITY_MODE_SYNCHRONOUS\x10\x01\x12 \n" + + "\x1cDURABILITY_MODE_ASYNCHRONOUS\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_rpc_proto_rawDescOnce sync.Once + file_actor_rpc_proto_rawDescData []byte +) + +func file_actor_rpc_proto_rawDescGZIP() []byte { + file_actor_rpc_proto_rawDescOnce.Do(func() { + file_actor_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc))) + }) + return file_actor_rpc_proto_rawDescData +} + +var file_actor_rpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_actor_rpc_proto_goTypes = []any{ + (DurabilityMode)(0), // 0: substrate.ateapi.v1.DurabilityMode + (*GetActorRequest)(nil), // 1: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 2: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 3: substrate.ateapi.v1.SuspendActorRequest + (*SuspendActorResponse)(nil), // 4: substrate.ateapi.v1.SuspendActorResponse + (*SuspendOptions)(nil), // 5: substrate.ateapi.v1.SuspendOptions + (*ResumeActorRequest)(nil), // 6: substrate.ateapi.v1.ResumeActorRequest + (*ResumeActorResponse)(nil), // 7: substrate.ateapi.v1.ResumeActorResponse + (*DeleteActorRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorRequest + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*DeleteOptions)(nil), // 11: substrate.ateapi.v1.DeleteOptions +} +var file_actor_rpc_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.GetActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 1: substrate.ateapi.v1.CreateActorRequest.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 2: substrate.ateapi.v1.SuspendActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 3: substrate.ateapi.v1.SuspendActorRequest.options:type_name -> substrate.ateapi.v1.SuspendOptions + 10, // 4: substrate.ateapi.v1.SuspendActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 0, // 5: substrate.ateapi.v1.SuspendOptions.mode:type_name -> substrate.ateapi.v1.DurabilityMode + 9, // 6: substrate.ateapi.v1.ResumeActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 7: substrate.ateapi.v1.ResumeActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 8: substrate.ateapi.v1.DeleteActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 11, // 9: substrate.ateapi.v1.DeleteActorRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_actor_rpc_proto_init() } +func file_actor_rpc_proto_init() { + if File_actor_rpc_proto != nil { + return + } + file_actor_proto_init() + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_rpc_proto_goTypes, + DependencyIndexes: file_actor_rpc_proto_depIdxs, + EnumInfos: file_actor_rpc_proto_enumTypes, + MessageInfos: file_actor_rpc_proto_msgTypes, + }.Build() + File_actor_rpc_proto = out.File + file_actor_rpc_proto_goTypes = nil + file_actor_rpc_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.proto b/src/crux/sandbox/substrateapi/actor_rpc.proto new file mode 100644 index 0000000000..1e81978bc1 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.proto @@ -0,0 +1,72 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message GetActorRequest { + ObjectRef actor = 1; +} + +message CreateActorRequest { + Actor actor = 1; +} + +message SuspendActorRequest { + ObjectRef actor = 1; + SuspendOptions options = 2; +} + +message SuspendActorResponse { + Actor actor = 1; +} + +message SuspendOptions { + DurabilityMode mode = 1; +} + +enum DurabilityMode { + DURABILITY_MODE_UNSPECIFIED = 0; + DURABILITY_MODE_SYNCHRONOUS = 1; + DURABILITY_MODE_ASYNCHRONOUS = 2; +} + +message ResumeActorRequest { + ObjectRef actor = 1; +} + +message ResumeActorResponse { + Actor actor = 1; + bool resumed = 2; +} + +message DeleteActorRequest { + ObjectRef actor = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/buf.gen.yaml b/src/crux/sandbox/substrateapi/buf.gen.yaml new file mode 100644 index 0000000000..c5ca49b889 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.gen.yaml @@ -0,0 +1,8 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: . + opt: paths=source_relative + - local: protoc-gen-go-grpc + out: . + opt: paths=source_relative diff --git a/src/crux/sandbox/substrateapi/buf.yaml b/src/crux/sandbox/substrateapi/buf.yaml new file mode 100644 index 0000000000..b8699818a0 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.yaml @@ -0,0 +1,3 @@ +version: v2 +modules: + - path: . diff --git a/src/crux/sandbox/substrateapi/common.pb.go b/src/crux/sandbox/substrateapi/common.pb.go new file mode 100644 index 0000000000..02d6dd3697 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.pb.go @@ -0,0 +1,428 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: common.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + MatchLabels map[string]string `protobuf:"bytes,1,rep,name=match_labels,json=matchLabels,proto3" json:"match_labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{0} +} + +func (x *Selector) GetMatchLabels() map[string]string { + if x != nil { + return x.MatchLabels + } + return nil +} + +type ResourceMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Uid string `protobuf:"bytes,3,opt,name=uid,proto3" json:"uid,omitempty"` + Version int64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"` + CreateTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` + UpdateTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty"` + DeleteTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=delete_time,json=deleteTime,proto3" json:"delete_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceMetadata) Reset() { + *x = ResourceMetadata{} + mi := &file_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceMetadata) ProtoMessage() {} + +func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceMetadata.ProtoReflect.Descriptor instead. +func (*ResourceMetadata) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceMetadata) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ResourceMetadata) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ResourceMetadata) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +func (x *ResourceMetadata) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ResourceMetadata) GetCreateTime() *timestamppb.Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *ResourceMetadata) GetUpdateTime() *timestamppb.Timestamp { + if x != nil { + return x.UpdateTime + } + return nil +} + +func (x *ResourceMetadata) GetDeleteTime() *timestamppb.Timestamp { + if x != nil { + return x.DeleteTime + } + return nil +} + +type ObjectRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectRef) Reset() { + *x = ObjectRef{} + mi := &file_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectRef) ProtoMessage() {} + +func (x *ObjectRef) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectRef.ProtoReflect.Descriptor instead. +func (*ObjectRef) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{2} +} + +func (x *ObjectRef) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ObjectRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type DeleteOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteOptions) Reset() { + *x = DeleteOptions{} + mi := &file_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteOptions) ProtoMessage() {} + +func (x *DeleteOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. +func (*DeleteOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{3} +} + +func (x *DeleteOptions) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *DeleteOptions) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +type ListOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListOptions) Reset() { + *x = ListOptions{} + mi := &file_common_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListOptions) ProtoMessage() {} + +func (x *ListOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListOptions.ProtoReflect.Descriptor instead. +func (*ListOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{4} +} + +func (x *ListOptions) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListOptions) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +var File_common_proto protoreflect.FileDescriptor + +const file_common_proto_rawDesc = "" + + "\n" + + "\fcommon.proto\x12\x13substrate.ateapi.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x9d\x01\n" + + "\bSelector\x12Q\n" + + "\fmatch_labels\x18\x01 \x03(\v2..substrate.ateapi.v1.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + + "\x10MatchLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa5\x02\n" + + "\x10ResourceMetadata\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x10\n" + + "\x03uid\x18\x03 \x01(\tR\x03uid\x12\x18\n" + + "\aversion\x18\x04 \x01(\x03R\aversion\x12;\n" + + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "createTime\x12;\n" + + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "updateTime\x12;\n" + + "\vdelete_time\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "deleteTime\";\n" + + "\tObjectRef\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\";\n" + + "\rDeleteOptions\x12\x18\n" + + "\aversion\x18\x01 \x01(\x03R\aversion\x12\x10\n" + + "\x03uid\x18\x02 \x01(\tR\x03uid\"I\n" + + "\vListOptions\x12\x1b\n" + + "\tpage_size\x18\x01 \x01(\x05R\bpageSize\x12\x1d\n" + + "\n" + + "page_token\x18\x02 \x01(\tR\tpageTokenBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_common_proto_rawDescOnce sync.Once + file_common_proto_rawDescData []byte +) + +func file_common_proto_rawDescGZIP() []byte { + file_common_proto_rawDescOnce.Do(func() { + file_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc))) + }) + return file_common_proto_rawDescData +} + +var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_common_proto_goTypes = []any{ + (*Selector)(nil), // 0: substrate.ateapi.v1.Selector + (*ResourceMetadata)(nil), // 1: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 2: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 3: substrate.ateapi.v1.DeleteOptions + (*ListOptions)(nil), // 4: substrate.ateapi.v1.ListOptions + nil, // 5: substrate.ateapi.v1.Selector.MatchLabelsEntry + (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp +} +var file_common_proto_depIdxs = []int32{ + 5, // 0: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry + 6, // 1: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 6, // 2: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 6, // 3: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_common_proto_init() } +func file_common_proto_init() { + if File_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc)), + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_common_proto_goTypes, + DependencyIndexes: file_common_proto_depIdxs, + MessageInfos: file_common_proto_msgTypes, + }.Build() + File_common_proto = out.File + file_common_proto_goTypes = nil + file_common_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/common.proto b/src/crux/sandbox/substrateapi/common.proto new file mode 100644 index 0000000000..9ca13d7696 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.proto @@ -0,0 +1,59 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Selector { + map match_labels = 1; +} + +message ResourceMetadata { + string atespace = 1; + string name = 2; + string uid = 3; + int64 version = 4; + google.protobuf.Timestamp create_time = 5; + google.protobuf.Timestamp update_time = 6; + google.protobuf.Timestamp delete_time = 7; +} + +message ObjectRef { + string atespace = 1; + string name = 2; +} + +message DeleteOptions { + int64 version = 1; + string uid = 2; +} + +message ListOptions { + int32 page_size = 1; + string page_token = 2; +} diff --git a/src/crux/sandbox/substrateapi/container.pb.go b/src/crux/sandbox/substrateapi/container.pb.go new file mode 100644 index 0000000000..7389ce3037 --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.pb.go @@ -0,0 +1,401 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: container.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Container struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Image string `protobuf:"bytes,2,opt,name=image,proto3" json:"image,omitempty"` + Command []string `protobuf:"bytes,3,rep,name=command,proto3" json:"command,omitempty"` + Args []string `protobuf:"bytes,4,rep,name=args,proto3" json:"args,omitempty"` + Env []*EnvVar `protobuf:"bytes,5,rep,name=env,proto3" json:"env,omitempty"` + VolumeMounts []*VolumeMount `protobuf:"bytes,7,rep,name=volume_mounts,json=volumeMounts,proto3" json:"volume_mounts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Container) Reset() { + *x = Container{} + mi := &file_container_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Container) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Container) ProtoMessage() {} + +func (x *Container) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Container.ProtoReflect.Descriptor instead. +func (*Container) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{0} +} + +func (x *Container) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Container) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + +func (x *Container) GetCommand() []string { + if x != nil { + return x.Command + } + return nil +} + +func (x *Container) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *Container) GetEnv() []*EnvVar { + if x != nil { + return x.Env + } + return nil +} + +func (x *Container) GetVolumeMounts() []*VolumeMount { + if x != nil { + return x.VolumeMounts + } + return nil +} + +type EnvVar struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvVar) Reset() { + *x = EnvVar{} + mi := &file_container_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvVar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvVar) ProtoMessage() {} + +func (x *EnvVar) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. +func (*EnvVar) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{1} +} + +func (x *EnvVar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EnvVar) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type Volume struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + DurableDir *DurableDirVolumeSource `protobuf:"bytes,2,opt,name=durable_dir,json=durableDir,proto3" json:"durable_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Volume) Reset() { + *x = Volume{} + mi := &file_container_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Volume) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Volume) ProtoMessage() {} + +func (x *Volume) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Volume.ProtoReflect.Descriptor instead. +func (*Volume) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{2} +} + +func (x *Volume) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Volume) GetDurableDir() *DurableDirVolumeSource { + if x != nil { + return x.DurableDir + } + return nil +} + +type DurableDirVolumeSource struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DurableDirVolumeSource) Reset() { + *x = DurableDirVolumeSource{} + mi := &file_container_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DurableDirVolumeSource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DurableDirVolumeSource) ProtoMessage() {} + +func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. +func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{3} +} + +type VolumeMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeMount) Reset() { + *x = VolumeMount{} + mi := &file_container_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeMount) ProtoMessage() {} + +func (x *VolumeMount) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. +func (*VolumeMount) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{4} +} + +func (x *VolumeMount) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *VolumeMount) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + +var File_container_proto protoreflect.FileDescriptor + +const file_container_proto_rawDesc = "" + + "\n" + + "\x0fcontainer.proto\x12\x13substrate.ateapi.v1\"\xd9\x01\n" + + "\tContainer\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + + "\acommand\x18\x03 \x03(\tR\acommand\x12\x12\n" + + "\x04args\x18\x04 \x03(\tR\x04args\x12-\n" + + "\x03env\x18\x05 \x03(\v2\x1b.substrate.ateapi.v1.EnvVarR\x03env\x12E\n" + + "\rvolume_mounts\x18\a \x03(\v2 .substrate.ateapi.v1.VolumeMountR\fvolumeMounts\"7\n" + + "\x06EnvVar\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + + "\x05value\x18\x02 \x01(\tB\x03\x80\x01\x01R\x05value\"j\n" + + "\x06Volume\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12L\n" + + "\vdurable_dir\x18\x02 \x01(\v2+.substrate.ateapi.v1.DurableDirVolumeSourceR\n" + + "durableDir\"\x18\n" + + "\x16DurableDirVolumeSource\"@\n" + + "\vVolumeMount\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\n" + + "mount_path\x18\x02 \x01(\tR\tmountPathBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_container_proto_rawDescOnce sync.Once + file_container_proto_rawDescData []byte +) + +func file_container_proto_rawDescGZIP() []byte { + file_container_proto_rawDescOnce.Do(func() { + file_container_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc))) + }) + return file_container_proto_rawDescData +} + +var file_container_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_container_proto_goTypes = []any{ + (*Container)(nil), // 0: substrate.ateapi.v1.Container + (*EnvVar)(nil), // 1: substrate.ateapi.v1.EnvVar + (*Volume)(nil), // 2: substrate.ateapi.v1.Volume + (*DurableDirVolumeSource)(nil), // 3: substrate.ateapi.v1.DurableDirVolumeSource + (*VolumeMount)(nil), // 4: substrate.ateapi.v1.VolumeMount +} +var file_container_proto_depIdxs = []int32{ + 1, // 0: substrate.ateapi.v1.Container.env:type_name -> substrate.ateapi.v1.EnvVar + 4, // 1: substrate.ateapi.v1.Container.volume_mounts:type_name -> substrate.ateapi.v1.VolumeMount + 3, // 2: substrate.ateapi.v1.Volume.durable_dir:type_name -> substrate.ateapi.v1.DurableDirVolumeSource + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_container_proto_init() } +func file_container_proto_init() { + if File_container_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_container_proto_goTypes, + DependencyIndexes: file_container_proto_depIdxs, + MessageInfos: file_container_proto_msgTypes, + }.Build() + File_container_proto = out.File + file_container_proto_goTypes = nil + file_container_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/container.proto b/src/crux/sandbox/substrateapi/container.proto new file mode 100644 index 0000000000..d4d2dbf37f --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.proto @@ -0,0 +1,54 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Container { + string name = 1; + string image = 2; + repeated string command = 3; + repeated string args = 4; + repeated EnvVar env = 5; + repeated VolumeMount volume_mounts = 7; +} + +message EnvVar { + string name = 1; + string value = 2 [ debug_redact = true ]; +} + +message Volume { + string name = 1; + DurableDirVolumeSource durable_dir = 2; +} + +message DurableDirVolumeSource {} + +message VolumeMount { + string name = 1; + string mount_path = 2; +} diff --git a/src/crux/sandbox/substrateapi/control.pb.go b/src/crux/sandbox/substrateapi/control.pb.go new file mode 100644 index 0000000000..5dab339ffb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.pb.go @@ -0,0 +1,135 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_control_proto protoreflect.FileDescriptor + +const file_control_proto_rawDesc = "" + + "\n" + + "\rcontrol.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\x0factor_rpc.proto\x1a\fegress.proto\x1a\x0etemplate.proto\x1a\fworker.proto2\xed\a\n" + + "\aControl\x12N\n" + + "\bGetActor\x12$.substrate.ateapi.v1.GetActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12T\n" + + "\vCreateActor\x12'.substrate.ateapi.v1.CreateActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12e\n" + + "\fSuspendActor\x12(.substrate.ateapi.v1.SuspendActorRequest\x1a).substrate.ateapi.v1.SuspendActorResponse\"\x00\x12b\n" + + "\vResumeActor\x12'.substrate.ateapi.v1.ResumeActorRequest\x1a(.substrate.ateapi.v1.ResumeActorResponse\"\x00\x12T\n" + + "\vDeleteActor\x12'.substrate.ateapi.v1.DeleteActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12s\n" + + "\x17CreateActorEgressPolicy\x123.substrate.ateapi.v1.CreateActorEgressPolicyRequest\x1a!.substrate.ateapi.v1.EgressPolicy\"\x00\x12b\n" + + "\vListWorkers\x12'.substrate.ateapi.v1.ListWorkersRequest\x1a(.substrate.ateapi.v1.ListWorkersResponse\"\x00\x12l\n" + + "\x13CreateActorTemplate\x12/.substrate.ateapi.v1.CreateActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12f\n" + + "\x10GetActorTemplate\x12,.substrate.ateapi.v1.GetActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12l\n" + + "\x13DeleteActorTemplate\x12/.substrate.ateapi.v1.DeleteActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var file_control_proto_goTypes = []any{ + (*GetActorRequest)(nil), // 0: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 1: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 2: substrate.ateapi.v1.SuspendActorRequest + (*ResumeActorRequest)(nil), // 3: substrate.ateapi.v1.ResumeActorRequest + (*DeleteActorRequest)(nil), // 4: substrate.ateapi.v1.DeleteActorRequest + (*CreateActorEgressPolicyRequest)(nil), // 5: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ListWorkersRequest)(nil), // 6: substrate.ateapi.v1.ListWorkersRequest + (*CreateActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 9: substrate.ateapi.v1.DeleteActorTemplateRequest + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*SuspendActorResponse)(nil), // 11: substrate.ateapi.v1.SuspendActorResponse + (*ResumeActorResponse)(nil), // 12: substrate.ateapi.v1.ResumeActorResponse + (*EgressPolicy)(nil), // 13: substrate.ateapi.v1.EgressPolicy + (*ListWorkersResponse)(nil), // 14: substrate.ateapi.v1.ListWorkersResponse + (*ActorTemplate)(nil), // 15: substrate.ateapi.v1.ActorTemplate +} +var file_control_proto_depIdxs = []int32{ + 0, // 0: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest + 1, // 1: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest + 2, // 2: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest + 3, // 3: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest + 4, // 4: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest + 5, // 5: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest + 6, // 6: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest + 7, // 7: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest + 8, // 8: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest + 9, // 9: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest + 10, // 10: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor + 10, // 11: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor + 11, // 12: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse + 12, // 13: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse + 10, // 14: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor + 13, // 15: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 14, // 16: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse + 15, // 17: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 18: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 19: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 10, // [10:20] is the sub-list for method output_type + 0, // [0:10] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_control_proto_init() } +func file_control_proto_init() { + if File_control_proto != nil { + return + } + file_actor_proto_init() + file_actor_rpc_proto_init() + file_egress_proto_init() + file_template_proto_init() + file_worker_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_control_proto_rawDesc), len(file_control_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_control_proto_goTypes, + DependencyIndexes: file_control_proto_depIdxs, + }.Build() + File_control_proto = out.File + file_control_proto_goTypes = nil + file_control_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/control.proto b/src/crux/sandbox/substrateapi/control.proto new file mode 100644 index 0000000000..e95d5a6d98 --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.proto @@ -0,0 +1,48 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "actor_rpc.proto"; +import "egress.proto"; +import "template.proto"; +import "worker.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +// Control is the Agent Substrate control-plane API served by ate-api-server. +service Control { + rpc GetActor(GetActorRequest) returns (Actor) {} + rpc CreateActor(CreateActorRequest) returns (Actor) {} + rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} + rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} + rpc DeleteActor(DeleteActorRequest) returns (Actor) {} + rpc CreateActorEgressPolicy(CreateActorEgressPolicyRequest) returns (EgressPolicy) {} + rpc ListWorkers(ListWorkersRequest) returns (ListWorkersResponse) {} + rpc CreateActorTemplate(CreateActorTemplateRequest) returns (ActorTemplate) {} + rpc GetActorTemplate(GetActorTemplateRequest) returns (ActorTemplate) {} + rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {} +} diff --git a/src/crux/sandbox/substrateapi/control_grpc.pb.go b/src/crux/sandbox/substrateapi/control_grpc.pb.go new file mode 100644 index 0000000000..cc4663c9fb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control_grpc.pb.go @@ -0,0 +1,490 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Control_GetActor_FullMethodName = "/substrate.ateapi.v1.Control/GetActor" + Control_CreateActor_FullMethodName = "/substrate.ateapi.v1.Control/CreateActor" + Control_SuspendActor_FullMethodName = "/substrate.ateapi.v1.Control/SuspendActor" + Control_ResumeActor_FullMethodName = "/substrate.ateapi.v1.Control/ResumeActor" + Control_DeleteActor_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActor" + Control_CreateActorEgressPolicy_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorEgressPolicy" + Control_ListWorkers_FullMethodName = "/substrate.ateapi.v1.Control/ListWorkers" + Control_CreateActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorTemplate" + Control_GetActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/GetActorTemplate" + Control_DeleteActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActorTemplate" +) + +// ControlClient is the client API for Control service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlClient interface { + GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) + SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) + ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) + DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) + ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) + CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) +} + +type controlClient struct { + cc grpc.ClientConnInterface +} + +func NewControlClient(cc grpc.ClientConnInterface) ControlClient { + return &controlClient{cc} +} + +func (c *controlClient) GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_GetActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_CreateActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SuspendActorResponse) + err := c.cc.Invoke(ctx, Control_SuspendActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ResumeActorResponse) + err := c.cc.Invoke(ctx, Control_ResumeActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_DeleteActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EgressPolicy) + err := c.cc.Invoke(ctx, Control_CreateActorEgressPolicy_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListWorkersResponse) + err := c.cc.Invoke(ctx, Control_ListWorkers_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_CreateActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_GetActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_DeleteActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ControlServer is the server API for Control service. +// All implementations must embed UnimplementedControlServer +// for forward compatibility. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlServer interface { + GetActor(context.Context, *GetActorRequest) (*Actor, error) + CreateActor(context.Context, *CreateActorRequest) (*Actor, error) + SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) + ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) + DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) + CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) + ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) + CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) + GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) + DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) + mustEmbedUnimplementedControlServer() +} + +// UnimplementedControlServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedControlServer struct{} + +func (UnimplementedControlServer) GetActor(context.Context, *GetActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method GetActor not implemented") +} +func (UnimplementedControlServer) CreateActor(context.Context, *CreateActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActor not implemented") +} +func (UnimplementedControlServer) SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SuspendActor not implemented") +} +func (UnimplementedControlServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ResumeActor not implemented") +} +func (UnimplementedControlServer) DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActor not implemented") +} +func (UnimplementedControlServer) CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorEgressPolicy not implemented") +} +func (UnimplementedControlServer) ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListWorkers not implemented") +} +func (UnimplementedControlServer) CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorTemplate not implemented") +} +func (UnimplementedControlServer) GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method GetActorTemplate not implemented") +} +func (UnimplementedControlServer) DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActorTemplate not implemented") +} +func (UnimplementedControlServer) mustEmbedUnimplementedControlServer() {} +func (UnimplementedControlServer) testEmbeddedByValue() {} + +// UnsafeControlServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ControlServer will +// result in compilation errors. +type UnsafeControlServer interface { + mustEmbedUnimplementedControlServer() +} + +func RegisterControlServer(s grpc.ServiceRegistrar, srv ControlServer) { + // If the following call panics, it indicates UnimplementedControlServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Control_ServiceDesc, srv) +} + +func _Control_GetActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActor(ctx, req.(*GetActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActor(ctx, req.(*CreateActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_SuspendActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SuspendActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).SuspendActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_SuspendActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).SuspendActor(ctx, req.(*SuspendActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ResumeActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResumeActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ResumeActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ResumeActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ResumeActor(ctx, req.(*ResumeActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActor(ctx, req.(*DeleteActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorEgressPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorEgressPolicyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorEgressPolicy_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, req.(*CreateActorEgressPolicyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ListWorkers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListWorkersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ListWorkers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ListWorkers_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ListWorkers(ctx, req.(*ListWorkersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorTemplate(ctx, req.(*CreateActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_GetActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActorTemplate(ctx, req.(*GetActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActorTemplate(ctx, req.(*DeleteActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Control_ServiceDesc is the grpc.ServiceDesc for Control service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Control_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "substrate.ateapi.v1.Control", + HandlerType: (*ControlServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetActor", + Handler: _Control_GetActor_Handler, + }, + { + MethodName: "CreateActor", + Handler: _Control_CreateActor_Handler, + }, + { + MethodName: "SuspendActor", + Handler: _Control_SuspendActor_Handler, + }, + { + MethodName: "ResumeActor", + Handler: _Control_ResumeActor_Handler, + }, + { + MethodName: "DeleteActor", + Handler: _Control_DeleteActor_Handler, + }, + { + MethodName: "CreateActorEgressPolicy", + Handler: _Control_CreateActorEgressPolicy_Handler, + }, + { + MethodName: "ListWorkers", + Handler: _Control_ListWorkers_Handler, + }, + { + MethodName: "CreateActorTemplate", + Handler: _Control_CreateActorTemplate_Handler, + }, + { + MethodName: "GetActorTemplate", + Handler: _Control_GetActorTemplate_Handler, + }, + { + MethodName: "DeleteActorTemplate", + Handler: _Control_DeleteActorTemplate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "control.proto", +} diff --git a/src/crux/sandbox/substrateapi/egress.pb.go b/src/crux/sandbox/substrateapi/egress.pb.go new file mode 100644 index 0000000000..170d873c8a --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.pb.go @@ -0,0 +1,553 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: egress.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EgressPolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Rules []*EgressRule `protobuf:"bytes,2,rep,name=rules,proto3" json:"rules,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressPolicy) Reset() { + *x = EgressPolicy{} + mi := &file_egress_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressPolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressPolicy) ProtoMessage() {} + +func (x *EgressPolicy) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. +func (*EgressPolicy) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{0} +} + +func (x *EgressPolicy) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *EgressPolicy) GetRules() []*EgressRule { + if x != nil { + return x.Rules + } + return nil +} + +type EgressRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Http *HttpRule `protobuf:"bytes,1,opt,name=http,proto3" json:"http,omitempty"` + Https *HttpsRule `protobuf:"bytes,2,opt,name=https,proto3" json:"https,omitempty"` + TlsPassthrough *TlsPassthroughRule `protobuf:"bytes,3,opt,name=tls_passthrough,json=tlsPassthrough,proto3" json:"tls_passthrough,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressRule) Reset() { + *x = EgressRule{} + mi := &file_egress_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressRule) ProtoMessage() {} + +func (x *EgressRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. +func (*EgressRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{1} +} + +func (x *EgressRule) GetHttp() *HttpRule { + if x != nil { + return x.Http + } + return nil +} + +func (x *EgressRule) GetHttps() *HttpsRule { + if x != nil { + return x.Https + } + return nil +} + +func (x *EgressRule) GetTlsPassthrough() *TlsPassthroughRule { + if x != nil { + return x.TlsPassthrough + } + return nil +} + +type HttpRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpRule) Reset() { + *x = HttpRule{} + mi := &file_egress_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpRule) ProtoMessage() {} + +func (x *HttpRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpRule.ProtoReflect.Descriptor instead. +func (*HttpRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{2} +} + +func (x *HttpRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type HttpsRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpsRule) Reset() { + *x = HttpsRule{} + mi := &file_egress_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpsRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpsRule) ProtoMessage() {} + +func (x *HttpsRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpsRule.ProtoReflect.Descriptor instead. +func (*HttpsRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{3} +} + +func (x *HttpsRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpsRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type TlsPassthroughRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TlsPassthroughRule) Reset() { + *x = TlsPassthroughRule{} + mi := &file_egress_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TlsPassthroughRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TlsPassthroughRule) ProtoMessage() {} + +func (x *TlsPassthroughRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TlsPassthroughRule.ProtoReflect.Descriptor instead. +func (*TlsPassthroughRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{4} +} + +func (x *TlsPassthroughRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *TlsPassthroughRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type Ports struct { + state protoimpl.MessageState `protogen:"open.v1"` + All *AllPorts `protobuf:"bytes,1,opt,name=all,proto3" json:"all,omitempty"` + Numbers []int32 `protobuf:"varint,2,rep,packed,name=numbers,proto3" json:"numbers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ports) Reset() { + *x = Ports{} + mi := &file_egress_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ports) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ports) ProtoMessage() {} + +func (x *Ports) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ports.ProtoReflect.Descriptor instead. +func (*Ports) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{5} +} + +func (x *Ports) GetAll() *AllPorts { + if x != nil { + return x.All + } + return nil +} + +func (x *Ports) GetNumbers() []int32 { + if x != nil { + return x.Numbers + } + return nil +} + +type AllPorts struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AllPorts) Reset() { + *x = AllPorts{} + mi := &file_egress_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AllPorts) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllPorts) ProtoMessage() {} + +func (x *AllPorts) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllPorts.ProtoReflect.Descriptor instead. +func (*AllPorts) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{6} +} + +type CreateActorEgressPolicyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + EgressPolicy *EgressPolicy `protobuf:"bytes,2,opt,name=egress_policy,json=egressPolicy,proto3" json:"egress_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorEgressPolicyRequest) Reset() { + *x = CreateActorEgressPolicyRequest{} + mi := &file_egress_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorEgressPolicyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorEgressPolicyRequest) ProtoMessage() {} + +func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. +func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{7} +} + +func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *CreateActorEgressPolicyRequest) GetEgressPolicy() *EgressPolicy { + if x != nil { + return x.EgressPolicy + } + return nil +} + +var File_egress_proto protoreflect.FileDescriptor + +const file_egress_proto_rawDesc = "" + + "\n" + + "\fegress.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\x88\x01\n" + + "\fEgressPolicy\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x125\n" + + "\x05rules\x18\x02 \x03(\v2\x1f.substrate.ateapi.v1.EgressRuleR\x05rules\"\xc7\x01\n" + + "\n" + + "EgressRule\x121\n" + + "\x04http\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.HttpRuleR\x04http\x124\n" + + "\x05https\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.HttpsRuleR\x05https\x12P\n" + + "\x0ftls_passthrough\x18\x03 \x01(\v2'.substrate.ateapi.v1.TlsPassthroughRuleR\x0etlsPassthrough\"Z\n" + + "\bHttpRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"[\n" + + "\tHttpsRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"d\n" + + "\x12TlsPassthroughRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"R\n" + + "\x05Ports\x12/\n" + + "\x03all\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.AllPortsR\x03all\x12\x18\n" + + "\anumbers\x18\x02 \x03(\x05R\anumbers\"\n" + + "\n" + + "\bAllPorts\"\x9e\x01\n" + + "\x1eCreateActorEgressPolicyRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12F\n" + + "\regress_policy\x18\x02 \x01(\v2!.substrate.ateapi.v1.EgressPolicyR\fegressPolicyBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_egress_proto_rawDescOnce sync.Once + file_egress_proto_rawDescData []byte +) + +func file_egress_proto_rawDescGZIP() []byte { + file_egress_proto_rawDescOnce.Do(func() { + file_egress_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc))) + }) + return file_egress_proto_rawDescData +} + +var file_egress_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_egress_proto_goTypes = []any{ + (*EgressPolicy)(nil), // 0: substrate.ateapi.v1.EgressPolicy + (*EgressRule)(nil), // 1: substrate.ateapi.v1.EgressRule + (*HttpRule)(nil), // 2: substrate.ateapi.v1.HttpRule + (*HttpsRule)(nil), // 3: substrate.ateapi.v1.HttpsRule + (*TlsPassthroughRule)(nil), // 4: substrate.ateapi.v1.TlsPassthroughRule + (*Ports)(nil), // 5: substrate.ateapi.v1.Ports + (*AllPorts)(nil), // 6: substrate.ateapi.v1.AllPorts + (*CreateActorEgressPolicyRequest)(nil), // 7: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ResourceMetadata)(nil), // 8: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef +} +var file_egress_proto_depIdxs = []int32{ + 8, // 0: substrate.ateapi.v1.EgressPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 1, // 1: substrate.ateapi.v1.EgressPolicy.rules:type_name -> substrate.ateapi.v1.EgressRule + 2, // 2: substrate.ateapi.v1.EgressRule.http:type_name -> substrate.ateapi.v1.HttpRule + 3, // 3: substrate.ateapi.v1.EgressRule.https:type_name -> substrate.ateapi.v1.HttpsRule + 4, // 4: substrate.ateapi.v1.EgressRule.tls_passthrough:type_name -> substrate.ateapi.v1.TlsPassthroughRule + 5, // 5: substrate.ateapi.v1.HttpRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 6: substrate.ateapi.v1.HttpsRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 7: substrate.ateapi.v1.TlsPassthroughRule.ports:type_name -> substrate.ateapi.v1.Ports + 6, // 8: substrate.ateapi.v1.Ports.all:type_name -> substrate.ateapi.v1.AllPorts + 9, // 9: substrate.ateapi.v1.CreateActorEgressPolicyRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 0, // 10: substrate.ateapi.v1.CreateActorEgressPolicyRequest.egress_policy:type_name -> substrate.ateapi.v1.EgressPolicy + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_egress_proto_init() } +func file_egress_proto_init() { + if File_egress_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_egress_proto_goTypes, + DependencyIndexes: file_egress_proto_depIdxs, + MessageInfos: file_egress_proto_msgTypes, + }.Build() + File_egress_proto = out.File + file_egress_proto_goTypes = nil + file_egress_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/egress.proto b/src/crux/sandbox/substrateapi/egress.proto new file mode 100644 index 0000000000..8922f27f97 --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.proto @@ -0,0 +1,68 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message EgressPolicy { + ResourceMetadata metadata = 1; + repeated EgressRule rules = 2; +} + +message EgressRule { + HttpRule http = 1; + HttpsRule https = 2; + TlsPassthroughRule tls_passthrough = 3; +} + +message HttpRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message HttpsRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message TlsPassthroughRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message Ports { + AllPorts all = 1; + repeated int32 numbers = 2; +} + +message AllPorts {} + +message CreateActorEgressPolicyRequest { + ObjectRef actor = 1; + EgressPolicy egress_policy = 2; +} diff --git a/src/crux/sandbox/substrateapi/generate.sh b/src/crux/sandbox/substrateapi/generate.sh new file mode 100755 index 0000000000..fbfed28814 --- /dev/null +++ b/src/crux/sandbox/substrateapi/generate.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# Regenerates the *.pb.go files from the trimmed *.proto subset. +# Requires buf, protoc-gen-go (v1.36.x) and protoc-gen-go-grpc on PATH. +set -eu +cd "$(dirname "$0")" +buf generate diff --git a/src/crux/sandbox/substrateapi/substrateapi_test.go b/src/crux/sandbox/substrateapi/substrateapi_test.go new file mode 100644 index 0000000000..68047d96be --- /dev/null +++ b/src/crux/sandbox/substrateapi/substrateapi_test.go @@ -0,0 +1,364 @@ +package substrateapi + +import ( + "context" + "errors" + "net" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" +) + +// The generated code is a wire-compatible subset of Substrate's Control API. +// These tests exercise every message's accessors — empty and populated — and +// every RPC through a real gRPC connection so a regeneration that drops or +// renames a field or method fails here rather than against a cluster. + +// allFiles is a function because the File_* descriptors are populated by +// the generated init functions, after package-level vars. +func allFiles() []protoreflect.FileDescriptor { + return []protoreflect.FileDescriptor{ + File_common_proto, File_actor_proto, File_actor_rpc_proto, File_container_proto, + File_template_proto, File_egress_proto, File_worker_proto, File_control_proto, + } +} + +// callZeroArgMethods invokes every method on v that takes no arguments and +// does not panic on a nil or populated receiver. +func callZeroArgMethods(t *testing.T, v reflect.Value, onlyGetters bool) { + t.Helper() + for i := 0; i < v.NumMethod(); i++ { + m := v.Type().Method(i) + if m.Type.NumIn() != 1 || m.Type.IsVariadic() { + continue + } + if onlyGetters && !strings.HasPrefix(m.Name, "Get") { + continue + } + v.Method(i).Call(nil) + } +} + +// populate sets every field of m to a non-default value, recursing into +// message fields one level so nested getters see data. +func populate(m protoreflect.Message, depth int) { + fields := m.Descriptor().Fields() + for i := 0; i < fields.Len(); i++ { + fd := fields.Get(i) + switch { + case fd.IsList(): + list := m.Mutable(fd).List() + list.Append(sampleValue(fd, list.NewElement, depth)) + case fd.IsMap(): + mp := m.Mutable(fd).Map() + mp.Set(protoreflect.ValueOfString("k").MapKey(), sampleValue(fd.MapValue(), mp.NewValue, depth)) + default: + m.Set(fd, sampleValue(fd, func() protoreflect.Value { return m.NewField(fd) }, depth)) + } + } +} + +func sampleValue(fd protoreflect.FieldDescriptor, newVal func() protoreflect.Value, depth int) protoreflect.Value { + switch fd.Kind() { + case protoreflect.BoolKind: + return protoreflect.ValueOfBool(true) + case protoreflect.StringKind: + return protoreflect.ValueOfString("s") + case protoreflect.BytesKind: + return protoreflect.ValueOfBytes([]byte("b")) + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind: + return protoreflect.ValueOfInt32(7) + case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + return protoreflect.ValueOfInt64(7) + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind: + return protoreflect.ValueOfUint32(7) + case protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + return protoreflect.ValueOfUint64(7) + case protoreflect.FloatKind: + return protoreflect.ValueOfFloat32(1.5) + case protoreflect.DoubleKind: + return protoreflect.ValueOfFloat64(1.5) + case protoreflect.EnumKind: + vals := fd.Enum().Values() + return protoreflect.ValueOfEnum(vals.Get(vals.Len() - 1).Number()) + case protoreflect.MessageKind, protoreflect.GroupKind: + v := newVal() + if depth > 0 { + populate(v.Message(), depth-1) + } + return v + } + return newVal() +} + +func TestEveryMessageAccessorAndRoundTrip(t *testing.T) { + seen := 0 + for _, file := range allFiles() { + msgs := file.Messages() + for i := 0; i < msgs.Len(); i++ { + md := msgs.Get(i) + mt, err := protoregistry.GlobalTypes.FindMessageByName(md.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", md.FullName(), err) + } + seen++ + // Nil receiver: every getter must return the zero value. + nilMsg := reflect.New(reflect.TypeOf(mt.Zero().Interface()).Elem()) + callZeroArgMethods(t, reflect.Zero(nilMsg.Type()), true) + + // Populated receiver, then a marshal/unmarshal round trip. + full := mt.New() + populate(full, 2) + goVal := reflect.ValueOf(full.Interface()) + callZeroArgMethods(t, goVal, false) + raw, err := proto.Marshal(full.Interface()) + if err != nil { + t.Fatalf("%s marshal: %v", md.FullName(), err) + } + back := mt.New().Interface() + if err := proto.Unmarshal(raw, back); err != nil { + t.Fatalf("%s unmarshal: %v", md.FullName(), err) + } + if !proto.Equal(full.Interface(), back) { + t.Errorf("%s round trip lost data", md.FullName()) + } + // Reset then String on the emptied message. + proto.Reset(back) + callZeroArgMethods(t, reflect.ValueOf(back), false) + } + enums := file.Enums() + for i := 0; i < enums.Len(); i++ { + ed := enums.Get(i) + et, err := protoregistry.GlobalTypes.FindEnumByName(ed.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", ed.FullName(), err) + } + for j := 0; j < ed.Values().Len(); j++ { + e := et.New(ed.Values().Get(j).Number()) + v := reflect.ValueOf(e) + callZeroArgMethods(t, v, false) + p := reflect.New(v.Type()) + p.Elem().Set(v) + callZeroArgMethods(t, p, false) + if s := v.MethodByName("String").Call(nil)[0].String(); s == "" { + t.Errorf("%s value %d has empty String()", ed.FullName(), j) + } + } + } + } + if seen < 40 { + t.Errorf("only %d messages found; the subset shrank unexpectedly", seen) + } +} + +func TestFieldNumbersMatchUpstream(t *testing.T) { + // A few load-bearing field numbers from Substrate's ateapi/v1 protos; + // renumbering any of these would silently talk past the server. + checks := map[string]map[string]int32{ + "substrate.ateapi.v1.ObjectRef": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.ResourceMetadata": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.Actor": {"metadata": 1, "actor_template": 2, "worker_selector": 3, "status": 5}, + "substrate.ateapi.v1.Container": {"name": 1, "image": 2, "command": 3, "env": 5, "volume_mounts": 7}, + } + for name, fields := range checks { + md, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + m := md.(protoreflect.MessageDescriptor) + for fname, want := range fields { + fd := m.Fields().ByName(protoreflect.Name(fname)) + if fd == nil { + t.Errorf("%s.%s missing", name, fname) + continue + } + if int32(fd.Number()) != want { + t.Errorf("%s.%s = %d, want %d", name, fname, fd.Number(), want) + } + } + } +} + +// fakeServer answers every RPC by echoing the request's target back. +type fakeServer struct { + UnimplementedControlServer + calls []string +} + +func (f *fakeServer) note(name string) { f.calls = append(f.calls, name) } + +func (f *fakeServer) GetActor(_ context.Context, r *GetActorRequest) (*Actor, error) { + f.note("GetActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}, Status: &ActorStatus{State: ActorState_ACTOR_STATE_RUNNING}}, nil +} +func (f *fakeServer) CreateActor(_ context.Context, r *CreateActorRequest) (*Actor, error) { + f.note("CreateActor") + return r.GetActor(), nil +} +func (f *fakeServer) SuspendActor(_ context.Context, r *SuspendActorRequest) (*SuspendActorResponse, error) { + f.note("SuspendActor:" + r.GetOptions().GetMode().String()) + return &SuspendActorResponse{}, nil +} +func (f *fakeServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + f.note("ResumeActor") + return &ResumeActorResponse{}, nil +} +func (f *fakeServer) DeleteActor(_ context.Context, r *DeleteActorRequest) (*Actor, error) { + f.note("DeleteActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}}, nil +} +func (f *fakeServer) CreateActorEgressPolicy(_ context.Context, r *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + f.note("CreateActorEgressPolicy") + return r.GetEgressPolicy(), nil +} +func (f *fakeServer) ListWorkers(_ context.Context, r *ListWorkersRequest) (*ListWorkersResponse, error) { + f.note("ListWorkers:" + r.GetOptions().GetPageToken()) + if r.GetOptions().GetPageToken() == "" { + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w1"}}}, NextPageToken: "p2"}, nil + } + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w2"}}}}, nil +} +func (f *fakeServer) CreateActorTemplate(_ context.Context, r *CreateActorTemplateRequest) (*ActorTemplate, error) { + f.note("CreateActorTemplate") + return r.GetActorTemplate(), nil +} +func (f *fakeServer) GetActorTemplate(_ context.Context, r *GetActorTemplateRequest) (*ActorTemplate, error) { + f.note("GetActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} +func (f *fakeServer) DeleteActorTemplate(_ context.Context, r *DeleteActorTemplateRequest) (*ActorTemplate, error) { + f.note("DeleteActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} + +func dialBufconn(t *testing.T, srv ControlServer) ControlClient { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return NewControlClient(conn) +} + +func TestControlClientRoundTripsEveryRPC(t *testing.T) { + srv := &fakeServer{} + c := dialBufconn(t, srv) + ctx := context.Background() + ref := &ObjectRef{Atespace: "hive", Name: "a1"} + + tpl, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{ActorTemplate: &ActorTemplate{Metadata: &ResourceMetadata{Name: "t1"}}}) + if err != nil || tpl.GetMetadata().GetName() != "t1" { + t.Fatalf("CreateActorTemplate: %v %v", tpl, err) + } + if got, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("GetActorTemplate: %v %v", got, err) + } + if got, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActorTemplate: %v %v", got, err) + } + actor, err := c.CreateActor(ctx, &CreateActorRequest{Actor: &Actor{Metadata: &ResourceMetadata{Name: "a1"}, ActorTemplate: ref}}) + if err != nil || actor.GetActorTemplate().GetName() != "a1" { + t.Fatalf("CreateActor: %v %v", actor, err) + } + if got, err := c.GetActor(ctx, &GetActorRequest{Actor: ref}); err != nil || got.GetStatus().GetState() != ActorState_ACTOR_STATE_RUNNING { + t.Fatalf("GetActor: %v %v", got, err) + } + if _, err := c.ResumeActor(ctx, &ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor: %v", err) + } + if _, err := c.SuspendActor(ctx, &SuspendActorRequest{Actor: ref, Options: &SuspendOptions{Mode: DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor: %v", err) + } + pol, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: &EgressPolicy{Metadata: &ResourceMetadata{Name: "default"}}}) + if err != nil || pol.GetMetadata().GetName() != "default" { + t.Fatalf("CreateActorEgressPolicy: %v %v", pol, err) + } + if got, err := c.DeleteActor(ctx, &DeleteActorRequest{Actor: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActor: %v %v", got, err) + } + page1, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{}}) + if err != nil || page1.GetNextPageToken() != "p2" || len(page1.GetWorkers()) != 1 { + t.Fatalf("ListWorkers page 1: %v %v", page1, err) + } + page2, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{PageToken: "p2"}}) + if err != nil || page2.GetNextPageToken() != "" || page2.GetWorkers()[0].GetMetadata().GetName() != "w2" { + t.Fatalf("ListWorkers page 2: %v %v", page2, err) + } + joined := strings.Join(srv.calls, ",") + for _, want := range []string{"CreateActorTemplate", "GetActorTemplate", "DeleteActorTemplate", "CreateActor", "GetActor", "ResumeActor", "SuspendActor:DURABILITY_MODE_ASYNCHRONOUS", "CreateActorEgressPolicy", "DeleteActor", "ListWorkers:", "ListWorkers:p2"} { + if !strings.Contains(joined, want) { + t.Errorf("server never saw %s: %s", want, joined) + } + } +} + +func TestUnimplementedServerAnswersUnimplemented(t *testing.T) { + c := dialBufconn(t, UnimplementedControlServer{}) + ctx := context.Background() + calls := []func() error{ + func() error { _, err := c.GetActor(ctx, &GetActorRequest{}); return err }, + func() error { _, err := c.CreateActor(ctx, &CreateActorRequest{}); return err }, + func() error { _, err := c.SuspendActor(ctx, &SuspendActorRequest{}); return err }, + func() error { _, err := c.ResumeActor(ctx, &ResumeActorRequest{}); return err }, + func() error { _, err := c.DeleteActor(ctx, &DeleteActorRequest{}); return err }, + func() error { _, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{}); return err }, + func() error { _, err := c.ListWorkers(ctx, &ListWorkersRequest{}); return err }, + func() error { _, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{}); return err }, + func() error { _, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{}); return err }, + func() error { _, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{}); return err }, + } + for i, call := range calls { + err := call() + if st, ok := status.FromError(err); !ok || st.Code() != codes.Unimplemented { + t.Errorf("call %d: %v, want Unimplemented", i, err) + } + } + var u UnimplementedControlServer + u.testEmbeddedByValue() + if _, err := u.GetActor(ctx, nil); err == nil { + t.Error("direct GetActor on the unimplemented server returned nil error") + } + var _ UnsafeControlServer = &fakeServer{} +} + +func TestRegisterControlServerRejectsNilEmbedding(t *testing.T) { + // Registering a server whose embedded UnimplementedControlServer is a + // nil pointer must panic, which is grpc-go's guard against a forgotten + // embedding. + defer func() { + if r := recover(); r == nil { + t.Error("no panic for a nil embedded UnimplementedControlServer") + } + }() + type nilEmbed struct{ *UnimplementedControlServer } + RegisterControlServer(grpc.NewServer(), nilEmbed{}) +} + +func TestServiceDescriptor(t *testing.T) { + if Control_ServiceDesc.ServiceName != "substrate.ateapi.v1.Control" { + t.Errorf("service name = %q", Control_ServiceDesc.ServiceName) + } + if len(Control_ServiceDesc.Methods) != 10 { + t.Errorf("methods = %d, want 10", len(Control_ServiceDesc.Methods)) + } + if !errors.Is(context.Canceled, context.Canceled) { + t.Fatal("unreachable") + } +} diff --git a/src/crux/sandbox/substrateapi/template.pb.go b/src/crux/sandbox/substrateapi/template.pb.go new file mode 100644 index 0000000000..b27724d605 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.pb.go @@ -0,0 +1,616 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: template.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GoldenSnapshotMode int32 + +const ( + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED GoldenSnapshotMode = 0 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED GoldenSnapshotMode = 1 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED GoldenSnapshotMode = 2 +) + +// Enum value maps for GoldenSnapshotMode. +var ( + GoldenSnapshotMode_name = map[int32]string{ + 0: "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED", + 1: "GOLDEN_SNAPSHOT_MODE_ENABLED", + 2: "GOLDEN_SNAPSHOT_MODE_DISABLED", + } + GoldenSnapshotMode_value = map[string]int32{ + "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED": 0, + "GOLDEN_SNAPSHOT_MODE_ENABLED": 1, + "GOLDEN_SNAPSHOT_MODE_DISABLED": 2, + } +) + +func (x GoldenSnapshotMode) Enum() *GoldenSnapshotMode { + p := new(GoldenSnapshotMode) + *p = x + return p +} + +func (x GoldenSnapshotMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GoldenSnapshotMode) Descriptor() protoreflect.EnumDescriptor { + return file_template_proto_enumTypes[0].Descriptor() +} + +func (GoldenSnapshotMode) Type() protoreflect.EnumType { + return &file_template_proto_enumTypes[0] +} + +func (x GoldenSnapshotMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GoldenSnapshotMode.Descriptor instead. +func (GoldenSnapshotMode) EnumDescriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +type ActorTemplate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,2,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + Containers []*Container `protobuf:"bytes,3,rep,name=containers,proto3" json:"containers,omitempty"` + Volumes []*Volume `protobuf:"bytes,4,rep,name=volumes,proto3" json:"volumes,omitempty"` + SnapshotConfig *SnapshotConfig `protobuf:"bytes,5,opt,name=snapshot_config,json=snapshotConfig,proto3" json:"snapshot_config,omitempty"` + Resources *ResourceRequirements `protobuf:"bytes,6,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorTemplate) Reset() { + *x = ActorTemplate{} + mi := &file_template_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorTemplate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorTemplate) ProtoMessage() {} + +func (x *ActorTemplate) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorTemplate.ProtoReflect.Descriptor instead. +func (*ActorTemplate) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +func (x *ActorTemplate) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ActorTemplate) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *ActorTemplate) GetContainers() []*Container { + if x != nil { + return x.Containers + } + return nil +} + +func (x *ActorTemplate) GetVolumes() []*Volume { + if x != nil { + return x.Volumes + } + return nil +} + +func (x *ActorTemplate) GetSnapshotConfig() *SnapshotConfig { + if x != nil { + return x.SnapshotConfig + } + return nil +} + +func (x *ActorTemplate) GetResources() *ResourceRequirements { + if x != nil { + return x.Resources + } + return nil +} + +type ResourceRequirements struct { + state protoimpl.MessageState `protogen:"open.v1"` + Limits *Resources `protobuf:"bytes,1,opt,name=limits,proto3" json:"limits,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceRequirements) Reset() { + *x = ResourceRequirements{} + mi := &file_template_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceRequirements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceRequirements) ProtoMessage() {} + +func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceRequirements.ProtoReflect.Descriptor instead. +func (*ResourceRequirements) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceRequirements) GetLimits() *Resources { + if x != nil { + return x.Limits + } + return nil +} + +type Resources struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuMillicores int64 `protobuf:"varint,1,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `protobuf:"varint,2,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Resources) Reset() { + *x = Resources{} + mi := &file_template_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Resources) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Resources) ProtoMessage() {} + +func (x *Resources) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Resources.ProtoReflect.Descriptor instead. +func (*Resources) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{2} +} + +func (x *Resources) GetCpuMillicores() int64 { + if x != nil { + return x.CpuMillicores + } + return 0 +} + +func (x *Resources) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +type SnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + GoldenSnapshotConfig *GoldenSnapshotConfig `protobuf:"bytes,3,opt,name=golden_snapshot_config,json=goldenSnapshotConfig,proto3" json:"golden_snapshot_config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotConfig) Reset() { + *x = SnapshotConfig{} + mi := &file_template_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotConfig) ProtoMessage() {} + +func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. +func (*SnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{3} +} + +func (x *SnapshotConfig) GetGoldenSnapshotConfig() *GoldenSnapshotConfig { + if x != nil { + return x.GoldenSnapshotConfig + } + return nil +} + +type GoldenSnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode GoldenSnapshotMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.GoldenSnapshotMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GoldenSnapshotConfig) Reset() { + *x = GoldenSnapshotConfig{} + mi := &file_template_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GoldenSnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GoldenSnapshotConfig) ProtoMessage() {} + +func (x *GoldenSnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GoldenSnapshotConfig.ProtoReflect.Descriptor instead. +func (*GoldenSnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{4} +} + +func (x *GoldenSnapshotConfig) GetMode() GoldenSnapshotMode { + if x != nil { + return x.Mode + } + return GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED +} + +type CreateActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ActorTemplate `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorTemplateRequest) Reset() { + *x = CreateActorTemplateRequest{} + mi := &file_template_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorTemplateRequest) ProtoMessage() {} + +func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{5} +} + +func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type GetActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorTemplateRequest) Reset() { + *x = GetActorTemplateRequest{} + mi := &file_template_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorTemplateRequest) ProtoMessage() {} + +func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{6} +} + +func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type DeleteActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorTemplateRequest) Reset() { + *x = DeleteActorTemplateRequest{} + mi := &file_template_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorTemplateRequest) ProtoMessage() {} + +func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *DeleteActorTemplateRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_template_proto protoreflect.FileDescriptor + +const file_template_proto_rawDesc = "" + + "\n" + + "\x0etemplate.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x0fcontainer.proto\"\xa8\x03\n" + + "\rActorTemplate\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12F\n" + + "\x0fworker_selector\x18\x02 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12>\n" + + "\n" + + "containers\x18\x03 \x03(\v2\x1e.substrate.ateapi.v1.ContainerR\n" + + "containers\x125\n" + + "\avolumes\x18\x04 \x03(\v2\x1b.substrate.ateapi.v1.VolumeR\avolumes\x12L\n" + + "\x0fsnapshot_config\x18\x05 \x01(\v2#.substrate.ateapi.v1.SnapshotConfigR\x0esnapshotConfig\x12G\n" + + "\tresources\x18\x06 \x01(\v2).substrate.ateapi.v1.ResourceRequirementsR\tresources\"N\n" + + "\x14ResourceRequirements\x126\n" + + "\x06limits\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ResourcesR\x06limits\"U\n" + + "\tResources\x12%\n" + + "\x0ecpu_millicores\x18\x01 \x01(\x03R\rcpuMillicores\x12!\n" + + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\"q\n" + + "\x0eSnapshotConfig\x12_\n" + + "\x16golden_snapshot_config\x18\x03 \x01(\v2).substrate.ateapi.v1.GoldenSnapshotConfigR\x14goldenSnapshotConfig\"S\n" + + "\x14GoldenSnapshotConfig\x12;\n" + + "\x04mode\x18\x01 \x01(\x0e2'.substrate.ateapi.v1.GoldenSnapshotModeR\x04mode\"g\n" + + "\x1aCreateActorTemplateRequest\x12I\n" + + "\x0eactor_template\x18\x01 \x01(\v2\".substrate.ateapi.v1.ActorTemplateR\ractorTemplate\"`\n" + + "\x17GetActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\"\xa1\x01\n" + + "\x1aDeleteActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*\x7f\n" + + "\x12GoldenSnapshotMode\x12$\n" + + " GOLDEN_SNAPSHOT_MODE_UNSPECIFIED\x10\x00\x12 \n" + + "\x1cGOLDEN_SNAPSHOT_MODE_ENABLED\x10\x01\x12!\n" + + "\x1dGOLDEN_SNAPSHOT_MODE_DISABLED\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_template_proto_rawDescOnce sync.Once + file_template_proto_rawDescData []byte +) + +func file_template_proto_rawDescGZIP() []byte { + file_template_proto_rawDescOnce.Do(func() { + file_template_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc))) + }) + return file_template_proto_rawDescData +} + +var file_template_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_template_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_template_proto_goTypes = []any{ + (GoldenSnapshotMode)(0), // 0: substrate.ateapi.v1.GoldenSnapshotMode + (*ActorTemplate)(nil), // 1: substrate.ateapi.v1.ActorTemplate + (*ResourceRequirements)(nil), // 2: substrate.ateapi.v1.ResourceRequirements + (*Resources)(nil), // 3: substrate.ateapi.v1.Resources + (*SnapshotConfig)(nil), // 4: substrate.ateapi.v1.SnapshotConfig + (*GoldenSnapshotConfig)(nil), // 5: substrate.ateapi.v1.GoldenSnapshotConfig + (*CreateActorTemplateRequest)(nil), // 6: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorTemplateRequest + (*ResourceMetadata)(nil), // 9: substrate.ateapi.v1.ResourceMetadata + (*Selector)(nil), // 10: substrate.ateapi.v1.Selector + (*Container)(nil), // 11: substrate.ateapi.v1.Container + (*Volume)(nil), // 12: substrate.ateapi.v1.Volume + (*ObjectRef)(nil), // 13: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 14: substrate.ateapi.v1.DeleteOptions +} +var file_template_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.ActorTemplate.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 10, // 1: substrate.ateapi.v1.ActorTemplate.worker_selector:type_name -> substrate.ateapi.v1.Selector + 11, // 2: substrate.ateapi.v1.ActorTemplate.containers:type_name -> substrate.ateapi.v1.Container + 12, // 3: substrate.ateapi.v1.ActorTemplate.volumes:type_name -> substrate.ateapi.v1.Volume + 4, // 4: substrate.ateapi.v1.ActorTemplate.snapshot_config:type_name -> substrate.ateapi.v1.SnapshotConfig + 2, // 5: substrate.ateapi.v1.ActorTemplate.resources:type_name -> substrate.ateapi.v1.ResourceRequirements + 3, // 6: substrate.ateapi.v1.ResourceRequirements.limits:type_name -> substrate.ateapi.v1.Resources + 5, // 7: substrate.ateapi.v1.SnapshotConfig.golden_snapshot_config:type_name -> substrate.ateapi.v1.GoldenSnapshotConfig + 0, // 8: substrate.ateapi.v1.GoldenSnapshotConfig.mode:type_name -> substrate.ateapi.v1.GoldenSnapshotMode + 1, // 9: substrate.ateapi.v1.CreateActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ActorTemplate + 13, // 10: substrate.ateapi.v1.GetActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 13, // 11: substrate.ateapi.v1.DeleteActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 14, // 12: substrate.ateapi.v1.DeleteActorTemplateRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_template_proto_init() } +func file_template_proto_init() { + if File_template_proto != nil { + return + } + file_common_proto_init() + file_container_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_template_proto_goTypes, + DependencyIndexes: file_template_proto_depIdxs, + EnumInfos: file_template_proto_enumTypes, + MessageInfos: file_template_proto_msgTypes, + }.Build() + File_template_proto = out.File + file_template_proto_goTypes = nil + file_template_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/template.proto b/src/crux/sandbox/substrateapi/template.proto new file mode 100644 index 0000000000..adcde1ee59 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.proto @@ -0,0 +1,76 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "container.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message ActorTemplate { + ResourceMetadata metadata = 1; + Selector worker_selector = 2; + repeated Container containers = 3; + repeated Volume volumes = 4; + SnapshotConfig snapshot_config = 5; + ResourceRequirements resources = 6; +} + +message ResourceRequirements { + Resources limits = 1; +} + +message Resources { + int64 cpu_millicores = 1; + int64 memory_bytes = 2; +} + +message SnapshotConfig { + GoldenSnapshotConfig golden_snapshot_config = 3; +} + +enum GoldenSnapshotMode { + GOLDEN_SNAPSHOT_MODE_UNSPECIFIED = 0; + GOLDEN_SNAPSHOT_MODE_ENABLED = 1; + GOLDEN_SNAPSHOT_MODE_DISABLED = 2; +} + +message GoldenSnapshotConfig { + GoldenSnapshotMode mode = 1; +} + +message CreateActorTemplateRequest { + ActorTemplate actor_template = 1; +} + +message GetActorTemplateRequest { + ObjectRef actor_template = 1; +} + +message DeleteActorTemplateRequest { + ObjectRef actor_template = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/worker.pb.go b/src/crux/sandbox/substrateapi/worker.pb.go new file mode 100644 index 0000000000..07611b1abc --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.pb.go @@ -0,0 +1,499 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: worker.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type WorkerState int32 + +const ( + WorkerState_WORKER_STATE_UNSPECIFIED WorkerState = 0 + WorkerState_WORKER_STATE_ACTIVE WorkerState = 1 + WorkerState_WORKER_STATE_DRAINING WorkerState = 2 + WorkerState_WORKER_STATE_UNAVAILABLE WorkerState = 3 +) + +// Enum value maps for WorkerState. +var ( + WorkerState_name = map[int32]string{ + 0: "WORKER_STATE_UNSPECIFIED", + 1: "WORKER_STATE_ACTIVE", + 2: "WORKER_STATE_DRAINING", + 3: "WORKER_STATE_UNAVAILABLE", + } + WorkerState_value = map[string]int32{ + "WORKER_STATE_UNSPECIFIED": 0, + "WORKER_STATE_ACTIVE": 1, + "WORKER_STATE_DRAINING": 2, + "WORKER_STATE_UNAVAILABLE": 3, + } +) + +func (x WorkerState) Enum() *WorkerState { + p := new(WorkerState) + *p = x + return p +} + +func (x WorkerState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (WorkerState) Descriptor() protoreflect.EnumDescriptor { + return file_worker_proto_enumTypes[0].Descriptor() +} + +func (WorkerState) Type() protoreflect.EnumType { + return &file_worker_proto_enumTypes[0] +} + +func (x WorkerState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use WorkerState.Descriptor instead. +func (WorkerState) EnumDescriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +type Worker struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerNamespace string `protobuf:"bytes,2,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,3,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,4,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,5,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + NodeName string `protobuf:"bytes,6,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,7,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Status *WorkerStatus `protobuf:"bytes,9,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Worker) Reset() { + *x = Worker{} + mi := &file_worker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Worker) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Worker) ProtoMessage() {} + +func (x *Worker) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Worker.ProtoReflect.Descriptor instead. +func (*Worker) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +func (x *Worker) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Worker) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *Worker) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *Worker) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *Worker) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *Worker) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *Worker) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +func (x *Worker) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *Worker) GetStatus() *WorkerStatus { + if x != nil { + return x.Status + } + return nil +} + +type WorkerStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State WorkerState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.WorkerState" json:"state,omitempty"` + SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,5,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerStatus) Reset() { + *x = WorkerStatus{} + mi := &file_worker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerStatus) ProtoMessage() {} + +func (x *WorkerStatus) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. +func (*WorkerStatus) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{1} +} + +func (x *WorkerStatus) GetState() WorkerState { + if x != nil { + return x.State + } + return WorkerState_WORKER_STATE_UNSPECIFIED +} + +func (x *WorkerStatus) GetSandboxClasses() []*WorkerSandboxClass { + if x != nil { + return x.SandboxClasses + } + return nil +} + +type WorkerSandboxClass struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerSandboxClass) Reset() { + *x = WorkerSandboxClass{} + mi := &file_worker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerSandboxClass) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerSandboxClass) ProtoMessage() {} + +func (x *WorkerSandboxClass) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerSandboxClass.ProtoReflect.Descriptor instead. +func (*WorkerSandboxClass) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{2} +} + +func (x *WorkerSandboxClass) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +type ListWorkersRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Options *ListOptions `protobuf:"bytes,1,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersRequest) Reset() { + *x = ListWorkersRequest{} + mi := &file_worker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersRequest) ProtoMessage() {} + +func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. +func (*ListWorkersRequest) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{3} +} + +func (x *ListWorkersRequest) GetOptions() *ListOptions { + if x != nil { + return x.Options + } + return nil +} + +type ListWorkersResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Workers []*Worker `protobuf:"bytes,1,rep,name=workers,proto3" json:"workers,omitempty"` + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersResponse) Reset() { + *x = ListWorkersResponse{} + mi := &file_worker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersResponse) ProtoMessage() {} + +func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. +func (*ListWorkersResponse) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{4} +} + +func (x *ListWorkersResponse) GetWorkers() []*Worker { + if x != nil { + return x.Workers + } + return nil +} + +func (x *ListWorkersResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +var File_worker_proto protoreflect.FileDescriptor + +const file_worker_proto_rawDesc = "" + + "\n" + + "\fworker.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\xda\x03\n" + + "\x06Worker\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12)\n" + + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x03 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x04 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12(\n" + + "\x10worker_spiffe_id\x18\a \x01(\tR\x0eworkerSpiffeId\x12?\n" + + "\x06labels\x18\b \x03(\v2'.substrate.ateapi.v1.Worker.LabelsEntryR\x06labels\x129\n" + + "\x06status\x18\t \x01(\v2!.substrate.ateapi.v1.WorkerStatusR\x06status\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x98\x01\n" + + "\fWorkerStatus\x126\n" + + "\x05state\x18\x01 \x01(\x0e2 .substrate.ateapi.v1.WorkerStateR\x05state\x12P\n" + + "\x0fsandbox_classes\x18\x05 \x03(\v2'.substrate.ateapi.v1.WorkerSandboxClassR\x0esandboxClasses\"9\n" + + "\x12WorkerSandboxClass\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\"P\n" + + "\x12ListWorkersRequest\x12:\n" + + "\aoptions\x18\x01 \x01(\v2 .substrate.ateapi.v1.ListOptionsR\aoptions\"t\n" + + "\x13ListWorkersResponse\x125\n" + + "\aworkers\x18\x01 \x03(\v2\x1b.substrate.ateapi.v1.WorkerR\aworkers\x12&\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken*}\n" + + "\vWorkerState\x12\x1c\n" + + "\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n" + + "\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n" + + "\x15WORKER_STATE_DRAINING\x10\x02\x12\x1c\n" + + "\x18WORKER_STATE_UNAVAILABLE\x10\x03BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_worker_proto_rawDescOnce sync.Once + file_worker_proto_rawDescData []byte +) + +func file_worker_proto_rawDescGZIP() []byte { + file_worker_proto_rawDescOnce.Do(func() { + file_worker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc))) + }) + return file_worker_proto_rawDescData +} + +var file_worker_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_worker_proto_goTypes = []any{ + (WorkerState)(0), // 0: substrate.ateapi.v1.WorkerState + (*Worker)(nil), // 1: substrate.ateapi.v1.Worker + (*WorkerStatus)(nil), // 2: substrate.ateapi.v1.WorkerStatus + (*WorkerSandboxClass)(nil), // 3: substrate.ateapi.v1.WorkerSandboxClass + (*ListWorkersRequest)(nil), // 4: substrate.ateapi.v1.ListWorkersRequest + (*ListWorkersResponse)(nil), // 5: substrate.ateapi.v1.ListWorkersResponse + nil, // 6: substrate.ateapi.v1.Worker.LabelsEntry + (*ResourceMetadata)(nil), // 7: substrate.ateapi.v1.ResourceMetadata + (*ListOptions)(nil), // 8: substrate.ateapi.v1.ListOptions +} +var file_worker_proto_depIdxs = []int32{ + 7, // 0: substrate.ateapi.v1.Worker.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 6, // 1: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry + 2, // 2: substrate.ateapi.v1.Worker.status:type_name -> substrate.ateapi.v1.WorkerStatus + 0, // 3: substrate.ateapi.v1.WorkerStatus.state:type_name -> substrate.ateapi.v1.WorkerState + 3, // 4: substrate.ateapi.v1.WorkerStatus.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass + 8, // 5: substrate.ateapi.v1.ListWorkersRequest.options:type_name -> substrate.ateapi.v1.ListOptions + 1, // 6: substrate.ateapi.v1.ListWorkersResponse.workers:type_name -> substrate.ateapi.v1.Worker + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_worker_proto_init() } +func file_worker_proto_init() { + if File_worker_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc)), + NumEnums: 1, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_worker_proto_goTypes, + DependencyIndexes: file_worker_proto_depIdxs, + EnumInfos: file_worker_proto_enumTypes, + MessageInfos: file_worker_proto_msgTypes, + }.Build() + File_worker_proto = out.File + file_worker_proto_goTypes = nil + file_worker_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/worker.proto b/src/crux/sandbox/substrateapi/worker.proto new file mode 100644 index 0000000000..13a527149e --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.proto @@ -0,0 +1,67 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Worker { + ResourceMetadata metadata = 1; + string worker_namespace = 2; + string worker_pool = 3; + string worker_pod = 4; + string worker_pod_uid = 5; + string node_name = 6; + string worker_spiffe_id = 7; + map labels = 8; + WorkerStatus status = 9; +} + +enum WorkerState { + WORKER_STATE_UNSPECIFIED = 0; + WORKER_STATE_ACTIVE = 1; + WORKER_STATE_DRAINING = 2; + WORKER_STATE_UNAVAILABLE = 3; +} + +message WorkerStatus { + WorkerState state = 1; + repeated WorkerSandboxClass sandbox_classes = 5; +} + +message WorkerSandboxClass { + string sandbox_class = 1; +} + +message ListWorkersRequest { + ListOptions options = 1; +} + +message ListWorkersResponse { + repeated Worker workers = 1; + string next_page_token = 2; +} diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go new file mode 100644 index 0000000000..389ffabb18 --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -0,0 +1,338 @@ +// Package substratebridge is the in-actor side of hive's Agent Substrate +// runtime (ADR-0026 phase 2). Substrate has no exec, attach or logs RPC and +// cannot mount an existing PersistentVolumeClaim, so the sandbox image runs +// this small HTTP server as the actor's entrypoint and hive drives the kick +// through it over the atenet-router mTLS CONNECT tunnel: push the workspace +// in, run the command, pull the workspace back out. The server listens on +// the actor's private network only; the router is the only way in and it +// admits only allowlisted SPIFFE identities. +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + // DefaultPort is the port the bridge serves on inside the actor and the + // port hive's CONNECT tunnel targets. + DefaultPort = 8477 + // DefaultWorkspace is where the bridge unpacks the kick workspace. + DefaultWorkspace = "/workspace" + + PathHealth = "/healthz" + PathWorkspace = "/workspace" + PathRun = "/run" + + // MaxOutput caps each of stdout and stderr in a RunResponse; the same + // 16 MiB the OpenShell launcher keeps. + MaxOutput = 16 << 20 + + // ExitCodeUnknown is reported when the command could not be started or + // ended without an exit status. + ExitCodeUnknown = -1 + + contentTypeTar = "application/x-tar+gzip" + contentTypeJSON = "application/json" + maxRequestJSON = 1 << 20 + maxTarEntry = 1 << 30 +) + +// RunRequest is the body of POST /run. +type RunRequest struct { + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` + WorkDir string `json:"workdir,omitempty"` +} + +// RunResponse is the body POST /run answers with once the command exits. +type RunResponse struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode int `json:"exit_code"` + Error string `json:"error,omitempty"` +} + +// Handler serves the bridge protocol for one workspace directory. +type Handler struct { + // Workspace is the directory PUT /workspace fills, GET /workspace packs + // and POST /run runs in. Defaults to DefaultWorkspace. + Workspace string + // RunTimeout bounds one POST /run; zero means the request context alone. + RunTimeout time.Duration +} + +// ServeHTTP implements http.Handler. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathHealth && r.Method == http.MethodGet: + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok\n") + case r.URL.Path == PathWorkspace && r.Method == http.MethodPut: + h.putWorkspace(w, r) + case r.URL.Path == PathWorkspace && r.Method == http.MethodGet: + h.getWorkspace(w, r) + case r.URL.Path == PathRun && r.Method == http.MethodPost: + h.run(w, r) + default: + http.Error(w, "substrate bridge: unknown route", http.StatusNotFound) + } +} + +func (h *Handler) workspace() string { + if h.Workspace != "" { + return h.Workspace + } + return DefaultWorkspace +} + +func (h *Handler) putWorkspace(w http.ResponseWriter, r *http.Request) { + dir := h.workspace() + if err := os.RemoveAll(dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: clearing %s: %v", dir, err), http.StatusInternalServerError) + return + } + if err := Unpack(r.Body, dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: unpacking workspace: %v", err), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) getWorkspace(w http.ResponseWriter, _ *http.Request) { + var buf bytes.Buffer + if err := Pack(h.workspace(), &buf); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: packing workspace: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", contentTypeTar) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, &buf) +} + +func (h *Handler) run(w http.ResponseWriter, r *http.Request) { + var req RunRequest + if err := json.NewDecoder(io.LimitReader(r.Body, maxRequestJSON)).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: decoding run request: %v", err), http.StatusBadRequest) + return + } + if len(req.Command) == 0 { + http.Error(w, "substrate bridge: command is required", http.StatusBadRequest) + return + } + ctx := r.Context() + if h.RunTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, h.RunTimeout) + defer cancel() + } + resp := Run(ctx, req, h.workspace()) + w.Header().Set("Content-Type", contentTypeJSON) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) +} + +// Run executes one RunRequest with the request env replacing the process +// env — the launcher already filtered it through hive's credential-free +// allowlist — and reports the exit code the way the Podman launcher does. +func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { + cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) + cmd.Dir = req.WorkDir + if cmd.Dir == "" { + cmd.Dir = workspace + } + keys := make([]string, 0, len(req.Env)) + for k := range req.Env { + keys = append(keys, k) + } + sort.Strings(keys) + env := make([]string, 0, len(keys)+1) + for _, k := range keys { + env = append(env, k+"="+req.Env[k]) + } + if _, ok := req.Env["PATH"]; !ok { + env = append(env, "PATH="+os.Getenv("PATH")) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + resp := RunResponse{Stdout: capString(stdout.Bytes()), Stderr: capString(stderr.Bytes())} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + resp.ExitCode = exitErr.ExitCode() + default: + resp.ExitCode = ExitCodeUnknown + resp.Error = err.Error() + } + if ctx.Err() != nil && resp.Error == "" { + resp.Error = ctx.Err().Error() + } + return resp +} + +func capString(b []byte) string { + if len(b) > MaxOutput { + b = b[:MaxOutput] + } + return string(b) +} + +// Pack writes dir as a gzip-compressed tar stream: regular files, directories +// and symlinks, paths relative to dir. Git workspaces are small enough that a +// single stream is simpler than any chunked protocol. +func Pack(dir string, w io.Writer) error { + gz := gzip.NewWriter(w) + tw := tar.NewWriter(gz) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + link := "" + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(rel) + if info.IsDir() { + hdr.Name += "/" + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + return gz.Close() +} + +// Unpack reads a Pack stream into dir, refusing entries that would escape it. +func Unpack(r io.Reader, dir string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip: %w", err) + } + defer gz.Close() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("tar: %w", err) + } + target, err := safeJoin(dir, hdr.Name) + if err != nil { + return err + } + if err := unpackEntry(tr, hdr, target); err != nil { + return err + } + } +} + +func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { + switch hdr.Typeflag { + case tar.TypeDir: + return os.MkdirAll(target, os.FileMode(hdr.Mode)&os.ModePerm|0o700) + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + return os.Symlink(hdr.Linkname, target) + case tar.TypeReg: + if hdr.Size > maxTarEntry { + return fmt.Errorf("tar: entry %q is too large (%d bytes)", hdr.Name, hdr.Size) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) + if err != nil { + return err + } + if _, err := io.Copy(f, io.LimitReader(tr, hdr.Size)); err != nil { + f.Close() + return err + } + return f.Close() + default: + // Devices, FIFOs and hard links have no place in a git workspace. + return fmt.Errorf("tar: entry %q has unsupported type %q", hdr.Name, hdr.Typeflag) + } +} + +func safeJoin(dir, name string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(name)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("tar: entry %q escapes the workspace", name) + } + return filepath.Join(dir, clean), nil +} + +// Serve runs the bridge on ln until ctx ends. The actor's command is +// expected to be the sandbox image's bridge entrypoint calling this. +func Serve(ctx context.Context, ln net.Listener, h *Handler) error { + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + errCh := make(chan error, 1) + go func() { errCh <- srv.Serve(ln) }() + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + return ctx.Err() + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go new file mode 100644 index 0000000000..e91754700a --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -0,0 +1,394 @@ +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestPackUnpackRoundTrip(t *testing.T) { + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "alpha") + writeFile(t, filepath.Join(src, "sub", "deep", "b.txt"), "beta") + if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(src, "a.txt"), 0o755); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatalf("Pack: %v", err) + } + dst := filepath.Join(t.TempDir(), "out") + if err := Unpack(&buf, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "a.txt")); string(got) != "alpha" { + t.Errorf("a.txt = %q", got) + } + if got, _ := os.ReadFile(filepath.Join(dst, "sub", "deep", "b.txt")); string(got) != "beta" { + t.Errorf("b.txt = %q", got) + } + if st, err := os.Stat(filepath.Join(dst, "empty")); err != nil || !st.IsDir() { + t.Errorf("empty dir not restored: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "a.txt" { + t.Errorf("symlink = %q, %v", target, err) + } + if st, _ := os.Stat(filepath.Join(dst, "a.txt")); st.Mode()&0o100 == 0 { + t.Errorf("executable bit lost: %v", st.Mode()) + } +} + +func TestPackMissingDir(t *testing.T) { + if err := Pack(filepath.Join(t.TempDir(), "nope"), io.Discard); err == nil { + t.Fatal("Pack of a missing dir succeeded") + } +} + +func tarStream(t *testing.T, build func(tw *tar.Writer)) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + build(tw) + // A deliberately truncated entry (header without body) makes Close + // complain; the header is already on the wire, which is what the + // test wants. + _ = tw.Close() + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return &buf +} + +func TestUnpackRejectsBadStreams(t *testing.T) { + dst := t.TempDir() + if err := Unpack(strings.NewReader("not gzip"), dst); err == nil || !strings.Contains(err.Error(), "gzip") { + t.Errorf("non-gzip accepted: %v", err) + } + escape := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "../evil", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(escape, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("escaping entry accepted: %v", err) + } + abs := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "/etc/passwd", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(abs, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("absolute entry accepted: %v", err) + } + fifo := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o644}) + }) + if err := Unpack(fifo, dst); err == nil || !strings.Contains(err.Error(), "unsupported type") { + t.Errorf("fifo accepted: %v", err) + } + huge := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "big", Typeflag: tar.TypeReg, Size: maxTarEntry + 1, Mode: 0o644}) + }) + if err := Unpack(huge, dst); err == nil || !strings.Contains(err.Error(), "too large") { + t.Errorf("oversized entry accepted: %v", err) + } + // Truncated tar body: gzip is fine, tar is not. + var truncated bytes.Buffer + gz := gzip.NewWriter(&truncated) + _, _ = gz.Write([]byte("garbage that is not a tar header at all")) + _ = gz.Close() + if err := Unpack(&truncated, dst); err == nil || !strings.Contains(err.Error(), "tar") { + t.Errorf("garbage tar accepted: %v", err) + } + // Destination that cannot be created. + blocker := filepath.Join(t.TempDir(), "file") + writeFile(t, blocker, "x") + empty := tarStream(t, func(*tar.Writer) {}) + if err := Unpack(empty, filepath.Join(blocker, "sub")); err == nil { + t.Errorf("unpack under a regular file succeeded") + } +} + +func TestUnpackSymlinkReplacesExistingAndNestedEntries(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "link"), "a file where the link goes") + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "target", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/link2", Typeflag: tar.TypeSymlink, Linkname: "../x", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/f.txt", Typeflag: tar.TypeReg, Size: 2, Mode: 0o600}) + _, _ = tw.Write([]byte("hi")) + }) + if err := Unpack(stream, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "target" { + t.Errorf("link = %q, %v", target, err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "d", "e", "f.txt")); string(got) != "hi" { + t.Errorf("f.txt = %q", got) + } +} + +func TestUnpackEntryErrorsWhenParentIsAFile(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "blocker"), "x") + for _, hdr := range []*tar.Header{ + {Name: "blocker/sub/file", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}, + {Name: "blocker/sub/link", Typeflag: tar.TypeSymlink, Linkname: "x", Mode: 0o777}, + {Name: "blocker/sub/", Typeflag: tar.TypeDir, Mode: 0o755}, + } { + h := hdr + stream := tarStream(t, func(tw *tar.Writer) { _ = tw.WriteHeader(h) }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("entry %q under a regular file succeeded", h.Name) + } + } + // A regular file whose target is an existing directory cannot be opened. + if err := os.Mkdir(filepath.Join(dst, "dir"), 0o755); err != nil { + t.Fatal(err) + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "dir", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("regular file over a directory succeeded") + } +} + +func TestRunReportsExitCodesAndErrors(t *testing.T) { + ws := t.TempDir() + ok := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo out; echo err >&2; pwd"}}, ws) + if ok.ExitCode != 0 || ok.Error != "" || !strings.Contains(ok.Stdout, "out") || !strings.Contains(ok.Stderr, "err") { + t.Errorf("ok run = %+v", ok) + } + if evalWS, _ := filepath.EvalSymlinks(ws); !strings.Contains(ok.Stdout, filepath.Base(evalWS)) { + t.Errorf("command did not run in the workspace: %q", ok.Stdout) + } + + fail := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "exit 7"}}, ws) + if fail.ExitCode != 7 || fail.Error != "" { + t.Errorf("exit 7 run = %+v", fail) + } + + missing := Run(context.Background(), RunRequest{Command: []string{"/definitely/not/a/binary"}}, ws) + if missing.ExitCode != ExitCodeUnknown || missing.Error == "" { + t.Errorf("missing binary run = %+v", missing) + } + + env := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $FOO:$BAR; env | grep -c ^HOME= || true"}, Env: map[string]string{"FOO": "1", "BAR": "2"}}, ws) + if !strings.HasPrefix(env.Stdout, "1:2\n0") { + t.Errorf("env not replaced wholesale: %q", env.Stdout) + } + withPath := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $PATH"}, Env: map[string]string{"PATH": "/custom/bin"}}, ws) + if strings.TrimSpace(withPath.Stdout) != "/custom/bin" { + t.Errorf("explicit PATH not honoured: %q", withPath.Stdout) + } + + other := t.TempDir() + wd := Run(context.Background(), RunRequest{Command: []string{"pwd"}, WorkDir: other}, ws) + if evalOther, _ := filepath.EvalSymlinks(other); !strings.Contains(wd.Stdout, filepath.Base(evalOther)) { + t.Errorf("workdir not honoured: %q", wd.Stdout) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cancelled := Run(ctx, RunRequest{Command: []string{"sh", "-c", "exit 0"}}, ws) + if cancelled.Error == "" { + t.Errorf("cancelled run reported no error: %+v", cancelled) + } +} + +func TestCapString(t *testing.T) { + if got := capString(bytes.Repeat([]byte("x"), MaxOutput+10)); len(got) != MaxOutput { + t.Errorf("len = %d", len(got)) + } +} + +func newBridgeServer(t *testing.T) (*httptest.Server, string) { + t.Helper() + ws := filepath.Join(t.TempDir(), "ws") + srv := httptest.NewServer(&Handler{Workspace: ws}) + t.Cleanup(srv.Close) + return srv, ws +} + +func TestHandlerRoutes(t *testing.T) { + srv, ws := newBridgeServer(t) + + resp, err := http.Get(srv.URL + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz: %v %v", err, resp) + } + resp.Body.Close() + + src := t.TempDir() + writeFile(t, filepath.Join(src, "hello.txt"), "hi") + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, &buf) + resp, err = http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != http.StatusNoContent { + t.Fatalf("put workspace: %v %v", err, resp) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(ws, "hello.txt")); string(got) != "hi" { + t.Errorf("workspace not unpacked: %q", got) + } + + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo changed > hello.txt; exit 3"}}) + resp, err = http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if out.ExitCode != 3 { + t.Errorf("run response = %+v", out) + } + + resp, err = http.Get(srv.URL + PathWorkspace) + if err != nil || resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != contentTypeTar { + t.Fatalf("get workspace: %v %v", err, resp) + } + dst := t.TempDir() + if err := Unpack(resp.Body, dst); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(dst, "hello.txt")); strings.TrimSpace(string(got)) != "changed" { + t.Errorf("pulled workspace = %q", got) + } +} + +func TestHandlerErrors(t *testing.T) { + srv, ws := newBridgeServer(t) + status := func(method, path string, body io.Reader) int { + req, _ := http.NewRequest(method, srv.URL+path, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + return resp.StatusCode + } + if got := status(http.MethodGet, "/nope", nil); got != http.StatusNotFound { + t.Errorf("unknown route = %d", got) + } + if got := status(http.MethodDelete, PathWorkspace, nil); got != http.StatusNotFound { + t.Errorf("unknown method = %d", got) + } + if got := status(http.MethodPut, PathWorkspace, strings.NewReader("not a tarball")); got != http.StatusBadRequest { + t.Errorf("bad tarball = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader("{")); got != http.StatusBadRequest { + t.Errorf("bad json = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader(`{"command":[]}`)); got != http.StatusBadRequest { + t.Errorf("empty command = %d", got) + } + // Workspace missing: GET cannot pack it. + if got := status(http.MethodGet, PathWorkspace, nil); got != http.StatusInternalServerError { + t.Errorf("get of missing workspace = %d", got) + } + // Workspace parent is a file: PUT cannot recreate it. + _ = ws + blocked := httptest.NewServer(&Handler{Workspace: filepath.Join(ws, "..", "ws-file", "sub")}) + defer blocked.Close() + writeFile(t, filepath.Join(ws, "..", "ws-file"), "x") + empty := tarStream(t, func(*tar.Writer) {}) + req, _ := http.NewRequest(http.MethodPut, blocked.URL+PathWorkspace, empty) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("put under a file = %d, want 500 from clearing the workspace", resp.StatusCode) + } +} + +func TestHandlerDefaultsAndRunTimeout(t *testing.T) { + h := &Handler{} + if h.workspace() != DefaultWorkspace { + t.Errorf("default workspace = %q", h.workspace()) + } + // RunTimeout already expired: the command is killed and the response + // says why, without the test waiting on it. + srv := httptest.NewServer(&Handler{Workspace: t.TempDir(), RunTimeout: 1}) + defer srv.Close() + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "sleep 30"}}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + if out.Error == "" { + t.Errorf("timed-out run reported no error: %+v", out) + } +} + +func TestServeStopsOnContextAndListenerClose(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- Serve(ctx, ln, &Handler{Workspace: t.TempDir()}) }() + resp, err := http.Get("http://" + ln.Addr().String() + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over Serve: %v %v", err, resp) + } + resp.Body.Close() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Errorf("Serve returned %v, want context.Canceled", err) + } + + ln2, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done2 := make(chan error, 1) + go func() { done2 <- Serve(context.Background(), ln2, &Handler{}) }() + // Closing the listener makes Serve return the listener error (not + // ErrServerClosed, which only Shutdown produces). + _ = ln2.Close() + if err := <-done2; err == nil { + t.Errorf("Serve returned nil after the listener closed") + } +} diff --git a/src/pkg/config/agent_sandbox_substrate_test.go b/src/pkg/config/agent_sandbox_substrate_test.go new file mode 100644 index 0000000000..3490aa1f61 --- /dev/null +++ b/src/pkg/config/agent_sandbox_substrate_test.go @@ -0,0 +1,115 @@ +package config + +import ( + "strings" + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" + "gopkg.in/yaml.v3" +) + +// sandbox.runtime: substrate (ADR-0026 phase 2) — parse, merge semantics and +// the boot warnings that keep a misconfigured runtime from failing silently. + +func TestSandboxSubstrateParsesAndMerges(t *testing.T) { + src := ` +agent_sandbox: + enabled: true + image: img + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 + atespace: hive + token_env: ATE_TOKEN + image: ghcr.io/hivecommons/sandbox@sha256:0000000000000000000000000000000000000000000000000000000000000000 + allow_hosts: [litellm.example.com] + timeout_s: 600 + snapshot_on_idle: true +agents: + deps: + sandbox: + enabled: true + substrate: + allow_hosts: [hub.example.org] + worker_selector: { class: gvisor } +` + var cfg Config + if err := yaml.Unmarshal([]byte(src), &cfg); err != nil { + t.Fatal(err) + } + a := cfg.Agents["deps"] + if got := a.SandboxRuntime(cfg.AgentSandbox); got != SandboxRuntimeSubstrate { + t.Fatalf("runtime = %q", got) + } + eff := a.SandboxSubstrate(cfg.AgentSandbox) + if eff.Endpoint != "api.ate-system.svc:443" || eff.Atespace != "hive" || eff.TokenEnv != "ATE_TOKEN" || eff.TimeoutS != 600 || !eff.SnapshotOnIdle || !strings.Contains(eff.Image, "@sha256:") { + t.Errorf("cluster facts must fall back to the global block: %+v", eff) + } + if len(eff.AllowHosts) != 1 || eff.AllowHosts[0] != "hub.example.org" { + t.Errorf("per-agent allow_hosts must replace the global list: %v", eff.AllowHosts) + } + if eff.WorkerSelector["class"] != "gvisor" { + t.Errorf("worker_selector = %v", eff.WorkerSelector) + } + other := AgentConfig{} + if got := other.SandboxSubstrate(cfg.AgentSandbox); len(got.AllowHosts) != 1 || got.AllowHosts[0] != "litellm.example.com" { + t.Errorf("agent without a block gets the global one: %+v", got) + } +} + +func TestSandboxSubstrateGateWarnings(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]AgentConfig{ + "deps": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + cruxconfig.SetSubstratePreflight(func([]SandboxSubstrateConfig) []string { return nil }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "digest-pinned") { + t.Errorf("warnings = %q, want exactly one about the digest-pinned image", got) + } + cfg.AgentSandbox.Substrate = &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)} + if got := AgentSandboxGateWarnings(cfg); len(got) != 0 { + t.Errorf("fully configured substrate runtime still warns: %q", got) + } +} + +func TestAgentSandboxGateWarningsIncludeSubstratePreflight(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{ + Enabled: true, + Image: "img", + Substrate: &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)}, + }, + Agents: map[string]AgentConfig{ + "a": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + "b": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + var calls, gotBlocks int + cruxconfig.SetSubstratePreflight(func(b []SandboxSubstrateConfig) []string { + calls++ + gotBlocks = len(b) + return []string{"substrate control API api.ate-system.svc:443 is unreachable"} + }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") || calls != 1 || gotBlocks != 2 { + t.Errorf("warnings = %q calls=%d blocks=%d, want the preflight warning from one call over both agents", got, calls, gotBlocks) + } + + cfg.Agents = map[string]AgentConfig{"p": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimePodman}}} + calls = 0 + AgentSandboxGateWarnings(cfg) + if calls != 0 { + t.Errorf("preflight ran with no substrate agent") + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 32da9850e6..92917c4108 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1592,8 +1592,9 @@ func AgentSandboxGateWarnings(cfg *Config) []string { return nil } - var optedIn, noImage, noClaim, noGateway, badRuntime []string + var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string var openshellBlocks []SandboxOpenShellConfig + var substrateBlocks []SandboxSubstrateConfig for name, a := range cfg.Agents { if !a.SandboxEnabled(cfg.AgentSandbox) { continue @@ -1618,12 +1619,21 @@ func AgentSandboxGateWarnings(cfg *Config) []string { if strings.TrimSpace(os.GatewayURL) != "" { openshellBlocks = append(openshellBlocks, os) } + case rt == SandboxRuntimeSubstrate: + sb := a.SandboxSubstrate(cfg.AgentSandbox) + // Substrate refuses tag-only images; the agent image is usually a + // tag, so the substrate block must carry the digest-pinned one. + if !strings.Contains(sb.Image, "@sha256:") { + noDigest = append(noDigest, name) + } + substrateBlocks = append(substrateBlocks, sb) } } sort.Strings(optedIn) sort.Strings(noImage) sort.Strings(noClaim) sort.Strings(noGateway) + sort.Strings(noDigest) sort.Strings(badRuntime) var out []string @@ -1658,6 +1668,12 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "set agent_sandbox.openshell.gateway_url (or the per-agent sandbox.openshell.gateway_url) to the OpenShell gateway's gRPC address (ADR-0026)", strings.Join(noGateway, ", "))) } + if len(noDigest) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s use the substrate sandbox runtime but resolve no digest-pinned substrate.image; Substrate refuses tag-only images and every kick will fail — "+ + "set agent_sandbox.substrate.image (or the per-agent sandbox.substrate.image) to the sandbox image's …@sha256:… reference (ADR-0026 phase 2)", + strings.Join(noDigest, ", "))) + } if len(optedIn) < len(cfg.Agents) { out = append(out, fmt.Sprintf( "agent_sandbox.enabled is true but only %d of %d agent(s) are opted in (%s); the rest still run unconfined on the tmux path (#4918)", @@ -1666,6 +1682,7 @@ func AgentSandboxGateWarnings(cfg *Config) []string { // Last, because it may reach the network (bounded and cached): the static // warnings above must not wait on a gateway. out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) + out = append(out, cruxconfig.SubstratePreflightWarnings(substrateBlocks)...) return out } diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 077db8fb3a..6e8998cc56 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -38,6 +38,7 @@ type ( SandboxJobToleration = cruxconfig.SandboxJobToleration SandboxJobVolume = cruxconfig.SandboxJobVolume SandboxOpenShellConfig = cruxconfig.SandboxOpenShellConfig + SandboxSubstrateConfig = cruxconfig.SandboxSubstrateConfig StatsDisplayEntry = cruxconfig.StatsDisplayEntry ToolApprovalConfig = cruxconfig.ToolApprovalConfig ToolApprovalRule = cruxconfig.ToolApprovalRule @@ -143,6 +144,7 @@ const ( RotationHeadroomSourceCCLeft = cruxconfig.RotationHeadroomSourceCCLeft SandboxRuntimeJob = cruxconfig.SandboxRuntimeJob SandboxRuntimeOpenShell = cruxconfig.SandboxRuntimeOpenShell + SandboxRuntimeSubstrate = cruxconfig.SandboxRuntimeSubstrate SandboxRuntimePodman = cruxconfig.SandboxRuntimePodman SelfMergeMinACMMLevel = cruxconfig.SelfMergeMinACMMLevel WritableBobAPIKeyFile = cruxconfig.WritableBobAPIKeyFile diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 718fbbf15f..eeee612fb8 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -29,6 +29,25 @@ agent_sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell runtime; agent_sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) +agent_sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) +agent_sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 @@ -147,6 +166,25 @@ agents..sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell r agents..sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) +agents..sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) +agents..sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) From 42c463c57bfdc499a5b712825804617cee210a07 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 14:45:36 -0700 Subject: [PATCH 006/166] =?UTF-8?q?=F0=9F=93=96=20docs:=20ADR-0026=20phase?= =?UTF-8?q?=202=20=E2=80=94=20Agent=20Substrate=20runtime=20operator=20and?= =?UTF-8?q?=20deploy=20notes=20(#17745)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: ADR-0026 phase 2 — Agent Substrate runtime operator and deploy notes Amend ADR-0026 with the Substrate implementation notes (contract mapping, deviations from the reference model, cluster prerequisites and which hive clusters qualify, when to choose it over OpenShell); add the operator section to sandbox-isolation.md, the integrations table row, deploy/README pointer and a new deploy/substrate.md with install and verification steps. Part of #17695 Signed-off-by: Andrew Anderson * changelog: fragment for #17745 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🌱 changelog: make the substrate docs fragment an entry bullet Signed-off-by: Andrew Anderson * 🌱 changelog: drop the auto-generated duplicate fragment Signed-off-by: Andrew Anderson * 📖 docs: fix the ADR-0002 link in the Substrate section Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-substrate-runtime-docs.md | 1 + src/deploy/README.md | 1 + src/deploy/substrate.md | 118 ++++++++++++++++++ .../adr/0026-sandbox-runtime-selection.md | 76 ++++++++++- src/docs/integrations.md | 1 + src/docs/sandbox-isolation.md | 45 ++++++- 6 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 changelog.d/changed-substrate-runtime-docs.md create mode 100644 src/deploy/substrate.md diff --git a/changelog.d/changed-substrate-runtime-docs.md b/changelog.d/changed-substrate-runtime-docs.md new file mode 100644 index 0000000000..fadfda564d --- /dev/null +++ b/changelog.d/changed-substrate-runtime-docs.md @@ -0,0 +1 @@ +- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. diff --git a/src/deploy/README.md b/src/deploy/README.md index 6b3fb54d45..db3234b79d 100644 --- a/src/deploy/README.md +++ b/src/deploy/README.md @@ -20,6 +20,7 @@ Use it indirectly through the dashboard terminal link. If a terminal pane says i - `k8s/` — the Kustomize base (`kustomization.yaml`) with namespace, deployment, service, PVC, Secret, ConfigMap and RBAC manifests, plus the custom error backend (`error-pages.yaml`), the hub backup CronJob (`backup-cronjob.yaml`) and the node-prep DaemonSet (`node-prep/`), which are applied separately. The OpenShift Route lives in `kustomize/overlays/openshift/`. - `inference/` — sample in-cluster OpenAI-compatible inference deployment and RBAC. - `openshell.md` — cluster-side notes for the opt-in `sandbox.runtime: openshell` (ADR-0026): prerequisites, the upstream Helm install, the gateway settings and PVC labels hive's runtime depends on, and how to verify. No manifests are vendored. +- `substrate.md` — cluster-side notes for the opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2): the Agent Substrate operator install, WorkerPool and SandboxConfig, the pod-certificate feature gates, the router client bundle hive needs, and how to verify. No manifests are vendored. - `docker-compose.architect.yaml`, `hive-quickstart.yaml`, `hive-level*.yaml`, `architect-only.yaml`, `hive.yaml` — example deployment/configuration manifests. - `blue-green-deploy.sh`, `bootstrap-lxc.sh`, `create-lxc.sh` — operational scripts for **Docker Compose** deployments (LXC bootstrap and a blue-green Compose upgrade). Not "non-Kubernetes" generally: none of the three shells out to `podman` or reads `HIVE_DEPLOY_RUNTIME`, so none of them applies to the Quadlet stack in `quadlet/` below. See [Deployment helper scripts](../docs/deployment-scripts.md) for each script's scope and where a Podman operator should go instead. - `quadlet/` — Podman Quadlet units for the standalone deployment (ADR-0017). `hive.container` and `hive-data.volume` start the Hive service under `systemd`, rootful or rootless; `hive.network` and `hive-gateway.container` put the authenticating gateway in front of it with the same published-port split as the Compose stack — 3001 published, 7681 never. See [the operator guide](../docs/podman-standalone-quadlet.md). Docker Compose remains the default runtime and is unaffected. diff --git a/src/deploy/substrate.md b/src/deploy/substrate.md new file mode 100644 index 0000000000..941d32d2c3 --- /dev/null +++ b/src/deploy/substrate.md @@ -0,0 +1,118 @@ +# Installing Agent Substrate for `sandbox.runtime: substrate` + +Cluster-side notes for the opt-in Substrate sandbox runtime +([ADR-0026 phase 2](../docs/adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), +epic [#17695](https://github.com/hivecommons/hive/issues/17695); operator view +in [sandbox-isolation.md](../docs/sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor)). +Hive installs nothing: Substrate is a cluster-admin install, once per cluster, +and hive only talks to its Control API and router. Upstream is the source of +truth for the install — [agent-substrate/substrate → docs/guides/operator-install.md](https://github.com/agent-substrate/substrate/blob/main/docs/guides/operator-install.md), +the [API guide](https://github.com/agent-substrate/substrate/blob/main/docs/reference/api-guide.md) +and [sandbox classes](https://github.com/agent-substrate/substrate/blob/main/docs/reference/sandbox-classes.md) — +and no manifests are vendored here; pin the release you install and record it +alongside the hive config. + +## Prerequisites to check before installing + +| Requirement | Why | How to check | +|---|---|---| +| Kubernetes with `ClusterTrustBundle`, `ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates and `certificates.k8s.io/v1beta1` enabled | actor and client identities are pod certificates (SPIFFE); the podcert controller does not start without them. Still off by default as of Kubernetes 1.36 | `kubectl get --raw /metrics \| grep 'kubernetes_feature_enabled{name="PodCertificateRequest"'` and `kubectl api-resources --api-group=certificates.k8s.io`. Managed control planes (OKE, LKE) need the provider to enable them; OpenShift needs a `FeatureGate` CR | +| Standard Linux nodes with **cgroup v2** for the `gvisor` class | `ateworker-gvisor` runs `runsc` inside the worker pod; there is no kernel floor beyond ≥ 5.8 | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`; `stat -fc %T /sys/fs/cgroup` on a node → `cgroup2fs` | +| `/dev/kvm` on nodes for the `microvm` class | Kata on Cloud Hypervisor guests; `atelet` advertises the `ate.dev/kvm` device only where it exists | `kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.allocatable.ate\.dev/kvm}{"\n"}{end}'` after install | +| Nodes trusted to run worker pods with `SYS_ADMIN` and `NET_ADMIN` | workers are `privileged: false` but keep those two for mounts and networking | on OpenShift, an SCC granting them to the `ate-system` worker ServiceAccount | +| A CNI that **enforces** egress `NetworkPolicy` | Substrate's `atenet` dataplane carries actor egress; the worker pod itself must not be a side door | create a deny-all policy in a scratch namespace and confirm a connection is refused | +| PostgreSQL and S3-compatible object storage | Control API state; actor snapshots (`snapshot_on_idle`) | `ate-setup` deploys bundled ones for a lab; use managed services in production | +| Cluster-admin for the install | CRDs (`workerpools`, `sandboxconfigs`, `egresspolicies`, …), ValidatingAdmissionPolicies, RBAC, PKI | — | + +### Which hive clusters qualify (survey 2026-11-11) + +| cluster | kernel / Kubernetes | `gvisor` | `microvm` | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes, once the podcert feature gates are enabled on the OKE control plane | needs KVM-capable shapes (bare metal or nested virtualisation) | +| lke648397, hive-ci | Debian 13, 6.12 | yes; confirm the feature gates — LKE exposes no control-plane flags | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: SCC for the worker capabilities + `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +## Install + +Build or download `ate-setup` from a pinned release of +`github.com/agent-substrate/substrate`, write the install document, and deploy: + +```yaml +# substrate-install.yaml +apiVersion: install.ate.dev/v1alpha1 +kind: SubstrateInstall +namespace: ate-system +atenet: + dataplane: envoy +images: + repo: /substrate + tag: +``` + +```shell +ate-setup deploy ate-system --config substrate-install.yaml \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns//sa/ +``` + +`--ingress-allowed-spiffe-ids` must name hive's own pod identity: that is what +lets hive open `CONNECT` tunnels through `atenet-router` to the bridge in each +actor. Unknown keys in the document are rejected, not ignored. + +Then create the pieces hive's runtime depends on, in Substrate terms: + +- **An atespace** for hive (`sandbox.substrate.atespace`, default `default`), + and a principal for hive's token (`token_env` / `token_file`) that may + create and delete `ActorTemplate`, `Actor` and `EgressPolicy` and list + workers in it. `kubectl ate` shows the exact RBAC verbs for the release. +- **A `WorkerPool`** with `spec.sandboxClasses[].name: gvisor` (or `microvm`) + resolving the default `SandboxConfig` (`gvisor-default`), labelled so that + `sandbox.substrate.worker_selector` matches it; hive only uses pools whose + workers report `ACTIVE` and advertise a sandbox class. Taint the node pool + `ate.dev/sandboxClass=:NoSchedule` to reserve it; worker pods + tolerate it. +- **Projected files in the hive pod**: a ServiceAccount token with audience + `api.ate-system.svc` (→ `token_file`), the Control API CA via + `clusterTrustBundle` with signer `servicedns.podcert.ate.dev/identity` + (→ `ca_file`), and a `podCertificate` with signer + `podidentity.podcert.ate.dev/identity` (→ `router_client_bundle`) plus the + router CA (→ `router_ca_file`). The authentication guide upstream has the + volume stanza. +- **A digest-pinned sandbox image** containing `crux` (for the + `substrate-bridge` entrypoint) and the agent CLI, set as + `sandbox.substrate.image`; tags are refused at boot and at kick. Publishing + it is tracked in [#17697](https://github.com/hivecommons/hive/issues/17697). +- **Egress**: hive writes its own `EgressPolicy` per actor (`github.com`, + `api.github.com`, `allow_hosts`). Do **not** also configure Substrate's + credential injection for GitHub — hive's MITM proxy brokers credentials and + the two would conflict. + +## Verify + +```shell +kubectl -n ate-system get deploy,ds # ateapi, atenet-router, podcert controller, atelet ready +kubectl get crd | grep ate.dev # workerpools, sandboxconfigs, egresspolicies, … +kubectl get workerpool -A # at least one pool with ACTIVE workers +kubectl get sandboxconfig gvisor-default -o yaml | grep defaultVersion +kubectl get nodes -o custom-columns=NAME:.metadata.name,KVM:.status.allocatable.ate\.dev/kvm +kubectl ate create actor smoke -a --template && kubectl ate delete actor smoke -a +``` + +From the hive side, start with `runtime: substrate` and watch the boot log: the +preflight warns when the Control API is unreachable, has no matching `ACTIVE` +worker, or its workers report no sandbox class. Then run one kick and confirm +in the router log that the actor's SPIFFE id made exactly the allowed +connections and that a connect to any other host was refused — that denial is +the point of the runtime. Latency and image numbers go back into this page +from the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); +the fuller prover (CRDs, operator, feature gates, KVM) is +[#17698](https://github.com/hivecommons/hive/issues/17698). + +## Upgrades + +Substrate is pre-1.0; follow its version-skew policy (N ↔ N+1 only) and pin +the release and the sandbox image by digest. A `SandboxConfig` upgrade adds a +`runsc` version and keeps the previous one so suspended actors still restore. +Hive's generated client (`crux/sandbox/substrateapi`) tracks the fields it +uses; bump it with `generate.sh` against the matching upstream tag when the +Control API changes. diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 3a6e062fde..153111ef6f 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Accepted (phase 1 implemented, runtime opt-in) — #17642, hivecommons/hive#17646 +Status: Accepted (phase 1 OpenShell implemented, runtime opt-in — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) ## Context @@ -218,6 +218,74 @@ runs on, read on 2026-11-10: LKE (`lke648397`, `hive-ci`) run 6.12 on Debian 13 with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters (5.14) do not. +## Implementation notes (phase 2: Agent Substrate) + +Phase 2 (#17695, hivecommons/hive#17741) adds a **fourth** runtime, +`sandbox.runtime: substrate`, behind the same `sandbox.Launcher` seam. It is +[Agent Substrate](https://github.com/agent-substrate/substrate) (CNCF Sandbox, +donated by Google, Apache-2.0): a Kubernetes-hosted *actor* runtime where an +`ActorTemplate` (image, command, env, limits, volumes) is instantiated as an +`Actor` through a gRPC Control API (`ate.control.v1`), scheduled on a +`WorkerPool` whose pods run the workload under gVisor (`runsc`, class +`gvisor`) or in a KVM microVM (Kata on Cloud Hypervisor, class `microvm`). All +egress leaves through `atenet-router` under an `EgressPolicy`, and every actor +holds a SPIFFE identity issued through Kubernetes pod certificates. It was not +in the 2026-10-25 comparison; the write-up that led here is the Substrate +vs. crux research note, and the operator view is in +[sandbox-isolation.md](../sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor) +and [deploy/substrate.md](../../deploy/substrate.md). Like OpenShell it is +opt-in and inert unless selected. + +### How it maps onto the contract + +| Contract item | Substrate | What phase 2 does | +|---|---|---| +| Kernel-enforced confinement | gVisor user-space kernel or a microVM; worker pods are `privileged: false` but keep `SYS_ADMIN`/`NET_ADMIN` | selects the class through `worker_selector`; no change to hive | +| Default-deny egress | `atenet-egress` `EgressPolicy` per actor; rules are TLS-passthrough or terminate per host:port, no per-method read-only mode | `SubstrateEgressPolicy`: `github.com` and `api.github.com` on 443 plus `sandbox.substrate.allow_hosts`, TLS passthrough; everything else (and all UDP) denied | +| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | unchanged hive brokering — the agent still gets `HTTPS_PROXY` to the hive MITM proxy (ADR-0002); Substrate's injection is **not** used, the two are complementary | +| Workspace in, commits out | `durableDir` volumes and object-storage snapshots; no exec/attach API | an in-actor HTTP **bridge** (`crux substrate-bridge`) reached through the router's mTLS `CONNECT` tunnel: push tar → `POST /run` → pull tar; the hive-side workspace stays authoritative | +| Cleanup and timeouts | `DeleteActor` / `SuspendActor` / `DeleteActorTemplate` | one template + actor per kick, both deleted on success, failure and timeout (fresh bounded context, `ExitCode -1`, 16 MiB cap, as `job`/`openshell`) | + +### Deviations from Substrate's reference model + +| Substrate assumes | Phase 2 does | +|---|---| +| long-lived actors and reusable templates | per-kick `ActorTemplate` + `Actor`, named `hive-kick--` | +| Go client `go.ate.dev/substrate` | **not imported**: it needs Go 1.27 (hive is on 1.26) and a very large module graph. A trimmed, wire-compatible proto subset is generated in-repo (`crux/sandbox/substrateapi`, field numbers pinned by tests); zero new Go modules, +578 KB on the `crux` binary | +| image by tag or digest | digest-pinned `sandbox.substrate.image` only, enforced at boot (`AgentSandboxGateWarnings`) and at kick | +| no `policy_profile` concept | passed to the actor as `HIVE_POLICY_PROFILE`; Substrate has no annotations | +| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; resume-on-next-kick is not implemented (#17697) | +| `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | + +### Cluster prerequisites and which hive clusters qualify + +Substrate's real requirement is **not** a kernel floor: the gVisor class runs +`runsc` inside the worker pod on any standard Linux node with cgroup v2; the +microVM class needs `/dev/kvm` (nested virtualisation on cloud shapes). Both +need the operator (`ate-setup deploy ate-system`: Control API, `atelet` +DaemonSet, `atenet-router`, podcert controller, PostgreSQL, snapshot object +storage) and Kubernetes with the `ClusterTrustBundle`, +`ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates — still +off by default as of 1.36 — because actor identities are pod certificates. +Survey on 2026-11-11: + +| cluster | kernel / Kubernetes | `gvisor` class | `microvm` class | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes once the feature gates are on (OKE control plane: needs a support request or a self-managed control plane) | needs KVM-capable (bare-metal or nested-virt) shapes | +| lke648397, hive-ci | Debian 13, 6.12 | yes; LKE exposes no control-plane flags, so the podcert gates are the blocker — check with `kubectl get --raw /metrics \| grep kubernetes_feature_enabled` | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: the kernel is fine, but the worker's `SYS_ADMIN`/`NET_ADMIN` needs an SCC and the gates a `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +### When to choose it over OpenShell + +- Choose **OpenShell** when the cluster already runs a ≥ 6.2 kernel with an + enforcing CNI and you want per-binary, per-method egress policy and the + shared-PVC workspace with no image changes. +- Choose **Substrate** when you want a stronger boundary than Landlock + + seccomp (gVisor or a VM), snapshot/suspend of idle agents, or SPIFFE + identities for the sandboxed workload, and you can run its operator, + PostgreSQL and object storage. Its egress policy is coarser (host:port, no + read-only API mode) and the workspace is copied, not shared. + ## Consequences - Hive gains a sandbox boundary that is checked before the agent runs, without @@ -233,8 +301,12 @@ with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters hive's policy. - Telemetry must be disabled explicitly. A preflight check should refuse a gateway that reports it enabled. +- Substrate (phase 2) adds a second operator-side stack (Control API, + atelet, atenet-router, PostgreSQL, object storage) and a copied workspace; + the hive-side credential model is untouched. Its preflight (#17698) and + prototype (#17697) follow the OpenShell ones. - If the prototype fails a gate, this record is rewritten to decide "no change" with the measured reason, and the Podman and Job runtimes remain the only ones. -Refs #2804, #6311, #13806, #14092. +Refs #2804, #6311, #13806, #14092, #17642, #17695. diff --git a/src/docs/integrations.md b/src/docs/integrations.md index f4479b2e3c..fed5bf8cad 100644 --- a/src/docs/integrations.md +++ b/src/docs/integrations.md @@ -94,6 +94,7 @@ per-backend confinement matrix. Podman logoRootless Podman agent sandboxOpt-insandbox-isolation.md Kubernetes logoKubernetes Job sandbox runtimeOpt-insandbox-isolation.md NVIDIA logoNVIDIA OpenShell sandbox runtimeIn progress (opt-in, phase 1)sandbox-isolation.md · #17642 +CNCF logoAgent Substrate sandbox runtime (CNCF Sandbox)In progress (opt-in, phase 2)sandbox-isolation.md · #17695 OpenSandboxAAIF OpenSandbox (evaluation)Planned#14092 diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 55edeb2784..507b404980 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -286,6 +286,49 @@ The gateway, workspace, token, TLS and claim fields are cluster facts and fall b **Limitations (phase 1).** Kubernetes driver only (no Podman-driver path for standalone hosts yet); no Secret references into the sandbox; `policy_profile` is advisory; kernel and claim-label prerequisites are not preflighted; latency and image size are not yet measured ([#17643](https://github.com/hivecommons/hive/issues/17643)). +## Running sandboxed kicks in a Substrate actor + +`sandbox.runtime: substrate` runs an agent's sandboxed kicks as an [Agent Substrate](https://github.com/agent-substrate/substrate) **actor** ([ADR-0026 phase 2](adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), epic [#17695](https://github.com/hivecommons/hive/issues/17695)). Substrate is a CNCF Sandbox project donated by Google: a `WorkerPool` of pods runs workloads under gVisor (`runsc`) or in a KVM microVM, every actor gets a SPIFFE identity, and all egress leaves through `atenet-router` under an `EgressPolicy`. It is the fourth runtime and, like OpenShell, opt-in; nothing changes for a hive that does not select it. + +Everything above about the kick path still holds; only step 2 changes. The launcher is `sandbox.SubstrateLauncher`, which talks to Substrate's gRPC **Control API** (`ate.control.v1`) with a trimmed, wire-compatible client generated in-repo (`crux/sandbox/substrateapi`; the upstream Go module needs a newer Go than hive builds with). + +```yaml +agent_sandbox: + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 # Control API, host:port or https:// + atespace: hive # Substrate's tenancy unit, default "default" + token_env: ATE_TOKEN # bearer token env var (or token_file for a projected SA token) + ca_file: /var/run/ate/ca.crt # Control API CA; insecure_tls: true only for a lab + router_url: https://atenet-router.ate-system.svc:8444 + router_client_bundle: /var/run/ate/credential-bundle.pem # hive's own mTLS client identity + router_ca_file: /var/run/ate/router-ca.crt + image: ghcr.io/hivecommons/crux-sandbox@sha256:… # digest-pinned; tags are refused + cpu_millicores: 2000 + memory_bytes: 4294967296 + worker_selector: {ate.dev/sandboxClass: gvisor} # pick the pool (gvisor | microvm) + snapshot_on_idle: false + allow_hosts: [models.example.com, hub.example.com:8443] + timeout_s: 1800 +agents: + reviewer: + sandbox: + substrate: + snapshot_on_idle: true # per-agent overrides merge onto the global block +``` + +**One actor per kick.** Hive creates an `ActorTemplate` (one container running the bridge command, the workspace as a `durableDir`, the operator's limits and `worker_selector`) and an `Actor` from it, both named `hive-kick--`, attaches the egress policy, waits for the actor to report `RUNNING`, and then talks to a small HTTP **bridge** inside the actor (`crux substrate-bridge`, overridable with `bridge_command`, port `bridge_port`) through the router's mTLS `CONNECT` tunnel: it pushes the workspace as a tar, runs the kick with the sanitized environment, and pulls the workspace back. The hive-side workspace remains the source of truth and feeds the push broker exactly as for Podman. Substrate has no exec API and hive shares no PVC with the worker nodes, which is why the workspace is copied rather than mounted; the copy is bounded by the kick timeout. + +**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. + +**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. + +**What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). + +**The terminal pane.** A substrate-runtime agent has no tmux session and, until [#17699](https://github.com/hivecommons/hive/issues/17699), no live pane: the ▶ terminal shows the last persisted transcript like the job viewer between kicks. + +**Limitations (phase 2).** Workspace copied, not shared; `crux substrate-bridge` entrypoint and a published sandbox image land with the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); no resume of suspended actors; `api.github.com` not read-only at the policy layer; latency and image size not yet measured. + ## Health and metrics for sandboxed agents A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. @@ -315,7 +358,7 @@ On-demand agents, paused agents and `pause`/`continuous` cadences are never over - The Job runtime is a first slice: one Job per kick, workspace shared through a PVC, no per-kick Secret minting. A clone-inside-the-Job transport for clusters without ReadWriteMany storage is future work. - Live Podman execution is covered by skip-when-absent tests; CI still needs a rootless-Podman runner lane for always-on integration coverage. - Sandboxed inference depends on an operator-provided restricted network/proxy policy. `none` is stronger but not yet usable for all model backends. -- A stronger runtime than plain containers is under evaluation: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Proposed) compares NVIDIA OpenShell and OpenSandbox against this contract. +- Stronger runtimes than plain containers ship opt-in: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Accepted) covers NVIDIA OpenShell (phase 1) and Agent Substrate (phase 2); both still need their prototype measurements and cluster preflights. ## Agent guardrails (defense in depth) From 30ac300cb5997d4cfea59a445af296378aa93b41 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 21:59:41 +0000 Subject: [PATCH 007/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.221.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.221.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md | 1 - changelog.d/fixed-discovery-binary-stderr.md | 1 - changelog.d/fixed-gitconfig-identity-warning-once.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md delete mode 100644 changelog.d/fixed-discovery-binary-stderr.md delete mode 100644 changelog.d/fixed-gitconfig-identity-warning-once.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f1a05a1eaa..c95a8aeea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.221.1) + +### Fixed + +- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). +- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). +- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). + ## 2026-11-11 (v5.221.0) ### Added diff --git a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md b/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md deleted file mode 100644 index 80a3ef1606..0000000000 --- a/changelog.d/fixed-17729-copilot-enoexec-ldso-shim.md +++ /dev/null @@ -1 +0,0 @@ -- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). diff --git a/changelog.d/fixed-discovery-binary-stderr.md b/changelog.d/fixed-discovery-binary-stderr.md deleted file mode 100644 index 18be9f2d91..0000000000 --- a/changelog.d/fixed-discovery-binary-stderr.md +++ /dev/null @@ -1 +0,0 @@ -- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). diff --git a/changelog.d/fixed-gitconfig-identity-warning-once.md b/changelog.d/fixed-gitconfig-identity-warning-once.md deleted file mode 100644 index 55515cc892..0000000000 --- a/changelog.d/fixed-gitconfig-identity-warning-once.md +++ /dev/null @@ -1 +0,0 @@ -- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). From 9bf4885cedbf57ff7c58e42e584d448da1206c0f Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:18:35 +0000 Subject: [PATCH 008/166] fix: stop read-only lanes retrying hive:unauthorized label removal (#17743) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...nauthorized-label-removal-readonly-lane.md | 1 + .../automerge/automerge_unauthorized.go | 6 ++ .../automerge_unauthorized_readonly_test.go | 58 +++++++++++++++++++ src/pkg/github/integration_read_only.go | 7 +++ 4 files changed, 72 insertions(+) create mode 100644 changelog.d/fixed-unauthorized-label-removal-readonly-lane.md create mode 100644 src/pkg/github/automerge/automerge_unauthorized_readonly_test.go diff --git a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md new file mode 100644 index 0000000000..9939b3a538 --- /dev/null +++ b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md @@ -0,0 +1 @@ +- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). diff --git a/src/pkg/github/automerge/automerge_unauthorized.go b/src/pkg/github/automerge/automerge_unauthorized.go index a872bf1ff8..d72e68fc7d 100644 --- a/src/pkg/github/automerge/automerge_unauthorized.go +++ b/src/pkg/github/automerge/automerge_unauthorized.go @@ -165,6 +165,12 @@ func (c *Engine) clearUnauthorized(ctx context.Context, displayRepo, owner, repo if !had && !labelled { return } + if labelled && hgithub.IntegrationReadOnly(ctx) { + // The read-only lane probe refuses the DELETE; a write-capable lane + // clears the label, so retrying here would only warn every cycle. + c.debug("lane unauthorized: label removal left to a write-capable lane", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "step", step) + return + } if labelled { if _, err := c.gh.Issues.RemoveLabelForIssue(hgithub.WithRESTCaller(ctx, unauthorizedRESTCaller), owner, repo, number, url.PathEscape(unauthorizedLabel)); err != nil && !isGitHubStatus(err, http.StatusNotFound) { c.warn("lane unauthorized: could not remove label", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "error", err) diff --git a/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go new file mode 100644 index 0000000000..765c435926 --- /dev/null +++ b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go @@ -0,0 +1,58 @@ +package automerge + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func TestClearUnauthorizedReadOnlyLaneSkipsLabelRemoval(t *testing.T) { + for _, tc := range []struct { + name string + readOnly bool + }{ + {"read-only lane", true}, + {"write-capable lane", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var deletes atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete && strings.HasSuffix(r.URL.Path, "/issues/7/labels/"+unauthorizedLabel) { + deletes.Add(1) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + c := newContributorLaneEngine(srv.URL, ContributorLanePolicy{}, nil, Options{}) + var logs bytes.Buffer + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo})) + pr := &gh.PullRequest{Labels: []*gh.Label{{Name: gh.Ptr(unauthorizedLabel)}}} + ctx := context.Background() + if tc.readOnly { + ctx = hgithub.WithIntegrationReadOnly(ctx) + } + for range 3 { + c.clearUnauthorized(ctx, "acme/widget", "acme", "widget", pr, 7, "approve-workflow-runs") + } + if tc.readOnly { + if got := deletes.Load(); got != 0 { + t.Fatalf("deletes = %d, want 0", got) + } + } else if deletes.Load() < 1 { + t.Fatal("write-capable lane did not remove the label") + } + if strings.Contains(logs.String(), "level=WARN") { + t.Fatalf("unexpected warning: %s", logs.String()) + } + }) + } +} diff --git a/src/pkg/github/integration_read_only.go b/src/pkg/github/integration_read_only.go index dace3f27ae..ab0fa0f93c 100644 --- a/src/pkg/github/integration_read_only.go +++ b/src/pkg/github/integration_read_only.go @@ -24,6 +24,13 @@ func WithIntegrationReadOnly(ctx context.Context) context.Context { return context.WithValue(ctx, integrationReadOnlyKey{}, true) } +// IntegrationReadOnly reports whether ctx carries WithIntegrationReadOnly, so +// callers can skip writes the transport would refuse anyway. +func IntegrationReadOnly(ctx context.Context) bool { + guarded, _ := ctx.Value(integrationReadOnlyKey{}).(bool) + return guarded +} + func integrationReadOnlyRequest(request *http.Request) error { if guarded, _ := request.Context().Value(integrationReadOnlyKey{}).(bool); !guarded { return nil From c7deb66b8730dad23f85be858ab43b9af2673e4e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:19:01 +0000 Subject: [PATCH 009/166] fix: reconcile and expire effect-ledger ops stuck in Unknown (#17746) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ixed-effect-ledger-unknown-op-reconcile.md | 1 + src/cmd/hive/selfreport_mutation_hooks.go | 3 +- src/crux/effects/effects.go | 7 + src/crux/effects/execute_reconciling_test.go | 29 ++++ src/docs/fleet-report.md | 2 +- src/docs/troubleshooting.md | 4 +- src/pkg/convergence/mutation/boundary.go | 51 ++++++ .../mutation/boundary_unresolved_ttl_test.go | 156 ++++++++++++++++++ src/pkg/effects/effects.go | 1 + src/pkg/github/issue_request_watcher.go | 2 + 10 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-effect-ledger-unknown-op-reconcile.md create mode 100644 src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go diff --git a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md new file mode 100644 index 0000000000..cd12604414 --- /dev/null +++ b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md @@ -0,0 +1 @@ +- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). diff --git a/src/cmd/hive/selfreport_mutation_hooks.go b/src/cmd/hive/selfreport_mutation_hooks.go index 48e7ac77d1..1d450607e4 100644 --- a/src/cmd/hive/selfreport_mutation_hooks.go +++ b/src/cmd/hive/selfreport_mutation_hooks.go @@ -12,7 +12,8 @@ import ( // hooks to the self-report tracker: a journaled external effect that stays // Unknown/Planned past mutation.DefaultStuckAfter while callers keep retrying // it is reported as fleetreport.ClassEffectLedgerOpStuck (key = the op's -// logical ID), and the condition clears when the op is applied or reconciled. +// logical ID), and the condition clears when the op is applied, reconciled or +// expired past its kind's unresolved TTL (Boundary.UnresolvedTTL). func installMutationSelfReportHooks(b *mutation.Boundary) { if b == nil { return diff --git a/src/crux/effects/effects.go b/src/crux/effects/effects.go index ed3491670f..6eb3c962a8 100644 --- a/src/crux/effects/effects.go +++ b/src/crux/effects/effects.go @@ -43,6 +43,12 @@ var ErrNotApplied = errors.New("external effect definitely not applied") // must be reconciled (see Reconcile) before any retry. var ErrNeedsReconciliation = errors.New("external effect needs reconciliation before retry") +// ErrUnresolvedExpired accompanies ErrNeedsReconciliation when the boundary +// refused this attempt but, finding the operation unresolved past its TTL, +// expired it to not-applied: the effect did not run, and the caller's next +// attempt (after its own duplicate check) may. +var ErrUnresolvedExpired = errors.New("unresolved external effect expired past TTL; retry authorized") + // NotApplied wraps err as a definite refusal (ErrNotApplied) while keeping // err's own message and chain, so callers still see the original error. func NotApplied(err error) error { @@ -146,6 +152,7 @@ func ExecuteReconciling(ctx context.Context, boundary Boundary, claim Claim, obs if expErr != nil { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + expired = expired || errors.Is(journalErr, ErrUnresolvedExpired) if !expired { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } diff --git a/src/crux/effects/execute_reconciling_test.go b/src/crux/effects/execute_reconciling_test.go index 24d473bf07..558ca6b3d4 100644 --- a/src/crux/effects/execute_reconciling_test.go +++ b/src/crux/effects/execute_reconciling_test.go @@ -3,6 +3,7 @@ package effects import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -192,3 +193,31 @@ func TestLoggingBoundaryLookupAppliedForwards(t *testing.T) { t.Fatal("LoggingBoundary over a non-Lookup must not report applied") } } + +// A boundary that already expired the operation while refusing (the refusal +// carries ErrUnresolvedExpired) authorizes the retry even though its own +// ExpireUnresolved now finds nothing unresolved to expire. +func TestExecuteReconcilingBoundaryExpiredRefusalRetries(t *testing.T) { + b := &expiredOnRefusalBoundary{} + ran := 0 + res, err := ExecuteReconciling(context.Background(), b, reconcilingClaim, + func(context.Context) (ExternalState, error) { return ExternalState{}, errors.New("api down") }, + time.Minute, okEffect(&ran)) + if err != nil || ran != 1 || res.Provenance != "ran" { + t.Fatalf("got (%+v, %v, ran=%d), want the effect to run once", res, err, ran) + } +} + +type expiredOnRefusalBoundary struct{ refused bool } + +func (b *expiredOnRefusalBoundary) Execute(ctx context.Context, _ Claim, effect func(context.Context) (Result, error)) (Result, error) { + if !b.refused { + b.refused = true + return Result{}, fmt.Errorf("%w: %w", ErrNeedsReconciliation, ErrUnresolvedExpired) + } + return effect(ctx) +} + +func (b *expiredOnRefusalBoundary) ExpireUnresolved(context.Context, Claim, time.Duration) (bool, error) { + return false, nil +} diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 134fada243..be45d81494 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -137,7 +137,7 @@ title is `[selfreport] : `. | `agent_crash_loop` | the watchdog latches an agent as crash-looping and pauses it (acting mode only — observe mode would re-count every sweep); the healthy-window reset clears it. Diagnostics carry the classifier detail, failure count/cap, backoff ladder and healthy-reset window (#13470, was "agent crash loop") | high | | `ci_poll_nudge` | the harness sent a non-monitoring agent the stop-polling-CI nudge (at most one per kick); reviewer, CI-maintainer and quality lanes are exempt because reading CI status across PRs and repositories is their assigned work. Only a command line run more than once since the kick counts as polling (#14016): a sweep that reads each of several PRs' checks once does not cross the count threshold. A self-report fires only after an eligible agent ran ≥2 such repeated polls since the kick — a single nudged poll or a sweep does not trigger a report. One per 24h window; message carries polls since kick / time since kick / nudges since boot, diagnostics the scrubbed pane tail plus the repeated CI-status command lines with their counts (#14289; #13470, was "CI-poll nudge") | low | | `issue_request_stuck` | an issue/comment/label request file in the relay queue (`issue-requests/`) has failed ≥3 attempts **or** has been retrying for ≥30 min. Key is the request file name; message carries kind, attempts and age, diagnostics the last error (scrubbed). Clears when the request completes or is quarantined (`.failed`/`.bad`) | medium | -| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | +| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, reconciled, or — for `issue_create` — expired to NotApplied once a refused retry finds it unresolved past 1h (the caller's next attempt re-runs its duplicate search, then creates; #17693) | medium | | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `pkg/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | | `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. The record carries `backend` (which provider to re-authenticate), `login_hint` (the CLI's login command) and `credential_valid`; re-login in the agent's tmux pane, then Resume. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index a94e34ef0b..b6ecc797a3 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -802,7 +802,9 @@ when self-reporting is enabled, as a `[selfreport]` issue. mutation journal (`/data/convergence/mutation/journal.json` in the pod). Check whether the effect exists on GitHub; the owning call site reconciles it on its next pass, or the op resolves once the request behind it is removed. A watcher retrying - a doomed request keeps the op pinned, so fix the request first. + a doomed request keeps the op pinned, so fix the request first. An + `issue_create` op still refused after 1h is expired to NotApplied by the + boundary itself and the report clears. - **`unclassified_error_repeated`** — any WARN/ERROR line repeated five times in ten minutes. The report's message is the normalised line; grep the log for it (`kubectl -n hive logs deploy/hive | grep -F ""`) to find diff --git a/src/pkg/convergence/mutation/boundary.go b/src/pkg/convergence/mutation/boundary.go index 9dd6fd9190..1411c30763 100644 --- a/src/pkg/convergence/mutation/boundary.go +++ b/src/pkg/convergence/mutation/boundary.go @@ -40,6 +40,34 @@ type Boundary struct { // leaves the unresolved state: applied, already applied, or reconciled. OnResolved func(logicalID string) StuckAfter time.Duration + // UnresolvedTTL bounds, per effect kind, how long a journaled operation + // may stay Planned/Unknown while Execute keeps refusing retries of it. + // Past it the refused call expires the operation to NotApplied (see + // ExpireUnresolved) and reports it resolved, so the stuck alert stops and + // the caller's NEXT attempt — which runs its own duplicate check first — + // may proceed. The refused call itself never runs the effect. Nil uses + // DefaultUnresolvedTTLs; a missing or non-positive entry keeps refusing + // until an explicit Reconcile. + UnresolvedTTL map[string]time.Duration +} + +// DefaultUnresolvedTTLs returns the per-kind unresolved TTLs a Boundary uses +// when UnresolvedTTL is nil. Only issue creates are listed: every issue +// create caller searches for an existing twin before it reaches Execute, so +// an Unknown create that is still refused an hour later was never found and +// a fresh attempt cannot duplicate a visible issue. Without this an advisory +// issue create left Unknown by an ambiguous response stayed refused and +// reported stuck forever (hivecommons/hive#17693). +func DefaultUnresolvedTTLs() map[string]time.Duration { + return map[string]time.Duration{effects.KindIssueCreate: effects.DefaultUnresolvedTTL} +} + +func (b *Boundary) unresolvedTTL(kind string) time.Duration { + ttls := b.UnresolvedTTL + if ttls == nil { + ttls = DefaultUnresolvedTTLs() + } + return ttls[kind] } // ExecuteJournaled preserves legacy Execute's off-mode passthrough while @@ -162,6 +190,10 @@ func (b *Boundary) Execute(ctx context.Context, claim effects.Claim, effect func return out, fmt.Errorf("%w: %v", effects.ErrDenied, execErr) } if errors.Is(execErr, ErrNeedsReconciliation) { + if b.expireRefused(ctx, claim) { + b.noteResolved(logicalID) + return out, fmt.Errorf("%w: %w: %w", effects.ErrNeedsReconciliation, execErr, effects.ErrUnresolvedExpired) + } b.noteStuck(executor, logicalID) return out, fmt.Errorf("%w: %w", effects.ErrNeedsReconciliation, execErr) } @@ -195,6 +227,25 @@ func (b *Boundary) noteStuck(executor Executor, logicalID string) { b.OnStuck(op, age) } +// expireRefused expires claim's refused operation to NotApplied once it has +// been unresolved for longer than its kind's UnresolvedTTL. It never runs the +// effect: the retry is left to the caller's next attempt. +func (b *Boundary) expireRefused(ctx context.Context, claim effects.Claim) bool { + ttl := b.unresolvedTTL(claim.Kind) + if ttl <= 0 { + return false + } + expired, err := b.ExpireUnresolved(ctx, claim, ttl) + if err != nil { + if b.Logger != nil { + b.Logger.Warn("expiring unresolved external mutation failed", + "repo", claim.Repo, "kind", claim.Kind, "target", claim.Target, "error", err) + } + return false + } + return expired +} + func (b *Boundary) noteResolved(logicalID string) { if b.OnResolved != nil && logicalID != "" { b.OnResolved(logicalID) diff --git a/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go new file mode 100644 index 0000000000..7ae2e659aa --- /dev/null +++ b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go @@ -0,0 +1,156 @@ +package mutation + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/effects" +) + +// hivecommons/hive#17693: an issue create left Unknown by an ambiguous +// response, whose caller only ever calls Execute (no Reconcile/Expire of its +// own), was refused and reported stuck every cycle for 15h. Past the kind's +// unresolved TTL the refused call must expire the op to NotApplied, clear the +// stuck alert and — without running the effect in that call — let the +// caller's next attempt create. +func TestBoundaryExpiresRefusedUnknownIssueCreatePastTTL(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + stuck := 0 + var resolved []string + b.OnStuck = func(Operation, time.Duration) { stuck++ } + b.OnResolved = func(id string) { resolved = append(resolved, id) } + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "advisory"} + id := boundaryEffect(claim).LogicalID() + mustNotRun := func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run on a refused attempt") + return effects.Result{}, nil + } + + ambiguous := errors.New("502 Bad Gateway") + if _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, ambiguous + }); !errors.Is(err, ambiguous) { + t.Fatalf("first execute err = %v", err) + } + + // Inside the TTL: refused and reported stuck, still Unknown. + now = now.Add(DefaultStuckAfter) + _, err := b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("inside TTL err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("inside TTL: status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + + // Past the TTL: still refused this call, but expired and resolved. + now = now.Add(effects.DefaultUnresolvedTTL) + _, err = b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || !errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("past TTL err = %v, want refusal carrying ErrUnresolvedExpired", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusNotApplied { + t.Fatalf("past TTL status = %s, want NotApplied", op.Status) + } + if stuck != 1 { + t.Fatalf("stuck = %d after expiry, want no further report", stuck) + } + if len(resolved) != 1 || resolved[0] != id { + t.Fatalf("resolved = %v, want [%s]", resolved, id) + } + + // The next attempt is a fresh one and runs the effect. + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/9"}, nil + }) + if err != nil || !ran || res.Provenance == "" { + t.Fatalf("retry = (%+v, %v, ran=%v), want the create to run", res, err, ran) + } + if stuck != 1 { + t.Fatalf("stuck = %d after successful retry", stuck) + } +} + +// Reconciliation still wins inside the TTL: an issue found externally +// resolves the op Applied (replayed without creating), one definitively +// absent releases a retry. +func TestBoundaryReconcileResolvesUnknownIssueCreate(t *testing.T) { + for _, tc := range []struct { + name string + state effects.ExternalState + status string + runs bool + }{ + {"found", effects.ExternalState{Applied: true, Provenance: "https://github.example/acme/widget/issues/7"}, StatusApplied, false}, + {"absent", effects.ExternalState{Applied: false}, StatusNotApplied, true}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "finding"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("connection reset") + }) + if err := b.Reconcile(context.Background(), claim, tc.state); err != nil { + t.Fatal(err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != tc.status { + t.Fatalf("status = %s, want %s", op.Status, tc.status) + } + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/8"}, nil + }) + if err != nil || ran != tc.runs { + t.Fatalf("after reconcile: err=%v ran=%v, want ran=%v", err, ran, tc.runs) + } + if !tc.runs && res.Provenance != tc.state.Provenance { + t.Fatalf("replay provenance = %q, want %q", res.Provenance, tc.state.Provenance) + } + }) + } +} + +// Kinds without an unresolved TTL keep refusing (and reporting stuck) until +// an explicit Reconcile, however old the entry; an explicit empty map +// disables the default for issue creates too. +func TestBoundaryUnresolvedTTLOnlyForConfiguredKinds(t *testing.T) { + for _, tc := range []struct { + name string + kind string + ttls map[string]time.Duration + }{ + {"merge default", effects.KindPullRequestMerge, nil}, + {"issue create disabled", effects.KindIssueCreate, map[string]time.Duration{}}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + b.UnresolvedTTL = tc.ttls + stuck := 0 + b.OnStuck = func(Operation, time.Duration) { stuck++ } + claim := effects.Claim{Repo: "acme/widget", Kind: tc.kind, Target: "7"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("timeout") + }) + now = now.Add(48 * time.Hour) + _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run") + return effects.Result{}, nil + }) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + }) + } +} diff --git a/src/pkg/effects/effects.go b/src/pkg/effects/effects.go index a7066d3756..f57ef11014 100644 --- a/src/pkg/effects/effects.go +++ b/src/pkg/effects/effects.go @@ -28,6 +28,7 @@ var ( ErrDenied = cruxeffects.ErrDenied ErrNotApplied = cruxeffects.ErrNotApplied ErrNeedsReconciliation = cruxeffects.ErrNeedsReconciliation + ErrUnresolvedExpired = cruxeffects.ErrUnresolvedExpired ) type ( diff --git a/src/pkg/github/issue_request_watcher.go b/src/pkg/github/issue_request_watcher.go index ff789da3a8..51a9de7fc7 100644 --- a/src/pkg/github/issue_request_watcher.go +++ b/src/pkg/github/issue_request_watcher.go @@ -1782,6 +1782,8 @@ func (c *Client) reconcileIssueCreate(ctx context.Context, boundary effects.Boun if expErr != nil { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + // The boundary may already have expired the entry while refusing. + expired = expired || errors.Is(journalErr, effects.ErrUnresolvedExpired) if !expired { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } From 53c442024344afba579d33b09175c40566616b95 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:20:04 +0000 Subject: [PATCH 010/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20guard=20nil=20con?= =?UTF-8?q?fig=20in=20stageEngineOptions=20before=20reading=20WorkflowsDir?= =?UTF-8?q?=20(#17765)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stageEngineOptions dereferenced cfg.Data.WorkflowsDir as an argument before any nil check. Because stageEngine is a package-level sync.Once, the panic only fired when a nil-config caller was first in the process, making it order-dependent. Build the workflow store only when cfg != nil. Closes #17736 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-stage-engine-nil-config.md | 1 + src/cmd/hive/stage_engine.go | 8 +++-- src/cmd/hive/stage_engine_nilcfg_test.go | 32 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-stage-engine-nil-config.md create mode 100644 src/cmd/hive/stage_engine_nilcfg_test.go diff --git a/changelog.d/fixed-stage-engine-nil-config.md b/changelog.d/fixed-stage-engine-nil-config.md new file mode 100644 index 0000000000..25490b8df1 --- /dev/null +++ b/changelog.d/fixed-stage-engine-nil-config.md @@ -0,0 +1 @@ +- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). diff --git a/src/cmd/hive/stage_engine.go b/src/cmd/hive/stage_engine.go index 6038b3da79..f856c9bfcb 100644 --- a/src/cmd/hive/stage_engine.go +++ b/src/cmd/hive/stage_engine.go @@ -39,6 +39,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op if logger == nil { logger = slog.Default() } + var wf *wfstore.Store + if cfg != nil { + wf = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + } stageEngine.opts = &stageengine.Options{ Mode: stageEngineConfiguredMode(cfg), Store: stagelabels.Shared(), @@ -46,10 +50,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op PR: stageengine.PRDeriver(), Logger: logger, Metrics: stageengine.DefaultCounters(), - WorkflowFor: stageEngineWorkflowFor(cfg, wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)), logger), + WorkflowFor: stageEngineWorkflowFor(cfg, wf, logger), } if cfg != nil { - stageEngine.opts.WorkflowProvisioned = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + stageEngine.opts.WorkflowProvisioned = wf stageEngine.opts.WorkflowLabelGC = func() bool { return cfg.Workflow.LabelGCEnabled() } } }) diff --git a/src/cmd/hive/stage_engine_nilcfg_test.go b/src/cmd/hive/stage_engine_nilcfg_test.go new file mode 100644 index 0000000000..951b091a47 --- /dev/null +++ b/src/cmd/hive/stage_engine_nilcfg_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "sync" + "testing" +) + +// A nil config must not panic while building the process-wide options +// (hivecommons/hive#17736); the sync.Once is reset so the test is the first caller. +func TestStageEngineOptions_NilConfigDoesNotPanic(t *testing.T) { + savedOpts := stageEngine.opts + stageEngine.once = sync.Once{} + stageEngine.opts = nil + t.Cleanup(func() { + stageEngine.once = sync.Once{} + stageEngine.opts = savedOpts + if savedOpts != nil { + stageEngine.once.Do(func() {}) + } + }) + + opts := stageEngineOptions(nil, nil) + if opts == nil { + t.Fatal("stageEngineOptions(nil, nil) returned nil Options") + } + if opts.WorkflowFor != nil { + t.Error("WorkflowFor should be nil without a config") + } + if opts.WorkflowProvisioned != nil { + t.Error("WorkflowProvisioned should be nil without a config") + } +} From 4a86281fad2b7c097df16025e41d06d2e91b792a Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:26:41 +0100 Subject: [PATCH 011/166] =?UTF-8?q?=F0=9F=90=9B=20Skip=20archived=20reposi?= =?UTF-8?q?tories=20during=20GFI=20curation=20(#17739)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix GFI curation for archived repositories Hive-Run: hivecommons/hive#17734 Hive-Plan: archived-repository GFI recovery Hive-Spec: good-first-issue#archived-repositories Signed-off-by: Karibhou * fix: correct changelog fragment category Signed-off-by: Karibhou --------- Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-gfi-archived-repositories.md | 1 + src/docs/good-first-issue.md | 5 ++ src/pkg/gfi/gfi.go | 29 +++++++++++ src/pkg/gfi/gfi_test.go | 48 +++++++++++++++++++ src/pkg/github/gfi_forge.go | 13 +++++ src/pkg/github/gfi_forge_test.go | 22 +++++++++ 6 files changed, 118 insertions(+) create mode 100644 changelog.d/fixed-gfi-archived-repositories.md diff --git a/changelog.d/fixed-gfi-archived-repositories.md b/changelog.d/fixed-gfi-archived-repositories.md new file mode 100644 index 0000000000..4d7c472e30 --- /dev/null +++ b/changelog.d/fixed-gfi-archived-repositories.md @@ -0,0 +1 @@ +- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. diff --git a/src/docs/good-first-issue.md b/src/docs/good-first-issue.md index ab7a63fdcf..cbee5f9d1f 100644 --- a/src/docs/good-first-issue.md +++ b/src/docs/good-first-issue.md @@ -121,6 +121,11 @@ wins, oldest first on ties. Issues with the `good first issue` label the hive did not apply are still given the welcome comment (once) so that their visitors get the relay pitch too. +If GitHub refuses a curation write because a repository is archived and +read-only, the sweep skips that repository until hive restarts. Remove archived +repositories from the configured repo list, or restart hive after unarchiving. +Other write failures are still reported and retried on later sweeps. + ## The welcome comment One comment per issue, found by its first-line marker ``. A diff --git a/src/pkg/gfi/gfi.go b/src/pkg/gfi/gfi.go index b1ae84d129..d05d8fcb21 100644 --- a/src/pkg/gfi/gfi.go +++ b/src/pkg/gfi/gfi.go @@ -447,6 +447,17 @@ type Forge interface { UpsertIssueMarkerComment(ctx context.Context, repo string, number int, marker, body string, create bool) (MarkerComment, error) } +// ArchivedWriteTracker remembers GitHub's explicit archived-repository write +// refusals. A nil error asks whether the repository was already recorded. +type ArchivedWriteTracker interface { + ArchivedWrite(repo string, err error) bool +} + +func archivedWrite(forge Forge, repo string, err error) bool { + tracker, ok := forge.(ArchivedWriteTracker) + return ok && tracker.ArchivedWrite(repo, err) +} + // LabelReader is the optional half of Forge the sweep uses to re-read an // issue's labels immediately before advertising it. Candidates come from // the last enumeration, which can be many minutes old; hivecommons/hive @@ -653,6 +664,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate labelledPerRepo := map[string]int{} for _, c := range candidates { res.Scanned++ + if archivedWrite(forge, c.Repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if lowerSet(c.Labels)[Label] { labelledPerRepo[c.Repo]++ } @@ -673,6 +688,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate if d.AlreadyLabelled && Withdrawable(d.Skip) { if _, released := s.releasedUntil(c.Key(), p.ReselectCooldown); !released { if err := s.Withdraw(ctx, forge, c, d.Skip, opts); err != nil { + if archivedWrite(forge, c.Repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", c.Repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } res.Errors++ } else { res.Withdrawn = append(res.Withdrawn, c.Key()) @@ -699,6 +719,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate }) picked := 0 for _, sc := range list { + if archivedWrite(forge, repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if picked >= p.MaxPerSweep { res.Skipped["sweep-cap"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "sweep-cap") @@ -729,6 +753,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate } added, err := s.apply(ctx, forge, sc.c, body) if err != nil { + if archivedWrite(forge, repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } if errors.Is(err, errSuperseded) { res.Skipped["previously-released"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "previously-released") diff --git a/src/pkg/gfi/gfi_test.go b/src/pkg/gfi/gfi_test.go index e69e31915b..c13ed34a3c 100644 --- a/src/pkg/gfi/gfi_test.go +++ b/src/pkg/gfi/gfi_test.go @@ -327,6 +327,54 @@ func quietOpts() SweepOptions { return SweepOptions{Links: Links{Origin: "https://hive.example", HiveID: "acme-hive"}, Now: now, Logger: slog.New(slog.DiscardHandler)} } +type archivedForge struct { + *fakeForge + archived bool + attempts int +} + +var archivedErr = errors.New("archived repository") + +func (f *archivedForge) ArchivedWrite(repo string, err error) bool { + if repo != "acme/archived" { + return false + } + if errors.Is(err, archivedErr) { + f.archived = true + } + return f.archived +} + +func (f *archivedForge) UpsertIssueMarkerComment(ctx context.Context, repo string, n int, marker, body string, create bool) (MarkerComment, error) { + if repo == "acme/archived" && create { + f.attempts++ + return MarkerComment{}, archivedErr + } + return f.fakeForge.UpsertIssueMarkerComment(ctx, repo, n, marker, body, create) +} + +func TestSweepSkipsArchivedRepoAfterRefusal(t *testing.T) { + forge := &archivedForge{fakeForge: newFakeForge()} + s := NewSweeper() + a := good() + a.Repo = "acme/archived" + b := a + b.Number++ + active := good() + candidates := []Candidate{a, b, active} + first := s.Sweep(context.Background(), forge, candidates, quietOpts()) + second := s.Sweep(context.Background(), forge, candidates, quietOpts()) + if forge.attempts != 1 || first.Errors != 0 || second.Errors != 0 { + t.Fatalf("attempts=%d first=%+v second=%+v", forge.attempts, first, second) + } + if first.Skipped["archived-repo"] != 2 || second.Skipped["archived-repo"] != 2 { + t.Fatalf("archived skips: first=%+v second=%+v", first, second) + } + if len(first.Selected) != 1 || first.Selected[0] != active.Key() { + t.Fatalf("active repo not selected: %+v", first) + } +} + func TestSweepSelectsLabelsAndCommentsIdempotently(t *testing.T) { forge := newFakeForge() s := NewSweeper() diff --git a/src/pkg/github/gfi_forge.go b/src/pkg/github/gfi_forge.go index 5f1c22d387..613a0537de 100644 --- a/src/pkg/github/gfi_forge.go +++ b/src/pkg/github/gfi_forge.go @@ -17,6 +17,19 @@ func HardSuppressIssueLabels() []string { return append([]string(nil), hardSuppressIssueLabels...) } +// ArchivedWrite shares the stage sweeps' process-lifetime archive cache. +func (c *Client) ArchivedWrite(repo string, err error) bool { + if c == nil { + return false + } + owner, name := c.splitRepo(repo) + key := strings.ToLower(owner + "/" + name) + if err != nil { + return c.noteArchivedWrite(key, err) + } + return c.stageArchived.has(key) +} + // UpsertIssueMarkerComment implements gfi.Forge: it finds the first comment // on repo#number containing marker, edits it to body when body is non-empty // and differs, and — only when create is set — posts body when none exists. diff --git a/src/pkg/github/gfi_forge_test.go b/src/pkg/github/gfi_forge_test.go index a577f1482c..1a954e73a4 100644 --- a/src/pkg/github/gfi_forge_test.go +++ b/src/pkg/github/gfi_forge_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/pkg/gfi" ) @@ -29,6 +30,27 @@ func TestHardSuppressIssueLabelsIsCopyWithoutGFI(t *testing.T) { } } +func TestGFIArchivedWriteRecognizesOnlyArchived403(t *testing.T) { + c := NewClientForTest("http://example.com", "acme", []string{"widgets"}, testLogger()) + for _, tc := range []struct { + status int + message string + want bool + }{ + {http.StatusForbidden, "Resource not accessible by integration", false}, + {http.StatusInternalServerError, "Repository was archived", false}, + {http.StatusForbidden, "Repository was archived so is read-only.", true}, + } { + err := &gh.ErrorResponse{Response: &http.Response{StatusCode: tc.status}, Message: tc.message} + if got := c.ArchivedWrite("widgets", err); got != tc.want { + t.Fatalf("status=%d message=%q: got %v", tc.status, tc.message, got) + } + } + if !c.ArchivedWrite("ACME/Widgets", nil) || c.ArchivedWrite("acme/other", nil) { + t.Fatal("archive cache must be case-insensitive and repository-scoped") + } +} + func TestUpsertIssueMarkerComment(t *testing.T) { var mu sync.Mutex comments := []map[string]any{ From acaa4813918b930633693903e9638a26cdcc370a Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 22:33:36 +0000 Subject: [PATCH 012/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.222.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.222.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-substrate-sandbox-runtime.md | 1 - changelog.d/changed-substrate-runtime-docs.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-substrate-sandbox-runtime.md delete mode 100644 changelog.d/changed-substrate-runtime-docs.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c95a8aeea6..cb85f26dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.222.0) + +### Added + +- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. + +### Changed + +- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. + ## 2026-11-11 (v5.221.1) ### Fixed diff --git a/changelog.d/added-substrate-sandbox-runtime.md b/changelog.d/added-substrate-sandbox-runtime.md deleted file mode 100644 index f9c190943b..0000000000 --- a/changelog.d/added-substrate-sandbox-runtime.md +++ /dev/null @@ -1 +0,0 @@ -- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. diff --git a/changelog.d/changed-substrate-runtime-docs.md b/changelog.d/changed-substrate-runtime-docs.md deleted file mode 100644 index fadfda564d..0000000000 --- a/changelog.d/changed-substrate-runtime-docs.md +++ /dev/null @@ -1 +0,0 @@ -- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. From a63b4bed201b78b1258df9c6bac621e9772be349 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:37:36 -0700 Subject: [PATCH 013/166] =?UTF-8?q?=F0=9F=90=9B=20marketplace:=20resume=20?= =?UTF-8?q?CNCF=20catalogue=20seeding=20across=20hub=20restarts=20and=20le?= =?UTF-8?q?t=20spokes=20trigger=20refresh=20(#17773)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 marketplace: resume CNCF catalogue seeding across hub restarts and let spokes trigger refresh Root cause of the one-entry catalogue on hive.hivecommons.dev: GitHub tarballs lead with a pax_global_header entry, which Go's tar reader surfaces as a header. ExtractTarGz counted it as a second top-level name, returned dest instead of the wrapped "--/" directory, and ingest.ReadTree then saw every path with the wrapper prefix — no GitHub Actions workflow matched. Only tektoncd/pipeline got through, via the content-based Tekton sniff. Skip pax headers when extracting. Seeding is now resumable: at startup the hub ingests every corpus repo that is due (no record, record older than 24h or in an older format, error older than 1h) instead of only when the catalogue is empty, so a restart mid-seed continues and a flapping hub does not re-crawl fresh repos. Records carry a format version; bumping it (format 2 for the tarball fix) re-ingests everything on the next boot. The last run summary is persisted so status.lastRefresh survives restarts; status gains pendingRepos/lastRefreshErrors and ?verbose=1 (spoke or admin) lists the failing repos with their errors. POST /api/marketplace/refresh accepts the same credentials as publish (hub admin session or spoke heartbeat bearer + X-Hive-ID); spokes are rate-limited (one full run/hour, 10 ?repo= runs/min → 429 + Retry-After), admins are not. The spoke proxy passes 429/409 through with the hub's message. The crawl sleeps on 403/429 Retry-After (secondary limits) and logs progress every 25 repos. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson * 🐛 marketplace: update refresh tooltip in static-src and regenerate index.html Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/fixed-marketplace-seed-resume.md | 1 + src/docs/api-reference/hub-server.md | 10 +- src/docs/design/workflow-models.md | 46 ++- .../dashboard/api_workflows_marketplace.go | 13 +- .../api_workflows_marketplace_test.go | 26 +- .../js/config-workflows-marketplace.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/evidence/complexity/runner_test.go | 12 + src/pkg/evidence/complexity/tarball.go | 8 + src/pkg/hub/marketplace.go | 112 ++++-- src/pkg/hub/marketplace_refresh.go | 147 +++++++- src/pkg/hub/marketplace_store.go | 42 +++ src/pkg/hub/marketplace_test.go | 344 +++++++++++++++++- 13 files changed, 690 insertions(+), 75 deletions(-) create mode 100644 changelog.d/fixed-marketplace-seed-resume.md diff --git a/changelog.d/fixed-marketplace-seed-resume.md b/changelog.d/fixed-marketplace-seed-resume.md new file mode 100644 index 0000000000..5321431ab7 --- /dev/null +++ b/changelog.d/fixed-marketplace-seed-resume.md @@ -0,0 +1 @@ +- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 5fd503f013..b79feb07b9 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -71,8 +71,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | | `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1802` | | `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1804` | -| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:34` | -| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:35` | -| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:36` | -| `POST` | `/api/marketplace/refresh` | Hub admin | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}` | `pkg/hub/marketplace.go:37` | -| `GET` | `/api/marketplace/status` | Public | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary and counts by tier | `pkg/hub/marketplace.go:38` | +| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | +| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | +| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | +| `POST` | `/api/marketplace/refresh` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}`. Spokes are rate-limited across all hives (one full run per hour, 10 `?repo=` runs per minute) → `429` + `Retry-After`; admins are not. `401` anonymous, `403` non-admin session or unknown hive | `pkg/hub/marketplace.go:39` | +| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts) and counts by tier. `?verbose=1` adds `failing:[{repo, error, checkedAt}]` and `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index d5e7bd53e5..10ed66c3ef 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -719,6 +719,7 @@ Storage is a directory under the hub data dir: workflows//.bundle.json immutable; refuse overwrite of name@version index.json derived; rebuilt from the bundles if missing refresh/__.json last ingest run per repository + last-refresh.json summary of the last completed run ``` Entries are keyed `name@version` plus hash. Publishing the identical bundle again is a @@ -734,8 +735,8 @@ Routes (see [hub-server.md](../api-reference/hub-server.md)): | `GET /api/marketplace/workflows` | public | index; `If-None-Match` → `304` | | `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | | `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | -| `POST /api/marketplace/refresh[?repo=]` | hub admin | on-demand ingest | -| `GET /api/marketplace/status` | public | last refresh, counts by tier | +| `POST /api/marketplace/refresh[?repo=]` | spoke bearer or hub admin | on-demand ingest; spokes rate-limited (`429`) | +| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier; `verbose` adds failing and pending repos | **Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the `X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may @@ -748,16 +749,37 @@ same-origin CSRF evidence. The hub never stores a bundle it did not itself `Veri falls back to it when no `projects.json` is given) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and -records `{sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → +records `{format, sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → `ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier `ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never touched; `marketplaceNormaliseIndex` flags a curated entry `newerIngestAvailable` when a -later ingest of the same name exists. The crawl is sequential with a short pause between -repositories and honours `X-RateLimit-Remaining`/`Reset`. It runs nightly at -`HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default `03:00`, empty disables), on -demand, and once at hub start when the catalogue is empty and a GitHub token is present -(in a goroutine; startup is not blocked). `HIVE_HUB_MARKETPLACE_DISABLED` turns the -routes off (`404`). +later ingest of the same name exists. The crawl is sequential in sorted repository order +with a short pause between repositories, logs progress every 25, honours +`X-RateLimit-Remaining`/`Reset` and sleeps on a `403`/`429` with `Retry-After` (secondary +limits). It runs nightly at `HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default +`03:00`, empty disables) and on demand. `HIVE_HUB_MARKETPLACE_DISABLED` turns the routes off +(`404`). + +**Seeding is resumable.** Each repository's record is persisted as the crawl goes, so a +run interrupted by a hub restart (releases roll several times a day) loses nothing. At +start, after a short delay and when a GitHub token is present, the hub ingests every corpus +repository that is *due*: no record, a record older than the nightly interval (24 h) or in +an older `format`, or an error older than one hour. A hub that restarts repeatedly +therefore only re-crawls what is missing instead of hammering GitHub, and a catalogue that +seeded one entry before a roll fills in on the next boot. Bumping +`marketplaceRecordFormat` forces a full re-ingest when the ingest itself changes (format 2: +tarball extraction recognises git's `pax_global_header`, without which the wrapped +directory was not detected and almost every repository read as having no pipeline files). +The last run's summary is persisted (`last-refresh.json`) so `status.lastRefresh` survives a +restart. + +**Refresh auth.** `POST /api/marketplace/refresh` takes the same credentials as publish: a +hub-admin session (same-origin CSRF evidence) or a registered spoke's heartbeat bearer + +`X-Hive-ID`, so a hive owner's **Refresh catalogue** button works through the spoke proxy. +Spokes are rate-limited across all hives — one full run per hour, ten `?repo=` runs per +minute — and get `429` with `Retry-After`; admins are not limited. A run already in flight +is `409`. The spoke relays the hub's `401/403` as `403`, keeps `429`/`409`, and maps other +failures to `502`, always with the hub's message in the body. ### Spoke — import, publish, browse @@ -785,8 +807,10 @@ Routes (`pkg/dashboard/api_workflows_marketplace.go`): - `POST /api/marketplace/workflows/publish {name, version}` (owner) — export and POST to the hub with the spoke's heartbeat credential; `201 {published, hash, tier}`. Audit `workflow_publish`. -- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub; a non-admin hive - surfaces the hub's `403`. Audit `workflow_marketplace_refresh`. +- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub with the spoke's + heartbeat credential; the hub's `429` (rate limit, with `Retry-After`) and `409` (run in + progress) pass through with its message, `401/403` → `403`. Audit + `workflow_marketplace_refresh`. Imported models are pinned `name@version`; a newer catalogue version is a new library entry and **bindings never move automatically** — rebinding stays an explicit act on Settings → diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go index 98d2390836..b7836e8b2a 100644 --- a/src/pkg/dashboard/api_workflows_marketplace.go +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -450,8 +450,11 @@ func (s *Server) handleMarketplacePublish(w http.ResponseWriter, r *http.Request } // handleMarketplaceRefresh asks the hub to re-ingest its corpus (or one -// ?repo=). Owner on the spoke; the hub applies its own admin policy and may -// refuse, which is surfaced as 403. +// ?repo=). Owner on the spoke; the request carries the spoke's heartbeat +// bearer + X-Hive-ID, which the hub accepts for refresh the way it does +// for publish. The hub's error body is relayed verbatim: 401/403 → 403, +// 429 (hive rate limit) and 409 (run in progress) keep their status, and +// anything else → 502. func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !requireOwnerRole(w, r) { return @@ -482,6 +485,12 @@ func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: jsonError(w, "hub refused refresh: "+strings.TrimSpace(hubErrorMessage(body)), http.StatusForbidden) return + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusConflict: + if ra := resp.Header.Get("Retry-After"); ra != "" { + w.Header().Set("Retry-After", ra) + } + jsonError(w, "hub refresh not started: "+strings.TrimSpace(hubErrorMessage(body)), resp.StatusCode) + return case resp.StatusCode >= 300: jsonError(w, fmt.Sprintf("hub refresh failed (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(body))), http.StatusBadGateway) return diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index 875fe0b7d0..eed2d9897b 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -108,6 +108,10 @@ func newFakeHub(t *testing.T) (*fakeHub, *httptest.Server) { _, _ = w.Write([]byte(`{"error":"nope"}`)) case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/refresh": h.refreshes.Add(1) + if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") || r.Header.Get(marketplaceHiveIDHeader) == "" { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"accepted":true,"repo":"` + r.URL.Query().Get("repo") + `"}`)) default: @@ -403,12 +407,18 @@ func TestMarketplaceRefreshProxy(t *testing.T) { t.Fatalf("refresh: %d %s", rec.Code, rec.Body.String()) } deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.URL.RawQuery, "boom") { + switch { + case strings.Contains(r.URL.RawQuery, "boom"): w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("plain text failure")) - return + case strings.Contains(r.URL.RawQuery, "slow"): + w.Header().Set("Retry-After", "42") + http.Error(w, `{"error":"full catalogue refresh was requested recently by a hive; try again in 42s"}`, http.StatusTooManyRequests) + case strings.Contains(r.URL.RawQuery, "busy"): + http.Error(w, `{"error":"marketplace: refresh already running"}`, http.StatusConflict) + default: + http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) } - http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) })) defer deny.Close() s.deps.Config.Workflow.Marketplace.URL = deny.URL @@ -416,6 +426,16 @@ func TestMarketplaceRefreshProxy(t *testing.T) { if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "admin only") { t.Fatalf("denied: %d %s", rec.Code, rec.Body.String()) } + // Hub rate limit and busy keep their status and message so the toast + // tells the owner when to retry. + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=slow/x", nil, true) + if rec.Code != http.StatusTooManyRequests || rec.Header().Get("Retry-After") != "42" || !strings.Contains(rec.Body.String(), "try again in 42s") { + t.Fatalf("rate limited: %d %s retry-after=%q", rec.Code, rec.Body.String(), rec.Header().Get("Retry-After")) + } + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=busy/x", nil, true) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "already running") { + t.Fatalf("busy: %d %s", rec.Code, rec.Body.String()) + } rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=boom", nil, true) if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "plain text failure") { t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) diff --git a/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js index fbc7d75f3e..180baf32bb 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js @@ -100,7 +100,7 @@ `` + `${shown.length} of ${entries.length}` + `` + - (owner ? `` : '') + + (owner ? `` : '') + ''; let body; if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index db978b8760..92bfc42a08 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -65926,7 +65926,7 @@

Agent Watchdog

`` + `${shown.length} of ${entries.length}` + `` + - (owner ? `` : '') + + (owner ? `` : '') + ''; let body; if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; diff --git a/src/pkg/evidence/complexity/runner_test.go b/src/pkg/evidence/complexity/runner_test.go index 9fbacb52b7..a91d5269e6 100644 --- a/src/pkg/evidence/complexity/runner_test.go +++ b/src/pkg/evidence/complexity/runner_test.go @@ -393,6 +393,18 @@ func TestExtractTarGz(t *testing.T) { t.Error("symlink must be skipped") } + // git archive leads with a pax_global_header entry; it must not be + // mistaken for a second top-level name. + pax := tarGz(t, map[string]string{"o-r-abc1234/p.go": "package p\n"}, + tar.Header{Name: "pax_global_header", Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": "abc1234"}}) + d2 := t.TempDir() + if top, err := ExtractTarGz(bytes.NewReader(pax), d2, 0); err != nil || top != filepath.Join(d2, "o-r-abc1234") { + t.Errorf("pax header: top=%s err=%v", top, err) + } + if _, err := os.Stat(filepath.Join(d2, "pax_global_header")); !errors.Is(err, os.ErrNotExist) { + t.Error("pax_global_header must not be written") + } + // Path traversal is refused. bad := tarGz(t, map[string]string{"../escape.go": "x"}) if _, err := ExtractTarGz(bytes.NewReader(bad), t.TempDir(), 0); err == nil { diff --git a/src/pkg/evidence/complexity/tarball.go b/src/pkg/evidence/complexity/tarball.go index 056d3a626c..2a3814cf93 100644 --- a/src/pkg/evidence/complexity/tarball.go +++ b/src/pkg/evidence/complexity/tarball.go @@ -46,6 +46,14 @@ func ExtractTarGz(r io.Reader, dest string, maxBytes int64) (string, error) { if err != nil { return "", err } + // git archive (and so GitHub's tarball endpoint) leads with a + // "pax_global_header" entry carrying the commit SHA; the reader + // surfaces it as a header. It is not a tree entry and must not + // count as a second top-level name, or the wrapped directory is + // never detected and callers read paths with the wrapper prefix. + if hdr.Typeflag == tar.TypeXGlobalHeader || hdr.Typeflag == tar.TypeXHeader { + continue + } name := filepath.Clean(filepath.FromSlash(hdr.Name)) if name == "." || name == "" { continue diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 3eb24ea673..73e5196ae1 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -17,8 +18,9 @@ import ( // GET /api/marketplace/workflows index (ETag / 304) // GET /api/marketplace/workflows/{name}/{version}/bundle one bundle // POST /api/marketplace/workflows publish (spoke bearer or admin) -// POST /api/marketplace/refresh[?repo=] on-demand ingest (admin) -// GET /api/marketplace/status last refresh + counts by tier +// POST /api/marketplace/refresh[?repo=] on-demand ingest (spoke bearer or admin) +// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier +// (verbose: failing/pending repos; spoke bearer or admin) // // Tier policy: a publisher may assert `local`, `local-published` or // `ingested-unreviewed`; `curated` and `built-in` require a hub admin @@ -34,7 +36,7 @@ func (s *HubServer) registerMarketplaceRoutes() { s.mux.HandleFunc("GET /api/marketplace/workflows", s.handleMarketplaceIndex) s.mux.HandleFunc("GET /api/marketplace/workflows/{name}/{version}/bundle", s.handleMarketplaceBundle) s.mux.HandleFunc("POST /api/marketplace/workflows", s.handleMarketplacePublish) - s.mux.HandleFunc("POST /api/marketplace/refresh", s.requireAdmin(s.handleMarketplaceRefresh)) + s.mux.HandleFunc("POST /api/marketplace/refresh", s.handleMarketplaceRefresh) s.mux.HandleFunc("GET /api/marketplace/status", s.handleMarketplaceStatus) } @@ -207,18 +209,37 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ // handleMarketplaceRefresh runs an on-demand ingest in the background and // answers 202 immediately; ?repo= limits it to one repository (and runs // synchronously, since a single repo is quick enough to await). +// +// Callers are the same as publish: a hub admin session or a registered +// spoke with its heartbeat bearer + X-Hive-ID, so a hive owner's "Refresh +// catalogue" button works through the spoke. Spokes are rate-limited (one +// full run per marketplaceFullRefreshInterval across all spokes, +// marketplaceRepoRefreshBurst single-repo runs per minute) → 429 with +// Retry-After; admins are not. func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return } + who, code, msg := s.marketplaceAuthenticate(r) + if code != 0 { + marketplaceJSONError(w, code, msg) + return + } repo := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("repo"))) - user := s.getRealAuthUser(r) + rf := s.refresher() + now := rf.now().UTC() if repo != "" { if !strings.Contains(repo, "/") { marketplaceJSONError(w, http.StatusBadRequest, "repo must be owner/name") return } - res, err := s.runMarketplaceRefresh(r.Context(), []string{repo}, "on-demand:"+user) + if !who.admin { + if ok, wait := rf.allowRepoRefresh(now); !ok { + marketplaceTooMany(w, wait, "single-repository refresh") + return + } + } + res, err := s.runMarketplaceRefresh(r.Context(), []string{repo}, "on-demand:"+who.name) if errors.Is(err, errMarketplaceRefreshBusy) { marketplaceJSONError(w, http.StatusConflict, err.Error()) return @@ -227,12 +248,11 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) return } - s.logger.Info("marketplace refresh (single repo)", "by", user, "repo", repo, "published", res.Published, "errors", res.Errors) + s.logger.Info("marketplace refresh (single repo)", "by", who.name, "repo", repo, "published", res.Published, "errors", res.Errors) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repo": repo, "result": res}) return } - rf := s.refresher() rf.mu.Lock() busy := rf.running rf.mu.Unlock() @@ -240,9 +260,15 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusConflict, errMarketplaceRefreshBusy.Error()) return } - s.logger.Info("marketplace refresh requested", "by", user) + if !who.admin { + if ok, wait := rf.allowFullRefresh(now); !ok { + marketplaceTooMany(w, wait, "full catalogue refresh") + return + } + } + s.logger.Info("marketplace refresh requested", "by", who.name) go func() { - if _, err := s.runMarketplaceRefresh(s.marketplaceBackgroundContext(), nil, "on-demand:"+user); err != nil { + if _, err := s.runMarketplaceRefresh(s.marketplaceBackgroundContext(), nil, "on-demand:"+who.name); err != nil { s.logger.Warn("marketplace on-demand refresh skipped", "error", err) } }() @@ -251,11 +277,39 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repos": len(marketplaceRepos())}) } +// marketplaceTooMany answers 429 with Retry-After and an honest message: +// the spoke relays the body into the operator's toast. +func marketplaceTooMany(w http.ResponseWriter, wait time.Duration, what string) { + secs := int(wait.Round(time.Second) / time.Second) + if secs < 1 { + secs = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(secs)) + marketplaceJSONError(w, http.StatusTooManyRequests, fmt.Sprintf("%s was requested recently by a hive; try again in %s (hub admins are not limited)", what, (time.Duration(secs)*time.Second).String())) +} + +// marketplaceFailingRepo is one verbose-status row for a repo whose last +// ingest failed. +type marketplaceFailingRepo struct { + Repo string `json:"repo"` + Error string `json:"error"` + CheckedAt time.Time `json:"checkedAt"` +} + // handleMarketplaceStatus reports the last refresh and tier counts. +// ?verbose=1 (spoke bearer or admin) adds the failing repositories with +// their error strings and the repositories still awaiting a first ingest. func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return } + verbose := r.URL.Query().Get("verbose") != "" + if verbose { + if _, code, msg := s.marketplaceAuthenticate(r); code != 0 { + marketplaceJSONError(w, code, msg) + return + } + } idx, _, etag, err := s.marketplace().Index() if err != nil { marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) @@ -272,31 +326,45 @@ func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Reque rf.mu.Unlock() records := s.marketplace().refreshRecords() var lastChecked time.Time - failing := 0 + failing := []marketplaceFailingRepo{} for _, rec := range records { if rec.CheckedAt.After(lastChecked) { lastChecked = rec.CheckedAt } if rec.Error != "" { - failing++ + failing = append(failing, marketplaceFailingRepo{Repo: rec.Repo, Error: rec.Error, CheckedAt: rec.CheckedAt}) } } + pending := s.marketplacePendingRepos() + lastErrors := 0 + if last != nil { + lastErrors = len(last.Errors) + } out := map[string]interface{}{ - "entries": len(idx.Entries), - "byTier": byTier, - "etag": etag, - "generatedAt": idx.GeneratedAt, - "corpusRepos": len(marketplaceRepos()), - "trackedRepos": len(records), - "failingRepos": failing, - "refreshing": running, - "schedule": marketplaceSchedule(), - "lastRefresh": last, - "lastCheckedAt": nil, + "entries": len(idx.Entries), + "byTier": byTier, + "etag": etag, + "generatedAt": idx.GeneratedAt, + "corpusRepos": len(marketplaceRepos()), + "trackedRepos": len(records), + "failingRepos": len(failing), + "pendingRepos": len(pending), + "lastRefreshErrors": lastErrors, + "refreshing": running, + "schedule": marketplaceSchedule(), + "lastRefresh": last, + "lastCheckedAt": nil, } if !lastChecked.IsZero() { out["lastCheckedAt"] = lastChecked } + if verbose { + if pending == nil { + pending = []string{} + } + out["failing"] = failing + out["pending"] = pending + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(out) } diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index d9951a30b4..ed164784ce 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -49,6 +49,22 @@ const ( marketplaceRateMaxWait = 10 * time.Minute marketplaceMaxTarball = 256 << 20 marketplaceSeedDelay = 30 * time.Second + // marketplaceStaleAfter is how old a repo's refresh record may be + // before the startup pass re-ingests it: the nightly interval, so a + // hub that restarts repeatedly only re-crawls what the nightly missed. + marketplaceStaleAfter = 24 * time.Hour + // marketplaceRetryErrorAfter is how soon the startup pass retries a + // repository whose last record is an error. + marketplaceRetryErrorAfter = time.Hour + // marketplaceProgressEvery is how many repos between progress log lines. + marketplaceProgressEvery = 25 + // marketplaceFullRefreshInterval bounds on-demand full runs from + // non-admin callers (spokes): one per interval across all callers. + marketplaceFullRefreshInterval = time.Hour + // marketplaceRepoRefreshBurst / Window bound on-demand single-repo + // runs from non-admin callers. + marketplaceRepoRefreshBurst = 10 + marketplaceRepoRefreshWindow = time.Minute ) // marketplaceRepoPause keeps the crawl polite: one repo at a time with a @@ -84,6 +100,39 @@ type marketplaceRefresher struct { // bgCtx bounds on-demand runs started from a request; set by // StartMarketplaceRefresh so they stop with the other pollers. bgCtx context.Context + // lastFullRequest and repoRequests back the on-demand rate limits. + lastFullRequest time.Time + repoRequests []time.Time +} + +// allowFullRefresh reports whether a non-admin full refresh may start now +// and, when not, how long until one may. Callers hold no lock. +func (rf *marketplaceRefresher) allowFullRefresh(now time.Time) (bool, time.Duration) { + rf.mu.Lock() + defer rf.mu.Unlock() + if wait := marketplaceFullRefreshInterval - now.Sub(rf.lastFullRequest); !rf.lastFullRequest.IsZero() && wait > 0 { + return false, wait + } + rf.lastFullRequest = now + return true, 0 +} + +// allowRepoRefresh is the sliding-window limit for non-admin ?repo= runs. +func (rf *marketplaceRefresher) allowRepoRefresh(now time.Time) (bool, time.Duration) { + rf.mu.Lock() + defer rf.mu.Unlock() + kept := rf.repoRequests[:0] + for _, t := range rf.repoRequests { + if now.Sub(t) < marketplaceRepoRefreshWindow { + kept = append(kept, t) + } + } + rf.repoRequests = kept + if len(kept) >= marketplaceRepoRefreshBurst { + return false, marketplaceRepoRefreshWindow - now.Sub(kept[0]) + } + rf.repoRequests = append(kept, now) + return true, 0 } // marketplaceBackgroundContext is the pollers' context when running, else @@ -106,6 +155,9 @@ func (s *HubServer) marketplace() *marketplaceStore { if s.marketplaceRefresher == nil { s.marketplaceRefresher = &marketplaceRefresher{fetch: s.marketplaceGitHubFetcher, now: time.Now} } + if s.marketplaceRefresher.last == nil { + s.marketplaceRefresher.last = s.marketplaceStore.loadLastRefresh() + } }) return s.marketplaceStore } @@ -168,6 +220,7 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t repos = marketplaceRepos() } res := &marketplaceRefreshResult{StartedAt: rf.now().UTC(), Trigger: trigger, Published: []string{}, Errors: []string{}} + s.logger.Info("marketplace refresh started", "trigger", trigger, "repos", len(repos)) for i, repo := range repos { if ctx.Err() != nil { res.Errors = append(res.Errors, "cancelled: "+ctx.Err().Error()) @@ -192,11 +245,17 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t default: res.Published = append(res.Published, outcome) } + if res.Repos%marketplaceProgressEvery == 0 && res.Repos < len(repos) { + s.logger.Info("marketplace refresh progress", "trigger", trigger, "done", res.Repos, "total", len(repos), "published", len(res.Published), "errors", len(res.Errors)) + } } res.FinishedAt = rf.now().UTC() rf.mu.Lock() rf.last = res rf.mu.Unlock() + if err := st.saveLastRefresh(res); err != nil { + s.logger.Warn("marketplace refresh: could not persist summary", "error", err) + } s.logger.Info("marketplace refresh finished", "trigger", trigger, "repos", res.Repos, "published", len(res.Published), "unchanged", res.Unchanged, "skipped", res.Skipped, "errors", len(res.Errors)) return res, nil } @@ -213,7 +272,7 @@ func (s *HubServer) refreshOneRepo(ctx context.Context, st *marketplaceStore, rf } hash := marketplaceFilesHash(files) rec := marketplaceRefreshRecord{Repo: repo, SourceSHA: sha, FilesHash: hash, Files: len(files), CheckedAt: rf.now().UTC()} - if prev, ok := st.refreshRecord(repo); ok && prev.FilesHash == hash && prev.Error == "" { + if prev, ok := st.refreshRecord(repo); ok && prev.Format == marketplaceRecordFormat && prev.FilesHash == hash && prev.Error == "" { rec.Published = prev.Published _ = st.saveRefreshRecord(rec) return "", nil @@ -313,6 +372,7 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( return nil, "", err } defer resp.Body.Close() + marketplaceHonourRateLimit(ctx, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return nil, "", fmt.Errorf("tarball: %s", resp.Status) @@ -334,8 +394,9 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( } // marketplaceGitHubJSON GETs a GitHub API resource, honouring rate-limit -// headers: when the remaining budget is under marketplaceRateFloor it -// sleeps until the reset (bounded) before returning. +// headers: when the remaining budget is under marketplaceRateFloor, or the +// reply is a 403/429 carrying Retry-After (secondary limit), it sleeps +// (bounded) before returning so the next repo does not hit the same wall. func marketplaceGitHubJSON(ctx context.Context, client *http.Client, url string, out interface{}) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -348,7 +409,7 @@ func marketplaceGitHubJSON(ctx context.Context, client *http.Client, url string, return err } defer resp.Body.Close() - marketplaceHonourRateLimit(ctx, resp.Header) + marketplaceHonourRateLimit(ctx, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return fmt.Errorf("%s: %s", url, resp.Status) @@ -364,7 +425,20 @@ var marketplaceSleep = func(ctx context.Context, d time.Duration) { } } -func marketplaceHonourRateLimit(ctx context.Context, h http.Header) { +func marketplaceHonourRateLimit(ctx context.Context, status int, h http.Header) { + if status == http.StatusForbidden || status == http.StatusTooManyRequests { + // Secondary rate limits answer 403/429 with Retry-After seconds and + // a non-zero X-RateLimit-Remaining, so the floor check below would + // not catch them. + if secs, err := strconv.Atoi(strings.TrimSpace(h.Get("Retry-After"))); err == nil && secs > 0 { + wait := time.Duration(secs) * time.Second + if wait > marketplaceRateMaxWait { + wait = marketplaceRateMaxWait + } + marketplaceSleep(ctx, wait) + return + } + } rem, err := strconv.Atoi(strings.TrimSpace(h.Get("X-RateLimit-Remaining"))) if err != nil || rem >= marketplaceRateFloor { return @@ -412,9 +486,12 @@ func marketplaceSchedule() string { return strings.TrimSpace(v) } -// StartMarketplaceRefresh runs the nightly ingest until ctx is cancelled, -// seeding the catalogue once at start when it is empty and the hub has a -// GitHub token. Started from StartBackgroundPollers. +// StartMarketplaceRefresh runs the nightly ingest until ctx is cancelled. +// At start (after marketplaceSeedDelay) it ingests every corpus repository +// that is due — no refresh record, a record older than marketplaceStaleAfter +// or in an older format, or an error older than marketplaceRetryErrorAfter — +// so a seed interrupted by a hub restart resumes where it stopped instead of +// waiting for the nightly. Started from StartBackgroundPollers. func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { if marketplaceDisabled() { s.logger.Info("marketplace disabled", "env", hubMarketplaceDisableEnv) @@ -426,11 +503,11 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { rf.mu.Unlock() schedule := marketplaceSchedule() s.logger.Info("marketplace refresh started", "schedule_utc", schedule, "corpus_repos", len(marketplaceRepos())) - if s.marketplaceShouldSeed() { - s.logger.Warn("marketplace catalogue is empty: seeding from the CNCF corpus in the background", "delay", marketplaceSeedDelay.String()) + if due := s.marketplaceDueRepos(); len(due) > 0 { + s.logger.Warn("marketplace catalogue has repositories due for ingest: resuming in the background", "due", len(due), "delay", marketplaceSeedDelay.String()) marketplaceSleep(ctx, marketplaceSeedDelay) if ctx.Err() == nil { - if _, err := s.runMarketplaceRefresh(ctx, nil, "seed"); err != nil { + if _, err := s.runMarketplaceRefresh(ctx, due, "seed"); err != nil { s.logger.Warn("marketplace seed did not run", "error", err) } } @@ -452,12 +529,48 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { } } -// marketplaceShouldSeed is true when the catalogue has no entries and the -// hub can reach GitHub with a token. -func (s *HubServer) marketplaceShouldSeed() bool { +// marketplaceRepoDue reports whether a repository needs ingesting at +// startup given its last record (ok=false when there is none). +func marketplaceRepoDue(rec marketplaceRefreshRecord, ok bool, now time.Time) bool { + switch { + case !ok: + return true + case rec.Format != marketplaceRecordFormat: + return true + case rec.Error != "": + return now.Sub(rec.CheckedAt) >= marketplaceRetryErrorAfter + default: + return now.Sub(rec.CheckedAt) >= marketplaceStaleAfter + } +} + +// marketplaceDueRepos is the sorted subset of marketplaceRepos the startup +// pass should ingest, or nil when the hub has no GitHub token (nothing can +// be fetched, so nothing is due). +func (s *HubServer) marketplaceDueRepos() []string { if hubGitHubAuthTokenForRequest(context.Background()) == "" { - return false + return nil + } + st := s.marketplace() + now := s.refresher().now().UTC() + var due []string + for _, repo := range marketplaceRepos() { + rec, ok := st.refreshRecord(repo) + if marketplaceRepoDue(rec, ok, now) { + due = append(due, repo) + } + } + return due +} + +// marketplacePendingRepos counts corpus repositories with no refresh record. +func (s *HubServer) marketplacePendingRepos() []string { + st := s.marketplace() + var pending []string + for _, repo := range marketplaceRepos() { + if _, ok := st.refreshRecord(repo); !ok { + pending = append(pending, repo) + } } - idx, _, _, err := s.marketplace().Index() - return err == nil && len(idx.Entries) == 0 + return pending } diff --git a/src/pkg/hub/marketplace_store.go b/src/pkg/hub/marketplace_store.go index 5864ca561d..5f83eb1775 100644 --- a/src/pkg/hub/marketplace_store.go +++ b/src/pkg/hub/marketplace_store.go @@ -24,6 +24,7 @@ import ( // workflows//.bundle.json immutable published bundles // index.json derived catalogue index // refresh/__.json last ingest run per corpus repo +// last-refresh.json summary of the last completed run // // Entries are keyed name@version and never overwritten: a republish with the // same key and hash is a no-op, a different hash is a 409. Curated versions @@ -302,9 +303,18 @@ func marketplaceNormaliseIndex(idx *bundle.Index) { } } +// marketplaceRecordFormat versions the refresh record. Bumping it makes +// every existing record due at the next startup pass, which is how a fix to +// the ingest itself (not a change in the repository) gets re-crawled: +// +// 2 — tarball extraction recognises pax_global_header, so the pipeline +// files are actually read (format 1 recorded 0 files for most repos). +const marketplaceRecordFormat = 2 + // marketplaceRefreshRecord is what one ingest run remembers about a repo. type marketplaceRefreshRecord struct { Repo string `json:"repo"` + Format int `json:"format,omitempty"` SourceSHA string `json:"sourceSHA"` FilesHash string `json:"filesHash"` Files int `json:"files"` @@ -334,6 +344,7 @@ func (st *marketplaceStore) refreshRecord(repo string) (marketplaceRefreshRecord } func (st *marketplaceStore) saveRefreshRecord(rec marketplaceRefreshRecord) error { + rec.Format = marketplaceRecordFormat data, err := json.MarshalIndent(rec, "", " ") if err != nil { return err @@ -343,6 +354,37 @@ func (st *marketplaceStore) saveRefreshRecord(rec marketplaceRefreshRecord) erro return marketplaceWriteAtomic(filepath.Join(st.refreshDir(), marketplaceRepoSlug(rec.Repo)+".json"), data) } +func (st *marketplaceStore) lastRefreshPath() string { + return filepath.Join(st.root, "last-refresh.json") +} + +// saveLastRefresh persists a run summary so /api/marketplace/status can +// report it across hub restarts. +func (st *marketplaceStore) saveLastRefresh(res *marketplaceRefreshResult) error { + data, err := json.MarshalIndent(res, "", " ") + if err != nil { + return err + } + st.mu.Lock() + defer st.mu.Unlock() + return marketplaceWriteAtomic(st.lastRefreshPath(), data) +} + +// loadLastRefresh returns the persisted run summary, or nil when none. +func (st *marketplaceStore) loadLastRefresh() *marketplaceRefreshResult { + st.mu.RLock() + defer st.mu.RUnlock() + data, err := os.ReadFile(st.lastRefreshPath()) + if err != nil { + return nil + } + var res marketplaceRefreshResult + if json.Unmarshal(data, &res) != nil || res.StartedAt.IsZero() { + return nil + } + return &res +} + // refreshRecords lists every recorded run. func (st *marketplaceStore) refreshRecords() []marketplaceRefreshRecord { st.mu.RLock() diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index 5a0230dacf..185a7596a6 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -524,11 +525,13 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { } s := marketplaceTestHub(t, fetch) - // Non-admin / anon are refused by requireAdmin. - rec := mktDo(s, httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh?repo=a/b", nil)) - if rec.Code != 401 && rec.Code != 403 { - t.Fatalf("anon refresh: %d", rec.Code) + // Anon is refused. + anon := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh?repo=a/b", nil) + anon.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, anon); rec.Code != 401 { + t.Fatalf("anon refresh: %d %s", rec.Code, rec.Body.String()) } + var rec *httptest.ResponseRecorder admin := func(target string) *http.Request { req := httptest.NewRequest(http.MethodPost, target, nil) req.AddCookie(testAuthCookie(hubAdminUsername)) @@ -588,12 +591,298 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if status["entries"].(float64) != 1 || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { t.Fatalf("status body: %s", rec.Body.String()) } + if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { + t.Fatalf("pending/lastErrors: %s", rec.Body.String()) + } + if _, ok := status["failing"]; ok { + t.Fatal("non-verbose status must not list failing repos") + } byTier := status["byTier"].(map[string]interface{}) if byTier[bundle.TierIngestedUnreviewed].(float64) != 1 { t.Fatalf("byTier: %v", byTier) } } +// Resume after restart: a store with a record for one repo and a published +// entry must still trigger the startup pass for the rest, and a second +// start with everything fresh must not crawl again. +func TestMarketplaceSeedResumesAfterRestart(t *testing.T) { + t.Setenv(hubGitHubTokenEnv, "pat-token") + oldDelay := marketplaceSleep + marketplaceSleep = func(ctx context.Context, d time.Duration) { + if d != marketplaceSeedDelay { + <-ctx.Done() // the nightly park + } + } + t.Cleanup(func() { marketplaceSleep = oldDelay }) + + var mu sync.Mutex + fetched := map[string]int{} + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + mu.Lock() + fetched[repo]++ + mu.Unlock() + if repo == "konflux-ci/konflux-ci" { + return mktFixtureTree(t, "konflux-ci__konflux-ci"), "k1", nil + } + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + t.Setenv(hubMarketplaceExtraReposEnv, "konflux-ci/konflux-ci,broken/repo") + all := marketplaceRepos() + + // "Before the restart": one repo ingested and published, nothing else. + if _, err := s.runMarketplaceRefresh(context.Background(), []string{"konflux-ci/konflux-ci"}, "seed"); err != nil { + t.Fatal(err) + } + if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1 { + t.Fatalf("precondition: %d entries", len(idx.Entries)) + } + due := s.marketplaceDueRepos() + if len(due) != len(all)-1 { + t.Fatalf("due after partial seed: %d, want %d", len(due), len(all)-1) + } + for _, r := range due { + if r == "konflux-ci/konflux-ci" { + t.Fatal("freshly ingested repo must not be due") + } + } + + // "After the restart": StartMarketplaceRefresh resumes the rest. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.StartMarketplaceRefresh(ctx); close(done) }() + rf := s.refresher() + testutil.Eventually(t, 10*time.Second, func() bool { + rf.mu.Lock() + defer rf.mu.Unlock() + return rf.last != nil && rf.last.Trigger == "seed" && rf.last.Repos == len(all)-1 + }, "startup pass did not resume the remaining repos") + cancel() + <-done + mu.Lock() + n := fetched["konflux-ci/konflux-ci"] + mu.Unlock() + if n != 1 { + t.Fatalf("konflux fetched %d times; the resume must skip fresh records", n) + } + if len(s.marketplace().refreshRecords()) != len(all) { + t.Fatalf("records after resume: %d, want %d", len(s.marketplace().refreshRecords()), len(all)) + } + + // Everything fresh except the failing repo, which is retried only after + // marketplaceRetryErrorAfter. + if due := s.marketplaceDueRepos(); len(due) != 0 { + t.Fatalf("fresh records must not be due: %v", due) + } + rec, _ := s.marketplace().refreshRecord("broken/repo") + rec.CheckedAt = rec.CheckedAt.Add(-marketplaceRetryErrorAfter) + _ = s.marketplace().saveRefreshRecord(rec) + if due := s.marketplaceDueRepos(); len(due) != 1 || due[0] != "broken/repo" { + t.Fatalf("stale error must be due: %v", due) + } + + // Old-format records are due regardless of age. + now := rf.now() + if !marketplaceRepoDue(marketplaceRefreshRecord{Format: 1, CheckedAt: now}, true, now) { + t.Error("old format must be due") + } + if marketplaceRepoDue(marketplaceRefreshRecord{Format: marketplaceRecordFormat, CheckedAt: now.Add(-marketplaceStaleAfter + time.Minute)}, true, now) { + t.Error("fresh record must not be due") + } + if !marketplaceRepoDue(marketplaceRefreshRecord{Format: marketplaceRecordFormat, CheckedAt: now.Add(-marketplaceStaleAfter)}, true, now) { + t.Error("stale record must be due") + } +} + +// The last-run summary survives a restart (a fresh refresher over the same +// store), and the status route reports it with the error count. +func TestMarketplaceLastRefreshPersists(t *testing.T) { + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + res, err := s.runMarketplaceRefresh(context.Background(), []string{"ok/repo", "broken/repo"}, "nightly") + if err != nil || len(res.Errors) != 1 { + t.Fatalf("run: %v %+v", err, res) + } + + // Simulate a restart: a new HubServer over the same marketplace dir. + s2 := newHandlerHub() + s2.marketplaceRefresher = &marketplaceRefresher{fetch: fetch, now: time.Now} + rf := s2.refresher() + rf.mu.Lock() + last := rf.last + rf.mu.Unlock() + if last == nil || last.Trigger != "nightly" || last.Repos != 2 || len(last.Errors) != 1 || !last.StartedAt.Equal(res.StartedAt) { + t.Fatalf("persisted last refresh: %+v", last) + } + + rec := mktDo(s2, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) + var status map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status["lastRefresh"] == nil || status["lastRefreshErrors"].(float64) != 1 || status["failingRepos"].(float64) != 1 { + t.Fatalf("status after restart: %s", rec.Body.String()) + } + // Corrupt or missing summary → nil, not a crash. + st := s2.marketplace() + if err := os.WriteFile(st.lastRefreshPath(), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if st.loadLastRefresh() != nil { + t.Error("corrupt summary must load as nil") + } + if err := os.Remove(st.lastRefreshPath()); err != nil { + t.Fatal(err) + } + if st.loadLastRefresh() != nil { + t.Error("missing summary must load as nil") + } +} + +// Auth matrix for POST /api/marketplace/refresh: anon 401, spoke bearer ok +// (rate-limited), admin ok and unlimited; and the verbose status gate. +func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + if repo == "broken/repo" { + return nil, "", errors.New("tarball: 404 Not Found") + } + return map[string][]byte{}, "none", nil + } + s := marketplaceTestHub(t, fetch) + clock := s.refresher().now() + s.refresher().now = func() time.Time { return clock } + t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo") + + spoke := func(target string) *http.Request { + req := httptest.NewRequest(http.MethodPost, target, nil) + req.Header.Set("Authorization", heartbeatBearer(testHubSecret, mktTestHive)) + req.Header.Set(marketplaceHiveIDHeader, mktTestHive) + return req + } + admin := func(method, target string) *http.Request { + req := httptest.NewRequest(method, target, nil) + req.AddCookie(testAuthCookie(hubAdminUsername)) + req.Header.Set("Origin", "http://localhost") + return req + } + + // Anon. + anon := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh", nil) + anon.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, anon); rec.Code != 401 { + t.Fatalf("anon: %d %s", rec.Code, rec.Body.String()) + } + // Non-admin session. + user := httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh", nil) + user.AddCookie(testAuthCookie("someuser")) + user.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, user); rec.Code != 403 { + t.Fatalf("non-admin session: %d", rec.Code) + } + // Spoke with a wrong bearer. + bad := spoke("/api/marketplace/refresh") + bad.Header.Set("Authorization", heartbeatBearer("wrong", mktTestHive)) + if rec := mktDo(s, bad); rec.Code != 401 { + t.Fatalf("bad bearer: %d", rec.Code) + } + + // Spoke single repo: ok, then burst-limited. + rec := mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) + if rec.Code != 200 || !strings.Contains(rec.Body.String(), "404 Not Found") { + t.Fatalf("spoke single: %d %s", rec.Code, rec.Body.String()) + } + for i := 1; i < marketplaceRepoRefreshBurst; i++ { + if rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("spoke single #%d: %d", i, rec.Code) + } + } + rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")) + if rec.Code != 429 || rec.Header().Get("Retry-After") == "" { + t.Fatalf("spoke burst: %d %s retry-after=%q", rec.Code, rec.Body.String(), rec.Header().Get("Retry-After")) + } + // Admin single repo is not limited. + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("admin single while spoke limited: %d", rec.Code) + } + // Window passes → spoke allowed again. + clock = clock.Add(marketplaceRepoRefreshWindow) + if rec = mktDo(s, spoke("/api/marketplace/refresh?repo=ok/repo")); rec.Code != 200 { + t.Fatalf("spoke after window: %d", rec.Code) + } + + // Spoke full run: 202 once per hour across callers; admin unaffected. + rf := s.refresher() + waitIdle := func() { + testutil.Eventually(t, 5*time.Second, func() bool { + rf.mu.Lock() + defer rf.mu.Unlock() + return !rf.running + }, "background refresh did not finish") + } + rec = mktDo(s, spoke("/api/marketplace/refresh")) + if rec.Code != 202 { + t.Fatalf("spoke full: %d %s", rec.Code, rec.Body.String()) + } + waitIdle() + rf.mu.Lock() + trigger := rf.last.Trigger + rf.mu.Unlock() + if trigger != "on-demand:hive:"+mktTestHive { + t.Fatalf("trigger = %q", trigger) + } + rec = mktDo(s, spoke("/api/marketplace/refresh")) + if rec.Code != 429 || !strings.Contains(rec.Body.String(), "try again in") { + t.Fatalf("spoke full again: %d %s", rec.Code, rec.Body.String()) + } + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh")); rec.Code != 202 { + t.Fatalf("admin full while spoke limited: %d %s", rec.Code, rec.Body.String()) + } + waitIdle() + clock = clock.Add(marketplaceFullRefreshInterval) + if rec = mktDo(s, spoke("/api/marketplace/refresh")); rec.Code != 202 { + t.Fatalf("spoke full after interval: %d", rec.Code) + } + waitIdle() + + // Verbose status: anon 401, spoke and admin get the failing list. + if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil)); rec.Code != 401 { + t.Fatalf("anon verbose: %d", rec.Code) + } + sv := httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil) + sv.Header.Set("Authorization", heartbeatBearer(testHubSecret, mktTestHive)) + sv.Header.Set(marketplaceHiveIDHeader, mktTestHive) + rec = mktDo(s, sv) + if rec.Code != 200 { + t.Fatalf("spoke verbose: %d %s", rec.Code, rec.Body.String()) + } + var verbose struct { + Failing []marketplaceFailingRepo `json:"failing"` + Pending []string `json:"pending"` + Pend int `json:"pendingRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &verbose); err != nil { + t.Fatal(err) + } + if len(verbose.Failing) != 1 || verbose.Failing[0].Repo != "broken/repo" || !strings.Contains(verbose.Failing[0].Error, "404") { + t.Fatalf("failing: %+v", verbose.Failing) + } + if verbose.Pending == nil || len(verbose.Pending) != verbose.Pend || verbose.Pend != 0 { + t.Fatalf("pending: %+v / %d", verbose.Pending, verbose.Pend) + } + if rec = mktDo(s, admin(http.MethodGet, "/api/marketplace/status?verbose=1")); rec.Code != 200 || !strings.Contains(rec.Body.String(), `"failing"`) { + t.Fatalf("admin verbose: %d %s", rec.Code, rec.Body.String()) + } +} + func TestMarketplaceReposMergesExtra(t *testing.T) { t.Setenv(hubMarketplaceExtraReposEnv, " Foo/Bar ,foo/bar, nope ,") repos := marketplaceRepos() @@ -645,26 +934,49 @@ func TestMarketplaceHonourRateLimit(t *testing.T) { h := http.Header{} h.Set("X-RateLimit-Remaining", "1000") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("plenty remaining must not sleep") } h.Set("X-RateLimit-Remaining", "3") h.Set("X-RateLimit-Reset", "garbage") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("unparseable reset must not sleep") } h.Set("X-RateLimit-Reset", "1") - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != 0 { t.Fatal("past reset must not sleep") } h.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(2*time.Hour).Unix(), 10)) - marketplaceHonourRateLimit(context.Background(), h) + marketplaceHonourRateLimit(context.Background(), 200, h) if slept != marketplaceRateMaxWait { t.Fatalf("far reset should cap at max wait, slept %v", slept) } + + // Secondary rate limit: 403/429 + Retry-After sleeps that long even + // with budget remaining; capped; a 403 without Retry-After does not. + slept = 0 + sec := http.Header{} + sec.Set("X-RateLimit-Remaining", "4000") + sec.Set("Retry-After", "7") + marketplaceHonourRateLimit(context.Background(), 403, sec) + if slept != 7*time.Second { + t.Fatalf("403 Retry-After: slept %v", slept) + } + slept = 0 + sec.Set("Retry-After", "99999") + marketplaceHonourRateLimit(context.Background(), 429, sec) + if slept != marketplaceRateMaxWait { + t.Fatalf("429 Retry-After cap: slept %v", slept) + } + slept = 0 + sec.Del("Retry-After") + marketplaceHonourRateLimit(context.Background(), 403, sec) + if slept != 0 { + t.Fatalf("403 without Retry-After and plenty remaining: slept %v", slept) + } } func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { @@ -680,8 +992,8 @@ func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { t.Setenv(hubMarketplaceDisableEnv, "") t.Setenv("GITHUB_TOKEN", "") t.Setenv("HIVE_HUB_GITHUB_TOKEN", "") - if s.marketplaceShouldSeed() { - t.Error("no token → no seed") + if due := s.marketplaceDueRepos(); due != nil { + t.Errorf("no token → nothing due, got %d", len(due)) } // Empty schedule: loop parks on ctx and exits on cancel. @@ -703,6 +1015,12 @@ func mktTarball(t *testing.T, files map[string]string) []byte { var buf bytes.Buffer gz := gzip.NewWriter(&buf) tw := tar.NewWriter(gz) + // GitHub tarballs (git archive) lead with a pax_global_header; the + // production bug behind #17620's one-entry catalogue was treating it as + // a second top-level directory. + if err := tw.WriteHeader(&tar.Header{Name: "pax_global_header", Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": "abc"}}); err != nil { + t.Fatal(err) + } for name, body := range files { if err := tw.WriteHeader(&tar.Header{Name: "repo-abc/" + name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { t.Fatal(err) @@ -778,8 +1096,8 @@ func TestMarketplaceGitHubFetcher(t *testing.T) { t.Error("branch lookup failure should fail") } - // With a token and an empty catalogue the seed gate opens. - if !s.marketplaceShouldSeed() { - t.Error("token + empty catalogue → should seed") + // With a token and no records every corpus repo is due. + if due := s.marketplaceDueRepos(); len(due) != len(marketplaceRepos()) { + t.Errorf("token + empty catalogue → all due, got %d", len(due)) } } From d077228f6ed846c33464b2b2fdd3971d3598e0b9 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:43:40 +0000 Subject: [PATCH 014/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20bound=20consecuti?= =?UTF-8?q?ve=20stale-snapshot=20drops=20in=20the=20status=20builder=20(#1?= =?UTF-8?q?7744)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: bound consecutive stale-snapshot drops in the status builder Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: update staleDropStarvationThreshold comment for bounded publish The constant's doc comment still described the pre-PR contract (drop logged at Warn). It now marks the point where the build is published anyway to bound staleness. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: resync api-reference citations after server.go route shift Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...fixed-status-snapshot-bounded-staleness.md | 1 + src/docs/api-reference/auth.md | 6 +-- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++++----- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/dashboard/server.go | 44 +++++++++++-------- src/pkg/dashboard/status_stale_drop_test.go | 31 ++++++++++--- 7 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 changelog.d/fixed-status-snapshot-bounded-staleness.md diff --git a/changelog.d/fixed-status-snapshot-bounded-staleness.md b/changelog.d/fixed-status-snapshot-bounded-staleness.md new file mode 100644 index 0000000000..30daf850a4 --- /dev/null +++ b/changelog.d/fixed-status-snapshot-bounded-staleness.md @@ -0,0 +1 @@ +- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 36be77b0ee..08dba0af9e 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1681` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1682` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1683` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1683` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1684` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1685` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 1bdc09aed9..c8255286a9 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1688` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1690` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index c59b00248b..e41ee91d74 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1661` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1662` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1663` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1669` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1674` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1675` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1677` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1663` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1664` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1665` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1671` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1676` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1677` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1682` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 8dae7348f7..b8e71aaf96 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1751` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1753` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 233bebed05..7a72a42eb2 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -97,8 +97,10 @@ func isOwnerRole(role string) bool { // A WRONG proof header is ALWAYS rejected regardless of this setting. var proxyProofRequired = true -// staleDropStarvationThreshold is the number of consecutive stale-snapshot -// drops (no publish between) at which the drop is logged at Warn. +// staleDropStarvationThreshold bounds consecutive stale-snapshot drops (no +// publish between): the build that would be the Nth consecutive drop is +// published anyway, logged at Warn, so sustained mutation churn cannot +// starve consumers. The counter resets on every publish. const staleDropStarvationThreshold = 5 type Server struct { @@ -2797,24 +2799,30 @@ func (s *Server) UpdateStatusIfFresh(status *StatusPayload, buildEpoch uint64) b s.statusMu.Lock() if buildEpoch < s.statusMutationEpoch { - // A mutation landed after this snapshot's build began: its data may - // predate the mutation. Drop it — the mutation's own refresh rebuild - // (which began after the epoch bump) publishes the fresh state. curEpoch := s.statusMutationEpoch - s.statusStaleDrops++ - drops := s.statusStaleDrops - s.statusConsecutiveStaleDrops++ - consecutive := s.statusConsecutiveStaleDrops - s.statusMu.Unlock() - // A single drop is normal optimistic-concurrency self-healing; only a - // run of drops with no publish in between is an anomaly. - level := slog.LevelInfo - if consecutive >= staleDropStarvationThreshold { - level = slog.LevelWarn + if s.statusConsecutiveStaleDrops+1 >= staleDropStarvationThreshold { + // Bounded staleness: under sustained mutation churn every build + // loses the race, so publish this one rather than starve + // consumers. The counter resets below with the publish. + drops := s.statusStaleDrops + consecutive := s.statusConsecutiveStaleDrops + s.logger.Warn("publishing possibly-stale status snapshot after consecutive stale drops", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + } else { + // A mutation landed after this snapshot's build began: its data + // may predate the mutation. Drop it — the mutation's own refresh + // rebuild (which began after the epoch bump) publishes the fresh + // state. + s.statusStaleDrops++ + drops := s.statusStaleDrops + s.statusConsecutiveStaleDrops++ + consecutive := s.statusConsecutiveStaleDrops + s.statusMu.Unlock() + // A drop is a normal optimistic-concurrency race. + s.logger.Info("dropping stale status snapshot built before a mutation", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + return false } - s.logger.Log(context.Background(), level, "dropping stale status snapshot built before a mutation", - "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) - return false } // Attach the advisory ACMM level-up recommendation (#5225) from the SAME diff --git a/src/pkg/dashboard/status_stale_drop_test.go b/src/pkg/dashboard/status_stale_drop_test.go index 73c7abecb9..3ef712a878 100644 --- a/src/pkg/dashboard/status_stale_drop_test.go +++ b/src/pkg/dashboard/status_stale_drop_test.go @@ -97,21 +97,38 @@ func TestUpdateStatusIfFresh_StaleDropCounterAccumulates(t *testing.T) { } } -func TestUpdateStatusIfFresh_StarvationWarnsAfterThreshold(t *testing.T) { +func TestUpdateStatusIfFresh_StarvationPublishesAfterThreshold(t *testing.T) { var logBuf bytes.Buffer srv := NewServer(0, slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelInfo}))) for i := 0; i < staleDropStarvationThreshold-1; i++ { epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) + if srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("drop %d below the bound must not publish", i+1) + } } - if staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("below the threshold must not WARN, got:\n%s", logBuf.String()) + if staleDropLoggedAt(logBuf.String(), "WARN") || strings.Contains(logBuf.String(), "level=WARN msg=\"publishing possibly-stale") { + t.Fatalf("below the bound must not WARN, got:\n%s", logBuf.String()) } + seq := srv.currentStatusSeq() epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) - if !staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("threshold-th consecutive drop must WARN, got:\n%s", logBuf.String()) + if !srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("the bound-th stale build must publish, got:\n%s", logBuf.String()) + } + if got := srv.currentStatusSeq(); got != seq+1 { + t.Fatalf("statusSeq = %d, want %d", got, seq+1) + } + if n := strings.Count(logBuf.String(), "level=WARN msg=\"publishing possibly-stale"); n != 1 { + t.Fatalf("want exactly one bounded-staleness WARN, got %d:\n%s", n, logBuf.String()) + } + if staleDropLoggedAt(logBuf.String(), "WARN") { + t.Fatalf("per-drop line must never be WARN, got:\n%s", logBuf.String()) + } + srv.statusMu.RLock() + consecutive := srv.statusConsecutiveStaleDrops + srv.statusMu.RUnlock() + if consecutive != 0 { + t.Fatalf("counter must reset after the forced publish, got %d", consecutive) } } From 3d319688131336125d582541b607ba10abf2a17d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:44:02 +0000 Subject: [PATCH 015/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20log=20Prow=20ok-t?= =?UTF-8?q?o-test=20rejection=20at=20WARN=20only=20when=20first=20noted=20?= =?UTF-8?q?(#17754)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...fixed-prow-oktotest-rejection-warn-once.md | 1 + .../automerge_contributor_prow_oktotest.go | 11 +++++--- ...utomerge_contributor_prow_oktotest_test.go | 27 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-prow-oktotest-rejection-warn-once.md diff --git a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md new file mode 100644 index 0000000000..6e33a4e8bb --- /dev/null +++ b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md @@ -0,0 +1 @@ +- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go index f1ebb2f69c..04e9252929 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go @@ -68,11 +68,14 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, } if state.rejected { c.markWorkflowApprovalUntrusted(owner, repo, number) - if !state.rejectionNoted { - c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + args := []any{"repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw} + const msg = "contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it" + if state.rejectionNoted { + c.debug(msg, args...) + return } - c.warn("contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it", - "repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw) + c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + c.warn(msg, args...) return } if state.postedForHead || !(hgithub.ProwManagedLabels(labels) || state.prowManaged) { diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go index b884c6cfd8..2a985bd06a 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go @@ -7,7 +7,9 @@ package automerge // that shape, never otherwise, and stops for good once Prow rejects the App. import ( + "bytes" "context" + "log/slog" "strings" "testing" "time" @@ -181,3 +183,28 @@ func TestContributorLaneRelayEvaluationDoesNotPostProwOkToTest(t *testing.T) { t.Fatalf("comments = %d %q, want none from a relay evaluation", fx.comments, fx.allComments) } } + +// A rejection the PR thread already records is a handled state: a restarted +// hive must not log it at WARN again. +func TestContributorLaneProwRejectionAlreadyNotedLogsBelowWarn(t *testing.T) { + fx := prowOkToTestFixture() + fx.prComments = []map[string]any{ + appComment(2, workflowApprovalCommandBody("oldhead")), + prowComment(3, "@"+testHiveAppBotLogin+": Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message."), + } + api := newContributorLaneAPI(t, &fx) + defer api.Close() + policy := defaultContributorLanePolicy() + clock := &contributorLaneClock{at: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + var logs bytes.Buffer + for run := 0; run < 2; run++ { + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorSoak: soakedTracker(clock)}) + engine.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, _, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy); err != nil { + t.Fatalf("run %d err = %v", run, err) + } + } + if got := strings.Count(logs.String(), "ok-to-test not posted again"); got != 1 { + t.Fatalf("WARN lines = %d in %q, want 1 (only the run that left the note)", got, logs.String()) + } +} From 47b72257f32115bf9d26b35d8c4441107bf051d0 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:51:44 -0700 Subject: [PATCH 016/166] =?UTF-8?q?=E2=9C=A8=20crux:=20Substrate=20preflig?= =?UTF-8?q?ht=20prover=20(operator,=20CRDs,=20gVisor=20RuntimeClass,=20fea?= =?UTF-8?q?ture=20gates)=20(#17766)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ crux: Substrate preflight prover (operator, CRDs, gVisor RuntimeClass, feature gates) Extends the sandbox.runtime: substrate preflight with the cluster facts behind a failing kick: operator CRDs and workloads, ready WorkerPools and their sandbox classes, gVisor (SandboxConfig or runsc RuntimeClass) and KVM availability, node kernels, the certificates.k8s.io APIs behind the PodCertificateRequest and ClusterTrustBundle feature gates, and an mTLS CONNECT probe of atenet-router. Skipped outside a cluster; one warning when RBAC is missing, with a manifest that grants it. Fixes #17698 Signed-off-by: Andrew Anderson * fix(crux/sandbox): egress policy atespace + in-place workspace clear found by #17697 kick The first real kick through Substrate v1.0.0 (kind, gVisor) surfaced two defects in the launcher path: CreateActorEgressPolicy was rejected with 'egress_policy.metadata.atespace: Required value', and the bridge's PUT /workspace failed with EBUSY because os.RemoveAll targeted the durableDir mount point itself. Set the atespace on the singleton policy and clear the workspace directory's contents in place. Signed-off-by: Andrew Anderson * test(crux/sandbox): run the kube/router preflight halves only in a Pod wired for Substrate CI runs in Pods, so /var/run/secrets/kubernetes.io/serviceaccount/token exists there and the kube/router halves of the prover ran during the Control-API-only tests (crux/sandbox and crux/agent). The prover now requires both the Kubernetes token and hive's projected Substrate token (opts.token_file, /var/run/secrets/ate/token by default) before probing the cluster; the Kubernetes token path is a package variable the sandbox test package pins to a nonexistent file, with a helper that flips both markers on for the tests covering the in-cluster branches. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../added-substrate-preflight-prover.md | 1 + src/crux/sandbox/substrate.go | 3 + src/crux/sandbox/substrate_preflight.go | 23 +- .../sandbox/substrate_preflight_cluster.go | 534 ++++++++++++++++++ .../substrate_preflight_cluster_test.go | 513 +++++++++++++++++ src/crux/sandbox/substratebridge/bridge.go | 21 +- .../sandbox/substratebridge/bridge_test.go | 47 ++ .../k8s/sandbox-substrate-preflight-rbac.yaml | 72 +++ 8 files changed, 1209 insertions(+), 5 deletions(-) create mode 100644 changelog.d/added-substrate-preflight-prover.md create mode 100644 src/crux/sandbox/substrate_preflight_cluster.go create mode 100644 src/crux/sandbox/substrate_preflight_cluster_test.go create mode 100644 src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml diff --git a/changelog.d/added-substrate-preflight-prover.md b/changelog.d/added-substrate-preflight-prover.md new file mode 100644 index 0000000000..dd9d5bd5d9 --- /dev/null +++ b/changelog.d/added-substrate-preflight-prover.md @@ -0,0 +1 @@ +- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index 1ed56c0ee9..9ad2d81f2d 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -303,6 +303,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e if spec.NetworkNone { policy.Rules = nil } + // The Control API requires the singleton policy's atespace to match the + // actor's (egress_policy.metadata.atespace is a required field). + policy.Metadata.Atespace = ref.Atespace if err := control.CreateActorEgressPolicy(ctx, ref, policy); err != nil { cleanupActor() return res, fmt.Errorf("substrate: setting egress policy on actor %s: %w", name, err) diff --git a/src/crux/sandbox/substrate_preflight.go b/src/crux/sandbox/substrate_preflight.go index 94964cc5e9..e166591dfa 100644 --- a/src/crux/sandbox/substrate_preflight.go +++ b/src/crux/sandbox/substrate_preflight.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "fmt" + "net" "sort" "strings" "sync" @@ -12,10 +13,11 @@ import ( ) // Substrate cluster-prerequisite preflight (ADR-0026 phase 2; the sibling -// of the OpenShell preflight in openshell_preflight.go). Hive cannot see the -// node kernel or the WorkerPool CRDs from its pod without cluster RBAC, but -// the Control API answers whether it is up and which workers — and so which -// sandbox classes — exist; that is what decides whether a kick can land. +// of the OpenShell preflight in openshell_preflight.go). The Control API +// answers whether it is up and which workers — and so which sandbox classes +// — exist; that is what decides whether a kick can land. The cluster and +// router halves in substrate_preflight_cluster.go (#17698) add the +// Kubernetes facts behind that answer when hive runs in the cluster. const substratePreflightTTL = 5 * time.Minute @@ -32,6 +34,10 @@ type SubstratePreflighter struct { newClient func(SubstrateOptions) (substrateClient, error) now func() time.Time + // kube and routerDial let tests point the cluster and router halves at + // fakes; nil means the in-cluster API and a plain TCP dial. + kube *substrateKube + routerDial func(ctx context.Context, network, addr string) (net.Conn, error) } // DefaultSubstratePreflighter backs the warnings AgentSandboxGateWarnings emits. @@ -58,7 +64,16 @@ func (p *SubstratePreflighter) Check(ctx context.Context, opts SubstrateOptions) return append([]string(nil), warnings...) } +// probe runs the three halves — Control API, cluster facts, router — and +// concatenates their warnings; a down control plane does not hide the +// cluster facts that explain it. func (p *SubstratePreflighter) probe(ctx context.Context, opts SubstrateOptions) []string { + out := p.probeControl(ctx, opts) + out = append(out, p.probeCluster(ctx, opts)...) + return append(out, p.probeRouter(ctx, opts)...) +} + +func (p *SubstratePreflighter) probeControl(ctx context.Context, opts SubstrateOptions) []string { ep := opts.endpoint() ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) defer cancel() diff --git a/src/crux/sandbox/substrate_preflight_cluster.go b/src/crux/sandbox/substrate_preflight_cluster.go new file mode 100644 index 0000000000..bf9de8e81a --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster.go @@ -0,0 +1,534 @@ +package sandbox + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster-level half of the Substrate preflight (hivecommons/hive#17698). +// The Control API half in substrate_preflight.go answers "is the control +// plane up and does it have workers"; this half reads the Kubernetes facts +// that explain *why* it is not, or why a kick would fail even though it is: +// the operator's CRDs and Deployments, a ready WorkerPool for the class the +// operator selected, gVisor (a SandboxConfig of class gvisor, or a runsc +// RuntimeClass) or KVM for microVMs, the certificates.k8s.io APIs behind the +// PodCertificateRequest and ClusterTrustBundle feature gates Substrate's +// SPIFFE identities need, the node kernels, and whether atenet-router +// accepts hive's client certificate. Everything is read-only, bounded by one +// request timeout per call, and skipped entirely when hive is not running in +// a cluster (no projected ServiceAccount token), where the Control API probe +// is all there is to check. + +const ( + // SubstrateDefaultOperatorNamespace is where ate-setup installs the + // control plane when the operator does not rename it. + SubstrateDefaultOperatorNamespace = "ate-system" + + substrateKubeAPIServer = "https://kubernetes.default.svc" + substrateKubeCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + substrateKubeMaxBody = 8 << 20 + substrateKubeNodeLimit = 500 + + substrateClassGVisor = "gvisor" + substrateClassMicroVM = "microvm" + substrateGVisorHandler = "runsc" + // substrateKVMResource is the extended resource atelet's device plugin + // advertises on nodes with /dev/kvm; the microVM class schedules on it. + substrateKVMResource = "ate.dev/kvm" + // substrateGVisorMinKernel is gVisor's documented host kernel floor. + substrateGVisorMinKernel = "4.14.77" + + substrateAteletDaemonSet = "atelet" + substrateCertsGroupPath = "/apis/certificates.k8s.io/" + substrateResourcePCR = "podcertificaterequests" + substrateResourceCTB = "clustertrustbundles" +) + +// substrateCRDs are the cluster-scoped CRDs ate-setup installs; Actors and +// ActorTemplates are Control API resources, not CRDs, so they are not here. +var substrateCRDs = []string{"workerpools.ate.dev", "sandboxconfigs.ate.dev"} + +// substrateOperatorDeployments must each have a ready replica for a kick to +// create an actor, schedule it, reach its bridge, and egress to GitHub. +var substrateOperatorDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"} + +// substrateCertsVersions are the certificates.k8s.io versions that may serve +// PodCertificateRequest and ClusterTrustBundle (beta in 1.34, GA later). +var substrateCertsVersions = []string{"v1", "v1beta1"} + +// substrateKubeTokenPath is the projected Kubernetes ServiceAccount token +// the kube client authenticates with. A variable so tests pin the +// environment instead of inheriting the runner's (CI itself runs in Pods). +var substrateKubeTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +// substrateInCluster reports whether hive runs in a Pod wired for Substrate: +// both the Kubernetes ServiceAccount token and hive's projected Substrate +// token exist. The kube and router halves of the preflight only run then; +// a laptop, or a CI Pod without the Substrate projection, gets the +// Control-API probe alone. +func substrateInCluster(opts SubstrateOptions) bool { + if _, err := os.Stat(substrateKubeTokenPath); err != nil { + return false + } + _, err := os.Stat(opts.tokenFile()) + return err == nil +} + +// substrateKube is the minimal read-only in-cluster API client the prover +// needs; kubejob has its own because importing it here would be a cycle. +type substrateKube struct { + server string + tokenPath string + caPath string + client *http.Client +} + +// errSubstrateKubeForbidden marks a 403: the probe stops after one warning +// rather than repeating the same RBAC gap per resource. +var errSubstrateKubeForbidden = errors.New("forbidden") + +// errSubstrateKubeNotFound marks a 404. +var errSubstrateKubeNotFound = errors.New("not found") + +func (k *substrateKube) get(ctx context.Context, path string) ([]byte, error) { + client, err := k.httpClient() + if err != nil { + return nil, err + } + reqCtx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, k.server+path, nil) + if err != nil { + return nil, err + } + if raw, err := os.ReadFile(k.tokenPath); err == nil { + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(raw))) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, substrateKubeMaxBody)) + if err != nil { + return nil, err + } + switch { + case resp.StatusCode == http.StatusForbidden: + return nil, errSubstrateKubeForbidden + case resp.StatusCode == http.StatusNotFound: + return nil, errSubstrateKubeNotFound + case resp.StatusCode/100 != 2: + return nil, fmt.Errorf("GET %s: HTTP %d", path, resp.StatusCode) + } + return body, nil +} + +func (k *substrateKube) httpClient() (*http.Client, error) { + if k.client != nil { + return k.client, nil + } + pem, err := os.ReadFile(k.caPath) + if err != nil { + return nil, fmt.Errorf("reading in-cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("in-cluster CA bundle contained no certificates") + } + k.client = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + return k.client, nil +} + +// substrateOperatorNamespace derives the operator namespace from the Control +// API endpoint (`api..svc[:port]`, with or without a scheme); any other +// shape means the default install. +func substrateOperatorNamespace(endpoint string) string { + addr, _, err := substrateAddress(endpoint) + if err != nil { + return SubstrateDefaultOperatorNamespace + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + parts := strings.Split(host, ".") + if len(parts) >= 3 && parts[0] == "api" && parts[2] == "svc" && parts[1] != "" { + return parts[1] + } + return SubstrateDefaultOperatorNamespace +} + +// Minimal projections of the API objects the prover reads. + +type substrateKubeList struct { + Items []json.RawMessage `json:"items"` +} + +type substrateKubeWorkload struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + SandboxClass string `json:"sandboxClass"` + SandboxClasses []struct { + Name string `json:"name"` + } `json:"sandboxClasses"` + } `json:"spec"` + Status struct { + ReadyReplicas int `json:"readyReplicas"` + NumberReady int `json:"numberReady"` + } `json:"status"` + Handler string `json:"handler"` +} + +type substrateKubeNode struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Allocatable map[string]string `json:"allocatable"` + NodeInfo struct { + KernelVersion string `json:"kernelVersion"` + } `json:"nodeInfo"` + } `json:"status"` +} + +type substrateKubeResourceList struct { + Resources []struct { + Name string `json:"name"` + } `json:"resources"` +} + +// clusterProbe holds one probe's state so the step functions stay short. +type substrateClusterProbe struct { + kube *substrateKube + ns string + sel map[string]string + out []string + // classes are the sandbox classes the ready WorkerPools offer; gvisor and + // kvm record whether the cluster can run each class. + classes map[string]bool + gvisor bool + kvm bool +} + +// substrateKubeItems GETs a collection and decodes each item into T. +func substrateKubeItems[T any](ctx context.Context, k *substrateKube, path string) ([]T, error) { + body, err := k.get(ctx, path) + if err != nil { + return nil, err + } + var list substrateKubeList + if err := json.Unmarshal(body, &list); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items := make([]T, 0, len(list.Items)) + for _, raw := range list.Items { + var item T + if err := json.Unmarshal(raw, &item); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items = append(items, item) + } + return items, nil +} + +// warnf appends one warning. +func (c *substrateClusterProbe) warnf(format string, args ...any) { + c.out = append(c.out, fmt.Sprintf(format, args...)) +} + +// probeCluster runs the Kubernetes checks and returns their warnings. A 403 +// anywhere ends the probe with one warning naming the RBAC the prover needs. +func (p *SubstratePreflighter) probeCluster(ctx context.Context, opts SubstrateOptions) []string { + kube := p.kube + if kube == nil { + if !substrateInCluster(opts) { + return nil + } + kube = &substrateKube{server: substrateKubeAPIServer, tokenPath: substrateKubeTokenPath, caPath: substrateKubeCAPath} + } + c := &substrateClusterProbe{kube: kube, ns: substrateOperatorNamespace(opts.endpoint()), sel: opts.WorkerSelector, classes: map[string]bool{}} + steps := []func(context.Context) error{c.crds, c.operator, c.pools, c.sandboxClasses, c.featureGates, c.nodes} + for _, step := range steps { + if err := step(ctx); err != nil { + if errors.Is(err, errSubstrateCRDsMissing) { + return c.out + } + if errors.Is(err, errSubstrateKubeForbidden) { + return append(c.out, "hive's ServiceAccount may not read the Substrate cluster facts (403); the operator, WorkerPool, gVisor and feature-gate checks were skipped — apply deploy/k8s/sandbox-substrate-preflight-rbac.yaml") + } + c.warnf("substrate cluster facts could not be read (%v); the operator, WorkerPool, gVisor and feature-gate checks were skipped", err) + return c.out + } + } + return c.out +} + +// crds checks the operator's CRDs are installed; missing ones end the probe +// early, because every later check would just restate "not installed". +func (c *substrateClusterProbe) crds(ctx context.Context) error { + missing := 0 + for _, name := range substrateCRDs { + _, err := c.kube.get(ctx, "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/"+name) + switch { + case err == nil: + case errors.Is(err, errSubstrateKubeNotFound): + missing++ + c.warnf("substrate CRD %s is not installed; the Substrate operator is absent and every kick on the substrate runtime will fail — run `ate-setup deploy ate-system` (see deploy/substrate.md)", name) + default: + return err + } + } + if missing > 0 { + return errSubstrateCRDsMissing + } + return nil +} + +// errSubstrateCRDsMissing stops the probe after the CRD warnings. +var errSubstrateCRDsMissing = errors.New("crds missing") + +// operator checks each control-plane Deployment and the atelet DaemonSet +// have a ready replica in the operator namespace. +func (c *substrateClusterProbe) operator(ctx context.Context) error { + deps, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/apps/v1/namespaces/"+c.ns+"/deployments") + if err != nil { + return err + } + ready := map[string]int{} + for _, d := range deps { + ready[d.Metadata.Name] = d.Status.ReadyReplicas + } + for _, name := range substrateOperatorDeployments { + n, ok := ready[name] + switch { + case !ok: + c.warnf("substrate operator deployment %s/%s is not installed; run `ate-setup deploy ate-system` (see deploy/substrate.md)", c.ns, name) + case n == 0: + c.warnf("substrate operator deployment %s/%s has no ready replica; kicks on the substrate runtime will fail until it is up (kubectl -n %s describe deploy %s)", c.ns, name, c.ns, name) + } + } + body, err := c.kube.get(ctx, "/apis/apps/v1/namespaces/"+c.ns+"/daemonsets/"+substrateAteletDaemonSet) + switch { + case errors.Is(err, errSubstrateKubeNotFound): + c.warnf("substrate node agent daemonset %s/%s is not installed; no node can host a worker — run `ate-setup deploy ate-system`", c.ns, substrateAteletDaemonSet) + return nil + case err != nil: + return err + } + var ds substrateKubeWorkload + if err := json.Unmarshal(body, &ds); err != nil { + return fmt.Errorf("daemonset %s: %w", substrateAteletDaemonSet, err) + } + if ds.Status.NumberReady == 0 { + c.warnf("substrate node agent daemonset %s/%s has no ready pod; workers cannot start (kubectl -n %s describe ds %s)", c.ns, substrateAteletDaemonSet, c.ns, substrateAteletDaemonSet) + } + return nil +} + +// pools checks at least one WorkerPool has a ready replica and records the +// sandbox classes those pools offer. +func (c *substrateClusterProbe) pools(ctx context.Context) error { + pools, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/workerpools") + if err != nil { + return err + } + ready := 0 + for _, p := range pools { + if p.Status.ReadyReplicas == 0 { + continue + } + ready++ + for _, sc := range p.Spec.SandboxClasses { + c.classes[strings.ToLower(sc.Name)] = true + } + } + if len(pools) == 0 { + c.warnf("no substrate WorkerPool exists; actors have nowhere to run — create one (see deploy/substrate.md)") + } else if ready == 0 { + c.warnf("none of the %d substrate WorkerPool(s) has a ready replica; actors will never be scheduled (kubectl get workerpools -A)", len(pools)) + } + return nil +} + +// sandboxClasses checks the cluster can actually run the classes the pools +// offer: gVisor via a SandboxConfig of class gvisor (atelet fetches runsc +// itself) or a runsc RuntimeClass; microVMs via atelet's ate.dev/kvm +// resource on at least one node (checked in nodes). +func (c *substrateClusterProbe) sandboxClasses(ctx context.Context) error { + cfgs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/sandboxconfigs") + if err != nil { + return err + } + for _, cfg := range cfgs { + if strings.EqualFold(cfg.Spec.SandboxClass, substrateClassGVisor) { + c.gvisor = true + } + } + rcs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/node.k8s.io/v1/runtimeclasses") + if err != nil && !errors.Is(err, errSubstrateKubeNotFound) { + return err + } + for _, rc := range rcs { + if rc.Handler == substrateGVisorHandler { + c.gvisor = true + } + } + wantGVisor := c.classes[substrateClassGVisor] || len(c.classes) == 0 + if wantGVisor && !c.gvisor { + c.warnf("cluster has no gVisor: no SandboxConfig with sandboxClass %s and no RuntimeClass with handler %s; Substrate refuses plain containers, so a gvisor WorkerPool cannot start workers — `ate-setup deploy ate-system` installs the gvisor-default SandboxConfig", substrateClassGVisor, substrateGVisorHandler) + } + return nil +} + +// featureGates checks the certificates.k8s.io APIs that only exist when the +// PodCertificateRequest and ClusterTrustBundle feature gates are on; without +// them atelet, atenet-router and hive's own router client get no SPIFFE +// certificates and every CONNECT to an actor fails. +func (c *substrateClusterProbe) featureGates(ctx context.Context) error { + served := map[string]bool{} + for _, v := range substrateCertsVersions { + body, err := c.kube.get(ctx, substrateCertsGroupPath+v) + if errors.Is(err, errSubstrateKubeNotFound) { + continue + } + if err != nil { + return err + } + var rl substrateKubeResourceList + if err := json.Unmarshal(body, &rl); err != nil { + return fmt.Errorf("certificates.k8s.io/%s: %w", v, err) + } + for _, r := range rl.Resources { + served[strings.ToLower(r.Name)] = true + } + } + if !served[substrateResourcePCR] { + c.warnf("the PodCertificateRequest API is not served; Substrate's SPIFFE identities (atelet, atenet-router, hive's router client) cannot be issued — enable the PodCertificateRequest feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + if !served[substrateResourceCTB] { + c.warnf("the ClusterTrustBundle API is not served; the Control API and router CA bundles cannot be projected into hive — enable the ClusterTrustBundle feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + return nil +} + +// nodes surveys the node kernels against gVisor's floor and records whether +// any node advertises KVM for the microVM class. +func (c *substrateClusterProbe) nodes(ctx context.Context) error { + nodes, err := substrateKubeItems[substrateKubeNode](ctx, c.kube, "/api/v1/nodes?limit="+strconv.Itoa(substrateKubeNodeLimit)) + if err != nil { + return err + } + var old []string + for _, n := range nodes { + if _, ok := n.Status.Allocatable[substrateKVMResource]; ok { + c.kvm = true + } + if kv := n.Status.NodeInfo.KernelVersion; kv != "" && substrateKernelLess(kv, substrateGVisorMinKernel) { + old = append(old, n.Metadata.Name+" ("+kv+")") + } + } + if len(old) > 0 { + sort.Strings(old) + c.warnf("%d node(s) run a kernel older than gVisor's %s floor and cannot host a gvisor worker: %s", len(old), substrateGVisorMinKernel, strings.Join(old, ", ")) + } + if c.classes[substrateClassMicroVM] && !c.kvm { + c.warnf("a WorkerPool offers the %s class but no node advertises %s; microVM workers need /dev/kvm (KVM-capable shapes) and atelet's device plugin", substrateClassMicroVM, substrateKVMResource) + } + return nil +} + +// substrateKernelLess reports whether kernel release a (e.g. 5.15.0-1 or +// 6.12.0-207.el9uek.aarch64) is older than the dotted floor b. A release the +// prover cannot parse is never reported as old. +func substrateKernelLess(a, b string) bool { + pa, oka := substrateKernelParts(a) + pb, okb := substrateKernelParts(b) + if !oka || !okb { + return false + } + for i := range pa { + if pa[i] != pb[i] { + return pa[i] < pb[i] + } + } + return false +} + +func substrateKernelParts(s string) ([3]int, bool) { + var out [3]int + s, _, _ = strings.Cut(s, "-") + parts := strings.Split(s, ".") + if len(parts) < 2 { + return out, false + } + for i := 0; i < 3 && i < len(parts); i++ { + n, err := strconv.Atoi(parts[i]) + if err != nil { + return out, false + } + out[i] = n + } + return out, true +} + +// substrateRouterProbeActor is the (nonexistent) actor the preflight names +// in its CONNECT: the router answers with an HTTP status once it has +// accepted hive's client certificate, and with a TLS alert when it has not. +const substrateRouterProbeActor = "hive-preflight" + +// probeRouter checks hive can reach atenet-router and that the router +// accepts its projected client bundle: the same mTLS CONNECT the kick's +// bridge tunnel makes, aimed at an actor that does not exist. Any HTTP +// answer means the certificate was accepted (TLS 1.3 reports a rejected +// client certificate only after the handshake, so the handshake alone is +// not proof). Skipped outside a cluster, where neither the bundle nor the +// router exist. +func (p *SubstratePreflighter) probeRouter(ctx context.Context, opts SubstrateOptions) []string { + if p.kube == nil && !substrateInCluster(opts) { + return nil + } + bridge, err := newRouterBridge(opts) + if err != nil { + return []string{fmt.Sprintf("atenet-router client credentials are not usable (%v); every kick on the substrate runtime will fail at the bridge tunnel", err)} + } + if p.routerDial != nil { + bridge.dial = p.routerDial + } + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + ref := &substrateapi.ObjectRef{Atespace: opts.atespace(), Name: substrateRouterProbeActor} + conn, err := bridge.tunnel(ctx, ref) + if conn != nil { + _ = conn.Close() + } + switch { + case err == nil, strings.Contains(err.Error(), "router refused connect"): + return nil + case strings.Contains(err.Error(), "router tls handshake") || strings.Contains(err.Error(), "router connect"): + return []string{fmt.Sprintf("atenet-router %s rejected hive's client certificate (%v); check the podCertificate projection at %s and --ingress-allowed-spiffe-ids on the router", bridge.routerAddr, err, substrateRouterBundlePath(opts))} + } + return []string{fmt.Sprintf("atenet-router %s is unreachable (%v); every kick on the substrate runtime will fail at the bridge tunnel — check the atenet-router Service and NetworkPolicies", bridge.routerAddr, err)} +} + +func substrateRouterBundlePath(opts SubstrateOptions) string { + if opts.RouterClientBundle != "" { + return opts.RouterClientBundle + } + return SubstrateDefaultRouterClientBundle +} diff --git a/src/crux/sandbox/substrate_preflight_cluster_test.go b/src/crux/sandbox/substrate_preflight_cluster_test.go new file mode 100644 index 0000000000..0cefac58f0 --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster_test.go @@ -0,0 +1,513 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster/router half of the Substrate preflight (#17698): a fake Kubernetes +// API serves the handful of read-only endpoints the prover walks, and a TLS +// listener plays atenet-router. + +// fakeKubeAPI is a path → JSON (or status code) table. +type fakeKubeAPI struct { + bodies map[string]any + codes map[string]int + calls []string +} + +func (f *fakeKubeAPI) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls = append(f.calls, r.URL.Path) + if code, ok := f.codes[r.URL.Path]; ok { + w.WriteHeader(code) + return + } + body, ok := f.bodies[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if raw, isRaw := body.(string); isRaw { + _, _ = w.Write([]byte(raw)) + return + } + _ = json.NewEncoder(w).Encode(body) + }) +} + +func kubeList(items ...any) map[string]any { return map[string]any{"items": items} } + +func kubeDeploy(name string, ready int) map[string]any { + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubePool(name string, ready int, classes ...string) map[string]any { + var sc []map[string]any + for _, c := range classes { + sc = append(sc, map[string]any{"name": c}) + } + return map[string]any{"metadata": map[string]any{"name": name}, "spec": map[string]any{"sandboxClasses": sc}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubeNode(name, kernel string, kvm bool) map[string]any { + alloc := map[string]string{"cpu": "4"} + if kvm { + alloc[substrateKVMResource] = "1" + } + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"allocatable": alloc, "nodeInfo": map[string]any{"kernelVersion": kernel}}} +} + +func kubeResources(names ...string) map[string]any { + var rs []map[string]any + for _, n := range names { + rs = append(rs, map[string]any{"name": n}) + } + return map[string]any{"resources": rs} +} + +const ( + pathCRDs = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + pathDeploys = "/apis/apps/v1/namespaces/ate-system/deployments" + pathAtelet = "/apis/apps/v1/namespaces/ate-system/daemonsets/atelet" + pathPools = "/apis/ate.dev/v1/workerpools" + pathSandboxes = "/apis/ate.dev/v1/sandboxconfigs" + pathRuntimes = "/apis/node.k8s.io/v1/runtimeclasses" + pathNodes = "/api/v1/nodes" +) + +// healthyKube is a cluster where every check passes. +func healthyKube() *fakeKubeAPI { + f := &fakeKubeAPI{bodies: map[string]any{}, codes: map[string]int{}} + for _, crd := range substrateCRDs { + f.bodies[pathCRDs+crd] = map[string]any{"metadata": map[string]any{"name": crd}} + } + var deps []any + for _, d := range substrateOperatorDeployments { + deps = append(deps, kubeDeploy(d, 1)) + } + f.bodies[pathDeploys] = kubeList(deps...) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 3}} + f.bodies[pathPools] = kubeList(kubePool("gv", 2, "gvisor")) + f.bodies[pathSandboxes] = kubeList(map[string]any{"spec": map[string]any{"sandboxClass": "gvisor"}}) + f.bodies[pathRuntimes] = kubeList() + f.bodies[substrateCertsGroupPath+"v1beta1"] = kubeResources("podcertificaterequests", "clustertrustbundles") + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0-207.el9uek.aarch64", false)) + return f +} + +// clusterPreflighter returns a preflighter whose Control API half is healthy +// and whose cluster half talks to the fake, with the router half stubbed to +// a refused dial unless the test overrides it. +func clusterPreflighter(t *testing.T, f *fakeKubeAPI) (*SubstratePreflighter, SubstrateOptions) { + t.Helper() + srv := httptest.NewServer(f.handler()) + t.Cleanup(srv.Close) + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + if err := os.WriteFile(token, []byte("sa-token"), 0o600); err != nil { + t.Fatal(err) + } + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{ + newClient: fakeSubstrateFactory{fake: fake}.new, + kube: &substrateKube{server: srv.URL, tokenPath: token, caPath: ca, client: srv.Client()}, + routerDial: func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dial stubbed") + }, + } + opts := SubstrateOptions{Endpoint: "api.ate-system.svc:443", Token: "t", RouterClientBundle: bundle, RouterCAFile: ca} + return p, opts +} + +func TestSubstratePreflightClusterHealthyOnlyWarnsAboutRouter(t *testing.T) { + f := healthyKube() + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "atenet-router") || !strings.Contains(got[0], "dial stubbed") { + t.Fatalf("healthy cluster warnings = %q", got) + } + if !strings.Contains(strings.Join(f.calls, " "), pathNodes) { + t.Errorf("node survey not reached: %v", f.calls) + } +} + +func TestSubstratePreflightClusterMissingCRDsStopsEarly(t *testing.T) { + f := healthyKube() + delete(f.bodies, pathCRDs+"workerpools.ate.dev") + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + var crd int + for _, w := range got { + if strings.Contains(w, "CRD workerpools.ate.dev is not installed") { + crd++ + } + if strings.Contains(w, "deployment") || strings.Contains(w, "WorkerPool") { + t.Errorf("operator checks ran after missing CRDs: %q", w) + } + } + if crd != 1 { + t.Errorf("CRD warnings = %d in %q", crd, got) + } +} + +func TestSubstratePreflightClusterForbiddenIsOneWarning(t *testing.T) { + f := healthyKube() + f.codes[pathDeploys] = http.StatusForbidden + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + n := 0 + for _, w := range got { + if strings.Contains(w, "403") && strings.Contains(w, "sandbox-substrate-preflight-rbac.yaml") { + n++ + } + } + if n != 1 || len(got) != 2 { + t.Errorf("forbidden warnings = %q", got) + } +} + +func TestSubstratePreflightClusterReadErrorIsAWarning(t *testing.T) { + f := healthyKube() + f.codes[pathPools] = http.StatusInternalServerError + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 2 || !strings.Contains(got[0], "could not be read") || !strings.Contains(got[0], "HTTP 500") { + t.Errorf("read error warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad json warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = kubeList("not an object") + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad item warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathAtelet] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "daemonset atelet") { + t.Errorf("bad daemonset warnings = %q", got) + } + + f = healthyKube() + f.bodies[substrateCertsGroupPath+"v1beta1"] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "certificates.k8s.io/v1beta1") { + t.Errorf("bad discovery warnings = %q", got) + } +} + +func TestSubstratePreflightClusterOperatorAndPoolWarnings(t *testing.T) { + f := healthyKube() + f.bodies[pathDeploys] = kubeList(kubeDeploy("ate-api-server", 0), kubeDeploy("ate-controller", 1), kubeDeploy("atenet-router", 1)) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 0}} + f.bodies[pathPools] = kubeList(kubePool("gv", 0, "gvisor")) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{ + "deployment ate-system/ate-api-server has no ready replica", + "deployment ate-system/atenet-egress is not installed", + "daemonset ate-system/atelet has no ready pod", + "none of the 1 substrate WorkerPool(s) has a ready replica", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if strings.Contains(got, "ate-controller") || strings.Contains(got, "atenet-router is") { + t.Errorf("healthy deployments warned:\n%s", got) + } + + f = healthyKube() + delete(f.bodies, pathAtelet) + f.bodies[pathPools] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "daemonset ate-system/atelet is not installed") || !strings.Contains(got, "no substrate WorkerPool exists") { + t.Errorf("missing atelet/pool warnings:\n%s", got) + } +} + +func TestSubstratePreflightClusterSandboxClassWarnings(t *testing.T) { + // No gvisor SandboxConfig and no runsc RuntimeClass → warning. + f := healthyKube() + f.bodies[pathSandboxes] = kubeList() + delete(f.bodies, pathRuntimes) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "cluster has no gVisor") { + t.Errorf("missing gVisor warning:\n%s", got) + } + + // A runsc RuntimeClass alone satisfies gVisor. + f = healthyKube() + f.bodies[pathSandboxes] = kubeList() + f.bodies[pathRuntimes] = kubeList(map[string]any{"handler": "runsc"}, map[string]any{"handler": "kata"}) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "gVisor") { + t.Errorf("runsc RuntimeClass not honoured:\n%s", got) + } + + // A microvm pool with no KVM node warns; a gvisor-less pool set does not + // demand gVisor. + f = healthyKube() + f.bodies[pathPools] = kubeList(kubePool("vm", 1, "microvm")) + f.bodies[pathSandboxes] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "no node advertises ate.dev/kvm") || strings.Contains(got, "no gVisor") { + t.Errorf("microvm warnings:\n%s", got) + } + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0", true)) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "kvm") { + t.Errorf("KVM node not honoured:\n%s", got) + } +} + +func TestSubstratePreflightClusterFeatureGatesAndKernels(t *testing.T) { + f := healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[pathNodes] = kubeList(kubeNode("old", "4.14.0-1", false), kubeNode("new", "5.15.0", false), kubeNode("odd", "weird", false)) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{"PodCertificateRequest API is not served", "ClusterTrustBundle API is not served", "1 node(s) run a kernel older", "old (4.14.0-1)"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + + // GA v1 serving both resources is enough. + f = healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[substrateCertsGroupPath+"v1"] = kubeResources("PodCertificateRequests", "ClusterTrustBundles") + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "not served") { + t.Errorf("v1 discovery not honoured:\n%s", got) + } +} + +// init pins the Kubernetes token path to a file that does not exist so the +// Control-API-only tests behave the same on a laptop and in a CI Pod (where +// the runner has a real ServiceAccount token). Tests that want the +// in-cluster halves inject kube/routerDial or call inClusterToken. +func init() { + substrateKubeTokenPath = filepath.Join(os.TempDir(), "hive-substrate-preflight-no-token-"+strconv.Itoa(os.Getpid())) +} + +// inClusterToken makes the prover believe it runs in a Pod wired for +// Substrate: a Kubernetes token and a projected Substrate token both exist. +// It returns the Substrate token path to set as opts.TokenFile. +func inClusterToken(t *testing.T) string { + t.Helper() + dir := t.TempDir() + prev := substrateKubeTokenPath + substrateKubeTokenPath = filepath.Join(dir, "k8s-token") + ate := filepath.Join(dir, "ate-token") + for _, f := range []string{substrateKubeTokenPath, ate} { + if err := os.WriteFile(f, []byte("t"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { substrateKubeTokenPath = prev }) + return ate +} + +func TestSubstratePreflightClusterSkippedOutsideCluster(t *testing.T) { + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(context.Background(), SubstrateOptions{Endpoint: "api:443"}); len(got) != 0 { + t.Errorf("outside a cluster the kube/router halves must be silent: %q", got) + } + + // In a Pod with no reachable API server the kube half reports once and + // the router half reports unusable credentials, instead of staying silent. + ate := inClusterToken(t) + // Only one of the two markers present is still "outside". + if substrateInCluster(SubstrateOptions{}) { + t.Error("a Kubernetes token alone must not count as in-cluster") + } + opts := SubstrateOptions{Endpoint: "api:443", TokenFile: ate, RouterClientBundle: filepath.Join(t.TempDir(), "missing.pem")} + got := p.probeCluster(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "cluster facts could not be read") { + t.Errorf("in-cluster without an API server: %q", got) + } + if got := p.probeRouter(context.Background(), opts); len(got) != 1 || !strings.Contains(got[0], "not usable") { + t.Errorf("in-cluster without a bundle: %q", got) + } +} + +func TestSubstratePreflightKubeClientErrors(t *testing.T) { + k := &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: "/nonexistent"} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "in-cluster CA") { + t.Errorf("missing CA: %v", err) + } + dir := t.TempDir() + bad := filepath.Join(dir, "ca.crt") + _ = os.WriteFile(bad, []byte("not pem"), 0o600) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: bad} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "no certificates") { + t.Errorf("bad CA: %v", err) + } + _, ca, _ := selfSignedBundle(t, dir) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: ca} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("refused dial did not error") + } + if k.client == nil { + t.Error("client not cached") + } + k = &substrateKube{server: "http://[::1]:namedport", client: http.DefaultClient} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("bad URL did not error") + } +} + +func TestSubstrateOperatorNamespace(t *testing.T) { + for in, want := range map[string]string{ + "api.ate-system.svc:443": "ate-system", + "https://api.substrate-prod.svc.cluster.local": "substrate-prod", + "grpc://api.hive-sub.svc:8080": "hive-sub", + "": "ate-system", + "control.example.com:443": "ate-system", + "api..svc:443": "ate-system", + "not a url": "ate-system", + } { + if got := substrateOperatorNamespace(in); got != want { + t.Errorf("%q → %q, want %q", in, got, want) + } + } +} + +func TestSubstrateKernelLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"4.14.0-1", "4.14.77", true}, + {"4.14.77", "4.14.77", false}, + {"5.15.0-1.el8uek", "4.14.77", false}, + {"6.12.0-207.el9uek.aarch64", "4.14.77", false}, + {"4.9", "4.14.77", true}, + {"weird", "4.14.77", false}, + {"4.x.1", "4.14.77", false}, + {"5.15", "x", false}, + } + for _, c := range cases { + if got := substrateKernelLess(c.a, c.b); got != c.want { + t.Errorf("substrateKernelLess(%q, %q) = %v", c.a, c.b, got) + } + } +} + +// fakeRouterListener accepts TLS connections requiring a client certificate +// signed by the test CA and answers any CONNECT with 404, so the probe sees +// exactly what atenet-router would show an accepted or rejected client. +func fakeRouterListener(t *testing.T, cert tls.Certificate) net.Listener { + t.Helper() + pool := x509.NewCertPool() + leaf, _ := x509.ParseCertificate(cert.Certificate[0]) + pool.AddCert(leaf) + cfg := &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool} + ln, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + if _, err := http.ReadRequest(bufio.NewReader(c)); err != nil { + return + } + _, _ = io.WriteString(c, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + }() + } + }() + return ln +} + +func TestSubstratePreflightRouter(t *testing.T) { + dir := t.TempDir() + bundle, ca, cert := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + _ = os.WriteFile(token, []byte("t"), 0o600) + ln := fakeRouterListener(t, cert) + kube := &substrateKube{server: "http://127.0.0.1:1", tokenPath: token, caPath: ca, client: http.DefaultClient} + p := &SubstratePreflighter{kube: kube} + ctx := context.Background() + + // Good bundle, router verifying it → quiet. + opts := SubstrateOptions{RouterURL: "https://" + ln.Addr().String(), RouterClientBundle: bundle, RouterCAFile: ca} + if got := p.probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("healthy router warned: %q", got) + } + + // Unusable bundle → credentials warning. + bad := opts + bad.RouterClientBundle = filepath.Join(dir, "missing.pem") + if got := p.probeRouter(ctx, bad); len(got) != 1 || !strings.Contains(got[0], "client credentials are not usable") { + t.Errorf("missing bundle: %q", got) + } + + // Router rejects the client certificate: a second CA the router does + // not trust. TLS 1.3 surfaces that on the first read, not the handshake. + otherDir := t.TempDir() + otherBundle, _, _ := selfSignedBundle(t, otherDir) + rejected := opts + rejected.RouterClientBundle = otherBundle + if got := p.probeRouter(ctx, rejected); len(got) != 1 || !strings.Contains(got[0], "rejected hive's client certificate") || !strings.Contains(got[0], otherBundle) { + t.Errorf("rejected cert: %q", got) + } + + // Unreachable router. + down := opts + down.RouterURL = "https://127.0.0.1:1" + if got := p.probeRouter(ctx, down); len(got) != 1 || !strings.Contains(got[0], "is unreachable") { + t.Errorf("unreachable: %q", got) + } + + // Injected dial wins over the real one; default bundle path is named + // when none is configured. + stub := &SubstratePreflighter{kube: kube, routerDial: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("stubbed") }} + if got := stub.probeRouter(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "stubbed") { + t.Errorf("stubbed dial: %q", got) + } + if substrateRouterBundlePath(SubstrateOptions{}) != SubstrateDefaultRouterClientBundle { + t.Error("default bundle path") + } + + // Outside a cluster the router half is silent. + if got := (&SubstratePreflighter{}).probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("outside cluster: %q", got) + } +} diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 389ffabb18..653616887e 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -100,9 +100,28 @@ func (h *Handler) workspace() string { return DefaultWorkspace } +// clearDir empties dir without removing dir itself: inside the actor the +// workspace is a durableDir mount point, and unlinking a mount point fails +// with EBUSY. +func clearDir(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + func (h *Handler) putWorkspace(w http.ResponseWriter, r *http.Request) { dir := h.workspace() - if err := os.RemoveAll(dir); err != nil { + if err := clearDir(dir); err != nil { http.Error(w, fmt.Sprintf("substrate bridge: clearing %s: %v", dir, err), http.StatusInternalServerError) return } diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index e91754700a..bc18a77a56 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -392,3 +392,50 @@ func TestServeStopsOnContextAndListenerClose(t *testing.T) { t.Errorf("Serve returned nil after the listener closed") } } + +// Inside the actor /workspace is a durableDir mount point; PUT must empty it +// in place rather than unlink it (which fails with EBUSY on a mount). +func TestPutWorkspaceClearsInPlace(t *testing.T) { + ws := t.TempDir() + writeFile(t, filepath.Join(ws, "stale.txt"), "old") + writeFile(t, filepath.Join(ws, "nested", "deep.txt"), "old") + before, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(&Handler{Workspace: ws}) + defer srv.Close() + body := tarStream(t, func(tw *tar.Writer) { + content := []byte("new") + _ = tw.WriteHeader(&tar.Header{Name: "fresh.txt", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg}) + _, _ = tw.Write(content) + }) + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + t.Fatalf("put = %d", resp.StatusCode) + } + after, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("workspace directory was recreated; it must be emptied in place") + } + if _, err := os.Stat(filepath.Join(ws, "stale.txt")); !os.IsNotExist(err) { + t.Error("stale.txt survived PUT") + } + if _, err := os.Stat(filepath.Join(ws, "nested")); !os.IsNotExist(err) { + t.Error("nested/ survived PUT") + } + if got, err := os.ReadFile(filepath.Join(ws, "fresh.txt")); err != nil || string(got) != "new" { + t.Errorf("fresh.txt = %q, %v", got, err) + } + if err := clearDir(filepath.Join(ws, "does-not-exist")); err != nil { + t.Errorf("clearDir of a missing dir = %v, want nil", err) + } +} diff --git a/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml new file mode 100644 index 0000000000..7a60c377ee --- /dev/null +++ b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml @@ -0,0 +1,72 @@ +# Lets the spoke's Substrate preflight (sandbox.runtime: substrate, #17698; +# crux/sandbox/substrate_preflight_cluster.go) read the cluster facts behind +# a failing kick: the operator's CRDs and workloads, WorkerPools and +# SandboxConfigs, RuntimeClasses, node kernels/KVM, and whether the +# certificates.k8s.io APIs behind the PodCertificateRequest and +# ClusterTrustBundle feature gates are served. Read-only, get/list only; +# without it the preflight emits one warning and skips these checks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["workerpools.ate.dev", "sandboxconfigs.ate.dev"] + verbs: ["get"] + - apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs"] + verbs: ["get", "list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hive-substrate-preflight +--- +# The operator namespace (ate-system by default; the preflight derives it +# from sandbox.substrate.endpoint's api..svc host). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hive-substrate-preflight From 0d5f171af3a09ce3e5cf292196648273f0d90a95 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 15:54:51 -0700 Subject: [PATCH 017/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20openshell=20sandb?= =?UTF-8?q?ox=20names=20and=20driver=5Fconfig=20fit=20gateway=200.1.8=20li?= =?UTF-8?q?mits=20(#17780)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prototype on LKE (#17643) hit four gateway rejections before a kick ran: sandbox names over MAX_ROUTABLE_NAME_LEN (19), the reserved 'workspace' volume name, default-denied pod labels/annotations, and the missing Agent Sandbox prerequisite. Names are now hk-- (fnv tag when the slug does not fit), the claim mounts as hive-workspace, and the deploy doc records the operator settings the measurements depended on (allowed_pod_labels, workspaceStorageClass, sandboxUid, auth modes) plus the landlock-in-LSM preflight lesson and the measured numbers. Refs #17643, #17642 Signed-off-by: Andrew Anderson --- changelog.d/fixed-openshell-sandbox-name.md | 1 + src/crux/sandbox/openshell.go | 42 +++++++++++--- src/crux/sandbox/openshell_logs.go | 2 +- src/crux/sandbox/openshell_logs_test.go | 2 +- src/crux/sandbox/openshell_test.go | 29 ++++++++-- src/deploy/openshell.md | 55 +++++++++++++++---- .../adr/0026-sandbox-runtime-selection.md | 2 +- src/docs/sandbox-isolation.md | 4 +- 8 files changed, 107 insertions(+), 30 deletions(-) create mode 100644 changelog.d/fixed-openshell-sandbox-name.md diff --git a/changelog.d/fixed-openshell-sandbox-name.md b/changelog.d/fixed-openshell-sandbox-name.md new file mode 100644 index 0000000000..6f902b3cbd --- /dev/null +++ b/changelog.d/fixed-openshell-sandbox-name.md @@ -0,0 +1 @@ +- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) diff --git a/src/crux/sandbox/openshell.go b/src/crux/sandbox/openshell.go index 5da7dfe449..f8d7cd0cdc 100644 --- a/src/crux/sandbox/openshell.go +++ b/src/crux/sandbox/openshell.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "net/url" "os" "path/filepath" @@ -50,14 +51,25 @@ const ( // on the created sandbox. OpenShellPolicyProfileAnnotation = "hive.dev/policy-profile" - openshellNamePrefix = "hive-kick-" - openshellNameMax = 63 + // openshellNamePrefix and openshellNameMax follow the gateway's + // MAX_ROUTABLE_NAME_LEN (19): a sandbox name is one of three segments + // in a 63-char DNS label (----), so the + // prefix is short and the timestamp is base36 (measured on 0.1.8, + // hivecommons/hive#17643). + openshellNamePrefix = "hk-" + openshellNameMax = 19 + openshellNameUnixBase = 36 + openshellNameTagLen = 3 + openshellNameTagMod = 36 * 36 * 36 openshellRequestWait = 30 * time.Second openshellReadyPoll = 2 * time.Second openshellMaxOutput = 16 << 20 openshellManagedLabel = "app.kubernetes.io/managed-by" openshellManagedValue = "hive-openshell" - openshellHTTPSPort = 443 + // openshellWorkspaceVolume names hive's claim in driver_config; the + // gateway reserves the bare name "workspace" for its own volumes. + openshellWorkspaceVolume = "hive-workspace" + openshellHTTPSPort = 443 ) // OpenShellAlwaysAllowedHosts are the egress hosts every hive sandbox gets: @@ -361,12 +373,12 @@ func (l *OpenShellLauncher) execEnv(spec LaunchSpec) map[string]string { func (l *OpenShellLauncher) driverConfig(subPath, mount string, readOnly bool) map[string]any { k8s := map[string]any{ "volumes": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "persistent_volume_claim": map[string]any{"claim_name": l.Options.WorkspaceClaim, "read_only": false}, }}, "containers": map[string]any{"agent": map[string]any{ "volume_mounts": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "mount_path": mount, "sub_path": subPath, "read_only": readOnly, @@ -550,16 +562,28 @@ func openshellWorkspaceSubPath(workspace, claimMount string) (string, error) { var openshellNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) // openshellSandboxName builds a unique, DNS-label-safe sandbox name from the -// launch name and a timestamp: hive-kick--. +// launch name and a timestamp: hk--, at most +// openshellNameMax characters. The launch name's hive-/-sandbox wrapping is +// dropped so the agent slug gets the room; a slug that still does not fit is +// cut and tagged with a short hash of the whole slug, so two agents sharing +// a long prefix never map to the same sandbox name. func openshellSandboxName(launchName string, now time.Time) string { - base := strings.Trim(openshellNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + base := strings.ToLower(strings.TrimSuffix(launchName, launchNameSuffix)) + base = strings.TrimPrefix(base, launchNamePrefix) + base = strings.Trim(openshellNameUnsafe.ReplaceAllString(base, "-"), "-") if base == "" { base = "agent" } - suffix := fmt.Sprintf("-%d", now.Unix()) + suffix := "-" + strconv.FormatInt(now.Unix(), openshellNameUnixBase) room := openshellNameMax - len(openshellNamePrefix) - len(suffix) if len(base) > room { - base = strings.TrimRight(base[:room], "-") + h := fnv.New32a() + _, _ = h.Write([]byte(base)) + tag := strconv.FormatUint(uint64(h.Sum32())%openshellNameTagMod, openshellNameUnixBase) + for len(tag) < openshellNameTagLen { + tag = "0" + tag + } + base = strings.TrimRight(base[:room-openshellNameTagLen], "-") + tag } return openshellNamePrefix + base + suffix } diff --git a/src/crux/sandbox/openshell_logs.go b/src/crux/sandbox/openshell_logs.go index 4913ebb4ed..191c1a0a65 100644 --- a/src/crux/sandbox/openshell_logs.go +++ b/src/crux/sandbox/openshell_logs.go @@ -153,7 +153,7 @@ func openshellSandboxNameUnix(launchName, name string) (int64, bool) { if i < 0 || !strings.HasPrefix(name, openshellNamePrefix) { return 0, false } - unix, err := strconv.ParseInt(name[i+1:], 10, 64) + unix, err := strconv.ParseInt(name[i+1:], openshellNameUnixBase, 64) if err != nil { return 0, false } diff --git a/src/crux/sandbox/openshell_logs_test.go b/src/crux/sandbox/openshell_logs_test.go index 2da1939117..b688e9b9e1 100644 --- a/src/crux/sandbox/openshell_logs_test.go +++ b/src/crux/sandbox/openshell_logs_test.go @@ -270,7 +270,7 @@ func TestOpenShellSandboxNameUnix(t *testing.T) { {good, true}, {"nodash", false}, {"other-deps-1700000000", false}, - {openshellNamePrefix + "deps-notanumber", false}, + {openshellNamePrefix + "deps-not.a.number", false}, {openshellSandboxName(LaunchName("deps-bot"), time.Unix(1700000000, 0)), false}, } for _, c := range cases { diff --git a/src/crux/sandbox/openshell_test.go b/src/crux/sandbox/openshell_test.go index 6f20cfc2d6..864eb64cf9 100644 --- a/src/crux/sandbox/openshell_test.go +++ b/src/crux/sandbox/openshell_test.go @@ -145,7 +145,7 @@ func TestOpenShellRunSuccess(t *testing.T) { if c.workspace != OpenShellDefaultWorkspace { t.Errorf("workspace = %q", c.workspace) } - if c.name != "hive-kick-deps-1700000000" { + if c.name != "hk-deps-s44we8" { t.Errorf("name = %q", c.name) } if c.spec.Template.Image != "registry.example/dev:sha256-abc" { @@ -186,12 +186,17 @@ func TestOpenShellRunDriverConfigMountsWorkspaceClaim(t *testing.T) { t.Fatalf("driver config = %v", dc) } vols := k8s["volumes"].([]any) - pvc := vols[0].(map[string]any)["persistent_volume_claim"].(map[string]any) + vol := vols[0].(map[string]any) + // The gateway rejects a caller volume named "workspace" (reserved). + if vol["name"] != openshellWorkspaceVolume || vol["name"] == "workspace" { + t.Errorf("volume name = %v", vol["name"]) + } + pvc := vol["persistent_volume_claim"].(map[string]any) if pvc["claim_name"] != "hive-data" { t.Errorf("claim = %v", pvc) } mount := k8s["containers"].(map[string]any)["agent"].(map[string]any)["volume_mounts"].([]any)[0].(map[string]any) - if mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { + if mount["name"] != vol["name"] || mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { t.Errorf("mount = %v", mount) } pod := k8s["pod"].(map[string]any) @@ -362,16 +367,28 @@ func TestOpenShellAddress(t *testing.T) { func TestOpenShellSandboxName(t *testing.T) { now := time.Unix(1700000000, 0) - if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hive-kick-deps-agent-1700000000" { + if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hk-deps-aftv-s44we8" { t.Errorf("name = %q", got) } long := openshellSandboxName(strings.Repeat("a", 100), now) - if len(long) > openshellNameMax || !strings.HasSuffix(long, "-1700000000") { + // The gateway rejects routable names over 19 characters + // (MAX_ROUTABLE_NAME_LEN); the suffix is unix seconds in base36. + if len(long) != openshellNameMax || !strings.HasSuffix(long, "-s44we8") { t.Errorf("long name = %q (%d)", long, len(long)) } - if got := openshellSandboxName("---", now); got != "hive-kick-agent-1700000000" { + if got := openshellSandboxName("---", now); got != "hk-agent-s44we8" { t.Errorf("empty base = %q", got) } + // The launch name's hive-/-sandbox wrapping is dropped, and two agents + // that share a long prefix get distinct, stable names. + if got := openshellSandboxName(LaunchName("deps"), now); got != "hk-deps-s44we8" { + t.Errorf("launch name = %q", got) + } + one := openshellSandboxName(LaunchName("reviewer-one"), now) + two := openshellSandboxName(LaunchName("reviewer-two"), now) + if one == two || len(one) != openshellNameMax || one != openshellSandboxName(LaunchName("reviewer-one"), now) { + t.Errorf("truncated names = %q %q", one, two) + } } func TestOpenShellRunRefusesClineProvider(t *testing.T) { diff --git a/src/deploy/openshell.md b/src/deploy/openshell.md index 7653c1ad9f..706461a648 100644 --- a/src/deploy/openshell.md +++ b/src/deploy/openshell.md @@ -16,7 +16,8 @@ record it alongside the hive config. | Requirement | Why | How to check | |---|---|---| -| Node kernel **≥ 6.2** with Landlock ABI ≥ 3 | OpenShell's sandbox qualification fails below it | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`. Of the clusters hive runs on, LKE (Debian 13, 6.12) qualifies; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| Node kernel **≥ 6.2** with Landlock **active** | OpenShell's sandbox qualification fails below it; a new-enough kernel built without Landlock in the LSM list fails the same way | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`, then from a privileged debug pod `cat /sys/kernel/security/lsm` must contain `landlock`. Of the clusters hive runs on, LKE (Debian 13, 6.12) and OKE on OL9/UEK8 (6.12) qualify; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| The **Agent Sandbox** controller (`sandboxes.agents.x-k8s.io`) | the gateway's Kubernetes driver creates `Sandbox` objects; the chart's preflight refuses to install without the API | `kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/sandbox.yaml`, then `kubectl api-resources \| grep agents.x-k8s.io` | | A CNI that **enforces** egress `NetworkPolicy` (Calico, Cilium) | the gateway proxies egress; the pod must not be able to go around it. Flannel accepts policies and ignores them | create a deny-all policy in a scratch namespace and confirm a connection is actually refused, not just that the object exists | | Cluster-admin for the install | CRDs, the gateway's RBAC, PKI secrets | — | | A **ReadWriteMany** claim for hive's data when sandboxes may land on other nodes | the workspace is shared by mounting the same PVC | same caveat as the Job runtime | @@ -29,9 +30,17 @@ helm upgrade --install openshell \ --version \ --namespace openshell --create-namespace \ --set server.drivers.kubernetes.allowDriverConfig=true \ - --set-string 'server.env.OPENSHELL_TELEMETRY_ENABLED=false' + --set server.telemetryEnabled=false \ + --set server.workspaceStorageClass= \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_labels={app.kubernetes.io/managed-by}' \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_annotations={hive.dev/*}' ``` +`gatewayConfig` is rendered to TOML: its top-level keys are table names +written as dotted strings (`openshell.drivers.kubernetes:` in a values file, +escaped as above on the command line), not nested maps; nesting +`openshell: {drivers: …}` breaks the config and crash-loops the gateway. + Then, for the workspace hive will use (hive's `agent_sandbox.openshell.namespace`, default `default`), install the workspace chart with driver config allowed so the gateway may read PVC metadata for admission: @@ -61,12 +70,30 @@ Settings hive's runtime depends on, in gateway terms: `kubectl label pvc hive-data openshell.ai/sandbox-attachable=true openshell.ai/sandbox-attachable-workspace=default` on the spoke's namespace is enough for the defaults. -- **`OPENSHELL_TELEMETRY_ENABLED=false`** on the gateway. Hive's threat model +- **`allowed_pod_labels` / `allowed_pod_annotations`** under + `[openshell.drivers.kubernetes]`. The gateway rejects any template label or + annotation it was not told about; hive labels sandboxes + `app.kubernetes.io/managed-by=hive-openshell` and annotates them + `hive.dev/policy-profile` when a profile is set. +- **`server.workspaceStorageClass`**. The gateway provisions a 2Gi + `workspace-` PVC for every sandbox even though hive mounts its own + claim. On a block-storage default class that provision + attach added ~20 s + to every kick on LKE (27 s vs 9 s warm start, [#17643](https://github.com/hivecommons/hive/issues/17643)); + point it at an NFS/RWX or otherwise fast class. +- **Sandbox uid**. The gateway runs every sandbox as `runAsUser`/`fsGroup` + `10001` by default, overriding the image's `USER`. The kick workspace hive + creates on the shared claim must be writable by that uid: either set + `server.sandboxUid`/`server.sandboxGid` to the uid crux runs as, or make the + claim's workspace directories group-writable for 10001. +- **`server.telemetryEnabled=false`** on the gateway. Hive's threat model treats sandbox metadata as spoke-private; keep upstream telemetry off. -- **A bearer token** for hive. Create one per the upstream access-control page - and hand it to the hive pod as an environment variable (a Secret-backed - `env` entry on the hive Deployment); name that variable in - `agent_sandbox.openshell.token_env`. The token never goes in `hive.yaml`. +- **A bearer token** for hive. Upstream authenticates clients by mTLS, OIDC + JWT, or a trusted reverse proxy; hive's client sends a static bearer token, + so "token" here means a JWT from the OIDC issuer the gateway trusts (see the + upstream access-control page). Hand it to the hive pod as an environment + variable (a Secret-backed `env` entry on the hive Deployment); name that + variable in `agent_sandbox.openshell.token_env`. The token never goes in + `hive.yaml`. `allowUnauthenticatedUsers` is for throwaway clusters only. - **Digest-pinned images**: the agent's `sandbox.image` should be `registry/image@sha256:…`. The gateway's own supervisor and sandbox runtime images are pinned by the chart version. @@ -91,11 +118,19 @@ Settings hive's runtime depends on, in gateway terms: `api.github.com` must be **denied** and `git fetch` must succeed. Sandboxes are labelled `app.kubernetes.io/managed-by=hive-openshell` and are deleted when the kick ends; one that is still there after a finished kick is a bug. + Sandbox names are `hk--` (the gateway caps names at + 19 characters) and the claim is mounted as volume `hive-workspace` + (`workspace` is reserved by the gateway). +5. Agent images must ship a glibc `sh`: debian/ubuntu/fedora/EL-based images + and hive's own image work, but busybox/musl images (alpine) fail at + `spawn sandbox entrypoint process '/bin/sh': Permission denied`. ## Not covered Hive does not preflight any of the above ([#17644](https://github.com/hivecommons/hive/issues/17644)) and has no Podman-driver path for standalone hosts yet. The hub provisioner -does not install the gateway for hosted spokes; that is a follow-up once the -prototype numbers ([#17643](https://github.com/hivecommons/hive/issues/17643)) -are in. +does not install the gateway for hosted spokes; that is a follow-up. The +prototype numbers are on [#17643](https://github.com/hivecommons/hive/issues/17643): +on LKE with the settings above, cold start 10.0 s vs 6.0 s for the Job runtime +(1.67×), warm 9.1 s vs 3.8 s (2.4×), +38 Mi per sandbox, egress policy +enforced. diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 153111ef6f..b2d0a8c838 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Accepted (phase 1 OpenShell implemented, runtime opt-in — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) +Status: Accepted (phase 1 OpenShell implemented, runtime opt-in; measured on LKE in #17643: cold 1.67× / warm 2.4× the Job runtime, egress enforced — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) ## Context diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 507b404980..13199d6fee 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -276,11 +276,11 @@ The gateway, workspace, token, TLS and claim fields are cluster facts and fall b **The credential boundary.** Same rule as the Job runtime: environment variables from the allowlist are copied in with the same credential-name filter, hive's brokered GitHub token never enters the sandbox, and hive pushes and opens the PR from the returned workspace. There is **no** Secret-reference path into the sandbox in phase 1, so model credentials have to arrive through an allowlisted non-credential variable pointing at a routable gateway named in `allow_hosts`. The Cline backend, which needs a provider file or Secret, is refused on this runtime. -**Lifecycle.** The sandbox is named `hive-kick--` and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. +**Lifecycle.** The sandbox is named `hk--` (at most 19 characters, the gateway's limit for DNS-routable names) and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. **What hive checks, and what it cannot.** Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when an openshell-runtime agent resolves no `gateway_url` or no `workspace_claim`, and refuses the kick with the same message before the agent is marked running. It also preflights each distinct gateway (`Health().Check` and `Config().GetGateway`, read-only, cached for five minutes, bounded to 30 s) and adds a warning when the gateway is unreachable or unhealthy, `allow_driver_config` is not enabled, or telemetry is on. It cannot see the node kernel, the CNI or the claim's labels from inside its pod; those remain the operator prerequisites in [deploy/openshell.md](../deploy/openshell.md). -**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hive-kick--` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. +**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hk--` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. **How to verify.** With a gateway reachable and the claim attachable, run the env-gated integration test from a pod that has the claim mounted: `OPENSHELL_GATEWAY_URL=… OPENSHELL_WORKSPACE_CLAIM=hive-data OPENSHELL_IMAGE= go test ./crux/sandbox -run TestOpenShellIntegration -v`. It writes a file through `/workspace`, reads it back on the hive side, and logs the round-trip latency. Then run a kick and confirm in the sandbox log that a `POST` to `api.github.com` is denied while `git fetch` succeeds — that denial is the whole point of the runtime. From 9cec3285f2bd2b63a14403ed3ec7e4c149c1e66e Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 23:09:29 +0000 Subject: [PATCH 018/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.222.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.222.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 9 +++++++++ changelog.d/fixed-effect-ledger-unknown-op-reconcile.md | 1 - changelog.d/fixed-gfi-archived-repositories.md | 1 - changelog.d/fixed-stage-engine-nil-config.md | 1 - .../fixed-unauthorized-label-removal-readonly-lane.md | 1 - 5 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/fixed-effect-ledger-unknown-op-reconcile.md delete mode 100644 changelog.d/fixed-gfi-archived-repositories.md delete mode 100644 changelog.d/fixed-stage-engine-nil-config.md delete mode 100644 changelog.d/fixed-unauthorized-label-removal-readonly-lane.md diff --git a/CHANGELOG.md b/CHANGELOG.md index cb85f26dab..8b3aa3ce8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.222.1) + +### Fixed + +- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). +- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. +- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). +- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). + ## 2026-11-11 (v5.222.0) ### Added diff --git a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md b/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md deleted file mode 100644 index cd12604414..0000000000 --- a/changelog.d/fixed-effect-ledger-unknown-op-reconcile.md +++ /dev/null @@ -1 +0,0 @@ -- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). diff --git a/changelog.d/fixed-gfi-archived-repositories.md b/changelog.d/fixed-gfi-archived-repositories.md deleted file mode 100644 index 4d7c472e30..0000000000 --- a/changelog.d/fixed-gfi-archived-repositories.md +++ /dev/null @@ -1 +0,0 @@ -- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. diff --git a/changelog.d/fixed-stage-engine-nil-config.md b/changelog.d/fixed-stage-engine-nil-config.md deleted file mode 100644 index 25490b8df1..0000000000 --- a/changelog.d/fixed-stage-engine-nil-config.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). diff --git a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md b/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md deleted file mode 100644 index 9939b3a538..0000000000 --- a/changelog.d/fixed-unauthorized-label-removal-readonly-lane.md +++ /dev/null @@ -1 +0,0 @@ -- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). From 4dcfc42bc9d594a22a1d4f8600607fac396b9c0f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:21:06 -0700 Subject: [PATCH 019/166] =?UTF-8?q?=F0=9F=90=9B=20marketplace:=20stream=20?= =?UTF-8?q?repository=20tarballs=20and=20keep=20only=20pipeline=20files=20?= =?UTF-8?q?(#17793)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marketplace fetcher extracted each default-branch tarball to disk via complexity.ExtractTarGz under a 256 MiB whole-tree cap, so large monorepos (kubernetes/kubernetes, kserve/kserve, meshery/meshery, shipwright-io/build, spidernet-io/spiderpool) failed with "archive exceeds size cap" and never reached the catalogue. ingest.ReadTarball now streams the archive and keeps only the paths ReadTree would select (shared selectTreeFile), 1 MiB per file and 32 MiB in total, discarding everything else without buffering; the complexity cap is untouched for its own callers. GET /api/marketplace/status now includes failing:[{repo,error,checkedAt}] without ?verbose=1 (repo names and fetch errors carry no secrets), so an operator with a plain PAT can see why repos fail. Four corpus entries that GitHub now redirects are updated to their current names. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../fixed-marketplace-stream-tarball.md | 1 + src/docs/api-reference/hub-server.md | 2 +- src/docs/design/workflow-models.md | 6 +- src/pkg/hub/marketplace.go | 13 +- src/pkg/hub/marketplace_refresh.go | 13 +- src/pkg/hub/marketplace_test.go | 7 +- .../workflowmodel/ingest/corpus/corpus.json | 8 +- src/pkg/workflowmodel/ingest/tree.go | 130 +++++++++++++++--- src/pkg/workflowmodel/ingest/tree_test.go | 69 ++++++++++ 9 files changed, 203 insertions(+), 46 deletions(-) create mode 100644 changelog.d/fixed-marketplace-stream-tarball.md diff --git a/changelog.d/fixed-marketplace-stream-tarball.md b/changelog.d/fixed-marketplace-stream-tarball.md new file mode 100644 index 0000000000..fd6a16ff29 --- /dev/null +++ b/changelog.d/fixed-marketplace-stream-tarball.md @@ -0,0 +1 @@ +- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index b79feb07b9..0c4f7404d3 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -75,4 +75,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | | `POST` | `/api/marketplace/refresh` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}`. Spokes are rate-limited across all hives (one full run per hour, 10 `?repo=` runs per minute) → `429` + `Retry-After`; admins are not. `401` anonymous, `403` non-admin session or unknown hive | `pkg/hub/marketplace.go:39` | -| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts) and counts by tier. `?verbose=1` adds `failing:[{repo, error, checkedAt}]` and `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | +| `GET` | `/api/marketplace/status` | Public; `?verbose=1` requires spoke heartbeat bearer + `X-Hive-ID` or Hub admin session | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, failing:[{repo, error, checkedAt}], pendingRepos, lastRefreshErrors, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary (persisted across restarts), counts by tier and the failing repos with their fetch errors (no secrets). `?verbose=1` adds `pending:[repo]` (corpus repos never ingested) | `pkg/hub/marketplace.go:40` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 10ed66c3ef..f025f9276b 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -736,7 +736,7 @@ Routes (see [hub-server.md](../api-reference/hub-server.md)): | `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | | `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | | `POST /api/marketplace/refresh[?repo=]` | spoke bearer or hub admin | on-demand ingest; spokes rate-limited (`429`) | -| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier; `verbose` adds failing and pending repos | +| `GET /api/marketplace/status[?verbose=1]` | public (`verbose`: spoke bearer or hub admin) | last refresh, counts by tier, failing repos with their errors; `verbose` adds pending repos | **Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the `X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may @@ -748,7 +748,9 @@ same-origin CSRF evidence. The hub never stores a bundle it did not itself `Veri `pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` falls back to it when no `projects.json` is given) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch -tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and +tarball through its GitHub identity and streams it (`ingest.ReadTarball`): only the paths +`ingest.ReadTree` would select are kept (per file ≤ 1 MiB, ≤ 32 MiB in total), everything +else is discarded as it streams, so a monorepo's source size never matters. It then records `{format, sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → `ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier `ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 73e5196ae1..19750e96aa 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -19,8 +19,8 @@ import ( // GET /api/marketplace/workflows/{name}/{version}/bundle one bundle // POST /api/marketplace/workflows publish (spoke bearer or admin) // POST /api/marketplace/refresh[?repo=] on-demand ingest (spoke bearer or admin) -// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier -// (verbose: failing/pending repos; spoke bearer or admin) +// GET /api/marketplace/status[?verbose=1] last refresh + counts by tier + failing repos +// (verbose: pending repos; spoke bearer or admin) // // Tier policy: a publisher may assert `local`, `local-published` or // `ingested-unreviewed`; `curated` and `built-in` require a hub admin @@ -296,9 +296,10 @@ type marketplaceFailingRepo struct { CheckedAt time.Time `json:"checkedAt"` } -// handleMarketplaceStatus reports the last refresh and tier counts. -// ?verbose=1 (spoke bearer or admin) adds the failing repositories with -// their error strings and the repositories still awaiting a first ingest. +// handleMarketplaceStatus reports the last refresh and tier counts, plus +// the failing repositories with their error strings (repo names and fetch +// errors carry no secrets, so any caller sees them). ?verbose=1 (spoke +// bearer or admin) adds the repositories still awaiting a first ingest. func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return @@ -358,11 +359,11 @@ func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Reque if !lastChecked.IsZero() { out["lastCheckedAt"] = lastChecked } + out["failing"] = failing if verbose { if pending == nil { pending = []string{} } - out["failing"] = failing out["pending"] = pending } w.Header().Set("Content-Type", "application/json") diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index ed164784ce..c6eb57af02 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/evidence/complexity" "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/bundle" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" @@ -47,7 +46,6 @@ const ( // continuing. marketplaceRateFloor = 50 marketplaceRateMaxWait = 10 * time.Minute - marketplaceMaxTarball = 256 << 20 marketplaceSeedDelay = 30 * time.Second // marketplaceStaleAfter is how old a repo's refresh record may be // before the startup pass re-ingests it: the nightly interval, so a @@ -377,16 +375,7 @@ func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) ( _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) return nil, "", fmt.Errorf("tarball: %s", resp.Status) } - dir, err := os.MkdirTemp("", "hive-marketplace-") - if err != nil { - return nil, "", err - } - defer os.RemoveAll(dir) - top, err := complexity.ExtractTarGz(resp.Body, dir, marketplaceMaxTarball) - if err != nil { - return nil, "", err - } - files, err := ingest.ReadTree(top) + files, err := ingest.ReadTarball(resp.Body) if err != nil { return nil, "", err } diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index 185a7596a6..c3672cd46e 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -594,8 +594,11 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { t.Fatalf("pending/lastErrors: %s", rec.Body.String()) } - if _, ok := status["failing"]; ok { - t.Fatal("non-verbose status must not list failing repos") + if f, ok := status["failing"].([]interface{}); !ok || len(f) != 0 { + t.Fatalf("non-verbose status must list failing repos (empty here): %s", rec.Body.String()) + } + if _, ok := status["pending"]; ok { + t.Fatal("non-verbose status must not list pending repos") } byTier := status["byTier"].(map[string]interface{}) if byTier[bundle.TierIngestedUnreviewed].(float64) != 1 { diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.json b/src/pkg/workflowmodel/ingest/corpus/corpus.json index cfe16ee3bf..c831ae57ee 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus.json +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.json @@ -306,7 +306,7 @@ "maturity": "sandbox", "category": "Runtime", "repos": [ - "containers/composefs" + "composefs/composefs" ] }, { @@ -1120,7 +1120,7 @@ "maturity": "sandbox", "category": "AI Native Infra", "repos": [ - "ray-project/kuberay" + "kuberay-project/kuberay" ] }, { @@ -1542,7 +1542,7 @@ "maturity": "archived", "category": "Observability and Analysis", "repos": [ - "OpenObservability/OpenMetrics" + "prometheus/OpenMetrics" ] }, { @@ -1738,7 +1738,7 @@ "maturity": "sandbox", "category": "Provisioning", "repos": [ - "ratify-project/ratify" + "notaryproject/ratify" ] }, { diff --git a/src/pkg/workflowmodel/ingest/tree.go b/src/pkg/workflowmodel/ingest/tree.go index fc9beafd7a..87546ba559 100644 --- a/src/pkg/workflowmodel/ingest/tree.go +++ b/src/pkg/workflowmodel/ingest/tree.go @@ -1,6 +1,11 @@ package ingest import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" "io/fs" "os" "path/filepath" @@ -50,28 +55,13 @@ func ReadTree(root string) (map[string][]byte, error) { if !d.Type().IsRegular() || len(files) >= MaxTreeFiles { return nil } - switch { - case IsIntakePath(rel): - case !isYAMLPath(rel): - return nil - case githubactions.IsWorkflowPath(rel), underTektonDir(rel): - case hintsPipeline(rel): - data, err := os.ReadFile(p) - if err != nil { - return err - } - if tekton.LooksTekton(data) { - files[rel] = data - } - return nil - default: - return nil - } - data, err := os.ReadFile(p) + data, keep, err := selectTreeFile(rel, func() ([]byte, error) { return os.ReadFile(p) }) if err != nil { return err } - files[rel] = data + if keep { + files[rel] = data + } return nil }) if err != nil { @@ -80,6 +70,108 @@ func ReadTree(root string) (map[string][]byte, error) { return files, nil } +// Tarball streaming limits: ReadTarball keeps at most MaxTarballFileBytes +// per matched file and MaxTarballKeptBytes across all kept files, so a +// monorepo's source tree never has to fit anywhere — only its CI files do. +const ( + MaxTarballFileBytes = 1 << 20 + MaxTarballKeptBytes = 32 << 20 +) + +// ErrTarballKeptTooLarge is returned when the CI files alone exceed +// MaxTarballKeptBytes. +var ErrTarballKeptTooLarge = errors.New("ingest: pipeline files exceed size cap") + +// ReadTarball streams a gzipped tar archive (GitHub's /tarball shape: one +// wrapping top-level directory, a leading pax_global_header) and returns the +// same selection ReadTree makes from a checkout, without extracting +// anything to disk. Entries outside the selection are discarded as they +// stream, so the archive's total size is unbounded; a matched file over +// MaxTarballFileBytes is skipped (and Repository never sees it), and the +// kept set is capped at MaxTarballKeptBytes. +func ReadTarball(r io.Reader) (map[string][]byte, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, fmt.Errorf("ingest: tarball gzip: %w", err) + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string][]byte{} + var kept int64 + for { + hdr, err := tr.Next() + if err == io.EOF { + return files, nil + } + if err != nil { + return nil, fmt.Errorf("ingest: tarball: %w", err) + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + continue + } + rel, ok := tarballRelPath(hdr.Name) + if !ok || len(files) >= MaxTreeFiles || hdr.Size > MaxTarballFileBytes { + continue + } + data, keep, err := selectTreeFile(rel, func() ([]byte, error) { + return io.ReadAll(io.LimitReader(tr, MaxTarballFileBytes+1)) + }) + if err != nil { + return nil, fmt.Errorf("ingest: tarball %s: %w", rel, err) + } + if !keep || int64(len(data)) > MaxTarballFileBytes { + continue + } + kept += int64(len(data)) + if kept > MaxTarballKeptBytes { + return nil, ErrTarballKeptTooLarge + } + files[rel] = data + } +} + +// tarballRelPath strips the wrapping top-level directory from a GitHub +// tarball entry and rejects anything under a skipDirs component or that +// escapes the tree. +func tarballRelPath(name string) (string, bool) { + name = strings.TrimPrefix(filepath.ToSlash(name), "./") + _, rel, ok := strings.Cut(name, "/") + if !ok || rel == "" || strings.HasSuffix(rel, "/") { + return "", false + } + for _, seg := range strings.Split(rel, "/") { + if seg == ".." || skipDirs[seg] { + return "", false + } + } + return rel, true +} + +// selectTreeFile applies the ReadTree selection to one repo-relative path, +// calling read only when the path is a candidate; keep reports whether +// the returned data belongs in the result. +func selectTreeFile(rel string, read func() ([]byte, error)) (data []byte, keep bool, err error) { + switch { + case IsIntakePath(rel): + case !isYAMLPath(rel): + return nil, false, nil + case githubactions.IsWorkflowPath(rel), underTektonDir(rel): + case hintsPipeline(rel): + data, err := read() + if err != nil { + return nil, false, err + } + return data, tekton.LooksTekton(data), nil + default: + return nil, false, nil + } + data, err = read() + if err != nil { + return nil, false, err + } + return data, true, nil +} + func underTektonDir(rel string) bool { for _, d := range tektonDirs { if strings.HasPrefix(rel, d) { diff --git a/src/pkg/workflowmodel/ingest/tree_test.go b/src/pkg/workflowmodel/ingest/tree_test.go index 6c25e1b936..3ee53d627e 100644 --- a/src/pkg/workflowmodel/ingest/tree_test.go +++ b/src/pkg/workflowmodel/ingest/tree_test.go @@ -1,6 +1,9 @@ package ingest import ( + "archive/tar" + "bytes" + "compress/gzip" "os" "path/filepath" "sort" @@ -64,3 +67,69 @@ func TestReadTreeFeedsLifecycle(t *testing.T) { t.Fatalf("pipelines %d", len(m.Spec.Pipelines)) } } + +func TestReadTarballStreamsSelection(t *testing.T) { + type entry struct { + name string + body []byte + flag byte + } + big := bytes.Repeat([]byte("x"), 3*MaxTarballFileBytes) + entries := []entry{ + {name: "pax_global_header", flag: tar.TypeXGlobalHeader}, + {name: "acme-widgets-abc123/", flag: tar.TypeDir}, + {name: "acme-widgets-abc123/.github/workflows/ci.yml", body: []byte("on: push\njobs: {}\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/.tekton/pr.yaml", body: []byte("apiVersion: tekton.dev/v1\nkind: PipelineRun\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/deploy/release-pipeline.yaml", body: []byte("apiVersion: tekton.dev/v1\nkind: Pipeline\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/deploy/other-pipeline.yaml", body: []byte("kind: Deployment\n"), flag: tar.TypeReg}, + {name: "acme-widgets-abc123/vendor/x/.github/workflows/v.yml", body: []byte("on: push\n"), flag: tar.TypeReg}, + // Source blobs far past any per-file cap stream through untouched. + {name: "acme-widgets-abc123/third_party/blob.bin", body: big, flag: tar.TypeReg}, + {name: "acme-widgets-abc123/assets/huge.tar", body: big, flag: tar.TypeReg}, + // A matched path over the per-file cap is skipped, not fatal. + {name: "acme-widgets-abc123/.github/workflows/huge.yml", body: big, flag: tar.TypeReg}, + {name: "acme-widgets-abc123/README.md", body: []byte("hi\n"), flag: tar.TypeReg}, + } + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, e := range entries { + hdr := &tar.Header{Name: e.name, Typeflag: e.flag, Mode: 0o644, Size: int64(len(e.body))} + if e.flag == tar.TypeXGlobalHeader { + hdr = &tar.Header{Name: e.name, Typeflag: e.flag, PAXRecords: map[string]string{"comment": "abc123"}} + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(e.body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + + files, err := ReadTarball(&buf) + if err != nil { + t.Fatal(err) + } + var got []string + for p := range files { + got = append(got, p) + } + sort.Strings(got) + want := ".github/workflows/ci.yml .tekton/pr.yaml deploy/release-pipeline.yaml" + if strings.Join(got, " ") != want { + t.Fatalf("got %v", got) + } + if string(files[".tekton/pr.yaml"]) != "apiVersion: tekton.dev/v1\nkind: PipelineRun\n" { + t.Fatalf("content: %q", files[".tekton/pr.yaml"]) + } + + if _, err := ReadTarball(strings.NewReader("not gzip")); err == nil { + t.Fatal("expected gzip error") + } +} From e1a5dbdbfc5c5c96bbcd63f264da56fa69437cb6 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:21:54 +0000 Subject: [PATCH 020/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20redact=20proxy=20?= =?UTF-8?q?userinfo=20in=20upstreamproxy=20URL=20errors=20(#17731)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: redact proxy userinfo in upstreamproxy URL errors parseProxyURL formatted the raw proxy URL into its rejection errors, and configureUpstreamProxy logs that error, so a malformed URL carrying the proxy's Basic credential put the password in the hive JSON log on every boot. logscrub's basic-auth category covers Authorization headers only, not URL userinfo, so nothing downstream masked it. Use url.Redacted() on the parsed form, strip the userinfo segment when url.Parse itself fails, and drop the *url.Error wrapper whose message repeats the full URL. Fixes #17730 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: re-key merge request after transient lint-pending refusal Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...security-17730-upstreamproxy-url-redact.md | 1 + src/crux/upstreamproxy/upstreamproxy.go | 28 +++++++++++++++++-- src/crux/upstreamproxy/upstreamproxy_test.go | 21 ++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-17730-upstreamproxy-url-redact.md diff --git a/changelog.d/security-17730-upstreamproxy-url-redact.md b/changelog.d/security-17730-upstreamproxy-url-redact.md new file mode 100644 index 0000000000..432e2a58ef --- /dev/null +++ b/changelog.d/security-17730-upstreamproxy-url-redact.md @@ -0,0 +1 @@ +- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). diff --git a/src/crux/upstreamproxy/upstreamproxy.go b/src/crux/upstreamproxy/upstreamproxy.go index 723dfa230b..52abb11359 100644 --- a/src/crux/upstreamproxy/upstreamproxy.go +++ b/src/crux/upstreamproxy/upstreamproxy.go @@ -144,17 +144,24 @@ func parseProxyURL(raw string) (*url.URL, error) { if !strings.Contains(raw, "://") { raw = "http://" + raw } + // Errors are logged by the caller; the URL's userinfo is the proxy's + // Basic credential, so never echo raw. u, err := url.Parse(raw) if err != nil { - return nil, fmt.Errorf("upstream proxy url %q: %w", raw, err) + // *url.Error.Error() repeats the full URL; keep only its inner cause. + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return nil, fmt.Errorf("upstream proxy url %q: %w", redactUserinfo(raw), err) } switch u.Scheme { case "http", "https": default: - return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", raw, u.Scheme) + return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", u.Redacted(), u.Scheme) } if u.Hostname() == "" { - return nil, fmt.Errorf("upstream proxy url %q: missing host", raw) + return nil, fmt.Errorf("upstream proxy url %q: missing host", u.Redacted()) } if u.Port() == "" { port := "80" @@ -166,6 +173,21 @@ func parseProxyURL(raw string) (*url.URL, error) { return u, nil } +// redactUserinfo masks everything between "://" and the first "@" of raw, for +// URLs url.Parse rejected and so cannot be redacted structurally. +func redactUserinfo(raw string) string { + i := strings.Index(raw, "://") + if i < 0 { + return raw + } + rest := raw[i+3:] + at := strings.Index(rest, "@") + if at < 0 { + return raw + } + return raw[:i+3] + "xxxxx@" + rest[at+1:] +} + func envList(getenv func(string) string, name string) ([]string, bool) { raw := getenv(name) if strings.TrimSpace(raw) == "" { diff --git a/src/crux/upstreamproxy/upstreamproxy_test.go b/src/crux/upstreamproxy/upstreamproxy_test.go index f3d45b8623..5c7d84dcab 100644 --- a/src/crux/upstreamproxy/upstreamproxy_test.go +++ b/src/crux/upstreamproxy/upstreamproxy_test.go @@ -144,6 +144,27 @@ func TestResolveRejectsBadInput(t *testing.T) { } } +func TestResolveErrorsRedactUserinfo(t *testing.T) { + // The caller logs the error; the URL's userinfo is the proxy's Basic + // credential and must not reach the log through any rejection branch. + for _, raw := range []string{ + "socks5://alice:s3cret-pw@proxy.corp:1080", // unsupported scheme + "http://alice:s3cret-pw@", // missing host + "http://alice:s3cret-pw@proxy.corp:%zz", // url.Parse failure + } { + _, err := Resolve(Config{URL: raw}, nil) + if err == nil { + t.Fatalf("%q: accepted", raw) + } + if strings.Contains(err.Error(), "s3cret-pw") { + t.Fatalf("%q: error leaks password: %v", raw, err) + } + if !strings.Contains(err.Error(), "alice") && !strings.Contains(err.Error(), "xxxxx") { + t.Fatalf("%q: error lost the redacted URL entirely: %v", raw, err) + } + } +} + func TestGitEnv(t *testing.T) { sel, _ := Resolve(Config{URL: "http://p:3128", Hosts: []string{"github.ibm.com", "*.github.ibm.com"}, NoProxy: []string{"github.com"}}, nil) got := strings.Join(sel.GitEnv(), "\n") From 7d122bafbe7b812601381403cacc38e50244fc05 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:22:05 -0700 Subject: [PATCH 021/166] =?UTF-8?q?=E2=9C=A8=20benchmarks:=20per-PR=20esti?= =?UTF-8?q?mated=20cost=20in=20/api/benchmarks/export=20(#17791)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records now carry cost.basis (tokens_x_pricing | unknown), cost.usd_est, token share, launches and price_table_date, attributed from the spoke's launch-cost collector: the swarm's PR ref for reviewer records, the governor's issue ref of an issue the PR closed for author records (both ref spellings). Vendored benchmarks schema refreshed; converter and dashboard lookup tests; docs cost section. Fixes #17790 Refs #17694 Signed-off-by: Andrew Anderson --- changelog.d/added-benchmarks-export-cost.md | 1 + src/docs/benchmarks.md | 22 ++++- src/pkg/dashboard/api_benchmarks.go | 1 + src/pkg/dashboard/benchmarks_cost.go | 80 +++++++++++++++++++ src/pkg/dashboard/benchmarks_cost_test.go | 43 ++++++++++ src/pkg/evidence/benchmarks/convert.go | 48 +++++++++++ src/pkg/evidence/benchmarks/convert_test.go | 67 +++++++++++++++- src/pkg/evidence/benchmarks/record.go | 35 ++++++++ .../benchmarks/testdata/result.schema.json | 46 +++++++---- 9 files changed, 323 insertions(+), 20 deletions(-) create mode 100644 changelog.d/added-benchmarks-export-cost.md create mode 100644 src/pkg/dashboard/benchmarks_cost.go create mode 100644 src/pkg/dashboard/benchmarks_cost_test.go diff --git a/changelog.d/added-benchmarks-export-cost.md b/changelog.d/added-benchmarks-export-cost.md new file mode 100644 index 0000000000..62f95173f7 --- /dev/null +++ b/changelog.d/added-benchmarks-export-cost.md @@ -0,0 +1 @@ +- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index be9cf68661..ef2ef3c072 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -28,11 +28,28 @@ One record per PR per role, conforming to | `task.acmm_level` | `policy.acmm_level` at review time. | | `outcome.*` | `pr_opened`, `pr_merged`, `merged_at`, `time_to_merge_s`, `ci_status_at_merge`, `ci_green_first_try`, `review_verdict`, `reverted`, `closed_unmerged`, `human_interventions` (human approvals / label changes). | | `quality.*` | Swarm verdict counts and confidence. | +| `cost.*` | Estimated spend — see [Cost](#cost-estimated-never-billed). | **What never leaves the spoke:** the hive id, repository and organization names (unless the repository is public — see below), PR numbers, commit SHAs, -agent prompts, diff content, reviewer comments, human logins, and cost/token -figures (not present in bundles today). +agent prompts, diff content, reviewer comments, human logins, per-session +token usage and any billed figure. Only the per-PR *estimated* USD and its +token share leave, rounded by the price table. + +## Cost: estimated, never billed + +Each record carries `cost.basis` and, when attributable, `cost.usd_est`: + +| `cost.basis` | Meaning | +|---|---| +| `tokens_x_pricing` | The spoke's launch-cost collector (`/api/launch-cost`, `/data/launch-cost.json`) saw a kick that named this PR — the swarm's `owner/repo#N` PR ref for reviewer records, or the governor's `repo#N` issue ref of an issue the PR closed (`issue_closures`) for author records. The kick's token usage is priced with `pkg/tokens/pricing.go` (public list prices, `price_table_date`) and split evenly across the refs the kick named — the same `share_usd` the Cost panel shows. `launches` counts the kicks folded in; `*_tokens` are the in-window token share, informational. | +| `unknown` | No kick with usage named the PR: kicked without a ref, older than the launch-cost store, or usage landed outside the 6 h ownership horizon. | + +The spoke never emits `fleet_model_median`; the benchmarks site imputes that +at index-build time from attributed records of the same model and role and +labels it as such. `reported` (a gateway-billed figure) is reserved. List +prices are not what subscription plans or self-hosted inference cost, so the +site shows every dollar as an estimate with its basis mix. ## Privacy model: default on, anonymized @@ -97,5 +114,6 @@ pods and a nightly workflow rebuilds the index. - `pkg/evidence/benchmarks` — record types, `HiveHash`, `Convert` (bundles → records), schema golden test. - `pkg/dashboard/api_benchmarks.go` — export endpoint, status block, hourly pusher, Settings panel data. +- `pkg/dashboard/benchmarks_cost.go` — `CostLookup` over the launch-cost collector (daily rollup USD + snapshot tokens). - `pkg/hub/benchmarks.go` — ingest, month-file store, hourly GitHub publisher, status endpoint. - `pkg/evidence` `Author.runtime` / `Author.task_kind` — stamped at bundle write via `github.EvidenceAgentContext`. diff --git a/src/pkg/dashboard/api_benchmarks.go b/src/pkg/dashboard/api_benchmarks.go index ae05bddc3c..c11c0608c0 100644 --- a/src/pkg/dashboard/api_benchmarks.go +++ b/src/pkg/dashboard/api_benchmarks.go @@ -147,6 +147,7 @@ func (s *Server) benchmarksOptions(ctx context.Context, since time.Time) benchma RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), RepoPublic: func(repo string) bool { return s.benchmarksRepoPublic(ctx, repo) }, SandboxRuntime: cfg.BenchmarkSandboxRuntime, + CostLookup: benchmarksCostLookup(s.deps.LaunchCost), } } diff --git a/src/pkg/dashboard/benchmarks_cost.go b/src/pkg/dashboard/benchmarks_cost.go new file mode 100644 index 0000000000..650523332f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost.go @@ -0,0 +1,80 @@ +package dashboard + +import ( + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +// benchmarksCostLookup attributes kick spend to a benchmark record from the +// launch-cost collector: USD from the persisted daily rollup (every day the +// store has seen, live days at their latest share) and token counts from the +// in-window snapshot, each launch split evenly across the refs it named — +// the same accounting as /api/launch-cost by_ref[].share_usd. Nil collector +// means nothing is attributed. +func benchmarksCostLookup(lc *collect.LaunchCostCollector) benchmarks.CostLookup { + if lc == nil { + return nil + } + return func(refs []benchmarks.CostRef) (benchmarks.Cost, bool) { + want := map[benchmarks.CostRef]bool{} + for _, r := range refs { + want[r] = true + } + var usd float64 + var launches int + for _, day := range lc.DailyRollup() { + for _, ref := range day.ByRef { + if want[benchmarks.CostRef{Ref: ref.Ref, Kind: ref.Kind}] { + usd += ref.ShareUSD + launches += ref.Launches + } + } + } + if usd <= 0 { + return benchmarks.Cost{Basis: benchmarks.CostBasisUnknown}, false + } + c := benchmarks.Cost{ + Basis: benchmarks.CostBasisTokensXPricing, + USDEst: &usd, + Launches: launches, + PriceTableDate: tokens.PriceTableDate(), + } + if snap, ok := lc.Snapshot(); ok { + addLaunchTokens(&c, snap.ByLaunch, want) + } + return c, true + } +} + +// addLaunchTokens adds the per-ref token share of every in-window launch +// that named one of the wanted refs. Tokens are informational: usd_est is +// already priced by the collector. +func addLaunchTokens(c *benchmarks.Cost, launches []collect.LaunchCostEntry, want map[benchmarks.CostRef]bool) { + var in, out, read, write int64 + seen := false + for _, l := range launches { + n := int64(len(l.Refs)) + if n == 0 { + continue + } + hit := 0 + for _, r := range l.Refs { + if want[benchmarks.CostRef{Ref: r.Ref, Kind: r.Kind}] { + hit++ + } + } + if hit == 0 { + continue + } + seen = true + k := int64(hit) + in += l.Input * k / n + out += l.Output * k / n + read += l.CacheRead * k / n + write += l.CacheCreate * k / n + } + if seen { + c.InputTokens, c.OutputTokens, c.CacheReadTokens, c.CacheWriteTokens = &in, &out, &read, &write + } +} diff --git a/src/pkg/dashboard/benchmarks_cost_test.go b/src/pkg/dashboard/benchmarks_cost_test.go new file mode 100644 index 0000000000..be62d6712f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost_test.go @@ -0,0 +1,43 @@ +package dashboard + +import ( + "context" + "testing" + + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +func TestBenchmarksCostLookupFromLaunchCost(t *testing.T) { + if benchmarksCostLookup(nil) != nil { + t.Fatal("nil collector must disable attribution") + } + summary, launches, _ := launchFixture(t) + lc := collect.NewLaunchCostCollector(&fakeLaunchTokens{summary: summary, launches: launches}, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + lc.Start(ctx) + + lookup := benchmarksCostLookup(lc) + // Reviewer record: PR ref as the swarm spells it. + c, ok := lookup([]benchmarks.CostRef{{Ref: "o/r#9", Kind: tokens.LaunchRefKindPR}, {Ref: "r#9", Kind: tokens.LaunchRefKindPR}}) + if !ok || c.Basis != benchmarks.CostBasisTokensXPricing || c.USDEst == nil || *c.USDEst <= 0 { + t.Fatalf("pr ref not attributed: %+v", c) + } + if c.Launches != 1 || c.PriceTableDate != tokens.PriceTableDate() { + t.Fatalf("launches/price table: %+v", c) + } + if c.InputTokens == nil || *c.InputTokens != 50 || c.OutputTokens == nil || *c.OutputTokens != 5 { + t.Fatalf("tokens from snapshot: in=%v out=%v", c.InputTokens, c.OutputTokens) + } + // Author record closing issue 1: governor ref, issue kind. + c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#1", Kind: tokens.LaunchRefKindIssue}}) + if !ok || *c.USDEst <= 0 || *c.InputTokens != 200 { + t.Fatalf("issue ref not attributed: %+v", c) + } + // Unknown ref: nothing landed. + if c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#99", Kind: tokens.LaunchRefKindIssue}}); ok || c.Basis != benchmarks.CostBasisUnknown { + t.Fatalf("unrelated ref must be unknown: %+v", c) + } +} diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 0f9f6c5966..6f52f14ab5 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -27,6 +27,9 @@ type Options struct { // SandboxRuntime resolves the runtime an agent's kicks ran under // ("none" when sandboxing is off). nil means "none" for every agent. SandboxRuntime func(agent string) string + // CostLookup attributes kick spend to a PR (author: the PR and the issues + // it closed; reviewer: the PR). nil leaves cost.basis "unknown". + CostLookup CostLookup } // Roles. @@ -133,6 +136,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} rec.SandboxRuntime = runtimeOf(opts, a, name) rec.Task.Kind = kindOf(a.TaskKind, name) + rec.Cost = costOf(opts.CostLookup, authorCostRefs(latest)) out = append(out, rec) } if backend, model, ok := dominantReviewer(latest.Verdicts); ok { @@ -142,6 +146,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(model), Name: model} rec.SandboxRuntime = SandboxRuntimeNone rec.Task.Kind = kindPRReview + rec.Cost = costOf(opts.CostLookup, prCostRefs(latest.Repo, latest.Number, costKindPR)) out = append(out, rec) } } @@ -149,6 +154,49 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { return out } +const ( + costKindIssue = "issue" + costKindPR = "pr" +) + +// prCostRefs spells one ref both ways the kick path records it. +func prCostRefs(repo string, number int, kind string) []CostRef { + full := repo + "#" + strconv.Itoa(number) + refs := []CostRef{{Ref: full, Kind: kind}} + if _, short, ok := strings.Cut(repo, "/"); ok && short != "" { + refs = append(refs, CostRef{Ref: short + "#" + strconv.Itoa(number), Kind: kind}) + } + return refs +} + +// authorCostRefs lists the refs an authoring kick may have been launched +// against: the PR itself and every same-repo issue it closed. +func authorCostRefs(b *evidence.Bundle) []CostRef { + refs := prCostRefs(b.Repo, b.Number, costKindIssue) + for _, c := range b.IssueClosures { + repo := c.Repo + if repo == "" { + repo = b.Repo + } + if c.Issue > 0 { + refs = append(refs, prCostRefs(repo, c.Issue, costKindIssue)...) + } + } + return refs +} + +func costOf(lookup CostLookup, refs []CostRef) Cost { + if lookup != nil { + if c, ok := lookup(refs); ok && c.USDEst != nil && *c.USDEst > 0 { + if c.Basis == "" { + c.Basis = CostBasisTokensXPricing + } + return c + } + } + return Cost{Basis: CostBasisUnknown} +} + // maxACMM is the highest ACMM level the schema accepts. const maxACMM = 6 diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index b70a87d042..6bef3b93b8 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -28,9 +28,10 @@ func fixtureBundles() []*evidence.Bundle { {Perspective: "security", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "approve", Confidence: 0.9}, {Perspective: "style", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "comment", Confidence: 0.7}, }, - HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, - Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, - CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), + HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, + IssueClosures: []evidence.IssueClosure{{Repo: "hivecommons/hive", Issue: 40, At: t0(3 * time.Hour)}}, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, + CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), } private := &evidence.Bundle{ SchemaVersion: evidence.SchemaVersion, ID: "c", Repo: "acme/secret", Number: 7, @@ -53,6 +54,66 @@ func testOptions() Options { RepoConsent: true, RepoPublic: func(repo string) bool { return strings.HasPrefix(repo, "hivecommons/") }, SandboxRuntime: func(agent string) string { return "none" }, + CostLookup: fixtureCostLookup, + } +} + +// fixtureCostLookup mimics the launch-cost store: the governor kicked short +// "hive#40" (issue the PR closed), the swarm reviewed full "hivecommons/hive#42". +func fixtureCostLookup(refs []CostRef) (Cost, bool) { + spend := map[CostRef]float64{ + {Ref: "hive#40", Kind: "issue"}: 1.25, + {Ref: "hivecommons/hive#42", Kind: "pr"}: 0.40, + {Ref: "acme/secret#7", Kind: "issue"}: 0, // kick landed no usage + } + var usd float64 + n := 0 + for _, r := range refs { + if v, ok := spend[r]; ok { + usd += v + n++ + } + } + if n == 0 { + return Cost{Basis: CostBasisUnknown}, false + } + return Cost{Basis: CostBasisTokensXPricing, USDEst: &usd, Launches: n, PriceTableDate: "2026-11-09"}, true +} + +func TestConvertAttributesCost(t *testing.T) { + recs := Convert(fixtureBundles(), testOptions()) + author := byRole(recs, "hivecommons/hive", RoleAuthor) + if author == nil || author.Cost.Basis != CostBasisTokensXPricing || author.Cost.USDEst == nil || *author.Cost.USDEst != 1.25 { + t.Fatalf("author cost via closed issue short ref: %+v", author) + } + reviewer := byRole(recs, "hivecommons/hive", RoleReviewer) + if reviewer == nil || reviewer.Cost.USDEst == nil || *reviewer.Cost.USDEst != 0.40 || reviewer.Cost.Launches != 1 { + t.Fatalf("reviewer cost via full pr ref: %+v", reviewer) + } + private := byRole(recs, "", RoleAuthor) + if private == nil || private.Cost.Basis != CostBasisUnknown || private.Cost.USDEst != nil { + t.Fatalf("zero-usage kick must be unknown: %+v", private) + } + noLookup := testOptions() + noLookup.CostLookup = nil + for _, r := range Convert(fixtureBundles(), noLookup) { + if r.Cost.Basis != CostBasisUnknown { + t.Fatalf("nil lookup must yield unknown, got %+v", r.Cost) + } + } +} + +func TestCostRefsSpellBothForms(t *testing.T) { + b := &evidence.Bundle{Repo: "o/r", Number: 5, IssueClosures: []evidence.IssueClosure{{Issue: 3}}} + got := authorCostRefs(b) + want := []CostRef{{"o/r#5", "issue"}, {"r#5", "issue"}, {"o/r#3", "issue"}, {"r#3", "issue"}} + if len(got) != len(want) { + t.Fatalf("refs %v", got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("ref %d = %v want %v", i, got[i], want[i]) + } } } diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go index 50b2aaf0f9..875297febd 100644 --- a/src/pkg/evidence/benchmarks/record.go +++ b/src/pkg/evidence/benchmarks/record.go @@ -49,6 +49,7 @@ type Record struct { Task Task `json:"task"` Outcome Outcome `json:"outcome"` Quality *Quality `json:"quality,omitempty"` + Cost Cost `json:"cost"` } // Source says which hive emitted the record. @@ -109,6 +110,40 @@ type Quality struct { SentinelFindings *int `json:"sentinel_findings,omitempty"` } +// Cost basis values (schema cost.basis). A spoke never emits +// fleet_model_median — the index builder imputes that from published records. +const ( + CostBasisReported = "reported" + CostBasisTokensXPricing = "tokens_x_pricing" + CostBasisUnknown = "unknown" +) + +// Cost is the record's estimated spend. USDEst is hive's launch attribution +// priced with its public list-price table; it is an estimate, never a bill. +type Cost struct { + Basis string `json:"basis"` + USDEst *float64 `json:"usd_est,omitempty"` + InputTokens *int64 `json:"input_tokens,omitempty"` + OutputTokens *int64 `json:"output_tokens,omitempty"` + CacheReadTokens *int64 `json:"cache_read_tokens,omitempty"` + CacheWriteTokens *int64 `json:"cache_write_tokens,omitempty"` + // Launches counts the kicks whose spend was attributed to the record. + Launches int `json:"launches,omitempty"` + PriceTableDate string `json:"price_table_date,omitempty"` +} + +// CostRef is one work reference a kick may have named. Ref carries the +// spelling the kick path uses: issue refs are "repo#N" (governor), PR refs +// are "owner/repo#N" (review swarm); Convert asks for both. +type CostRef struct { + Ref string + Kind string // "issue" or "pr" +} + +// CostLookup sums the attributed spend of every ref and reports whether any +// usage landed. Spend of zero counts as no attribution. +type CostLookup func(refs []CostRef) (Cost, bool) + // recordID derives the stable id of one (hive, PR, role) triple so re-exports // overwrite rather than duplicate. func recordID(hiveHash, repo string, number int, role string) string { diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json index c4ecd39ec6..87f9edd6bb 100644 --- a/src/pkg/evidence/benchmarks/testdata/result.schema.json +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -357,7 +357,28 @@ "cost": { "type": "object", "additionalProperties": false, + "description": "Estimated spend for this record. usd_est is ALWAYS an estimate (list prices, apportioned kicks); basis says how it was derived. Records with basis=unknown carry no usd_est and are excluded from cost aggregates.", "properties": { + "usd_est": { + "type": "number", + "minimum": 0, + "description": "Estimated USD for this PR/review." + }, + "basis": { + "type": "string", + "enum": [ + "reported", + "tokens_x_pricing", + "fleet_model_median", + "unknown" + ], + "description": "reported: the harness/gateway returned a billed figure. tokens_x_pricing: hive's per-kick token counts \u00d7 schema/pricing.json list prices, apportioned to this PR via launch attribution (kick refs \u2192 issue_closures/PR). fleet_model_median: no attribution for this PR; median tokens_x_pricing cost for the same (model, role) across the fleet. unknown: nothing available." + }, + "usd": { + "type": "number", + "minimum": 0, + "description": "Billed USD when basis=reported." + }, "input_tokens": { "type": "integer", "minimum": 0 @@ -374,25 +395,20 @@ "type": "integer", "minimum": 0 }, - "usd": { - "type": "number", - "minimum": 0 - }, - "basis": { - "type": "string", - "description": "How usd was computed (GET /api/cost: estimated from public price table, or apportioned from per-agent totals).", - "enum": [ - "estimated", - "apportioned", - "billed", - "unpriced" - ] + "launches": { + "type": "integer", + "minimum": 0, + "description": "Agent kicks attributed to this PR (tokens_x_pricing only)." }, "price_table_date": { "type": "string", - "format": "date" + "format": "date", + "description": "as_of of the price table used." } - } + }, + "required": [ + "basis" + ] }, "quality": { "type": "object", From 3642bde118851d9940f79fc34c1e32807570e7d4 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Wed, 11 Nov 2026 23:46:33 +0000 Subject: [PATCH 022/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.223.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.223.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 13 +++++++++++++ changelog.d/added-substrate-preflight-prover.md | 1 - changelog.d/fixed-marketplace-seed-resume.md | 1 - changelog.d/fixed-openshell-sandbox-name.md | 1 - .../fixed-prow-oktotest-rejection-warn-once.md | 1 - .../fixed-status-snapshot-bounded-staleness.md | 1 - 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-substrate-preflight-prover.md delete mode 100644 changelog.d/fixed-marketplace-seed-resume.md delete mode 100644 changelog.d/fixed-openshell-sandbox-name.md delete mode 100644 changelog.d/fixed-prow-oktotest-rejection-warn-once.md delete mode 100644 changelog.d/fixed-status-snapshot-bounded-staleness.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b3aa3ce8c..541d4ef742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-11 (v5.223.0) + +### Added + +- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. + +### Fixed + +- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. +- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) +- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). +- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). + ## 2026-11-11 (v5.222.1) ### Fixed diff --git a/changelog.d/added-substrate-preflight-prover.md b/changelog.d/added-substrate-preflight-prover.md deleted file mode 100644 index dd9d5bd5d9..0000000000 --- a/changelog.d/added-substrate-preflight-prover.md +++ /dev/null @@ -1 +0,0 @@ -- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. diff --git a/changelog.d/fixed-marketplace-seed-resume.md b/changelog.d/fixed-marketplace-seed-resume.md deleted file mode 100644 index 5321431ab7..0000000000 --- a/changelog.d/fixed-marketplace-seed-resume.md +++ /dev/null @@ -1 +0,0 @@ -- Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. diff --git a/changelog.d/fixed-openshell-sandbox-name.md b/changelog.d/fixed-openshell-sandbox-name.md deleted file mode 100644 index 6f902b3cbd..0000000000 --- a/changelog.d/fixed-openshell-sandbox-name.md +++ /dev/null @@ -1 +0,0 @@ -- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) diff --git a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md b/changelog.d/fixed-prow-oktotest-rejection-warn-once.md deleted file mode 100644 index 6e33a4e8bb..0000000000 --- a/changelog.d/fixed-prow-oktotest-rejection-warn-once.md +++ /dev/null @@ -1 +0,0 @@ -- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). diff --git a/changelog.d/fixed-status-snapshot-bounded-staleness.md b/changelog.d/fixed-status-snapshot-bounded-staleness.md deleted file mode 100644 index 30daf850a4..0000000000 --- a/changelog.d/fixed-status-snapshot-bounded-staleness.md +++ /dev/null @@ -1 +0,0 @@ -- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). From ee3986344e470d6c874b8efad05b2ffe00ddcf5d Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:47:06 -0700 Subject: [PATCH 023/166] =?UTF-8?q?=E2=9C=A8=20marketplace:=20seed=20curat?= =?UTF-8?q?ed=20models=20and=20ship=20the=20CNCF=20TechDocs=20assessment?= =?UTF-8?q?=20workflow=20(#17798)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curated marketplace tier gets a source of truth in the binary: pkg/hub/marketplace/curated/*.yaml is embedded, strictly decoded, checked for the curated invariants (provenance curated, a version, nothing left inferred), compiled through bundle.Export and published at tier curated on hub startup and at the start of every refresh run when that name@version is absent. Bad files are logged and skipped, never fatal; edits without a version bump are refused as conflicts, so a new version is a new file. Seeded keys are reported in lastRefresh.curated. The first curated model is cncf-techdocs-assessment@curated-2026.11.1: Nate W.'s AI-assisted TechDocs assessment process (spec at nate-double-u/cncf-techdocs@882be6f, intake template @ad91530), scoped to cncf/techdocs. Issues travel requested → accepted → draft-open (→ escalated) → closed/declined; deliverable PRs travel drafting → in-review → stakeholder-review → awaiting-merge → merged with checks-failing and discarded branches. Every gate (/accept, /decline, /verify, /ready, /confirm, /approve, /discard, /abort) is released by a human role; the drafter and verifier agents stop at gates. Checks are the repo's format/link/spell-check workflows plus the manual provenance block. The rules evaluator is exercised over hand-written Facts for each step. cncf/techdocs is not a landscape project, so the corpus package gains an operator-added extra.json merged into Repos(); the ingested-unreviewed version now sits next to the curated one. Docs: marketplace section of docs/design/workflow-models.md (curated seeding, where a model goes, how versions work) and docs/workflows/techdocs-assessment.md for the TechDocs team. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../added-marketplace-curated-techdocs.md | 1 + src/docs/design/workflow-models.md | 32 +- src/docs/workflows/techdocs-assessment.md | 148 ++++ .../curated/cncf-techdocs-assessment.yaml | 634 ++++++++++++++++++ src/pkg/hub/marketplace/curated/curated.go | 96 +++ .../hub/marketplace/curated/curated_test.go | 212 ++++++ src/pkg/hub/marketplace/curated/load_test.go | 46 ++ src/pkg/hub/marketplace_curated.go | 56 ++ src/pkg/hub/marketplace_curated_test.go | 159 +++++ src/pkg/hub/marketplace_refresh.go | 22 +- src/pkg/hub/marketplace_test.go | 29 +- src/pkg/workflowmodel/ingest/corpus/corpus.go | 83 ++- .../ingest/corpus/corpus_test.go | 48 +- .../workflowmodel/ingest/corpus/extra.json | 8 + 14 files changed, 1534 insertions(+), 40 deletions(-) create mode 100644 changelog.d/added-marketplace-curated-techdocs.md create mode 100644 src/docs/workflows/techdocs-assessment.md create mode 100644 src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml create mode 100644 src/pkg/hub/marketplace/curated/curated.go create mode 100644 src/pkg/hub/marketplace/curated/curated_test.go create mode 100644 src/pkg/hub/marketplace/curated/load_test.go create mode 100644 src/pkg/hub/marketplace_curated.go create mode 100644 src/pkg/hub/marketplace_curated_test.go create mode 100644 src/pkg/workflowmodel/ingest/corpus/extra.json diff --git a/changelog.d/added-marketplace-curated-techdocs.md b/changelog.d/added-marketplace-curated-techdocs.md new file mode 100644 index 0000000000..4a22b467ee --- /dev/null +++ b/changelog.d/added-marketplace-curated-techdocs.md @@ -0,0 +1 @@ +- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index f025f9276b..08d1e7f5cd 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -744,9 +744,34 @@ assert `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`. `curated` and `built-in` require a hub-admin session with same-origin CSRF evidence. The hub never stores a bundle it did not itself `Verify`. +**Curated seeding.** Reviewed models ship with the hub binary under +`pkg/hub/marketplace/curated/*.yaml` (`go:embed`; loader `curated.Load`). On startup — +before any ingest, and without a GitHub token — and again at the start of every refresh +run, `seedCuratedModels` decodes each file strictly (unknown fields refused), checks the +curated invariants (`spec.meta.provenance: curated`, a `spec.meta.version`, zero +`inferred` elements), compiles it through `bundle.Export`, sets `Tier=curated` and +`Publish`es it when that `name@version` is absent. A file that fails any step is logged +(`marketplace: curated model refused` / `does not compile` / `not published`) and +skipped; the rest of the catalogue still seeds and the error appears in the run's +`errors` as `curated: …`. Seeded keys are reported in `lastRefresh.curated`, not +`published`. Entries are immutable: re-running is a no-op, and editing a file without +bumping its version is a `409` conflict at seed time, so **a new version is a new file** +(`cncf-techdocs-assessment.yaml` carries `curated-2026.11.1`; the next revision is a +second file with `curated-2026.12.1` or similar — the catalogue keeps both and the spoke +pins what it imported). To add a curated model: write the YAML (start from +`ingest.Lifecycle` + `spec.Ingest` output, confirm every inferred element, cite the +source document in `workflow.hivecommons.dev/source*` annotations and the element +`citations`), drop it in that directory, and add rules-evaluator cases to +`curated_test.go` — `TestLoadEmbedded` already refuses anything that does not compile +clean. The first curated model is the CNCF TechDocs assessment, +[docs/workflows/techdocs-assessment.md](../workflows/techdocs-assessment.md). + **Refresh.** The corpus is the CNCF survey list, now embedded as -`pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` -falls back to it when no `projects.json` is given) plus +`pkg/workflowmodel/ingest/corpus` (`corpus.json`, 262 projects / 299 repositories; +`cmd/workflow-survey` falls back to it when no `projects.json` is given) plus the +operator-added non-landscape list `extra.json` in the same package (`{"name", +"category", "repos", "why"}` per entry; `cncf/techdocs` is there so its +`ingested-unreviewed` version exists next to the curated one) plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch tarball through its GitHub identity and streams it (`ingest.ReadTarball`): only the paths `ingest.ReadTree` would select are kept (per file ≤ 1 MiB, ≤ 32 MiB in total), everything @@ -831,4 +856,5 @@ served cached data. - Hub-side marketplace settings are environment variables, not hub YAML. - The hub logs publishes/refreshes through its structured logger; it has no audit store. - Ingest versions are per-day sequence numbers, not semantic; curation (promoting an ingest - to `curated`) is a manual admin publish of a reviewed model. + to `curated`) is a manual admin publish of a reviewed model or a new file under + `pkg/hub/marketplace/curated/`. diff --git a/src/docs/workflows/techdocs-assessment.md b/src/docs/workflows/techdocs-assessment.md new file mode 100644 index 0000000000..a849b775eb --- /dev/null +++ b/src/docs/workflows/techdocs-assessment.md @@ -0,0 +1,148 @@ +# CNCF TechDocs AI-assisted assessment — curated workflow model + +`cncf-techdocs-assessment@curated-2026.11.1` is the first curated model in the hub +marketplace (https://hive.hivecommons.dev → Settings → Workflows → Marketplace on any +hive). It encodes the CNCF TechDocs team's AI-assisted documentation assessment process +so a hive bound to `cncf/techdocs` runs intake, drafting, verification, stakeholder review +and approval as a visible lifecycle, with every agent action followed by a human gate. + +Source of truth (the model cites both by SHA in its `workflow.hivecommons.dev/source*` +annotations and in each element's `citations`): + +- Specification: [`specs/ai-assisted-techdocs-assessment-spec.md`](https://github.com/nate-double-u/cncf-techdocs/blob/ai-assisted-assessment-spec/specs/ai-assisted-techdocs-assessment-spec.md) + at `nate-double-u/cncf-techdocs@882be6fef28794433bc3a6a34ac1401391c1d3e4`. +- Intake template: [`.github/ISSUE_TEMPLATE/assessment-request.yml`](https://github.com/nate-double-u/cncf-techdocs/blob/ad915303f9774b9aa6b7646969897fada7975a00/.github/ISSUE_TEMPLATE/assessment-request.yml) + at `nate-double-u/cncf-techdocs@ad915303f9774b9aa6b7646969897fada7975a00`. +- Checks and labels: the `cncf/techdocs` `.github` tree (format, link and spell-check + workflows; `settings.yml` labels), as snapshotted in + `pkg/workflowmodel/ingest/testdata/techdocs/`. + +The model's scope is `github.com cncf/techdocs` — the upstream repository where the +process runs; the fork above is where the spec was written. Model file: +`pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml` +(`apiVersion: workflow.hivecommons.dev/v1alpha1`, `derivation.mode: rules`, +`meta.provenance: curated`). The same repository is also in the ingest corpus +(`pkg/workflowmodel/ingest/corpus/extra.json`), so an `ingested-unreviewed` version built +from the pipeline files alone sits next to this one in the catalogue. + +## How the spec maps onto the model + +The spec (§5 Lifecycle, §6 Phases, §12 Lifecycle bindings) describes one six-step loop — +request → accept → draft → review → stakeholder review → merge — run once per phase +(A analysis, B implementation plan, C issue backlog). The model encodes the loop once and +carries the phase as a label (`phase/assessment`, `phase/implementation`, +`phase/backlog`; `skip-implementation` makes the A merge open C). Each phase's tracking +issue and the intake issue itself travel the issue states; each AI-drafted deliverable is +a pull request that travels the PR states. + +### Columns + +| Column | States | Spec | +|---|---|---| +| Requested | `requested` | §5 step 1, §12 Request | +| Accepted | `accepted`, `escalated` | §5 step 2, §12 Accept; §5 failure path | +| Drafting | `draft-open` (issue), `drafting` (PR) | §5 step 3, §12 Draft | +| Review | `in-review`, `checks-failing` | §5 step 4, §12 Review | +| Stakeholders | `stakeholder-review` | §5 step 5, §12 Stakeholder review | +| Approval | `awaiting-merge` | §5 step 6, §12 Merge | +| Done (terminal) | `merged`, `closed`, `declined`, `discarded` | §12 Merge / Failure and abort | + +### States and rules (first match wins) + +Issues (intake and per-phase tracking issues): + +| State | Stage | Rule | Owner | +|---|---|---|---| +| `declined` | terminal | closed and `triage/declined` | — | +| `closed` | terminal | closed | — | +| `escalated` | 45 needs-decision | a hold label is present | administrator | +| `draft-open` | 40 covered-by-PR | `triage/accepted` and a covering PR exists | registered agent | +| `accepted` | 20 claimed | `triage/accepted` | reviewer (technical writer) | +| `requested` | 10 actionable | `needs-triage` | reviewer | + +Pull requests (deliverables carry the `Docs analysis` label; anything else abstains): + +| State | Stage | Rule | +|---|---|---| +| `merged` | terminal | merged | +| `discarded` | terminal | closed unmerged | +| `in-review` | 50 review-pending | draft and `verified` | +| `drafting` | 70 fixing | draft | +| `checks-failing` | 71 needs-author | checks red | +| `awaiting-merge` | 90 ready-for-human-merge | `confirmed` | +| `stakeholder-review` | 60 reviewed | ready and `verified` | +| `in-review` | 50 review-pending | ready but not yet `verified` | + +### Transitions + +| From | To | Event | Gate (who) | Spec | +|---|---|---|---|---| +| requested | accepted | `accepted` | `/accept` (reviewer) | §12 Accept | +| requested | declined | `declined` | `/decline` (reviewer) | §12 Accept | +| requested | closed | `closed` | — | withdrawn | +| accepted | draft-open | `draft-opened` | — (drafter opens the PR) | §12 Draft | +| accepted | escalated | `held` | — | §5 failure path | +| accepted | closed | `closed` | — | — | +| draft-open | accepted | `draft-withdrawn` | — | restart / hand-write / stop | +| draft-open | escalated | `held` | — | §5 failure path | +| draft-open | closed | `closed` | — | phase merged or aborted | +| escalated | accepted | `released` | — (administrator) | — | +| escalated | closed | `closed` | `/abort` (administrator) | §12 Failure and abort | +| drafting | in-review | `verified` | `/verify` (reviewer; needs `verified`) | §12 Review | +| in-review | stakeholder-review | `ready` | `/ready` (reviewer; needs `verified` + green checks) | §12 Stakeholder review | +| stakeholder-review | awaiting-merge | `confirmed` | `/confirm` (stakeholder) | §12 Stakeholder review | +| stakeholder-review / awaiting-merge | checks-failing | `checks-red` | — | — | +| checks-failing | stakeholder-review | `checks-green` | — | — | +| awaiting-merge | merged | `merged` | `/approve` (approver; needs `confirmed` + green) | §5 step 6, §12 Merge | +| drafting / in-review / checks-failing / stakeholder-review / awaiting-merge | discarded | `closed` | `/discard` (reviewer) | — | + +Every gate is released by a human role; the two agent profiles (`techdocs-drafter` in +`accepted`/`drafting`, `techdocs-verifier` in `drafting`) can only move work up to a gate, +never through one — the compiler's `agent-no-human-gate` refusal enforces this. + +### Roles + +| Role | Selector | Spec | +|---|---|---| +| requester | issue author (human) | §4 | +| reviewer | repository write permission (technical writer who `/accept`ed) | §4 | +| approver | repository write permission | §4 | +| stakeholder | set frozen at `/accept` from the intake's validated contacts | §4, §12 | +| administrator | repository admin | §4, §12 Failure and abort | +| drafter, verifier | the hive bot (non-human) | §4, §6 | + +### Labels + +From `settings.yml`: `Docs analysis`. Proposed for the assessment: `needs-triage`, +`triage/accepted`, `triage/declined`, `phase/assessment`, `phase/implementation`, +`phase/backlog`, `verified`, `confirmed`, `skip-implementation`. + +### Checks + +Required CI: format-check, link-check, spell-check (the three `cncf/techdocs` workflows, +ingested as pipelines with their source hashes). Required manual: provenance block present +and the strict check passes (spec §15). + +### Intake + +`assessment-request.yml` fields map to facts as recorded in the +`workflow.hivecommons.dev/intake-fields` annotation: project name, maturity and +moving-levels → assessment metadata; website/docs/repo/other-scope/out-of-scope → scope; +domains → the agent's egress allowlist; contacts → the frozen stakeholder set (validated +at `/accept`); tooling/context → drafter input; the HC-6 disclosure acknowledgement is +required. + +## Not representable today + +These parts of the spec are documented in the model's annotations but not enforced by +the engine: + +- Close *reason* (completed vs. not planned) — no fact distinguishes them. +- Approver independence from the drafter and reviewer — no fact records who drafted or + reviewed. +- Phase as a state dimension and the A→C skip — carried by labels; the + `open-next-phase-tracking-issue`, `notify-contacts` and `mention-stakeholders` + bookkeeping actions are declarative only. +- `/verify` dispatching the verifier agent and membership of the frozen stakeholder set + — the gates exist, the side effects do not yet. +- Data-collection scripts and the strict provenance check — a manual check. diff --git a/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml new file mode 100644 index 0000000000..adb197f9a2 --- /dev/null +++ b/src/pkg/hub/marketplace/curated/cncf-techdocs-assessment.yaml @@ -0,0 +1,634 @@ +# Curated marketplace model: the CNCF TechDocs AI-assisted assessment +# lifecycle (hivecommons/hive#17620). Hand-curated from Nate W.'s written +# spec and the cncf/techdocs .github snapshot; every element is confirmed +# (no "inferred" confidence remains) and cites the spec section or file it +# came from. A new version is a new file; this one is never edited in place. +apiVersion: workflow.hivecommons.dev/v1alpha1 +kind: WorkflowModel +metadata: + name: cncf-techdocs-assessment + labels: + workflow.hivecommons.dev/tier: curated + annotations: + workflow.hivecommons.dev/source: nate-double-u/cncf-techdocs@882be6fef28794433bc3a6a34ac1401391c1d3e4 specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/source-url: https://github.com/nate-double-u/cncf-techdocs/blob/ai-assisted-assessment-spec/specs/ai-assisted-techdocs-assessment-spec.md + workflow.hivecommons.dev/intake-template: nate-double-u/cncf-techdocs@ad915303f9774b9aa6b7646969897fada7975a00 .github/ISSUE_TEMPLATE/assessment-request.yml + workflow.hivecommons.dev/intake-template-url: https://github.com/nate-double-u/cncf-techdocs/blob/ad915303f9774b9aa6b7646969897fada7975a00/.github/ISSUE_TEMPLATE/assessment-request.yml + # Intake form field id -> the fact or record it feeds (spec §7, §12 Accept). + workflow.hivecommons.dev/intake-fields: "project-name=assessment.project; maturity=assessment.maturity; moving-levels=assessment.criteria-level; website,docs-url,repo,other-scope,out-of-scope=assessment.scope; domains=agent.egress-allowlist; contacts=stakeholder frozen set (validated at /accept); tooling,context=drafter input; disclosure=HC-6 acknowledgement (required)" + workflow.hivecommons.dev/curated-by: Andrew Anderson + workflow.hivecommons.dev/curated-from: ingest.Lifecycle(cncf/techdocs .github snapshot) + spec.Ingest(deterministic) skeleton, then hand-curated + workflow.hivecommons.dev/spec-sections: "4. Roles; 5. Lifecycle; 6. Phases and deliverables; 7. Inputs: the assessment request; 12. Lifecycle bindings; 13. Components and repository layout; 14. Agent definitions; 15. The provenance block" +spec: + meta: + version: curated-2026.11.1 + author: CNCF TechDocs (Nate W.); curated for Hive by Andrew Anderson + description: >- + AI-assisted, expert-supervised CNCF TechDocs assessment: an intake issue is + triaged and accepted by a technical writer, an agent drafts each phase's + deliverable as a draft PR, a verifier fact-checks it, the reviewer marks it + ready, the project's stakeholders confirm factual accuracy and an + independent approver merges. Phases A/B/C repeat the same six steps and are + carried by the phase/* labels. + license: CC-BY-4.0 + source: spec + provenance: curated + scope: + forge: github.com + repository: cncf/techdocs + workTypes: [documentation-assessment] + sources: + - system: github-actions + path: .github/workflows/format-check.yml + sha256: bf8cbb5896ad92f1e88f1051998450b35d0c080b27dba24f03c5d58d54e4a67e + - system: github-actions + path: .github/workflows/link-check.yml + sha256: 1bcef2093eda4ef4ca90d5766985e7710fee8926e0e1d85e5e2055593e679e94 + - system: github-actions + path: .github/workflows/spell-check.yml + sha256: d144101194175c7382e093d176019815fbe460c3633ef892350bc2a2635d7f87 + initial: requested + columns: + - id: requested + label: Requested + hint: Intake or phase tracking issue filed; a technical writer triages it (spec §5 step 1, §12 Request) + states: [requested] + - id: accepted + label: Accepted + hint: A technical writer accepted (/accept) and owns the phase as reviewer; delegate the drafter (spec §5 step 2) + states: [accepted, escalated] + - id: drafting + label: Drafting + hint: AI-drafted deliverable as a draft PR; verifier pass runs here (spec §5 step 3, §12 Draft) + states: [draft-open, drafting] + - id: review + label: Reviewer check + hint: Verified; the reviewer refines in conversation and checks findings against source (spec §5 step 4, HC-7) + states: [in-review, checks-failing] + - id: stakeholders + label: Stakeholder review + hint: Ready for review; project stakeholders confirm factual accuracy with /confirm (spec §5 step 5, HC-2) + states: [stakeholder-review] + - id: approval + label: Approval + hint: Confirmed; an approver who was neither drafter nor reviewer signs off and merges (spec §5 step 6) + states: [awaiting-merge] + - id: done + label: Done + hint: Merged, discarded, declined or closed (spec §12 Merge, Failure and abort) + states: [merged, discarded, declined, closed] + terminal: true + states: + # ---- issues: the intake (umbrella) issue and per-phase tracking issues ---- + - id: requested + label: Requested + owner: human + origin: human + lifecycle: + column: requested + number: 10 + kinds: [issue] + waiting: [awaiting triage by a technical writer] + replies: ["/accept", "/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Request. Phase A is triggered by the intake issue; the next phase becomes eligible once the previous one is merged" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A request issue opens labeled needs-triage ... An issue may sit unassigned indefinitely; eligibility is not work." + - path: .github/ISSUE_TEMPLATE/assessment-request.yml + line: 4 + text: "labels: Docs analysis, needs-triage" + - id: accepted + label: Accepted + owner: human + origin: human + lifecycle: + column: accepted + number: 20 + kinds: [issue] + waiting: [reviewer to delegate the phase drafter, or restart/hand-write after a discard] + replies: ["/decline"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Accept. A technical writer triages and explicitly accepts the request ... eligibility alone does not start work (P-1)" + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/accept ... assigns the issue to them, recording who accepted and who acts as the phase's reviewer, and swaps needs-triage for triage/accepted" + - path: .github/settings.yml + line: 132 + text: triage/accepted + - id: escalated + label: Escalated + owner: human + origin: human + lifecycle: + column: accepted + number: 45 + kinds: [issue] + waiting: [administrator/platform owner to abort or release the hold] + replies: ["/abort"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Repeated failure is escalated to the administrator/platform owner, who may abort the run." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned, the rationale recorded on both" + - id: draft-open + label: Draft under way + owner: registered-agent + origin: human + lifecycle: + column: drafting + number: 40 + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "Draft (step 3): a draft pull request. The agent works on its own branch in cncf/techdocs and opens a draft PR linked to the tracking issue" + - id: declined + label: Declined + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "a writer declines an unaccepted request with /decline [reason] ... labels it triage/declined ... and closes the issue as not planned" + - path: .github/settings.yml + line: 135 + text: triage/declined + - id: closed + label: Closed + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [issue] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The final phase's merge closes the intake issue as completed ... the close reason and labels say how it ended, completed at the final merge, not planned on a decline or an abort." + # ---- pull requests: the phase deliverable ---- + - id: drafting + label: Drafting + owner: registered-agent + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: drafting + number: 70 + kinds: [pr] + waiting: [drafter to finish, then the reviewer's /verify (or a recorded skip)] + replies: ["/verify", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Draft. The drafter produces the deliverable as a draft PR." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer triggers the verifier's pass by commenting /verify on the draft PR; its report lands as a PR comment ... the workflow applies the verified label" + - path: "spec:14. Agent definitions#L716-L792" + text: "Assessment drafter (Phase A) ... Plan drafter (Phase B) ... Backlog drafter (Phase C) ... Verifier (every phase)" + - id: in-review + label: Reviewer check + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 50 + kinds: [pr] + waiting: [reviewer to verify findings against source and comment /ready] + replies: ["/ready", "/skip-implementation", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Review. The verifier's fact-check pass runs first ... The reviewer then refines the draft in conversation with the drafter, verifies findings against source (HC-7), and marks it ready." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label is the gate: /ready refuses without it" + - path: .github/settings.yml + line: 147 + text: verified + - id: checks-failing + label: Checks failing + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: review + number: 71 + kinds: [pr] + waiting: [format, link or spelling check to go green] + replies: ["/discard"] + citations: + - path: .github/workflows/format-check.yml + text: Format checks + - path: .github/workflows/link-check.yml + text: Link checks + - path: .github/workflows/spell-check.yml + text: Spelling checks + - id: stakeholder-review + label: Stakeholder review + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: stakeholders + number: 60 + kinds: [pr] + waiting: [a stakeholder from the set frozen at acceptance to comment /confirm] + replies: ["/confirm", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Stakeholder review. Stakeholders confirm factual accuracy (required to advance) and record any disagreement with the conclusions (not required; HC-2)." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer comments /ready: the workflow marks the PR ready for review and mentions the stakeholders from the set frozen at acceptance" + - id: awaiting-merge + label: Awaiting approver + owner: human + origin: human + pipelines: [gha-github-workflows-format-check, gha-github-workflows-link-check, gha-github-workflows-spell-check] + lifecycle: + column: approval + number: 90 + kinds: [pr] + waiting: [an approver who was neither drafter nor reviewer of this phase to sign off and merge] + replies: ["/approve", "/discard"] + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "Merge. The approver signs off and merges the PR into cncf/techdocs, which makes the next phase eligible." + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The approver merges, first checking the confirmation behind the confirmed label" + - path: .github/settings.yml + line: 150 + text: confirmed + - id: merged + label: Merged + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "A merge-triggered GitHub Actions workflow then opens the next phase's tracking issue, labeled for its phase and needs-triage" + - id: discarded + label: Discarded + owner: external-observation + origin: human + type: final + lifecycle: + column: done + kinds: [pr] + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The reviewer discards a draft with /discard , which closes the PR with the reason recorded ... the tracking issue stays open for a restart or a hand-written phase." + roles: + - id: requester + label: Requester + kind: author-kind + params: {kind: human} + human: true + - id: reviewer + label: Reviewer (technical writer) + kind: permission + params: {permission: write} + human: true + - id: approver + label: Approver (independent technical writer) + kind: permission + params: {permission: write} + human: true + - id: stakeholder + label: Project stakeholder + kind: frozen-set + params: {frozenAt: accepted, source: "intake contacts validated at /accept"} + human: true + - id: administrator + label: Administrator / platform owner + kind: permission + params: {permission: admin} + human: true + - id: drafter + label: Phase drafter (agent) + kind: bot + params: {login: hive} + - id: verifier + label: Verifier (agent) + kind: bot + params: {login: hive} + gates: + - id: accept + releasedBy: reviewer + reply: "/accept" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "the workflow refuses /accept from a commenter without write access" + - id: decline + releasedBy: reviewer + reply: "/decline" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/decline [reason], under the same authorization as /accept" + - id: verified + conditions: + - {fact: labels, op: has, value: verified} + releasedBy: reviewer + reply: "/verify" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "The label means the verification this PR needs is satisfied, not that the agent ran: a reviewer who deliberately skips the pass ... applies the label by hand with the reason in a comment" + - id: ready + conditions: + - {fact: labels, op: has, value: verified} + - {fact: pr.checks, op: is, value: green} + releasedBy: reviewer + reply: "/ready" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/ready refuses without it [the verified label]" + - id: confirm + conditions: + - {fact: labels, op: has, value: confirmed} + releasedBy: stakeholder + reply: "/confirm" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "/confirm, which the workflow checks against the stakeholder set frozen at acceptance ... never the requester-editable intake body" + - id: approve + conditions: + - {fact: labels, op: has, value: confirmed} + - {fact: pr.checks, op: is, value: green} + releasedBy: approver + reply: "/approve" + citations: + - path: "spec:4. Roles#L166-L218" + text: "Approver. Gives each phase its independent quality sign-off and merges the PR. An experienced technical writer who was neither the drafter nor a reviewer of that phase" + - id: discard + releasedBy: reviewer + reply: "/discard" + citations: + - path: "spec:5. Lifecycle#L219-L254" + text: "the reviewer may discard it and restart or write by hand ... retries are chosen, never automatic" + - id: abort + releasedBy: administrator + reply: "/abort" + citations: + - path: "spec:12. Lifecycle bindings#L482-L636" + text: "An administrator/platform owner abort closes the tracking issue and the intake with it as not planned" + events: + - id: accepted + label: Request accepted + fact: {fact: labels, op: has, value: triage/accepted} + actor: reviewer + - id: declined + label: Request declined + fact: {fact: labels, op: has, value: triage/declined} + actor: reviewer + - id: draft-opened + label: Draft PR opened + fact: {fact: issue.covered, op: "true"} + - id: draft-withdrawn + label: Draft discarded; issue uncovered + fact: {fact: issue.covered, op: "false"} + - id: verified + label: Verification satisfied + fact: {fact: labels, op: has, value: verified} + actor: reviewer + - id: ready + label: Marked ready for review + fact: {fact: draft, op: "false"} + actor: reviewer + - id: confirmed + label: Stakeholder confirmed factual accuracy + fact: {fact: labels, op: has, value: confirmed} + actor: stakeholder + - id: checks-red + label: A check failed + fact: {fact: pr.checks, op: is, value: red} + - id: checks-green + label: Checks green + fact: {fact: pr.checks, op: is, value: green} + - id: merged + label: Deliverable merged + fact: {fact: merged, op: "true"} + actor: approver + - id: closed + label: Closed + fact: {fact: open, op: "false"} + - id: held + label: Hold applied (escalation) + fact: {fact: hold, op: "true"} + - id: released + label: Hold released + fact: {fact: hold, op: "false"} + actor: administrator + agents: + - profile: techdocs-drafter + states: [accepted, drafting] + tools: [read-repo, web-research, write-draft-pr] + produces: [pull-request, push] + - profile: techdocs-verifier + states: [drafting] + tools: [read-repo, web-research, comment] + produces: [comment] + checks: + - id: format-check + name: Format checks + kind: ci + required: true + source: .github/workflows/format-check.yml + - id: link-check + name: Link checks + kind: ci + required: true + source: .github/workflows/link-check.yml + - id: spell-check + name: Spelling checks + kind: ci + required: true + source: .github/workflows/spell-check.yml + - id: provenance-block + name: Provenance block present and strict check passes + kind: manual + required: true + source: "spec:15. The provenance block#L793-L877" + bookkeeping: + - on: accepted + action: notify-contacts + params: {note: "mention the validated project contacts on the intake issue before drafting (HC-6)", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: ready + action: mention-stakeholders + params: {note: "mention the stakeholder set frozen at acceptance with the participant guide link", source: "spec:12. Lifecycle bindings#L482-L636"} + - on: merged + action: open-next-phase-tracking-issue + params: {note: "open the next phase's tracking issue labeled phase/* and needs-triage (phase C when skip-implementation is set); close the intake as completed after the final phase", source: "spec:12. Lifecycle bindings#L482-L636"} + labels: + entries: + - {name: Docs analysis, color: c2e0c6, description: CNCF technical documentation assistance} + - {name: needs-triage, color: EDEDED, description: Assessment request awaiting triage, state: requested} + - {name: triage/accepted, color: C2E0C6, description: Assessment request accepted, state: accepted} + - {name: triage/declined, color: F9D0C4, description: "Assessment request declined, reason posted", state: declined} + - {name: phase/assessment, color: C5DEF5, description: "Assessment phase A: analysis"} + - {name: phase/implementation, color: 84B6EB, description: "Assessment phase B: implementation plan"} + - {name: phase/backlog, color: 1D76DB, description: "Assessment phase C: issue backlog"} + - {name: verified, color: 0E8A16, description: Verification complete for this deliverable PR, state: in-review} + - {name: confirmed, color: 006B75, description: Stakeholder confirmation recorded for this deliverable PR, state: awaiting-merge} + - {name: skip-implementation, color: D4C5F9, description: Phase B skipped; phase A merge opens phase C} + transitions: + # issues + - {from: requested, to: accepted, on: accepted, gateRef: accept, note: "spec §12 Accept"} + - {from: requested, to: declined, on: declined, gateRef: decline, note: "spec §12 Accept: /decline"} + - {from: requested, to: closed, on: closed, note: "withdrawn by the project"} + - {from: accepted, to: draft-open, on: draft-opened, note: "spec §12 Draft"} + - {from: accepted, to: escalated, on: held, note: "spec §5 failure path: escalated to the administrator"} + - {from: accepted, to: closed, on: closed, note: "tracking issue declined or assessment ended"} + - {from: draft-open, to: accepted, on: draft-withdrawn, note: "draft discarded: restart, hand-write or stop"} + - {from: draft-open, to: escalated, on: held, note: "spec §5 failure path"} + - {from: draft-open, to: closed, on: closed, note: "phase merged (next phase opens) or aborted"} + - {from: escalated, to: accepted, on: released, note: "administrator released the hold"} + - {from: escalated, to: closed, on: closed, gateRef: abort, note: "spec §12 Failure and abort"} + # pull requests + - {from: drafting, to: in-review, on: verified, gateRef: verified, note: "spec §12 Review: verified label is the gate"} + - {from: drafting, to: discarded, on: closed, gateRef: discard} + - {from: in-review, to: stakeholder-review, on: ready, gateRef: ready, note: "spec §12 Stakeholder review: /ready"} + - {from: in-review, to: discarded, on: closed, gateRef: discard} + - {from: checks-failing, to: stakeholder-review, on: checks-green} + - {from: checks-failing, to: discarded, on: closed, gateRef: discard} + - {from: stakeholder-review, to: awaiting-merge, on: confirmed, gateRef: confirm, note: "spec §12: /confirm from the frozen stakeholder set"} + - {from: stakeholder-review, to: checks-failing, on: checks-red} + - {from: stakeholder-review, to: discarded, on: closed, gateRef: discard} + - {from: awaiting-merge, to: merged, on: merged, gateRef: approve, note: "spec §5 step 6 / §12 Merge"} + - {from: awaiting-merge, to: checks-failing, on: checks-red} + - {from: awaiting-merge, to: discarded, on: closed, gateRef: discard} + derivation: + mode: rules + rules: + # issues (intake umbrella and phase tracking issues) + - id: declined + kind: issue + state: declined + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: triage/declined} + reason: declined by a technical writer (triage/declined) + - id: closed + kind: issue + state: closed + when: + - {fact: open, op: "false"} + reason: assessment ended (completed at the final merge, or aborted) + - id: escalated + kind: issue + state: escalated + when: + - {fact: hold, op: "true"} + reason: on hold; escalated to the administrator/platform owner + - id: draft-open + kind: issue + state: draft-open + when: + - {fact: labels, op: has, value: triage/accepted} + - {fact: issue.covered, op: "true"} + reason: a draft PR covers this phase + - id: accepted + kind: issue + state: accepted + when: + - {fact: labels, op: has, value: triage/accepted} + reason: accepted; awaiting a draft + - id: requested + kind: issue + state: requested + when: + - {fact: labels, op: has, value: needs-triage} + reason: filed; awaiting triage + # pull requests (the deliverable carries the Docs analysis label) + - id: merged + kind: pr + state: merged + when: + - {fact: merged, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: deliverable merged by the approver + - id: discarded + kind: pr + state: discarded + when: + - {fact: open, op: "false"} + - {fact: labels, op: has, value: Docs analysis} + reason: draft discarded (closed unmerged) + - id: in-review-draft + kind: pr + state: in-review + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: verified; the reviewer refines and checks findings against source + - id: drafting + kind: pr + state: drafting + when: + - {fact: draft, op: "true"} + - {fact: labels, op: has, value: Docs analysis} + reason: AI-drafted deliverable in progress; verification not yet recorded + - id: checks-failing + kind: pr + state: checks-failing + when: + - {fact: pr.checks, op: is, value: red} + - {fact: labels, op: has, value: Docs analysis} + reason: a format, link or spelling check is red + - id: awaiting-merge + kind: pr + state: awaiting-merge + when: + - {fact: labels, op: has, value: confirmed} + - {fact: labels, op: has, value: Docs analysis} + reason: stakeholder confirmation recorded; awaiting the approver + - id: stakeholder-review + kind: pr + state: stakeholder-review + when: + - {fact: labels, op: has, value: verified} + - {fact: labels, op: has, value: Docs analysis} + reason: ready for review; awaiting /confirm from a stakeholder + - id: in-review-unverified + kind: pr + state: in-review + when: + - {fact: labels, op: has, value: Docs analysis} + reason: ready without the verified label; verification is still owed + pipelines: + - id: gha-github-workflows-format-check + name: Format checks + system: github-actions + path: .github/workflows/format-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: format-check, name: FILE FORMAT, kind: job, runsOn: [ubuntu-latest], level: 0} + - {id: markdown-linter, name: MARKDOWN linter, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-link-check + name: Link checks + system: github-actions + path: .github/workflows/link-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: link-check, name: LINK checking, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + - id: gha-github-workflows-spell-check + name: Spelling checks + system: github-actions + path: .github/workflows/spell-check.yml + role: pr-verify + triggers: + - event: pull_request + nodes: + - {id: spelling-check, name: SPELLING check, kind: job, runsOn: [ubuntu-latest], level: 0} + levels: 1 + coverage: + files: 3 + parsed: 3 diff --git a/src/pkg/hub/marketplace/curated/curated.go b/src/pkg/hub/marketplace/curated/curated.go new file mode 100644 index 0000000000..7a415b43de --- /dev/null +++ b/src/pkg/hub/marketplace/curated/curated.go @@ -0,0 +1,96 @@ +// Package curated embeds the hub marketplace's curated workflow models +// (hivecommons/hive#17620 phase 7). Each *.yaml here is one reviewed model +// at one version; the hub seeds the catalogue from them at startup and on +// every refresh run, publishing each name@version that is not yet present. +// Entries are immutable: a change is a new file with a new spec.meta.version, +// never an edit in place. +package curated + +import ( + "bytes" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +//go:embed *.yaml +var files embed.FS + +// Errors a curated file can fail with. +var ( + ErrNotCurated = errors.New("curated: spec.meta.provenance must be curated") + ErrNoVersion = errors.New("curated: spec.meta.version is empty") + ErrInferred = errors.New("curated: model still carries inferred elements") +) + +// Entry is one embedded curated model. +type Entry struct { + Path string + Model *workflowmodel.Model +} + +// Decode parses one curated model document (YAML or JSON) strictly and +// checks the curated invariants: provenance curated, a version, and no +// element left at confidence inferred. It does not compile the model; the +// seed does that through bundle.Export. +func Decode(data []byte) (*workflowmodel.Model, error) { + var doc interface{} + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("curated: yaml: %w", err) + } + raw, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("curated: yaml to json: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + var m workflowmodel.Model + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("curated: not a WorkflowModel: %w", err) + } + if m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + return nil, fmt.Errorf("%w: got %q", ErrNotCurated, m.Spec.Meta.Provenance) + } + if strings.TrimSpace(m.Spec.Meta.Version) == "" { + return nil, ErrNoVersion + } + if n := workflowmodel.Inferred(&m); n > 0 { + return nil, fmt.Errorf("%w: %d unconfirmed", ErrInferred, n) + } + return &m, nil +} + +// Load decodes every embedded *.yaml in path order. Files that fail are +// reported in errs (one per file, prefixed with the path) and skipped, so +// one bad file never hides the others. +func Load() (entries []Entry, errs []error) { + return load(files) +} + +func load(fsys fs.FS) (entries []Entry, errs []error) { + // The pattern is a constant, so Glob cannot fail on it. + names, _ := fs.Glob(fsys, "*.yaml") + sort.Strings(names) + for _, name := range names { + data, err := fs.ReadFile(fsys, name) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", name, err)) + continue + } + m, err := Decode(data) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", name, err)) + continue + } + entries = append(entries, Entry{Path: name, Model: m}) + } + return entries, errs +} diff --git a/src/pkg/hub/marketplace/curated/curated_test.go b/src/pkg/hub/marketplace/curated/curated_test.go new file mode 100644 index 0000000000..81700c3232 --- /dev/null +++ b/src/pkg/hub/marketplace/curated/curated_test.go @@ -0,0 +1,212 @@ +package curated_test + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/authorkind" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/stageengine" + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// TestLoadEmbedded is the gate every curated file must pass: strict +// decode, provenance curated, a version, zero compile diagnostics of any +// severity, nothing left inferred, and a clean bundle export at tier +// curated. +func TestLoadEmbedded(t *testing.T) { + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatalf("load errors: %v", errs) + } + if len(entries) == 0 { + t.Fatal("no curated models embedded") + } + seen := map[string]bool{} + for _, e := range entries { + m := e.Model + key := m.Metadata.Name + "@" + m.Spec.Meta.Version + if seen[key] { + t.Fatalf("%s: duplicate %s", e.Path, key) + } + seen[key] = true + if m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("%s: provenance %q", e.Path, m.Spec.Meta.Provenance) + } + if n := workflowmodel.Inferred(m); n != 0 { + t.Fatalf("%s: %d inferred elements unconfirmed", e.Path, n) + } + _, diags, err := workflowmodel.Compile(m) + if err != nil || len(diags) != 0 { + t.Fatalf("%s: compile err=%v diags=%+v", e.Path, err, diags) + } + b, err := bundle.Export(m, "test", testNow) + if err != nil { + t.Fatalf("%s: export: %v", e.Path, err) + } + if b.Tier != bundle.TierCurated { + t.Fatalf("%s: tier %q", e.Path, b.Tier) + } + } + if !seen["cncf-techdocs-assessment@curated-2026.11.1"] { + t.Fatalf("techdocs model missing: %v", seen) + } +} + +func TestDecodeRejects(t *testing.T) { + cases := []struct { + name string + doc string + want error + }{ + {"bad yaml", "a: [", nil}, + {"unknown field", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nbogus: 1\n", nil}, + {"not curated", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: ingested}}\n", curated.ErrNotCurated}, + {"no version", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {provenance: curated}}\n", curated.ErrNoVersion}, + {"inferred", "apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: curated}, states: [{id: x, lifecycle: {confidence: inferred}}]}\n", curated.ErrInferred}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := curated.Decode([]byte(tc.doc)) + if err == nil { + t.Fatal("expected error") + } + if tc.want != nil && !errors.Is(err, tc.want) { + t.Fatalf("err %v, want %v", err, tc.want) + } + }) + } +} + +func techdocs(t *testing.T) *workflowmodel.Profile { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + for _, e := range entries { + if e.Model.Metadata.Name == "cncf-techdocs-assessment" { + p, _, err := workflowmodel.Compile(e.Model) + if err != nil { + t.Fatal(err) + } + return p + } + } + t.Fatal("cncf-techdocs-assessment not embedded") + return nil +} + +func issue(labels ...string) stageengine.Facts { + return stageengine.Facts{ + Ref: stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 1, Kind: stagelabels.IssueItem}, + Open: true, + Labels: labels, + Author: stageengine.AuthorFacts{Kind: authorkind.Human, Login: "maintainer"}, + Issue: &stageengine.IssueFacts{}, + } +} + +func pr(labels ...string) stageengine.Facts { + return stageengine.Facts{ + Ref: stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 2, Kind: stagelabels.PRItem}, + Open: true, + Labels: append([]string{"Docs analysis"}, labels...), + Author: stageengine.AuthorFacts{Kind: authorkind.Hive, Login: "hive"}, + PR: &stageengine.PRFacts{Checks: stagelabels.CheckSummary{Known: true, Total: 3, Green: 3}, DCO: stagelabels.DCOYes}, + } +} + +// TestTechDocsRules walks an assessment through the rules evaluator the +// way Nate's spec describes it: request → accepted → draft → verified → +// stakeholder review → confirmed → merged, plus the declined and held +// branches. +func TestTechDocsRules(t *testing.T) { + p := techdocs(t) + issues := stageengine.DeriveFromModel(p, stagelabels.IssueItem) + prs := stageengine.DeriveFromModel(p, stagelabels.PRItem) + + cases := []struct { + name string + d stageengine.Deriver + f func() stageengine.Facts + stage stagelabels.Stage + term bool + }{ + {"new request", issues, func() stageengine.Facts { return issue("needs-triage") }, stagelabels.Actionable, false}, + {"accepted", issues, func() stageengine.Facts { return issue("triage/accepted") }, stagelabels.Claimed, false}, + {"draft posted covers the request", issues, func() stageengine.Facts { + f := issue("triage/accepted", "phase/analysis") + f.Issue.CoveringPR = &stageengine.Ref{Owner: "cncf", Repo: "techdocs", Number: 2, Kind: stagelabels.PRItem} + return f + }, stagelabels.CoveredByPR, false}, + {"escalated holds", issues, func() stageengine.Facts { + f := issue("triage/accepted") + f.Park.Holds = []string{"hold"} + return f + }, stagelabels.IssueNeedsDecision, false}, + {"declined", issues, func() stageengine.Facts { f := issue("triage/declined"); f.Open = false; return f }, stagelabels.None, true}, + {"closed", issues, func() stageengine.Facts { f := issue("triage/accepted"); f.Open = false; return f }, stagelabels.None, true}, + {"drafting", prs, func() stageengine.Facts { f := pr(); f.Draft = true; return f }, stagelabels.Fixing, false}, + {"verified draft in review", prs, func() stageengine.Facts { f := pr("verified"); f.Draft = true; return f }, stagelabels.ReviewPending, false}, + {"ready but unverified stays in review", prs, func() stageengine.Facts { return pr() }, stagelabels.ReviewPending, false}, + {"checks failing", prs, func() stageengine.Facts { + f := pr("verified") + f.PR.Checks.Red = []string{"spell-check"} + return f + }, stagelabels.NeedsAuthor, false}, + {"stakeholder review", prs, func() stageengine.Facts { return pr("verified") }, stagelabels.Reviewed, false}, + {"approved awaits merge", prs, func() stageengine.Facts { return pr("verified", "confirmed") }, stagelabels.ReadyForHumanMerge, false}, + {"merged", prs, func() stageengine.Facts { f := pr(); f.Open, f.Merged = false, true; return f }, stagelabels.None, true}, + {"discarded", prs, func() stageengine.Facts { f := pr(); f.Open = false; return f }, stagelabels.None, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := tc.d.Derive(tc.f()) + if got.Abstain { + t.Fatalf("abstained: %s", got.Reason) + } + if got.Terminal != tc.term || got.Stage != tc.stage { + t.Fatalf("got %+v; want stage %s terminal %v", got, tc.stage, tc.term) + } + }) + } + t.Run("unrelated issue abstains", func(t *testing.T) { + if got := issues.Derive(issue("bug")); !got.Abstain { + t.Fatalf("%+v", got) + } + }) + t.Run("unlabelled PR abstains", func(t *testing.T) { + f := pr() + f.Labels = nil + if got := prs.Derive(f); !got.Abstain { + t.Fatalf("%+v", got) + } + }) + t.Run("replies resolve to gates", func(t *testing.T) { + for state, verb := range map[string]string{"requested": "/accept", "stakeholder-review": "/confirm", "awaiting-merge": "/approve"} { + r, ok := p.Reply(state, verb) + if !ok || r.To == "" { + t.Fatalf("%s %s: ok=%v %+v", state, verb, ok, r) + } + } + if _, ok := p.Reply("requested", "/approve"); ok { + t.Fatal("approve offered out of requested") + } + }) +} + +var testNow = time.Date(2026, 3, 14, 0, 0, 0, 0, time.UTC) + +func TestEntriesSorted(t *testing.T) { + entries, _ := curated.Load() + for i := 1; i < len(entries); i++ { + if strings.Compare(entries[i-1].Path, entries[i].Path) > 0 { + t.Fatalf("not sorted: %s > %s", entries[i-1].Path, entries[i].Path) + } + } +} diff --git a/src/pkg/hub/marketplace/curated/load_test.go b/src/pkg/hub/marketplace/curated/load_test.go new file mode 100644 index 0000000000..d59018c33f --- /dev/null +++ b/src/pkg/hub/marketplace/curated/load_test.go @@ -0,0 +1,46 @@ +package curated + +import ( + "errors" + "io/fs" + "testing" + "testing/fstest" +) + +func TestLoadSkipsBadFiles(t *testing.T) { + good, err := files.ReadFile("cncf-techdocs-assessment.yaml") + if err != nil { + t.Fatal(err) + } + fsys := fstest.MapFS{ + "a-bad.yaml": {Data: []byte("a: [")}, + "b-good.yaml": {Data: good}, + "c-wrong.yaml": {Data: []byte("apiVersion: workflow.hivecommons.dev/v1alpha1\nkind: WorkflowModel\nspec: {meta: {version: v1, provenance: ingested}}\n")}, + "ignored.json": {Data: []byte("{")}, + } + entries, errs := load(fsys) + if len(entries) != 1 || entries[0].Path != "b-good.yaml" { + t.Fatalf("entries = %+v", entries) + } + if len(errs) != 2 { + t.Fatalf("errs = %v", errs) + } +} + +// unreadableFS globs one file whose read fails. +type unreadableFS struct{ fstest.MapFS } + +func (u unreadableFS) Open(name string) (fs.File, error) { + if name == "x.yaml" { + return nil, errors.New("boom") + } + return u.MapFS.Open(name) +} + +func TestLoadReadError(t *testing.T) { + fsys := unreadableFS{fstest.MapFS{"x.yaml": {Data: []byte("a: 1")}}} + entries, errs := load(fsys) + if len(entries) != 0 || len(errs) != 1 { + t.Fatalf("entries=%v errs=%v", entries, errs) + } +} diff --git a/src/pkg/hub/marketplace_curated.go b/src/pkg/hub/marketplace_curated.go new file mode 100644 index 0000000000..6ca8deb850 --- /dev/null +++ b/src/pkg/hub/marketplace_curated.go @@ -0,0 +1,56 @@ +package hub + +import ( + "time" + + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// RFC #17620 phase 7: curated seeding. +// +// The hub ships reviewed models under pkg/hub/marketplace/curated/*.yaml. +// seedCuratedModels compiles each, wraps it in a bundle at tier curated and +// publishes it when that name@version is not yet in the catalogue. Entries +// are immutable, so a changed file with an unchanged version is reported as +// a conflict rather than overwriting; a new version is a new file. Ingest +// never touches curated versions; marketplaceNormaliseIndex flags them +// newerIngestAvailable when a later ingested-unreviewed version exists. + +// marketplaceCuratedBy is the publisher recorded on seeded bundles. +const marketplaceCuratedBy = "curated-seed" + +// marketplaceCuratedLoad is swapped in tests to inject bad or extra models. +var marketplaceCuratedLoad = curated.Load + +// seedCuratedModels publishes every embedded curated model that is absent +// from the catalogue. It returns the keys it published and the errors it +// met; nothing here fails the caller — a bad curated file is logged and +// skipped so the rest of the catalogue still seeds. +func (s *HubServer) seedCuratedModels(st *marketplaceStore, now time.Time) (published []string, errs []error) { + entries, loadErrs := marketplaceCuratedLoad() + for _, err := range loadErrs { + s.logger.Error("marketplace: curated model refused", "error", err) + errs = append(errs, err) + } + for _, e := range entries { + b, err := bundle.Export(e.Model, marketplaceCuratedBy, now) + if err != nil { + s.logger.Error("marketplace: curated model does not compile", "path", e.Path, "error", err) + errs = append(errs, err) + continue + } + b.Tier = bundle.TierCurated + entry, created, err := st.Publish(b, marketplaceCuratedBy, now, "") + if err != nil { + s.logger.Error("marketplace: curated model not published", "path", e.Path, "key", b.Key(), "error", err) + errs = append(errs, err) + continue + } + if created { + s.logger.Info("marketplace: curated model published", "path", e.Path, "key", entry.Key()) + published = append(published, entry.Key()) + } + } + return published, errs +} diff --git a/src/pkg/hub/marketplace_curated_test.go b/src/pkg/hub/marketplace_curated_test.go new file mode 100644 index 0000000000..b08a0786cb --- /dev/null +++ b/src/pkg/hub/marketplace_curated_test.go @@ -0,0 +1,159 @@ +package hub + +import ( + "bytes" + "context" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +const mktTechDocsKey = "cncf-techdocs-assessment@curated-2026.11.1" + +// TestMarketplaceCuratedSeedPublishesAndIsIdempotent: the embedded curated +// models land at tier curated on the first seed; a second seed publishes +// nothing and leaves exactly one version per model. +func TestMarketplaceCuratedSeedPublishesAndIsIdempotent(t *testing.T) { + s := marketplaceTestHub(t, nil) + st := s.marketplace() + now := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + + published, errs := s.seedCuratedModels(st, now) + if len(errs) != 0 { + t.Fatalf("seed errors: %v", errs) + } + if len(published) != mktCuratedCount(t) || published[0] != mktTechDocsKey { + t.Fatalf("published = %v", published) + } + raw, err := st.Get("cncf-techdocs-assessment", "curated-2026.11.1") + if err != nil { + t.Fatal(err) + } + b, err := bundle.Decode(raw) + if err != nil { + t.Fatal(err) + } + if b.Tier != bundle.TierCurated || b.CreatedBy != marketplaceCuratedBy || b.Model.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("bundle: tier=%s by=%s prov=%s", b.Tier, b.CreatedBy, b.Model.Spec.Meta.Provenance) + } + if b.Model.Spec.Scope.Repository != "cncf/techdocs" { + t.Fatalf("scope: %+v", b.Model.Spec.Scope) + } + + published, errs = s.seedCuratedModels(st, now.Add(time.Hour)) + if len(published) != 0 || len(errs) != 0 { + t.Fatalf("second seed: published=%v errs=%v", published, errs) + } + vs, err := st.Versions("cncf-techdocs-assessment") + if err != nil || len(vs) != 1 { + t.Fatalf("versions: %v %d", err, len(vs)) + } + idx, _, _, err := st.Index() + if err != nil { + t.Fatal(err) + } + for _, e := range idx.Entries { + if e.Name == "cncf-techdocs-assessment" && (e.Tier != bundle.TierCurated || e.NewerIngestAvailable) { + t.Fatalf("index entry: %+v", e) + } + } +} + +// TestMarketplaceCuratedSeedRefusesBadModels: a loader error and a model +// that fails to compile are logged and skipped; the good model still seeds +// and nothing panics. +func TestMarketplaceCuratedSeedRefusesBadModels(t *testing.T) { + s := marketplaceTestHub(t, nil) + var buf bytes.Buffer + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + good, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + broken := *good[0].Model + broken.Metadata.Name = "broken-curated" + broken.Spec.Columns = nil // compile refuses: states without columns + old := marketplaceCuratedLoad + t.Cleanup(func() { marketplaceCuratedLoad = old }) + marketplaceCuratedLoad = func() ([]curated.Entry, []error) { + return append([]curated.Entry{{Path: "broken.yaml", Model: &broken}}, good...), + []error{errors.New("bad.yaml: curated: yaml: oops")} + } + + published, seedErrs := s.seedCuratedModels(s.marketplace(), time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC)) + if len(published) != len(good) || len(seedErrs) != 2 { + t.Fatalf("published=%v errs=%v", published, seedErrs) + } + logs := buf.String() + for _, want := range []string{"curated model refused", "bad.yaml", "does not compile", "broken.yaml", "curated model published"} { + if !strings.Contains(logs, want) { + t.Errorf("log missing %q:\n%s", want, logs) + } + } + if vs, err := s.marketplace().Versions("broken-curated"); err == nil && len(vs) != 0 { + t.Fatalf("broken model must not be published: %v", vs) + } + + // A refresh run surfaces the same errors without aborting ingest. + res, err := s.runMarketplaceRefresh(context.Background(), []string{}, "test") + if err != nil { + t.Fatal(err) + } + if len(res.Curated) != 0 || len(res.Errors) != 2 || !strings.HasPrefix(res.Errors[0], "curated: ") { + t.Fatalf("refresh: %+v", res) + } +} + +// TestMarketplaceCuratedSeedConflictIsImmutable: editing a curated file +// without bumping its version is refused as a conflict; the stored bundle +// stays as first published. +func TestMarketplaceCuratedSeedConflictIsImmutable(t *testing.T) { + s := marketplaceTestHub(t, nil) + st := s.marketplace() + now := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + if _, errs := s.seedCuratedModels(st, now); len(errs) != 0 { + t.Fatal(errs) + } + good, _ := curated.Load() + edited := *good[0].Model + edited.Spec.Meta.Description = "edited in place without a version bump" + old := marketplaceCuratedLoad + t.Cleanup(func() { marketplaceCuratedLoad = old }) + marketplaceCuratedLoad = func() ([]curated.Entry, []error) { + return []curated.Entry{{Path: good[0].Path, Model: &edited}}, nil + } + published, errs := s.seedCuratedModels(st, now) + if len(published) != 0 || len(errs) != 1 || !errors.Is(errs[0], errMarketplaceConflict) { + t.Fatalf("published=%v errs=%v", published, errs) + } +} + +// TestMarketplaceStartRefreshSeedsCurated: startup seeds curated models +// before any ingest, with no GitHub token configured. +func TestMarketplaceStartRefreshSeedsCurated(t *testing.T) { + s := marketplaceTestHub(t, nil) + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("HIVE_HUB_GITHUB_TOKEN", "") + t.Setenv(hubMarketplaceScheduleEnv, "") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.StartMarketplaceRefresh(ctx); close(done) }() + testutil.Eventually(t, 5*time.Second, func() bool { + vs, err := s.marketplace().Versions("cncf-techdocs-assessment") + return err == nil && len(vs) == 1 + }, "curated model not seeded at startup") + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("StartMarketplaceRefresh did not stop on cancel") + } +} diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index c6eb57af02..baf5d208a7 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -22,8 +22,9 @@ import ( // RFC #17620 phase 7: the marketplace's ingest refresh. // -// Every run walks the CNCF survey corpus (pkg/workflowmodel/ingest/corpus) -// plus any operator-added repos, fetches each default branch, hashes the +// Every run first seeds the curated models (marketplace_curated.go), then +// walks the CNCF survey corpus (pkg/workflowmodel/ingest/corpus, including +// its operator-added extra.json) plus HIVE_HUB_MARKETPLACE_EXTRA_REPOS, fetches each default branch, hashes the // pipeline-file set and compares it with the last recorded run for that // repository. Unchanged → no-op. Changed → Lifecycle → Compile → bundle → // publish as `ingested-YYYYMMDD.N`, tier ingested-unreviewed. Curated @@ -82,9 +83,12 @@ type marketplaceRefreshResult struct { Repos int `json:"repos"` Unchanged int `json:"unchanged"` Published []string `json:"published"` - Skipped int `json:"skipped"` - Errors []string `json:"errors"` - Trigger string `json:"trigger"` + // Curated lists the curated name@version keys this run seeded (empty + // once they are all present; curated entries are immutable). + Curated []string `json:"curated,omitempty"` + Skipped int `json:"skipped"` + Errors []string `json:"errors"` + Trigger string `json:"trigger"` } // marketplaceRefresher owns the run lock and the last-run summary. @@ -219,6 +223,11 @@ func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, t } res := &marketplaceRefreshResult{StartedAt: rf.now().UTC(), Trigger: trigger, Published: []string{}, Errors: []string{}} s.logger.Info("marketplace refresh started", "trigger", trigger, "repos", len(repos)) + seeded, seedErrs := s.seedCuratedModels(st, rf.now()) + res.Curated = seeded + for _, err := range seedErrs { + res.Errors = append(res.Errors, "curated: "+err.Error()) + } for i, repo := range repos { if ctx.Err() != nil { res.Errors = append(res.Errors, "cancelled: "+ctx.Err().Error()) @@ -492,6 +501,9 @@ func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { rf.mu.Unlock() schedule := marketplaceSchedule() s.logger.Info("marketplace refresh started", "schedule_utc", schedule, "corpus_repos", len(marketplaceRepos())) + // Curated models need no GitHub token and no delay: seed them first so + // the catalogue is never empty while the ingest crawl is pending. + s.seedCuratedModels(s.marketplace(), rf.now()) if due := s.marketplaceDueRepos(); len(due) > 0 { s.logger.Warn("marketplace catalogue has repositories due for ingest: resuming in the background", "due", len(due), "delay", marketplaceSeedDelay.String()) marketplaceSleep(ctx, marketplaceSeedDelay) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index c3672cd46e..c3b8e424de 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/hub/marketplace/curated" "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/bundle" ) @@ -48,6 +49,16 @@ func marketplaceTestHub(t *testing.T, fetch marketplaceTreeFetcher) *HubServer { return s } +// mktCuratedCount is how many curated models the hub embeds and seeds. +func mktCuratedCount(t *testing.T) int { + t.Helper() + entries, errs := curated.Load() + if len(errs) != 0 { + t.Fatal(errs) + } + return len(entries) +} + func mktModel(t *testing.T) *workflowmodel.Model { t.Helper() raw, err := os.ReadFile("../workflowmodel/testdata/models/techdocs.json") @@ -431,11 +442,17 @@ func TestMarketplaceRefreshNoopVsNewVersion(t *testing.T) { t.Errorf("version naming: %s", k) } } + if len(res.Curated) != mktCuratedCount(t) { + t.Fatalf("curated seeded on first run: %v", res.Curated) + } idx, _, _, err := s.marketplace().Index() - if err != nil || len(idx.Entries) != 2 { + if err != nil || len(idx.Entries) != 2+mktCuratedCount(t) { t.Fatalf("index after first run: %v %d", err, len(idx.Entries)) } for _, e := range idx.Entries { + if e.Tier == bundle.TierCurated { + continue + } if e.Tier != bundle.TierIngestedUnreviewed || e.SourceSHA == "" { t.Errorf("entry: %+v", e) } @@ -446,7 +463,7 @@ func TestMarketplaceRefreshNoopVsNewVersion(t *testing.T) { if err != nil { t.Fatal(err) } - if len(res.Published) != 0 || res.Unchanged != 3 || len(res.Errors) != 1 { + if len(res.Published) != 0 || len(res.Curated) != 0 || res.Unchanged != 3 || len(res.Errors) != 1 { t.Fatalf("second run: %+v", res) } @@ -588,9 +605,12 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { t.Fatal(err) } - if status["entries"].(float64) != 1 || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { + if int(status["entries"].(float64)) != 1+mktCuratedCount(t) || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { t.Fatalf("status body: %s", rec.Body.String()) } + if byTier, _ := status["byTier"].(map[string]interface{}); int(byTier[bundle.TierCurated].(float64)) != mktCuratedCount(t) { + t.Fatalf("curated tier not counted: %s", rec.Body.String()) + } if status["pendingRepos"].(float64) != 0 || status["lastRefreshErrors"].(float64) != 0 { t.Fatalf("pending/lastErrors: %s", rec.Body.String()) } @@ -641,7 +661,8 @@ func TestMarketplaceSeedResumesAfterRestart(t *testing.T) { if _, err := s.runMarketplaceRefresh(context.Background(), []string{"konflux-ci/konflux-ci"}, "seed"); err != nil { t.Fatal(err) } - if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1 { + // One ingested entry plus the curated models every run seeds. + if idx, _, _, _ := s.marketplace().Index(); len(idx.Entries) != 1+mktCuratedCount(t) { t.Fatalf("precondition: %d entries", len(idx.Entries)) } due := s.marketplaceDueRepos() diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.go b/src/pkg/workflowmodel/ingest/corpus/corpus.go index a9139905f8..0c87d12c58 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus.go +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.go @@ -4,11 +4,19 @@ // refresh seeds its `ingested-unreviewed` tier from this list, and // cmd/workflow-survey falls back to it when a corpus directory carries no // projects.json of its own. +// +// corpus.json is the survey's projects.json and is regenerated from the +// CNCF landscape; do not hand-edit it. extra.json is the operator-added +// list of repositories that are not landscape projects but whose workflow +// the marketplace should still ingest (cncf/techdocs, for one). Add a repo +// there: {"name", "category", "repos": ["owner/repo"], "why"}. Extra +// entries carry no maturity. package corpus import ( _ "embed" "encoding/json" + "fmt" "sort" "strings" "sync" @@ -17,6 +25,9 @@ import ( //go:embed corpus.json var corpusJSON []byte +//go:embed extra.json +var extraJSON []byte + // Project is one surveyed CNCF project. type Project struct { Name string `json:"name"` @@ -25,29 +36,44 @@ type Project struct { Repos []string `json:"repos"` } +// Extra is one operator-added, non-landscape repository group (extra.json). +type Extra struct { + Name string `json:"name"` + Category string `json:"category,omitempty"` + Repos []string `json:"repos"` + // Why records the reason the entry is in the corpus. + Why string `json:"why,omitempty"` +} + var ( - loadOnce sync.Once - loaded []Project - loadErr error + loadOnce sync.Once + loaded []Project + loadedExtra []Extra + loadErr error ) -func load() ([]Project, error) { +func load() ([]Project, []Extra, error) { loadOnce.Do(func() { var ps []Project if err := json.Unmarshal(corpusJSON, &ps); err != nil { - loadErr = err + loadErr = fmt.Errorf("corpus.json: %w", err) return } - loaded = ps + var xs []Extra + if err := json.Unmarshal(extraJSON, &xs); err != nil { + loadErr = fmt.Errorf("extra.json: %w", err) + return + } + loaded, loadedExtra = ps, xs }) - return loaded, loadErr + return loaded, loadedExtra, loadErr } // Projects returns a copy of the embedded project list, sorted by name. func Projects() []Project { - ps, err := load() + ps, _, err := load() if err != nil { - panic("corpus: embedded corpus.json is invalid: " + err.Error()) + panic("corpus: embedded corpus is invalid: " + err.Error()) } out := make([]Project, len(ps)) for i, p := range ps { @@ -57,19 +83,42 @@ func Projects() []Project { return out } -// Repos returns every distinct `owner/repo` in the corpus, lower-cased and -// sorted, so callers iterate deterministically. +// Extras returns a copy of the operator-added non-landscape entries. +func Extras() []Extra { + _, xs, err := load() + if err != nil { + panic("corpus: embedded corpus is invalid: " + err.Error()) + } + out := make([]Extra, len(xs)) + for i, x := range xs { + out[i] = x + out[i].Repos = append([]string(nil), x.Repos...) + } + return out +} + +// Repos returns every distinct `owner/repo` in the corpus — survey +// projects plus extras — lower-cased and sorted, so callers iterate +// deterministically. func Repos() []string { seen := map[string]bool{} var out []string + add := func(r string) { + r = strings.ToLower(strings.TrimSpace(r)) + if r == "" || seen[r] { + return + } + seen[r] = true + out = append(out, r) + } for _, p := range Projects() { for _, r := range p.Repos { - r = strings.ToLower(strings.TrimSpace(r)) - if r == "" || seen[r] { - continue - } - seen[r] = true - out = append(out, r) + add(r) + } + } + for _, x := range Extras() { + for _, r := range x.Repos { + add(r) } } sort.Strings(out) diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus_test.go b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go index 2892626047..787778cbb5 100644 --- a/src/pkg/workflowmodel/ingest/corpus/corpus_test.go +++ b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go @@ -38,6 +38,20 @@ func TestProjectsAndRepos(t *testing.T) { if m["project-akri/akri"] != "sandbox" { t.Fatalf("maturity = %q", m["project-akri/akri"]) } + if !seen["cncf/techdocs"] { + t.Fatal("expected the extra.json repo cncf/techdocs in Repos()") + } + xs := Extras() + if len(xs) == 0 || xs[0].Repos[0] != "cncf/techdocs" || xs[0].Why == "" { + t.Fatalf("extras = %+v", xs) + } + xs[0].Repos[0] = "tampered" + if Extras()[0].Repos[0] == "tampered" { + t.Fatal("Extras leaked internal slice") + } + if _, ok := m["cncf/techdocs"]; ok { + t.Fatal("extras must not carry a maturity") + } // Projects() hands out copies. ps[0].Repos[0] = "tampered" if Projects()[0].Repos[0] == "tampered" { @@ -46,15 +60,27 @@ func TestProjectsAndRepos(t *testing.T) { } func TestLoadInvalid(t *testing.T) { - old := corpusJSON - t.Cleanup(func() { corpusJSON = old; loadOnce = sync.Once{}; loaded, loadErr = nil, nil }) - loadOnce = sync.Once{} - loaded, loadErr = nil, nil - corpusJSON = []byte("{not json") - defer func() { - if recover() == nil { - t.Fatal("expected panic on invalid embedded corpus") - } - }() - Projects() + reset := func() { loadOnce = sync.Once{}; loaded, loadedExtra, loadErr = nil, nil, nil } + oldCorpus, oldExtra := corpusJSON, extraJSON + t.Cleanup(func() { corpusJSON, extraJSON = oldCorpus, oldExtra; reset() }) + expectPanic := func(t *testing.T, f func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Fatal("expected panic on invalid embedded corpus") + } + }() + f() + } + t.Run("corpus.json", func(t *testing.T) { + reset() + corpusJSON = []byte("{not json") + expectPanic(t, func() { Projects() }) + }) + t.Run("extra.json", func(t *testing.T) { + reset() + corpusJSON = oldCorpus + extraJSON = []byte("{not json") + expectPanic(t, func() { Extras() }) + }) } diff --git a/src/pkg/workflowmodel/ingest/corpus/extra.json b/src/pkg/workflowmodel/ingest/corpus/extra.json new file mode 100644 index 0000000000..bb2cfcaab1 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/corpus/extra.json @@ -0,0 +1,8 @@ +[ + { + "name": "CNCF TechDocs", + "category": "non-landscape", + "repos": ["cncf/techdocs"], + "why": "CNCF TechDocs team repository; home of the AI-assisted documentation assessment workflow (curated model cncf-techdocs-assessment). Not a landscape project, so the survey never listed it." + } +] From 9321c027c2af4e3378f08d997805817129eab5d9 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:51:38 +0000 Subject: [PATCH 024/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20classify=20repeat?= =?UTF-8?q?ed=20agent-lifecycle=20log=20lines=20instead=20of=20filing=20un?= =?UTF-8?q?classified=20selfreports=20(#17757)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: classify repeated agent-lifecycle log lines instead of filing unclassified selfreports Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * test: expect agent_lifecycle_repeated class for non-held kick failures TestPostRestartKickHoldLogsWithoutFleetReport asserted the generic unclassified class; 'failed to send kick' is now classified as agent_lifecycle_repeated. Held kicks still produce no observation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/fixed-agent-lifecycle-repeated-class.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 6 ++++++ src/pkg/fleetreport/slogfeed/slogfeed_test.go | 2 +- 5 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-agent-lifecycle-repeated-class.md diff --git a/changelog.d/fixed-agent-lifecycle-repeated-class.md b/changelog.d/fixed-agent-lifecycle-repeated-class.md new file mode 100644 index 0000000000..fda13f7ad7 --- /dev/null +++ b/changelog.d/fixed-agent-lifecycle-repeated-class.md @@ -0,0 +1 @@ +- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index be45d81494..bd96ad2143 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -160,6 +160,7 @@ title is `[selfreport] : `. | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`), or a request the translator could not translate (`inference translate request failed`, logged at ERROR by `logTranslateFailure`), all in `crux/proxy/github_proxy_inference.go`, repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. A request refused because the agent has neither an agent model nor a gateway `default_model` logs `inference model not configured; request refused` at WARN instead and is not counted here (a benign configuration condition; see below). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (`SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller keeps serving the SHA without a message. A throttled 403/429 is treated as client-wide: every SHA is parked until `Retry-After` / `X-RateLimit-Reset` elapses and a single WARN is logged per hour per status (`warnGitHubFetchOnce`, key `sha-msg:throttled:`), so a rate limit or revoked token cannot trip this class. Other statuses (e.g. 404 for a force-pushed-away commit) back off per SHA and are deduped to once per hour per (SHA, status), so one stuck commit logs one line; the class is reported only when ≥5 distinct SHA/status pairs fail within 10 min (many branches whose commits GitHub no longer serves). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `agent_lifecycle_repeated` | known agent-lifecycle log lines (`failed to send kick`, `agent CLI crashed (bare shell detected)`, `tmux kill-session before restart failed`, `agent tmux session missing`, watchdog `would restart dead agent` in observe mode) repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout`; replaces the catch-all `unclassified_error_repeated` for these lines | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | | `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently stale or unreadable). Diagnostics count stale snapshots, closed items, deferred reads and other read errors for the last withheld kick, without including refs or raw errors. See local per-ref logs for details. Deferred API-budget refusals and refs the re-read found closed or merged log at INFO, so a kick withheld only by those never trips it. Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 0318cd1d7c..c0b7a6779d 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -92,6 +92,7 @@ const ( ClassLabelProvisioningFailed = "label_provisioning_failed" ClassAgentCLIRestartLoop = "agent_cli_restart_loop" ClassCopilotCLIHung = "copilot_cli_hung" + ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" ClassTranscriptScanSlow = "transcript_scan_slow" ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" ClassIssueClaimEffectFailed = "issue_claim_effect_failed" @@ -212,6 +213,7 @@ var Classes = map[string]ClassSpec{ ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, + ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index be1fa01910..1593918264 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -441,6 +441,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassIssueRequestStuck, message: "issue-request stuck: issue request failed 3 attempts over 7m"}, {class: ClassTmuxSessionCheckFailed, message: "tmux has-session failed"}, {class: ClassAgentPaneTimeout, message: "pane content at timeout"}, + {class: ClassAgentLifecycleRepeated, message: "agent tmux session missing"}, {class: ClassStartupKickInterrupted, message: "startup kick dropped: agent restarted or stopped while waiting"}, {class: ClassProxyTLSHandshakeFailed, message: "proxy client TLS handshake failed"}, {class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"}, @@ -1196,6 +1197,11 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, "tmux has-session failed": ClassTmuxSessionCheckFailed, + "failed to send kick": ClassAgentLifecycleRepeated, + "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, + "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, + "agent tmux session missing": ClassAgentLifecycleRepeated, + "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, "pane content at timeout": ClassAgentPaneTimeout, "prompt timeout — dumping pane": ClassAgentPaneTimeout, "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index f49266b3de..4d2757e831 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -488,7 +488,7 @@ func TestPostRestartKickHoldLogsWithoutFleetReport(t *testing.T) { } // Other kick send failures must remain reportable. logger.Warn("failed to send kick", "agent", "reviewer-2", "error", "tmux send-keys exit 1") - if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated { + if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassAgentLifecycleRepeated { t.Fatalf("other kick failures must remain reportable: %+v", *seen) } } From 6e33719bc2265a2e41881ef67f4cfa559504fb95 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:02 +0000 Subject: [PATCH 025/166] =?UTF-8?q?=F0=9F=90=9B=20fix(marketplace):=20refu?= =?UTF-8?q?se=20and=20downgrade=20model-provenance=20curated=20claims=20fr?= =?UTF-8?q?om=20non-admin=20bundles=20(#17764)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(marketplace): refuse and downgrade model-provenance curated claims from non-admin bundles The hub checked only the envelope Tier when deciding whether a publisher may assert curated/built-in; the model's own spec.meta.provenance is hash-covered and stored verbatim. Spoke import preferred the model's provenance over the envelope tier, so a registered spoke could publish a bundle every importer badged as curated. The hub now refuses such bundles from non-admins (it cannot rewrite the hash-covered model) and importers honour a vouched model provenance only when the envelope asserts the same tier. Fixes #17763 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * fix(marketplace): express importedProvenance as an if so the mutation gate sees its coverage Go's cover tool does not instrument the case expressions of a tagless switch, only the case bodies, so gremlins reported the three conditionals on the switch as NOT COVERED even though TestImportedProvenance exercises every branch. Fold the switch into a single if with identical semantics; if conditions sit inside the preceding coverage block and are attributed correctly. No behaviour change: empty provenance still takes the envelope tier, and a curated/built-in claim the envelope does not assert still falls back to it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../security-marketplace-model-provenance.md | 1 + .../dashboard/api_workflows_marketplace.go | 26 +++++++-- .../api_workflows_marketplace_test.go | 55 +++++++++++++++++++ src/pkg/hub/marketplace.go | 7 +++ src/pkg/hub/marketplace_test.go | 12 ++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 changelog.d/security-marketplace-model-provenance.md diff --git a/changelog.d/security-marketplace-model-provenance.md b/changelog.d/security-marketplace-model-provenance.md new file mode 100644 index 0000000000..2f6054f7a6 --- /dev/null +++ b/changelog.d/security-marketplace-model-provenance.md @@ -0,0 +1 @@ +- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go index b7836e8b2a..f9ffcdea91 100644 --- a/src/pkg/dashboard/api_workflows_marketplace.go +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -193,15 +193,29 @@ func (s *Server) importBundle(st *wfstore.Store, b *bundle.Bundle, by string) (w version = next } m.Spec.Meta.Version = version - if m.Spec.Meta.Provenance == "" { - m.Spec.Meta.Provenance = workflowmodel.Provenance(b.Tier) - } - if m.Spec.Meta.Provenance == workflowmodel.ProvenanceBuiltIn || string(m.Spec.Meta.Provenance) == bundle.TierLocalPublished { - m.Spec.Meta.Provenance = workflowmodel.ProvenanceLocal - } + m.Spec.Meta.Provenance = importedProvenance(m.Spec.Meta.Provenance, b.Tier) return st.Save(m, by, time.Now().UTC()) } +// importedProvenance decides what provenance an imported model is stored +// with. The envelope tier is the field the hub polices (curated/built-in +// need a hub admin); the model's own provenance is hash-covered and set by +// whoever authored the model, so a vouched tier claimed there is honoured +// only when the envelope asserts the same tier. built-in and +// local-published always land as local: this hive did not ship the model +// and has not published it. +func importedProvenance(model workflowmodel.Provenance, tier string) workflowmodel.Provenance { + p := model + vouched := p == workflowmodel.ProvenanceCurated || p == workflowmodel.ProvenanceBuiltIn + if p == "" || (vouched && string(p) != tier) { + p = workflowmodel.Provenance(tier) + } + if p == workflowmodel.ProvenanceBuiltIn || string(p) == bundle.TierLocalPublished || p == "" { + p = workflowmodel.ProvenanceLocal + } + return p +} + // hubRequest builds an authenticated spoke→hub request. func (s *Server) hubRequest(ctx context.Context, method, rawURL string, body []byte) (*http.Request, error) { var rd io.Reader diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index eed2d9897b..735b3c0205 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -458,3 +458,58 @@ func TestMarketplaceBaseFallsBackToHubURL(t *testing.T) { t.Fatal("server without deps should be enabled but have no base") } } + +func TestImportedProvenance(t *testing.T) { + cases := []struct { + model workflowmodel.Provenance + tier string + want workflowmodel.Provenance + }{ + // Empty model provenance takes the envelope tier. + {"", bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {"", bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {"", bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + {"", "", workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope agrees with is kept (built-in still lands as local). + {workflowmodel.ProvenanceCurated, bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope does not assert is laundering: fall back to the tier. + {workflowmodel.ProvenanceCurated, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceCurated, bundle.TierIngestedUnreviewed, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceCurated, "", workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + // Lower tiers in the model are honoured as-is. + {workflowmodel.ProvenanceIngestedUnreviewed, bundle.TierLocalPublished, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceLocal, bundle.TierCurated, workflowmodel.ProvenanceLocal}, + } + for _, tc := range cases { + if got := importedProvenance(tc.model, tc.tier); got != tc.want { + t.Errorf("importedProvenance(%q, %q) = %q, want %q", tc.model, tc.tier, got, tc.want) + } + } +} + +// A bundle whose envelope the hub normalised to local-published but whose +// hash-covered model still claims curated must not import as curated. +func TestWorkflowImportDoesNotLaunderCuratedProvenance(t *testing.T) { + s := marketplaceServer(t, "") + m := techdocsModel(t) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + b, err := bundle.Export(m, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + b.Tier = bundle.TierLocalPublished + raw, _ := bundle.Encode(b) + rec := wfReq(s, http.MethodPost, "/api/workflows/import", string(raw), true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + got, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil { + t.Fatal(err) + } + if got.Spec.Meta.Provenance != workflowmodel.ProvenanceLocal { + t.Fatalf("laundered provenance = %q, want local", got.Spec.Meta.Provenance) + } +} diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 19750e96aa..0388838bb4 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -180,6 +180,13 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusBadRequest, "unknown tier "+b.Tier) return } + // The model's own provenance is hash-covered, so it cannot be normalised + // like the envelope tier; a vouched tier claimed there needs the same + // admin the envelope does, otherwise importers would store it as vouched. + if p := string(b.Model.Spec.Meta.Provenance); (p == bundle.TierCurated || p == bundle.TierBuiltIn) && !who.admin { + marketplaceJSONError(w, http.StatusForbidden, fmt.Sprintf("model provenance %q requires a hub admin", p)) + return + } if err := bundle.Verify(b); err != nil { marketplaceJSONError(w, http.StatusUnprocessableEntity, "bundle verification failed: "+err.Error()) return diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index c3b8e424de..d62f22c3f0 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -320,6 +320,18 @@ func TestMarketplacePublishAuth(t *testing.T) { if rec := mktDo(s, mktSpokeReq(mktBody(t, mktBundle(t, bundle.TierCurated)))); rec.Code != 403 { t.Fatalf("spoke curated: %d %s", rec.Code, rec.Body.String()) } + // Spoke asserting curated inside the hash-covered model (envelope tier + // legitimate) → 403: the hub cannot normalise it, so it must refuse. + launder := mktModel(t) + launder.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + lb, err := bundle.Export(launder, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + lb.Tier = bundle.TierLocalPublished + if rec := mktDo(s, mktSpokeReq(mktBody(t, lb))); rec.Code != 403 || !strings.Contains(rec.Body.String(), "model provenance") { + t.Fatalf("spoke model-curated: %d %s", rec.Code, rec.Body.String()) + } // Spoke OK: `local` is normalised to local-published. rec := mktDo(s, mktSpokeReq(body)) if rec.Code != 201 { From 66fe03c3e9d2e98467bf923bbc4232656f21fcb8 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:26 +0000 Subject: [PATCH 026/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hub=20digest=20re?= =?UTF-8?q?ads=20a=20stale=20stage-tracker=20body=20instead=20of=20blockin?= =?UTF-8?q?g=20on=20a=20rebuild=20(#17785)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: let the hub activity digest read a stale stage-tracker body instead of blocking on a rebuild Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🧪 test: wait for the hub digest rebuild with testutil.Eventually instead of a sleep loop TestSleepRatchet failed on CI (196 time.Sleep calls in tests, baseline 195): the new stale-read test polled stageTrackerBuild with a hand-rolled time.Sleep loop. Use testutil.Eventually, which is the helper the ratchet points to. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-hub-digest-stale-tracker-read.md | 1 + src/pkg/dashboard/hub_activity.go | 2 +- src/pkg/dashboard/hub_activity_test.go | 54 +++++++++++++++++++ src/pkg/dashboard/stage_tracker_build.go | 28 +++++++--- 4 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 changelog.d/fixed-hub-digest-stale-tracker-read.md diff --git a/changelog.d/fixed-hub-digest-stale-tracker-read.md b/changelog.d/fixed-hub-digest-stale-tracker-read.md new file mode 100644 index 0000000000..12611376c5 --- /dev/null +++ b/changelog.d/fixed-hub-digest-stale-tracker-read.md @@ -0,0 +1 @@ +- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. diff --git a/src/pkg/dashboard/hub_activity.go b/src/pkg/dashboard/hub_activity.go index a278242a58..0010590830 100644 --- a/src/pkg/dashboard/hub_activity.go +++ b/src/pkg/dashboard/hub_activity.go @@ -63,7 +63,7 @@ func (s *Server) HubActivitySnapshot(ctx context.Context) (hubactivity.Snapshot, if status == nil { return hubactivity.Snapshot{}, false } - body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false) + body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false, true) if body == nil { return hubactivity.Snapshot{}, false } diff --git a/src/pkg/dashboard/hub_activity_test.go b/src/pkg/dashboard/hub_activity_test.go index ea8f730c73..206ecc34a4 100644 --- a/src/pkg/dashboard/hub_activity_test.go +++ b/src/pkg/dashboard/hub_activity_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/hubactivity" "github.com/hivecommons/hive/pkg/stagelabels" @@ -110,3 +111,56 @@ func TestOnStageTrackerPublishFiresOnInvalidation(t *testing.T) { var nilServer *Server nilServer.OnStageTrackerPublish(func() {}) } + +// TestHubActivityItemsServesStaleBodyWithoutBuilding pins +// hivecommons/hive#17776: once the tracker has a body, an invalidated cache +// must not make the digest read wait on a rebuild. +func TestHubActivityItemsServesStaleBodyWithoutBuilding(t *testing.T) { + t.Setenv("HIVE_PUBLIC_REPOS", "") + now := time.Date(2026, 10, 31, 9, 0, 0, 0, time.UTC) + s := newTestServer() + s.stageStore = stagelabels.NewStore("", func() time.Time { return now }) + s.statusMu.Lock() + s.status = minimalPayload() + s.statusMu.Unlock() + + release := make(chan struct{}) + started := make(chan struct{}, 2) + s.stageTrackerBuildFn = func(_ *StatusPayload, _ *stagelabels.Store, at time.Time) StageTrackerResponse { + started <- struct{}{} + <-release + return StageTrackerResponse{GeneratedAt: at, Items: []StageTrackerItem{{Repo: "acme/widget", Number: 7, Kind: "pr", Modifiers: []string{}}}} + } + close(release) + if _, ok := s.HubActivitySnapshot(context.Background()); !ok { + t.Fatal("first read must build a body when none exists") + } + <-started + + release = make(chan struct{}) + s.invalidateStageTrackerCache() + type result struct { + snap hubactivity.Snapshot + ok bool + } + done := make(chan result, 1) + go func() { + snap, ok := s.HubActivitySnapshot(context.Background()) + done <- result{snap, ok} + }() + select { + case r := <-done: + if !r.ok || len(r.snap.Items) != 1 { + t.Fatalf("stale read = %+v, %v; want the previous body", r.snap, r.ok) + } + case <-time.After(2 * time.Second): + t.Fatal("digest read blocked on the rebuild instead of serving the stale body") + } + <-started + close(release) + testutil.Eventually(t, 2*time.Second, func() bool { + s.stageTrackerCacheMu.Lock() + defer s.stageTrackerCacheMu.Unlock() + return s.stageTrackerBuild == nil + }, "background rebuild never finished") +} diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index 2a99c9406e..b6f57dcefd 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -887,13 +887,16 @@ func (s *Server) stageTrackerPayload(status *StatusPayload, now time.Time) (body // came from (stageTrackerSource*Name; "" with a nil body when the build // failed or ctx ended while waiting on an in-flight build). func (s *Server) stageTrackerPayloadSource(ctx context.Context, status *StatusPayload, now time.Time) (body []byte, etag string, generatedAt time.Time, source string) { - return s.stageTrackerPayloadFrom(ctx, status, now, true) + return s.stageTrackerPayloadFrom(ctx, status, now, true, false) } // stageTrackerPayloadFrom is stageTrackerPayloadSource; markServed false // reads the tracker without counting as a client that asked for it, so an -// in-process reader (the hub activity digest) adds no SSE nudges. -func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed bool) (body []byte, etag string, generatedAt time.Time, source string) { +// in-process reader (the hub activity digest) adds no SSE nudges. preferStale +// serves a stale body at once when no build is in flight and rebuilds in the +// background, so a latency-bound reader is never the synchronous builder +// (hivecommons/hive#17776); with no body at all it still builds in place. +func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed, preferStale bool) (body []byte, etag string, generatedAt time.Time, source string) { ttl := s.stageTrackerCacheTTL() s.stageTrackerCacheMu.Lock() if markServed { @@ -927,6 +930,20 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl gen := s.stageTrackerGen s.stageTrackerCacheMu.Unlock() + if preferStale && len(c.body) > 0 { + go s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + return c.body, c.etag, c.generatedAt, stageTrackerSourceStaleName + } + built, ok := s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + if !ok { + return nil, "", now, "" + } + return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName +} + +// finishStageTrackerBuild runs the claimed in-flight build, stores the +// result and releases waiters. +func (s *Server) finishStageTrackerBuild(inflight *stageTrackerBuild, status *StatusPayload, now time.Time, ttl time.Duration, gen uint64) (stageTrackerCache, bool) { built, ok := s.encodeStageTracker(status, now, ttl) s.stageTrackerCacheMu.Lock() @@ -939,10 +956,7 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl s.stageTrackerBuild = nil close(inflight.done) s.stageTrackerCacheMu.Unlock() - if !ok { - return nil, "", now, "" - } - return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName + return built, ok } // encodeStageTracker builds and encodes the tracker for status. It holds no From 2dffb678b36046d945457e7fe79805bac45ce1e5 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:48 +0000 Subject: [PATCH 027/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20log=20issue-claim?= =?UTF-8?q?s=20GitHub=20side-effect=20retries=20below=20WARN=20(#17786)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-issue-claims-retry-warn-dedupe.md | 1 + src/cmd/hive/claimswire.go | 2 +- src/cmd/hive/claimswire_retry_test.go | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-issue-claims-retry-warn-dedupe.md diff --git a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md new file mode 100644 index 0000000000..0942f81c03 --- /dev/null +++ b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md @@ -0,0 +1 @@ +- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index 58b15080e5..ddb4cdd1f2 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -93,7 +93,7 @@ func retryClaimEffect(ctx context.Context, logger *slog.Logger, what, key string return n } if logger != nil { - logger.Warn("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) + logger.Info("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) } if !wait(ctx, backoff) { if logger != nil { diff --git a/src/cmd/hive/claimswire_retry_test.go b/src/cmd/hive/claimswire_retry_test.go index 3792af3928..b3ac7d9559 100644 --- a/src/cmd/hive/claimswire_retry_test.go +++ b/src/cmd/hive/claimswire_retry_test.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "context" "errors" + "log/slog" "net/http" + "strings" "testing" "time" @@ -138,3 +141,23 @@ func TestClaimEffectRetryable(t *testing.T) { } } } + +// An expected transient retry is logged below WARN so the fleet-report +// classifier does not file it; only exhaustion warns, once +// (hivecommons/hive#17782). +func TestRetryClaimEffect_RetryLoggedBelowWarn(t *testing.T) { + swapClaimsEffectRetry(t, 3, nil) + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + retryClaimEffect(context.Background(), logger, "claimed stage", "acme/widgets#7", func() error { + return mutation.ErrNeedsReconciliation + }) + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + if strings.Contains(line, "retrying GitHub side effect") && strings.Contains(line, "level=WARN") { + t.Fatalf("retry logged at WARN: %s", line) + } + } + if got := strings.Count(buf.String(), "GitHub side effect failed"); got != 1 { + t.Fatalf("exhaustion warnings = %d, want 1", got) + } +} From a25c4a7c994ad9b3463102e40271a436393dfa2d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:54:20 +0000 Subject: [PATCH 028/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detect=20orphaned?= =?UTF-8?q?=20queued=20workflow=20runs=20in=20automerge=20held-run=20resol?= =?UTF-8?q?ution=20(#17787)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-automerge-orphaned-queued-runs.md | 1 + .../github/automerge/automerge_held_runs.go | 46 +++++++++++++++- .../automerge/automerge_held_runs_test.go | 55 +++++++++++++++++++ .../github/automerge/automerge_sweep_test.go | 14 +++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-automerge-orphaned-queued-runs.md diff --git a/changelog.d/fixed-automerge-orphaned-queued-runs.md b/changelog.d/fixed-automerge-orphaned-queued-runs.md new file mode 100644 index 0000000000..6e7f72b821 --- /dev/null +++ b/changelog.d/fixed-automerge-orphaned-queued-runs.md @@ -0,0 +1 @@ +- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). diff --git a/src/pkg/github/automerge/automerge_held_runs.go b/src/pkg/github/automerge/automerge_held_runs.go index 3f871225f5..c09935e9f0 100644 --- a/src/pkg/github/automerge/automerge_held_runs.go +++ b/src/pkg/github/automerge/automerge_held_runs.go @@ -115,13 +115,19 @@ const ( // is still treated as "GitHub has not scheduled them yet". heldRunNeverStartedGrace = 30 * time.Minute workflowRunsPageSize = 100 + // workflowRunStatusQueued is the status of a run GitHub has not started. + workflowRunStatusQueued = "queued" + // orphanedRunWarnKey keys the once-per-head orphaned-run warning in + // terminalHoldLogged. + orphanedRunWarnKey = "orphaned-queued-runs:" ) // resolveMissingRequiredCheck refines a `required-check-missing:` skip // for a trusted-lane PR. It returns the original reason when nothing is known, // a workflow-approval-required reason when GitHub is waiting for approval and // Hive has posted `/ok-to-test`, or a never-started/never-ran reason when -// polling cannot progress. It never approves a run with the App token. +// polling cannot progress. Runs GitHub orphaned in `queued` with zero jobs +// count as never started (#17775). It never approves a run with the App token. func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, owner, repo string, pr *gh.PullRequest, number int, headSHA, reason string, headPushedAt time.Time) string { if c == nil || c.gh == nil || !strings.HasPrefix(reason, requiredCheckMissingPrefix) { return reason @@ -141,6 +147,14 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o } held := heldSameRepoWorkflowRuns(runs.WorkflowRuns, owner+"/"+repo) if len(held) == 0 { + startedRuns := len(runs.WorkflowRuns) + if orphaned := c.orphanedQueuedWorkflowRuns(ctx, owner, repo, runs.WorkflowRuns); orphaned > 0 && orphaned == startedRuns { + startedRuns = 0 + if c.markTerminalHoldLogged(owner, repo, number, headSHA, orphanedRunWarnKey+missing) { + c.warn("automerge workflow runs for head are orphaned in queued with no jobs; treating them as never started", + "repo", displayRepo, "pr", number, "sha", headSHA, "orphaned", orphaned, "context", missing) + } + } if c.rebaseNudged(owner, repo, number, headSHA) || c.botTakeoverAttempted(owner, repo, number, headSHA) { return requiredCheckNeverRanPrefix + missing } @@ -148,10 +162,10 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o c.nudgeDependabotRebase(ctx, displayRepo, owner, repo, number, headSHA, missing) return requiredCheckNeverRanPrefix + missing } - if c.botTakeoverEligible(pr, len(runs.WorkflowRuns), headPushedAt) { + if c.botTakeoverEligible(pr, startedRuns, headPushedAt) { return c.takeOverBotPR(ctx, displayRepo, owner, repo, pr, number, headSHA, missing) } - if len(runs.WorkflowRuns) == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { + if startedRuns == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { return requiredCheckNeverStartedPrefix + missing } return reason @@ -582,6 +596,32 @@ func heldSameRepoWorkflowRuns(runs []*gh.WorkflowRun, fullName string) []*gh.Wor return held } +// orphanedQueuedWorkflowRuns counts same-repo runs GitHub created but left in +// `queued` past heldRunNeverStartedGrace without materializing a single job. +// Such runs never progress, so they are no evidence CI is coming (#17775). +func (c *Engine) orphanedQueuedWorkflowRuns(ctx context.Context, owner, repo string, runs []*gh.WorkflowRun) int { + orphaned := 0 + for _, run := range runs { + if run == nil || !strings.EqualFold(run.GetStatus(), workflowRunStatusQueued) { + continue + } + if run.GetHeadRepository() == nil || !strings.EqualFold(run.GetHeadRepository().GetFullName(), owner+"/"+repo) { + continue + } + if !headOlderThan(c.now, run.GetCreatedAt().Time, heldRunNeverStartedGrace) { + continue + } + jobs, _, err := c.gh.Actions.ListWorkflowJobs(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), owner, repo, run.GetID(), &gh.ListWorkflowJobsOptions{ + ListOptions: gh.ListOptions{PerPage: 1}, + }) + if err != nil || jobs == nil || jobs.GetTotalCount() > 0 || len(jobs.Jobs) > 0 { + continue + } + orphaned++ + } + return orphaned +} + func headOlderThan(now func() time.Time, headPushedAt time.Time, grace time.Duration) bool { if headPushedAt.IsZero() { return false diff --git a/src/pkg/github/automerge/automerge_held_runs_test.go b/src/pkg/github/automerge/automerge_held_runs_test.go index 2cf239b1ff..f713c241f0 100644 --- a/src/pkg/github/automerge/automerge_held_runs_test.go +++ b/src/pkg/github/automerge/automerge_held_runs_test.go @@ -131,6 +131,61 @@ func TestTrySweepSelfAuthoredPRNamesNeverStartedRequiredCheck(t *testing.T) { } } +// TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted pins +// #17775: GitHub can leave a head's runs `queued` with zero jobs forever, and +// such runs must not keep the PR at a bare required-check-missing reason. +func TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted(t *testing.T) { + now := time.Date(2026, 11, 11, 22, 10, 0, 0, time.UTC) + old := now.Add(-2 * heldRunNeverStartedGrace).Format(time.RFC3339) + fresh := now.Add(-time.Minute).Format(time.RFC3339) + queuedRun := func(id int, headRepo, createdAt string) map[string]any { + return map[string]any{ + "id": id, "name": "v2 CI", "status": "queued", "created_at": createdAt, + "head_repository": map[string]any{"full_name": headRepo}, + } + } + for _, tc := range []struct { + name string + runs []map[string]any + jobs map[int]int + want string + }{ + {name: "old queued runs with zero jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old), queuedRun(2, "acme/widget", old)}, want: requiredCheckNeverStartedPrefix + "ci/build"}, + {name: "queued run with jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old)}, jobs: map[int]int{1: 1}, want: "required-check-missing:ci/build"}, + {name: "freshly queued run", runs: []map[string]any{queuedRun(1, "acme/widget", fresh)}, want: "required-check-missing:ci/build"}, + {name: "fork head queued run", runs: []map[string]any{queuedRun(1, "mallory/widget", old)}, want: "required-check-missing:ci/build"}, + { + name: "orphaned run beside an in-progress run", + runs: []map[string]any{queuedRun(1, "acme/widget", old), {"id": 2, "name": "Tests", "status": "in_progress", "head_repository": map[string]any{"full_name": "acme/widget"}}}, + want: "required-check-missing:ci/build", + }, + } { + t.Run(tc.name, func(t *testing.T) { + pr := missingRequiredPR(func(p *selfAuthoredPR) { + p.workflowRuns = tc.runs + p.workflowRunJobs = tc.jobs + p.updatedAt = old + }) + var merged []int + api := newSelfAuthoredAutoMergeAPI(t, []selfAuthoredPR{pr}, &merged) + defer api.Close() + c := newAutoMergeSweepClient(api.URL) + c.now = func() time.Time { return now } + + _, reason, err := c.trySweepSelfAuthoredPR(context.Background(), "widget", "acme", "widget", 23, true) + if err != nil { + t.Fatalf("trySweepSelfAuthoredPR returned error: %v", err) + } + if reason != tc.want { + t.Fatalf("reason = %q, want %q", reason, tc.want) + } + if len(merged) != 0 { + t.Fatalf("merged = %v, want no merge", merged) + } + }) + } +} + func TestTerminalRequiredCheckHoldLoggedOncePerHeadAndReason(t *testing.T) { tests := []struct { name string diff --git a/src/pkg/github/automerge/automerge_sweep_test.go b/src/pkg/github/automerge/automerge_sweep_test.go index 408fbc68e6..a139bcc2f9 100644 --- a/src/pkg/github/automerge/automerge_sweep_test.go +++ b/src/pkg/github/automerge/automerge_sweep_test.go @@ -1822,6 +1822,9 @@ type selfAuthoredPR struct { updatedAt string // workflowRuns are returned by GET /actions/runs?head_sha=. workflowRuns []map[string]any + // workflowRunJobs is the job count GET /actions/runs/{id}/jobs reports + // per run id (absent = zero jobs). + workflowRunJobs map[int]int // headCommitAuthor is the login GET /commits/{head} reports as author; // commentBodies collects POST /issues/{n}/comments bodies. headCommitAuthor string @@ -1898,6 +1901,17 @@ func newSelfAuthoredAutoMergeAPI(t *testing.T, prs []selfAuthoredPR, merged *[]i runs = []map[string]any{} } json.NewEncoder(w).Encode(map[string]any{"total_count": len(runs), "workflow_runs": runs}) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/actions/runs/") && strings.HasSuffix(r.URL.Path, "/jobs"): + id := pathNumber(t, r.URL.Path, "/repos/acme/widget/actions/runs/", "/jobs") + count := 0 + for _, pr := range prs { + count += pr.workflowRunJobs[id] + } + jobs := []map[string]any{} + for i := 0; i < count; i++ { + jobs = append(jobs, map[string]any{"id": i + 1, "run_id": id, "status": "queued"}) + } + json.NewEncoder(w).Encode(map[string]any{"total_count": count, "jobs": jobs}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main": json.NewEncoder(w).Encode(map[string]any{"commit": map[string]string{"sha": "basesha"}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main/protection/required_status_checks": From 8663b9f93dfb9590be4e1202d68d4137316479b1 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:54:43 +0000 Subject: [PATCH 029/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20surface=20agen?= =?UTF-8?q?t=5Fsandbox.substrate=20and=20per-agent=20sandbox.substrate=20s?= =?UTF-8?q?ettings=20(#17788)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-settings-substrate-leaves.md | 1 + src/pkg/dashboard/api_agent_config.go | 29 +++++ src/pkg/dashboard/api_governor_security.go | 101 ++++++++++++++---- .../dashboard/settings_parity_allowlist.txt | 38 ------- .../dashboard/static-src/js/config-agent.js | 3 + .../static-src/js/config-watchdog-security.js | 66 ++++++++++++ src/pkg/dashboard/static/index.html | 69 ++++++++++++ 7 files changed, 248 insertions(+), 59 deletions(-) create mode 100644 changelog.d/added-settings-substrate-leaves.md diff --git a/changelog.d/added-settings-substrate-leaves.md b/changelog.d/added-settings-substrate-leaves.md new file mode 100644 index 0000000000..59d543165c --- /dev/null +++ b/changelog.d/added-settings-substrate-leaves.md @@ -0,0 +1 @@ +- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. diff --git a/src/pkg/dashboard/api_agent_config.go b/src/pkg/dashboard/api_agent_config.go index 081f6a677f..17ab41cf89 100644 --- a/src/pkg/dashboard/api_agent_config.go +++ b/src/pkg/dashboard/api_agent_config.go @@ -184,6 +184,7 @@ func (s *Server) handleAgentConfigGet(w http.ResponseWriter, r *http.Request) { "jevMode": agentCfg.JevMode, "jevReady": s.deps.Config.JevReady(), "explainMode": agentCfg.ExplainMode, + "sandboxSubstrate": agentSandboxSubstrateOrEmpty(agentCfg), "sandboxEnabled": agentCfg.Sandbox != nil && agentCfg.Sandbox.Enabled != nil && *agentCfg.Sandbox.Enabled, "sandboxEffective": agentCfg.SandboxEnabled(s.deps.Config.AgentSandbox), "replicas": agentCfg.Replicas, @@ -661,6 +662,27 @@ func (s *Server) handleAgentConfigGeneral(w http.ResponseWriter, r *http.Request agentCfg.Sandbox.Enabled = &bb } } + if v, ok := body["sandboxSubstrate"]; ok { + raw, err := json.Marshal(v) + var in config.SandboxSubstrateConfig + if err == nil { + err = json.Unmarshal(raw, &in) + } + var sub *config.SandboxSubstrateConfig + if err == nil { + sub, err = sanitizeSubstrateConfig(in) + } + if err != nil { + jsonError(w, "invalid sandboxSubstrate: "+err.Error(), http.StatusBadRequest) + return + } + if sub != nil && agentCfg.Sandbox == nil { + agentCfg.Sandbox = &config.AgentSandboxOverride{} + } + if agentCfg.Sandbox != nil { + agentCfg.Sandbox.Substrate = sub + } + } if v, ok := body["cavemanMode"]; ok { if s, ok := v.(string); ok { s = sanitizeString(s) @@ -1915,3 +1937,10 @@ func (s *Server) getAgentHooks(name string) map[string][]any { } return map[string][]any{"preKick": {}, "postIdle": {}} } + +func agentSandboxSubstrateOrEmpty(a config.AgentConfig) config.SandboxSubstrateConfig { + if a.Sandbox == nil { + return config.SandboxSubstrateConfig{} + } + return substrateOrEmpty(a.Sandbox.Substrate) +} diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index 85b3928712..99889a117c 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -1,8 +1,10 @@ package dashboard import ( + "errors" "net/http" "os" + "reflect" "strings" "github.com/hivecommons/hive/pkg/config" @@ -33,27 +35,28 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) IntentAlignmentModel *string `json:"intentAlignmentModel"` // Intent classifier lists. An empty list restores the built-in // defaults (nil), matching how the intent package treats them. - IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` - IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` - IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` - IntentFeatureSignals *[]string `json:"intentFeatureSignals"` - ReviewRequire *bool `json:"reviewRequireApproval"` - ReviewFanOut *bool `json:"reviewFanOut"` - ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` - ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` - ReviewFixerAgent *string `json:"reviewFixerAgent"` - AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` - SandboxImage *string `json:"agentSandboxImage"` - SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` - SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` - SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` - SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` - SandboxRuntime *string `json:"agentSandboxRuntime"` - SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` - SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` - SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` - SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` - SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` + IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` + IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` + IntentFeatureSignals *[]string `json:"intentFeatureSignals"` + ReviewRequire *bool `json:"reviewRequireApproval"` + ReviewFanOut *bool `json:"reviewFanOut"` + ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` + ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` + ReviewFixerAgent *string `json:"reviewFixerAgent"` + AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` + SandboxImage *string `json:"agentSandboxImage"` + SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` + SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` + SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` + SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` + SandboxRuntime *string `json:"agentSandboxRuntime"` + SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` + SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` + SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` + SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` + SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + SandboxSubstrate *config.SandboxSubstrateConfig `json:"agentSandboxSubstrate"` // Sentinel (suspicious-activity alerts) is pointer-typed per field // so an absent key means "unchanged" — a save that only toggled a // behavior must not wipe the sensitive-path list. @@ -263,6 +266,14 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) job.TTLSeconds = *body.SandboxJobTTLSeconds } } + if body.SandboxSubstrate != nil { + sub, err := sanitizeSubstrateConfig(*body.SandboxSubstrate) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + cfg.AgentSandbox.Substrate = sub + } if nextSentinel != nil { cfg.Sentinel = *nextSentinel } @@ -361,6 +372,7 @@ func securitySectionResponse(cfg *config.Config) map[string]interface{} { "agentSandboxJobTtlSeconds": sandboxJobOrEmpty(cfg).TTLSeconds, "agentSandboxJobWorkspaceClaim": sandboxJobOrEmpty(cfg).WorkspaceClaim, "agentSandboxJobWorkspaceClaimMount": sandboxJobOrEmpty(cfg).WorkspaceClaimMount, + "agentSandboxSubstrate": substrateOrEmpty(cfg.AgentSandbox.Substrate), "sandboxedAgents": sandboxed, "totalAgents": len(cfg.Agents), "sandboxWarnings": sandboxWarnings, @@ -458,3 +470,50 @@ func sandboxJobOrEmpty(cfg *config.Config) config.SandboxJobConfig { } return *cfg.AgentSandbox.Job } + +func substrateOrEmpty(sub *config.SandboxSubstrateConfig) config.SandboxSubstrateConfig { + if sub == nil { + return config.SandboxSubstrateConfig{} + } + return *sub +} + +// sanitizeSubstrateConfig trims and validates a Substrate block from the +// Settings dialog. An all-empty block returns nil so the YAML key is dropped. +func sanitizeSubstrateConfig(in config.SandboxSubstrateConfig) (*config.SandboxSubstrateConfig, error) { + if in.TimeoutS < 0 || in.BridgePort < 0 || in.CPUMillicores < 0 || in.MemoryBytes < 0 { + return nil, errors.New("substrate numeric settings must be >= 0") + } + clean := func(s string) string { return sanitizeString(strings.TrimSpace(s)) } + out := config.SandboxSubstrateConfig{ + Endpoint: clean(in.Endpoint), + Atespace: clean(in.Atespace), + TokenEnv: clean(in.TokenEnv), + TokenFile: clean(in.TokenFile), + CAFile: clean(in.CAFile), + InsecureTLS: in.InsecureTLS, + TimeoutS: in.TimeoutS, + RouterURL: clean(in.RouterURL), + RouterClientBundle: clean(in.RouterClientBundle), + RouterCAFile: clean(in.RouterCAFile), + BridgePort: in.BridgePort, + Image: clean(in.Image), + BridgeCommand: sanitizeStringSlice(in.BridgeCommand), + CPUMillicores: in.CPUMillicores, + MemoryBytes: in.MemoryBytes, + WorkerSelector: sanitizedHeaderMap(in.WorkerSelector), + SnapshotOnIdle: in.SnapshotOnIdle, + AllowHosts: sanitizeStringSlice(in.AllowHosts), + PolicyProfile: clean(in.PolicyProfile), + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = nil + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = nil + } + if reflect.DeepEqual(out, config.SandboxSubstrateConfig{}) { + return nil, nil + } + return &out, nil +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index eeee612fb8..718fbbf15f 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -29,25 +29,6 @@ agent_sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell runtime; agent_sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agent_sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) -agent_sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) -agent_sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agent_sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) agent_sandbox.publish.verify_timeout_s # TODO #14999 (policy-file knob; Settings control pending) agents..acmm_levels[] # TODO #11854 @@ -166,25 +147,6 @@ agents..sandbox.openshell.tolerations[].operator # TODO #17645 (OpenShell r agents..sandbox.openshell.tolerations[].value # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim # TODO #17645 (OpenShell runtime; Settings control pending) agents..sandbox.openshell.workspace_claim_mount # TODO #17645 (OpenShell runtime; Settings control pending) -agents..sandbox.substrate.allow_hosts[] # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.atespace # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.bridge_command[] # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.bridge_port # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.cpu_millicores # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.endpoint # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.image # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.insecure_tls # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.memory_bytes # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.policy_profile # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_ca_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_client_bundle # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.router_url # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.snapshot_on_idle # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.timeout_s # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.token_env # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.token_file # TODO #17699 (Substrate runtime; Settings control pending) -agents..sandbox.substrate.worker_selector. # TODO #17699 (Substrate runtime; Settings control pending) agents..sandbox.runtime # TODO #11854 agents..sandbox.timeout_s # TODO #11854 agents..sandbox.publish.verify[] # TODO #14999 (policy-file knob; Settings control pending) diff --git a/src/pkg/dashboard/static-src/js/config-agent.js b/src/pkg/dashboard/static-src/js/config-agent.js index f4cfd4b815..ef85124de9 100644 --- a/src/pkg/dashboard/static-src/js/config-agent.js +++ b/src/pkg/dashboard/static-src/js/config-agent.js @@ -123,6 +123,9 @@
Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} +
Substrate sandbox overrides + ${substrateSandboxFields('general', g.sandboxSubstrate)} +
Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)} diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 3a74632df6..8ac65300be 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -526,6 +526,69 @@ markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents..sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents..sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents..sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents..sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents..sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents..sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents..sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents..sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents..sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents..sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents..sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents..sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents..sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '
'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '
'; + } + if (f.kind === 'int') { + return head + '
'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + ''; + } + return head + ''; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -927,6 +990,9 @@
+
Substrate sandbox + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} +
`; /* Backup encryption key: painted asynchronously from presence-only diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 92bfc42a08..68c2d1d5ce 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59612,6 +59612,9 @@

Hub settings

Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''} +
Substrate sandbox overrides + ${substrateSandboxFields('general', g.sandboxSubstrate)} +
Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)} @@ -66556,6 +66559,69 @@

Escalation Breaker

markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents..sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents..sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents..sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents..sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents..sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents..sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents..sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents..sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents..sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents..sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents..sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents..sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents..sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '
'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '
'; + } + if (f.kind === 'int') { + return head + '
'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + ''; + } + return head + ''; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -66957,6 +67023,9 @@

Escalation Breaker

+
Substrate sandbox + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} +
`; /* Backup encryption key: painted asynchronously from presence-only From a46309a75b9e2b3323d0111962e6eee9f0498c65 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 16:55:11 -0700 Subject: [PATCH 030/166] =?UTF-8?q?=F0=9F=90=9B=20claude=20auth:=20share?= =?UTF-8?q?=20refreshed=20logins=20across=20per-agent=20.claude=20homes=20?= =?UTF-8?q?without=20sharing=20memory=20(#17767)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 claude auth: share refreshed logins across per-agent .claude homes without sharing memory Since the per-agent CLI homes landed, each per-UID agent has a real ~/.claude directory with only .credentials.json symlinked to the shared login at /data/home/.claude/.credentials.json. Claude Code refreshes its token by writing a temp file in ~/.claude and renaming it over .credentials.json, then chmod 0600. The rename replaces the symlink with a private, agent-owned file. bridgeHomeEntry refuses to clobber a regular file, so nothing ever re-links it; the refreshed access token and the rotated refresh grant stay in that agent's private file, the shared credential ages out, and the rest of the fleet logs out every ~8h (Discord report by Danathar, 2026-11-11). Andy's direction: keep per-agent memory isolated, make an exception for the login only. So no opt-in whole-directory sharing. Instead the hive repairs the bridge in-process: - repairClaudeCredentialBridge: when the agent's credential path is a regular file, read it (directly, or as its owner through the existing su-exec seam when the server cannot open an agent's 0600 file — the per-UID case), move it aside as .credentials.json.stale-, promote it into the shared login only if it is complete, usable and expires later than the shared token (a failed refresh's empty-token file never wins; ties keep shared), preserving the previous shared file, then restore the symlink. Legacy whole-directory ~/.claude symlinks and unreadable private files are left untouched, with a once-per-version log. - Triggered at agent launch (setupPerAgentCLIHome), every 5s from the credential watchdog (HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL) so a refresh is fleet-visible before siblings try the rotated grant, and right after a dashboard sign-in (ReloadClaudeToken). - Audited as claude_credential_rebridged (no token material). - claude.ReadTokens/ParseTokens/TokensUsable split out of LoadTokens so "cannot open" is distinguishable from "no token". Tests reproduce the CLI's tmp+rename+chmod and show the shared file did not update before the fix and does after; cover newer-shared, failed refresh, su-exec owner read, unreadable fail-safe, legacy layout, stale name collisions, watchdog cycle and sign-in trigger. Refs hivecommons/hive#17039, hivecommons/hive#17253, hivecommons/hive#17678, hivecommons/hive#17752, hivecommons/hive#17753 Signed-off-by: Andrew Anderson * 🌱 keep manager_tokens.go at its file-size ratchet baseline Move the sign-in re-bridge body into claude_credential_bridge.go and defer it from ReloadClaudeToken (LIFO: after the Unlock). Signed-off-by: Andrew Anderson * 🐛 harden shared-home permission guard and credential promotion against non-regular files Shared logins and CLI configs written by the hive (.claude/.credentials.json, .copilot/config.json, the durable Copilot token) now go through a new crux/sharedfile package: a randomly named temporary beside the target, mode and owner set and verified on the open descriptor, rename, directory fsync, and the write is refused unless the target is absent or a regular file in a regular directory the server owns. Credential promotion writes a canonical re-encoding of the parsed token block and checks the moved-aside source is the same regular file before and after. Parity with the v6 fail-safe behaviour: an unreadable private credential is left in place except on a dashboard sign-in with a usable shared login, the refusal is logged once per file version, WriteCredentials refuses a credential without an access token, and ReadTokens treats malformed JSON as nothing usable. entrypoint.sh: the shared dot-dirs get the sticky bit (not under HIVE_SHARED_AGENT_HOME=1); the single-file repairs act only on a regular file in a regular directory and WARN once otherwise; the remaining root chmod/chown sites under agent-writable directories gained the same checks. The Go permissions watcher keeps setgid/sticky when widening a directory's group bits. Shell tests drive the shipped guard functions with non-regular entries. Refs hivecommons/hive#17039, hivecommons/hive#17253, hivecommons/hive#17678, hivecommons/hive#17748 Signed-off-by: Andrew Anderson * 🌱 wait for the watchdog re-bridge with testutil.Eventually instead of a polling sleep Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../fixed-claude-shared-credentials.md | 1 + src/crux/agent/audit.go | 2 + src/crux/agent/claude_credential_bridge.go | 636 +++++++++++++ .../agent/claude_credential_bridge_test.go | 853 ++++++++++++++++++ src/crux/agent/copilottoken/copilottoken.go | 20 +- src/crux/agent/coverage_boost_test.go | 26 + src/crux/agent/credentials_watchdog.go | 35 + src/crux/agent/interactive_home.go | 6 + src/crux/agent/manager_credentials.go | 25 +- src/crux/agent/manager_tokens.go | 10 +- src/crux/agent/permissions_watcher.go | 4 +- src/crux/audit.go | 9 + src/crux/claude/oauth.go | 63 +- .../claude/oauth_write_error_branches_test.go | 62 +- src/crux/claude/usable_token_test.go | 65 ++ src/crux/sharedfile/sharedfile.go | 278 ++++++ src/crux/sharedfile/sharedfile_test.go | 480 ++++++++++ src/deploy/entrypoint.sh | 108 ++- src/deploy/test_entrypoint_perm_guard.sh | 84 ++ src/docs/agent-configuration.md | 14 + src/docs/env-vars.md | 1 + src/pkg/dashboard/audit.go | 1 + 22 files changed, 2725 insertions(+), 58 deletions(-) create mode 100644 changelog.d/fixed-claude-shared-credentials.md create mode 100644 src/crux/agent/claude_credential_bridge.go create mode 100644 src/crux/agent/claude_credential_bridge_test.go create mode 100644 src/crux/sharedfile/sharedfile.go create mode 100644 src/crux/sharedfile/sharedfile_test.go diff --git a/changelog.d/fixed-claude-shared-credentials.md b/changelog.d/fixed-claude-shared-credentials.md new file mode 100644 index 0000000000..db3fa495ed --- /dev/null +++ b/changelog.d/fixed-claude-shared-credentials.md @@ -0,0 +1 @@ +- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. diff --git a/src/crux/agent/audit.go b/src/crux/agent/audit.go index b0572db5ba..41dc7b0b09 100644 --- a/src/crux/agent/audit.go +++ b/src/crux/agent/audit.go @@ -27,6 +27,8 @@ const ( AuditCopilotTokenMissing = crux.AuditCopilotTokenMissing AuditClaudeTokenMissing = crux.AuditClaudeTokenMissing + + AuditClaudeCredentialRebridged = crux.AuditClaudeCredentialRebridged ) // auditActorSystem attributes an event to the hive process itself rather than diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go new file mode 100644 index 0000000000..b511e8d955 --- /dev/null +++ b/src/crux/agent/claude_credential_bridge.go @@ -0,0 +1,636 @@ +package agent + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "sync" + "syscall" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Claude credential bridge: share the LOGIN across per-agent .claude homes +// without sharing anything else. +// +// Since hivecommons/hive#17039 each per-UID claude agent has a REAL ~/.claude +// (its own transcripts, todos, history and auto-memory) with one symlink in +// it, ~/.claude/.credentials.json -> /data/home/.claude/.credentials.json, so +// one operator sign-in serves the fleet. Claude Code does not write through +// that link. When it refreshes the OAuth token (about every 8 hours) it writes +// a temp file in ~/.claude and renames it over .credentials.json, which +// REPLACES the symlink with a regular file owned by the agent's uid, mode +// 0600. The refresh also rotates the grant: the shared file now holds a dead +// refresh token, the fresh one is private to that agent, and every other +// agent fails its own refresh within the access token's lifetime. On per-UID +// hives the whole fleet logged out about every 8 hours. +// +// Claude Code offers no supported way to keep the link: CLAUDE_CONFIG_DIR +// moves the WHOLE config directory (settings, transcripts, auto-memory) and +// would undo the per-agent memory split; a bind mount cannot be renamed over +// (EBUSY) and needs mount privileges the pod does not have. So hive cannot +// prevent the replacement and instead DETECTS and PUBLISHES it: at every agent +// launch, on a short poll from the credential watchdog, and right after a +// dashboard sign-in. +// +// The repair never deletes a token. It moves the private file aside, keeps +// whichever of the two credentials is newer AND usable in the shared file, and +// restores the symlink: +// +// - the private copy wins only when it is complete and usable +// (claudeBridgePromotable: access and refresh tokens present, an expiresAt +// recorded, and claude.TokensUsable) and either the shared credential is +// not usable or the private access token expires LATER than the shared +// one. A refresh mints a later expiresAt, so this is what promotes one +// agent's fresh refresh to the whole fleet instead of stranding everyone +// else on the grant it just rotated. A failed refresh leaves an +// empty-token, expiresAt-0 file; it never qualifies; +// - otherwise the shared file stays as it is (ties keep the shared file, so +// a private copy identical to it is simply moved aside). +// +// Who reads the private file. The hive server runs as its own uid (dev) and +// cannot open another uid's 0600 file directly. It CAN run a helper as that +// uid: the same su-exec seam seedClaudeSessionForAgent uses to adopt an +// agent-owned .claude.json (runAsAgentUser, gated to the hive-launch group). +// So the judgement happens in-process, as the file's owner, with no root +// helper and no widening of the private file's mode. When neither read works +// (no su-exec on this deployment) the repair fails SAFE: the private file is +// left exactly where it is, the agent keeps running on the token it already +// had, and one ERROR per agent and file version says so. +// +// The private file is renamed to .credentials.json.stale- in +// the agent's own .claude directory and keeps its owner and mode; when it is +// promoted, the shared file it replaces is preserved the same way beside the +// shared credential. +// +// The repair never acts on a link path that resolves to the shared credential +// itself (an agent whose whole ~/.claude is still the legacy directory symlink +// to /data/home/.claude): that "private file" is the fleet's login. Such an +// agent is skipped with one warning; its next launch retires the directory +// link (setupPerAgentCLIHome) before the repair runs. + +// Repair triggers, recorded in the audit detail and the log. +const ( + claudeBridgeTriggerLaunch = "launch" + claudeBridgeTriggerWatchdog = "watchdog" + claudeBridgeTriggerSignIn = "sign-in" +) + +// claudeCredentialStaleInfix names a credential moved aside by the repair. +const claudeCredentialStaleInfix = ".stale-" + +// claudeSharedCredentialMode is the mode of a shared credential the repair +// writes: owner and group read/write, the same as claude.WriteCredentials (the +// dashboard sign-in) and what the entrypoint guard asserts, so every agent uid +// reaches it through the node group. +const claudeSharedCredentialMode = 0o660 + +// claudeBridgeStaleNameAttempts bounds the search for a free +// .stale-[-N] name beside a credential. +const claudeBridgeStaleNameAttempts = 100 + +// claudeBridgeMaxCredentialBytes caps a credential file the repair will read. +// A real credentials file is well under 4 KiB. +const claudeBridgeMaxCredentialBytes = 64 << 10 + +// claudeBridgeNow stamps the moved-aside file names; a seam for tests. +var claudeBridgeNow = time.Now + +// claudeBridgeMu serializes repairs: the launch path, the watchdog and the +// sign-in hook can all reach the same agent's link. It is never held while +// taking m.mu, so it cannot invert against the launch path, which already +// holds m.mu when it provisions the home. +var claudeBridgeMu sync.Mutex + +// claudeBridgeRepair describes one repaired bridge, for the audit and log. +type claudeBridgeRepair struct { + agent string + // kept is "shared" when the shared credential was kept as it was, or + // "agent" when the agent's private copy was newer and usable and was + // promoted into the shared file. + kept string + // staleAgentPath is where the private file was moved. + staleAgentPath string + // staleSharedPath is where the replaced shared file was preserved; empty + // unless kept == "agent" and a shared file existed. + staleSharedPath string +} + +// claudeCredentialBridgePaths returns the agent's bridged credential path and +// the shared target it must link to. ok is false when the agent has no +// per-agent home (no uid, an inference backend, or HIVE_SHARED_AGENT_HOME=1), +// i.e. exactly when setupInteractiveHome provisions no bridge. +func claudeCredentialBridgePaths(agentName string, uid int, backend string) (link, target string, ok bool) { + if uid <= 0 || IsInferenceBackend(backend) || sharedAgentHomeForced() { + return "", "", false + } + link = filepath.Join(interactiveHomePath(agentName), claudeHomeDirName, claudeCredentialsFileName) + return link, sharedClaudeBridgeTarget(), true +} + +// sharedClaudeBridgeTarget is the shared credential every bridge links to. +// Derived from sharedAgentHome (not the claude.CredentialsPath constant) so it +// follows the same test seam as the links themselves. +func sharedClaudeBridgeTarget() string { + return filepath.Join(sharedAgentHome, claudeHomeDirName, claudeCredentialsFileName) +} + +// errClaudeBridgeLinkIsTarget: the agent's credential path IS the shared +// credential, so there is no private copy to move aside and no link to +// restore (a legacy whole-directory ~/.claude symlink). Moving that file aside +// would strand the fleet. +var errClaudeBridgeLinkIsTarget = errors.New("credential link path resolves to the shared credential it would link to") + +// errClaudeBridgeAgentUnreadable: the agent's private credential exists but +// neither this process nor a helper running as the file's owner could read +// it. Its token cannot be judged, so the repair leaves it exactly where it is. +var errClaudeBridgeAgentUnreadable = errors.New("agent's private credential cannot be read") + +// errClaudeBridgeSourceChanged: the file renamed aside is not the regular +// file that was inspected a moment earlier. It is put back and not read. +var errClaudeBridgeSourceChanged = errors.New("private credential changed under the repair") + +// errClaudeBridgeNotPromotable: promoteClaudeCredential re-checks the bytes it +// is about to write and refuses an incomplete or unusable credential. +var errClaudeBridgeNotPromotable = errors.New("refusing to promote an incomplete or unusable credential over the shared login") + +// claudeBridgeLegacyWarned holds the credential links the legacy-layout skip +// has already been logged for, so the watchdog, which walks every managed +// agent each cycle, logs it once per agent instead of every cycle. Keyed by +// the link path, which names the agent. +var claudeBridgeLegacyWarned sync.Map + +// claudeBridgeUnreadableWarned holds, per credential link, the modification +// time of the unreadable private file last logged, so the poll logs once per +// agent and file version instead of every cycle. A new refresh (a new mtime) +// is logged again. +var claudeBridgeUnreadableWarned sync.Map + +// claudeBridgeUnsafeWarned holds the shared credential paths whose location +// sharedfile.Replace refused, so the refusal is logged once, not per cycle. +var claudeBridgeUnsafeWarned sync.Map + +// claudeBridgeLinkIsTarget reports whether link names the shared credential +// itself: the same path, or a file of the same name in a directory that +// resolves to the target's directory (a legacy whole-directory ~/.claude +// symlink). An error means a directory could not be resolved (absent, or not +// reachable by this process); callers treat that as nothing to repair. +func claudeBridgeLinkIsTarget(link, target string) (bool, error) { + if filepath.Clean(link) == filepath.Clean(target) { + return true, nil + } + if filepath.Base(link) != filepath.Base(target) { + return false, nil + } + linkDir, err := filepath.EvalSymlinks(filepath.Dir(link)) + if err != nil { + return false, err + } + targetDir, err := filepath.EvalSymlinks(filepath.Dir(target)) + if err != nil { + return false, err + } + if linkDir == targetDir { + return true, nil + } + // Different resolved paths can still be one directory (a bind mount). + li, err := os.Stat(linkDir) + if err != nil { + return false, err + } + ti, err := os.Stat(targetDir) + if err != nil { + return false, err + } + return os.SameFile(li, ti), nil +} + +// claudeBridgePromotable reports whether t may replace the shared login: a +// complete credential (access token, refresh token and a recorded expiresAt) +// that is usable now. A failed refresh leaves an empty-token, expiresAt-0 +// file, which never qualifies; neither does a token with no recorded expiry, +// which cannot be judged newer than anything. +func claudeBridgePromotable(t *claude.OAuthTokens, now time.Time) bool { + return t != nil && t.AccessToken != "" && t.RefreshToken != "" && t.ExpiresAt > 0 && + claude.TokensUsable(t, now) +} + +// preferAgentClaudeCredential decides which credential the shared file keeps. +// See the file comment for the rule. +func preferAgentClaudeCredential(agent, shared *claude.OAuthTokens, now time.Time) bool { + if !claudeBridgePromotable(agent, now) { + return false + } + if !claude.TokensUsable(shared, now) { + return true + } + return agent.ExpiresAt > shared.ExpiresAt +} + +// readClaudeCredentialAsOwner reads an agent's private credential: directly +// when this process may, otherwise through su-exec as the file's owner (the +// per-UID case: an agent-owned 0600 file and a server running as dev). +// claudeSessionOwnerSpec fails closed — a symlink, a root-owned or an +// oversized file never gets a helper — so a planted link at the credential +// path cannot make the helper read something else. +func readClaudeCredentialAsOwner(path string) ([]byte, error) { + data, err := sharedfile.ReadNoFollow(path, claudeBridgeMaxCredentialBytes) + if err == nil || !errors.Is(err, fs.ErrPermission) { + return data, err + } + spec, serr := claudeSessionOwnerSpec(path) + if serr != nil { + return nil, errors.Join(err, serr) + } + owned, oerr := runAsAgentUser(spec, nil, "cat", path) + if oerr != nil { + return nil, errors.Join(err, oerr) + } + if len(owned) > claudeBridgeMaxCredentialBytes { + return nil, fmt.Errorf("%w: %s returned %d bytes as owner", sharedfile.ErrTooLarge, path, len(owned)) + } + return owned, nil +} + +// canonicalClaudeCredential decodes data as a credentials file and re-encodes +// only its typed token block. What reaches the shared file is this encoding, +// never the bytes read from an agent's directory: a file that is not a +// credential in the shape claude.Credentials describes is not promotable. +func canonicalClaudeCredential(data []byte, now time.Time) ([]byte, *claude.OAuthTokens, error) { + if len(data) > claudeBridgeMaxCredentialBytes { + return nil, nil, errClaudeBridgeNotPromotable + } + tok := claude.ParseTokens(data) + if !claudeBridgePromotable(tok, now) { + return nil, nil, errClaudeBridgeNotPromotable + } + out, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: tok}) + if err != nil { + return nil, nil, errClaudeBridgeNotPromotable + } + return out, tok, nil +} + +// repairClaudeCredentialBridge re-bridges one agent's credential when a regular +// file has replaced the symlink. It returns nil, nil when there is nothing to +// repair (a symlink, an absent link, a directory, which is not this failure, +// or a link or target directory that cannot be resolved). It returns an error +// wrapping errClaudeBridgeLinkIsTarget, and touches nothing, when the link +// path resolves to the shared credential itself, and, unless trigger is +// claudeBridgeTriggerSignIn, one wrapping errClaudeBridgeAgentUnreadable, +// again touching nothing, when the private file cannot be read. +// +// The sign-in trigger is the exception: a dashboard sign-in has just written +// a new shared login, newer than any private copy by construction, so on that +// trigger an unreadable private file is still moved aside (unread) and the +// agent relinked to the shared login. +func (m *Manager) repairClaudeCredentialBridge(agentName, link, target, trigger string) (*claudeBridgeRepair, error) { + claudeBridgeMu.Lock() + defer claudeBridgeMu.Unlock() + + info, err := os.Lstat(link) + if err != nil || !info.Mode().IsRegular() { + return nil, nil + } + + // Never operate on the shared credential itself. Checked BEFORE anything + // is moved: when the link path resolves to the target, the "private file" + // below IS the shared login, moving it aside strands the fleet, and the + // symlink restored afterwards would point at its own path. + same, err := claudeBridgeLinkIsTarget(link, target) + if err != nil { + m.logger.Debug("claude credential bridge: cannot resolve the link or shared directory; not repairing", + "agent", agentName, "link", link, "target", target, "error", err) + return nil, nil + } + if same { + return nil, fmt.Errorf("%w: %s", errClaudeBridgeLinkIsTarget, link) + } + + // Never move aside a file whose token cannot be judged: it may be the + // only fresh token in the fleet. + if _, rerr := readClaudeCredentialAsOwner(link); rerr != nil { + if errors.Is(rerr, fs.ErrNotExist) { + // Gone since the Lstat: the CLI is mid-rename. Next cycle. + return nil, nil + } + if trigger == claudeBridgeTriggerSignIn && claude.HasUsableToken(target) { + return m.relinkUnreadableAfterSignIn(agentName, link, target) + } + return nil, fmt.Errorf("%w: %w", errClaudeBridgeAgentUnreadable, rerr) + } + now := claudeBridgeNow() + + // Move the private file aside FIRST, then judge the moved file: after the + // rename the CLI can no longer change what is being judged (its next + // write lands at link), so the decision works on a stable snapshot. + staleAgent, err := moveAsideClaudeCredential(link, now) + if err != nil { + return nil, fmt.Errorf("move private credential aside: %w", err) + } + agentData, rerr := readClaudeCredentialAsOwner(staleAgent) + if rerr != nil { + // The file renamed is not the one read above (the CLI replaced it in + // between) and this one cannot be read either. Put it back unless the + // CLI has already written yet another file there. + if _, lerr := os.Lstat(link); errors.Is(lerr, fs.ErrNotExist) { + if berr := os.Rename(staleAgent, link); berr != nil { + m.logger.Error("claude credential bridge: an unreadable private credential could not be put back; it is kept at the stale path", + "agent", agentName, "stale", staleAgent, "error", berr) + } + } + return nil, fmt.Errorf("%w: %w", errClaudeBridgeAgentUnreadable, rerr) + } + rep := &claudeBridgeRepair{agent: agentName, kept: "shared", staleAgentPath: staleAgent} + + if preferAgentClaudeCredential(claude.ParseTokens(agentData), claude.LoadTokens(target), now) { + staleShared, perr := promoteClaudeCredential(agentData, target, now) + if perr != nil { + // Put the private file back: the agent's copy was the only usable + // token, and linking the agent to the older shared one would + // strand it. The next cycle tries again. + if berr := os.Rename(staleAgent, link); berr != nil { + m.logger.Error("claude credential bridge: promotion failed and the private credential could not be restored; it is kept at the stale path", + "agent", agentName, "stale", staleAgent, "error", perr, "restore_error", berr) + } + return nil, fmt.Errorf("promote private credential to shared: %w", perr) + } + rep.kept = "agent" + rep.staleSharedPath = staleShared + } + + if err := os.Symlink(target, link); err != nil { + // EEXIST: the CLI wrote a fresh file between the rename and now. The + // next cycle repairs that one; nothing is lost either way. + return rep, fmt.Errorf("restore credential symlink: %w", err) + } + return rep, nil +} + +// claudeBridgeStaleName returns a free .stale-[-N] name +// beside path, or "" when none is free. exists reports whether a candidate is +// taken; callers that create the file atomically pass a function that tries +// the create and reports EEXIST. +func claudeBridgeStaleName(path string, now time.Time, exists func(string) (bool, error)) (string, error) { + base := path + claudeCredentialStaleInfix + now.UTC().Format("20060102T150405Z") + for i := 0; i < claudeBridgeStaleNameAttempts; i++ { + dst := base + if i > 0 { + dst = fmt.Sprintf("%s-%d", base, i) + } + taken, err := exists(dst) + if err != nil { + return "", err + } + if !taken { + return dst, nil + } + } + return "", fmt.Errorf("no free stale name for %s", path) +} + +// moveAsideClaudeCredential renames path to a unique stale name beside it and +// returns the new path. A rename keeps the file's owner and mode: the private +// copy stays readable only by the agent that wrote it. The file is identified +// before the rename and checked after it (same device and inode, still a +// regular file); anything else is put back and reported as +// errClaudeBridgeSourceChanged, unread. +func moveAsideClaudeCredential(path string, now time.Time) (string, error) { + before, err := os.Lstat(path) + if err != nil { + return "", err + } + if !before.Mode().IsRegular() { + return "", fmt.Errorf("%w: %s is %s", errClaudeBridgeSourceChanged, path, before.Mode()) + } + dst, err := claudeBridgeStaleName(path, now, func(dst string) (bool, error) { + _, lerr := os.Lstat(dst) + if lerr == nil { + return true, nil + } + if errors.Is(lerr, fs.ErrNotExist) { + return false, nil + } + return false, lerr + }) + if err != nil { + return "", err + } + if err := os.Rename(path, dst); err != nil { + return "", err + } + after, err := os.Lstat(dst) + if err != nil { + return "", err + } + if !after.Mode().IsRegular() || !os.SameFile(before, after) { + if _, lerr := os.Lstat(path); errors.Is(lerr, fs.ErrNotExist) { + _ = os.Rename(dst, path) + } + return "", fmt.Errorf("%w: %s", errClaudeBridgeSourceChanged, path) + } + return dst, nil +} + +// relinkUnreadableAfterSignIn is the sign-in exception described on +// repairClaudeCredentialBridge: the private file is moved aside without being +// read and the agent is relinked to the shared login just written. +func (m *Manager) relinkUnreadableAfterSignIn(agentName, link, target string) (*claudeBridgeRepair, error) { + staleAgent, err := moveAsideClaudeCredential(link, claudeBridgeNow()) + if err != nil { + return nil, fmt.Errorf("move private credential aside: %w", err) + } + rep := &claudeBridgeRepair{agent: agentName, kept: "shared", staleAgentPath: staleAgent} + if err := os.Symlink(target, link); err != nil { + return rep, fmt.Errorf("restore credential symlink: %w", err) + } + return rep, nil +} + +// promoteClaudeCredential replaces the shared credential with the token block +// decoded from data, atomically and at claudeSharedCredentialMode, owned like +// the file it replaces. The replaced file is preserved beside it first; its +// path is returned ("" when there was no shared file). The bytes are decoded +// and re-checked here (canonicalClaudeCredential), so a credential that +// changed after it was judged can never put an incomplete token over the +// shared login, and nothing but a typed token block is ever written. The +// write itself is sharedfile.Replace: a random O_EXCL|O_NOFOLLOW temporary, +// metadata by descriptor, and a refusal (sharedfile.ErrUnsafeLocation) when +// the shared directory or file is not what it should be. +func promoteClaudeCredential(data []byte, target string, now time.Time) (string, error) { + canonical, _, err := canonicalClaudeCredential(data, now) + if err != nil { + return "", err + } + // Refuse before the backup copy, not only at the write: the copy lands in + // the same directory, and the directory is part of what is being checked. + info, err := sharedfile.CheckLocation(target) + if err != nil { + return "", err + } + staleShared := "" + if info != nil { + staleShared, err = preserveClaudeCredential(target, now) + if err != nil { + return "", fmt.Errorf("preserve shared credential: %w", err) + } + } + if err := sharedfile.Replace(target, canonical, claudeSharedCredentialMode, sharedfile.Options{PreserveOwner: true}); err != nil { + return staleShared, err + } + return staleShared, nil +} + +// preserveClaudeCredential copies path to a unique stale name beside it, +// leaving path itself in place (agents keep reading it until the rename that +// replaces it). The copy is 0600: it is an archive, not a live credential. +// Neither the read nor the create follows a symlink. +func preserveClaudeCredential(path string, now time.Time) (string, error) { + data, err := sharedfile.ReadNoFollow(path, claudeBridgeMaxCredentialBytes) + if err != nil { + return "", err + } + var f *os.File + dst, err := claudeBridgeStaleName(path, now, func(dst string) (bool, error) { + var oerr error + f, oerr = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if errors.Is(oerr, fs.ErrExist) { + return true, nil + } + return false, oerr + }) + if err != nil { + return "", err + } + _, werr := f.Write(data) + cerr := f.Close() + if werr != nil || cerr != nil { + _ = os.Remove(dst) + return "", errors.Join(werr, cerr) + } + return dst, nil +} + +// rebridgeClaudeCredential runs the repair for one agent and records the +// outcome. It is the single entry point the launch path, the watchdog poll and +// the sign-in hook share, so all three log and audit identically. +func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger string) *claudeBridgeRepair { + rep, err := m.repairClaudeCredentialBridge(agentName, link, target, trigger) + if errors.Is(err, errClaudeBridgeLinkIsTarget) { + // The legacy layout: skipped, and said once per agent rather than on + // every cycle. Not audited (nothing changed). + if _, seen := claudeBridgeLegacyWarned.LoadOrStore(link, true); !seen { + m.logger.Warn("claude credential bridge: this agent's .claude directory resolves to the shared login directory (a legacy whole-directory symlink), so its credential path IS the shared login; not re-bridging. The agent's next launch replaces the directory link with a per-agent directory", + "agent", agentName, "trigger", trigger, "dir", filepath.Dir(link), "shared", target) + } + return nil + } + // Any other outcome clears the mark, so a legacy layout that shows up + // again later is logged again. + claudeBridgeLegacyWarned.Delete(link) + if errors.Is(err, errClaudeBridgeAgentUnreadable) { + m.logClaudeBridgeUnreadable(agentName, link, trigger, err) + return nil + } + claudeBridgeUnreadableWarned.Delete(link) + if errors.Is(err, sharedfile.ErrUnsafeLocation) { + // The shared location is not what the repair expects; nothing was + // written and the agent keeps its private copy. Said once per target. + if _, seen := claudeBridgeUnsafeWarned.LoadOrStore(target, true); !seen { + m.logger.Error("claude credential bridge: the shared credential location was refused and nothing was written; the agent keeps its private copy", + "agent", agentName, "trigger", trigger, "shared", target, "error", err) + } + return nil + } + claudeBridgeUnsafeWarned.Delete(target) + if err != nil { + // Logged, not audited: a repair that keeps failing would otherwise add + // an audit row every cycle. + m.logger.Warn("claude credential bridge: repair failed", + "agent", agentName, "link", link, "trigger", trigger, "error", err) + return nil + } + if rep == nil { + return nil + } + m.logger.Warn("claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged", + "agent", agentName, + "trigger", trigger, + "kept", rep.kept, + "stale_agent_file", rep.staleAgentPath, + "stale_shared_file", rep.staleSharedPath, + "cause", "Claude Code token refresh writes a temp file and renames it over the link") + m.audit(AuditClaudeCredentialRebridged, agentName, auditFields( + "outcome", "success", + "trigger", trigger, + "kept", rep.kept, + "stale_agent_file", rep.staleAgentPath, + "stale_shared_file", rep.staleSharedPath, + )) + return rep +} + +// logClaudeBridgeUnreadable says, once per agent and private-file version, +// that the repair left an agent on a private credential it could not read. +// Not audited: nothing changed. +func (m *Manager) logClaudeBridgeUnreadable(agentName, link, trigger string, err error) { + var stamp int64 + if info, lerr := os.Lstat(link); lerr == nil { + stamp = info.ModTime().UnixNano() + } + if prev, loaded := claudeBridgeUnreadableWarned.Swap(link, stamp); loaded && prev == stamp { + return + } + m.logger.Error("claude credential bridge: this agent's private credential could not be read, directly or as its owner through su-exec, so its token cannot be judged; leaving the agent on its private copy (not moved aside, not re-linked). Other agents will not see this agent's token refresh until it is readable", + "agent", agentName, "path", link, "trigger", trigger, "error", err) +} + +// RepairClaudeCredentialBridges checks every per-agent Claude credential +// bridge and repairs each one a regular file has replaced. It returns the +// names of the agents it re-bridged, sorted. Agents are snapshotted under m.mu +// and repaired after it is released. +func (m *Manager) RepairClaudeCredentialBridges(trigger string) []string { + type candidate struct { + name string + uid int + backend string + } + m.mu.RLock() + cands := make([]candidate, 0, len(m.agents)) + for name, a := range m.agents { + if a == nil { + continue + } + cands = append(cands, candidate{name: name, uid: a.UID, backend: effectiveBackend(a)}) + } + m.mu.RUnlock() + sort.Slice(cands, func(i, j int) bool { return cands[i].name < cands[j].name }) + + var repaired []string + for _, c := range cands { + link, target, ok := claudeCredentialBridgePaths(c.name, c.uid, c.backend) + if !ok { + continue + } + if rep := m.rebridgeClaudeCredential(c.name, link, target, trigger); rep != nil { + repaired = append(repaired, c.name) + } + } + return repaired +} + +// rebridgeClaudeCredentialsAfterSignIn runs from ReloadClaudeToken: a sign-in +// wrote the shared file, and agents whose link a refresh replaced would never +// see it. Relink them now rather than on the next poll. A logout leaves +// nothing usable, so nothing is moved. +func (m *Manager) rebridgeClaudeCredentialsAfterSignIn() { + if claude.HasUsableToken(sharedClaudeBridgeTarget()) { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerSignIn) + } +} diff --git a/src/crux/agent/claude_credential_bridge_test.go b/src/crux/agent/claude_credential_bridge_test.go new file mode 100644 index 0000000000..67928f5207 --- /dev/null +++ b/src/crux/agent/claude_credential_bridge_test.go @@ -0,0 +1,853 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Fake tokens. Every token string carries this sentinel so a test can prove +// no log line or audit field ever leaks token material. +const bridgeTestTokenSentinel = "SENTINEL-TOKEN-MATERIAL" + +func bridgeTestCreds(t *testing.T, tag string, expiresAt int64) []byte { + t.Helper() + data, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: &claude.OAuthTokens{ + AccessToken: "acc-" + tag + "-" + bridgeTestTokenSentinel, + RefreshToken: "ref-" + tag + "-" + bridgeTestTokenSentinel, + ExpiresAt: expiresAt, + }}) + if err != nil { + t.Fatal(err) + } + return data +} + +// bridgeTestFixture provisions a per-agent home for "scanner" with the shared +// login in place and the credential symlink bridged, and returns the manager, +// the audit sink, the link path and the shared path. +func bridgeTestFixture(t *testing.T) (*Manager, *fakeAuditSink, *AgentProcess, string, string) { + t.Helper() + shared := withSharedAgentHome(t) + m, sink := testManagerWithSink(t) + ap := &AgentProcess{Name: "scanner", UID: 1001, Config: config.AgentConfig{Backend: "claude"}} + m.mu.Lock() + m.agents["scanner"] = ap + m.mu.Unlock() + target := filepath.Join(shared, ".claude", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + m.setupInteractiveHome(ap, "claude") + link := filepath.Join(interactiveHomePath("scanner"), ".claude", ".credentials.json") + if got, err := os.Readlink(link); err != nil || got != target { + t.Fatalf("fixture: credential bridge not provisioned: target=%q err=%v", got, err) + } + return m, sink, ap, link, target +} + +// simulateClaudeRefresh does what Claude Code's credential store does on a +// token refresh: write a temp file in the agent's .claude and rename it over +// .credentials.json, then chmod 0600. The rename replaces the symlink. +func simulateClaudeRefresh(t *testing.T, link string, data []byte) { + t.Helper() + tmp := link + ".tmp.12345" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmp, link); err != nil { + t.Fatal(err) + } + if err := os.Chmod(link, 0o600); err != nil { + t.Fatal(err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("simulated refresh did not replace the link: info=%v err=%v", info, err) + } +} + +func readTokensAt(t *testing.T, path string) *claude.OAuthTokens { + t.Helper() + tok := claude.LoadTokens(path) + if tok == nil { + t.Fatalf("no tokens at %s", path) + } + return tok +} + +func staleFilesBeside(t *testing.T, path string) []string { + t.Helper() + matches, err := filepath.Glob(path + claudeCredentialStaleInfix + "*") + if err != nil { + t.Fatal(err) + } + return matches +} + +func assertLinkRestored(t *testing.T, link, target string) { + t.Helper() + got, err := os.Readlink(link) + if err != nil || got != target { + t.Fatalf("credential link not restored: target=%q err=%v", got, err) + } +} + +// TestSetupInteractiveHome_RefreshReplacedLinkIsPublishedToSharedLogin is the +// reported failure (Discord, 2026-11-11): on a per-UID hive one agent's token +// refresh replaced its credential link, the refresh never reached the shared +// file, and the fleet logged out when the shared token expired. The refreshed +// token must reach the shared login and the agent must be re-linked, with +// memory untouched. +func TestSetupInteractiveHome_RefreshReplacedLinkIsPublishedToSharedLogin(t *testing.T) { + m, sink, ap, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + memory := filepath.Join(filepath.Dir(link), "projects", "-data-agents-scanner", "memory", "MEMORY.md") + if err := os.MkdirAll(filepath.Dir(memory), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(memory, []byte("# mine"), 0o644); err != nil { + t.Fatal(err) + } + + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + // The defect: the shared file did not move. + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-old-") { + t.Fatalf("precondition: shared file should still hold the old token, got %q", tok.AccessToken) + } + + m.setupInteractiveHome(ap, "claude") + + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-fresh-") { + t.Errorf("refresh not published to the shared login: shared access token = %q", tok.AccessToken) + } + assertLinkRestored(t, link, target) + info, err := os.Stat(target) + if err != nil || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("shared credential mode = %v err=%v, want %#o", info.Mode(), err, claudeSharedCredentialMode) + } + // Private copy moved aside, not deleted, still owner-only. + stale := staleFilesBeside(t, link) + if len(stale) != 1 { + t.Fatalf("stale agent files = %v, want exactly one", stale) + } + if info, err := os.Stat(stale[0]); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("stale private copy mode = %v err=%v, want 0600", info.Mode(), err) + } + // Old shared file preserved beside the shared credential. + if got := staleFilesBeside(t, target); len(got) != 1 || !bytes.Contains(mustRead(t, got[0]), []byte("acc-old-")) { + t.Errorf("replaced shared file not preserved: %v", got) + } + if got, err := os.ReadFile(memory); err != nil || string(got) != "# mine" { + t.Errorf("per-agent memory disturbed: %q err=%v", got, err) + } + // Audited, kept=agent, no token material. + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil { + t.Fatal("no claude_credential_rebridged audit entry") + } + if rec.Agent != "scanner" || rec.Fields["kept"] != "agent" || rec.Fields["trigger"] != claudeBridgeTriggerLaunch { + t.Errorf("audit entry = %+v", rec) + } + if strings.Contains(fmt.Sprint(rec.Fields), bridgeTestTokenSentinel) { + t.Error("audit fields carry token material") + } + // Idempotent: a second launch finds a healthy link and changes nothing. + m.setupInteractiveHome(ap, "claude") + if sink.count() != 1 { + t.Errorf("second launch audited again: %d entries", sink.count()) + } +} + +func mustRead(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data +} + +// TestRepairClaudeCredentialBridge_KeepsNewerShared: a dashboard sign-in after +// the refresh wrote a newer shared credential; the agent's stale private copy +// is moved aside and the agent re-linked, the shared file untouched. +func TestRepairClaudeCredentialBridge_KeepsNewerShared(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + now := time.Now() + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", now.Add(1*time.Hour).UnixMilli())) + sharedBytes := bridgeTestCreds(t, "signin", now.Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "shared" || rep.staleSharedPath != "" { + t.Fatalf("repair = %+v, want kept=shared with no preserved shared file", rep) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential was rewritten") + } + assertLinkRestored(t, link, target) + if got := staleFilesBeside(t, link); len(got) != 1 || !bytes.Contains(mustRead(t, got[0]), []byte("acc-agent-")) { + t.Errorf("private copy not moved aside: %v", got) + } + if rec := sink.find(AuditClaudeCredentialRebridged); rec == nil || rec.Fields["kept"] != "shared" { + t.Errorf("audit = %+v", rec) + } +} + +// TestRepairClaudeCredentialBridge_FailedRefreshNeverWins: an agent whose +// refresh failed on a rotated grant writes an empty-token, expiresAt-0 file. +// It must never replace the shared login, usable or not. +func TestRepairClaudeCredentialBridge_FailedRefreshNeverWins(t *testing.T) { + for _, sharedUsable := range []bool{true, false} { + t.Run(fmt.Sprintf("sharedUsable=%v", sharedUsable), func(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + exp := time.Now().Add(-time.Hour).UnixMilli() + if sharedUsable { + exp = time.Now().Add(time.Hour).UnixMilli() + } + sharedBytes := bridgeTestCreds(t, "shared", exp) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, []byte(`{"claudeAiOauth":{"accessToken":"","refreshToken":"","expiresAt":0}}`)) + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "shared" { + t.Fatalf("repair = %+v, want kept=shared", rep) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("empty-token file replaced the shared login") + } + assertLinkRestored(t, link, target) + }) + } +} + +// TestRepairClaudeCredentialBridge_ReadsAgentOwnedFileAsOwner is the per-UID +// case: the private file is the agent's, 0600, and the hive process cannot +// open it. The read must go through the su-exec seam as the file's owner, and +// the fresh token must still be promoted. +func TestRepairClaudeCredentialBridge_ReadsAgentOwnedFileAsOwner(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, _, _, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + + var mu sync.Mutex + var calls []string + old := runAsAgentUser + runAsAgentUser = func(spec string, stdin []byte, argv ...string) ([]byte, error) { + mu.Lock() + defer mu.Unlock() + calls = append(calls, spec+" "+strings.Join(argv, " ")) + if len(argv) != 2 || argv[0] != "cat" { + return nil, fmt.Errorf("unexpected argv %v", argv) + } + // Stand in for the owner: the file is ours, only its mode blocks us. + if err := os.Chmod(argv[1], 0o600); err != nil { + return nil, err + } + defer func() { _ = os.Chmod(argv[1], 0) }() + return os.ReadFile(argv[1]) + } + t.Cleanup(func() { runAsAgentUser = old }) + + rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep == nil || rep.kept != "agent" { + t.Fatalf("repair = %+v, want kept=agent", rep) + } + if !bytes.Equal(mustRead(t, target), fresh) { + t.Error("agent-owned refresh was not promoted to the shared login") + } + assertLinkRestored(t, link, target) + mu.Lock() + defer mu.Unlock() + wantSpec := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) + if len(calls) < 1 || !strings.HasPrefix(calls[0], wantSpec+" cat "+link) { + t.Errorf("su-exec calls = %v, want first read as %s of %s", calls, wantSpec, link) + } +} + +// TestRepairClaudeCredentialBridge_UnreadablePrivateLeftInPlace: when neither +// a direct read nor the owner helper can read the private file, nothing is +// moved or relinked (the agent keeps the token it has), nothing is audited, +// and the ERROR is logged once per file version. +func TestRepairClaudeCredentialBridge_UnreadablePrivateLeftInPlace(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, sink, _, link, target := bridgeTestFixture(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + private := bridgeTestCreds(t, "private", time.Now().Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, private) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(link, 0o600) }) + old := runAsAgentUser + runAsAgentUser = func(string, []byte, ...string) ([]byte, error) { + return nil, errors.New("su-exec unavailable") + } + t.Cleanup(func() { runAsAgentUser = old }) + + for i := 0; i < 3; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("cycle %d: repair = %+v, want none", i, rep) + } + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("private file was moved or relinked: info=%v err=%v", info, err) + } + if got := staleFilesBeside(t, link); len(got) != 0 { + t.Errorf("stale files created: %v", got) + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential changed") + } + if sink.count() != 0 { + t.Errorf("audited %d entries, want none (nothing changed)", sink.count()) + } + logs := logBuf.String() + if n := strings.Count(logs, "cannot be judged"); n != 1 { + t.Errorf("unreadable ERROR logged %d times over three cycles, want once:\n%s", n, logs) + } + if strings.Contains(logs, bridgeTestTokenSentinel) { + t.Error("log carries token material") + } + if !strings.Contains(logs, link) { + t.Error("log does not name the path") + } + // A new file version is logged again. + if err := os.Chmod(link, 0o600); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, private) + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(link, future, future); err != nil { + t.Fatal(err) + } + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog) + if n := strings.Count(logBuf.String(), "cannot be judged"); n != 2 { + t.Errorf("new file version not logged again: %d", n) + } +} + +// TestRepairClaudeCredentialBridge_LegacyLayoutIsNeverTouched: an agent whose +// whole ~/.claude is still the legacy directory symlink has the SHARED login +// at its credential path. Moving it aside would log the fleet out. +func TestRepairClaudeCredentialBridge_LegacyLayoutIsNeverTouched(t *testing.T) { + shared := withSharedAgentHome(t) + m, sink := testManagerWithSink(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + target := filepath.Join(shared, ".claude", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + home := interactiveHomePath("scanner") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(shared, ".claude"), filepath.Join(home, ".claude")); err != nil { + t.Fatal(err) + } + link := filepath.Join(home, ".claude", ".credentials.json") + + for i := 0; i < 2; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want none", rep) + } + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential disturbed") + } + if got := staleFilesBeside(t, target); len(got) != 0 { + t.Errorf("shared login moved aside: %v", got) + } + if sink.count() != 0 { + t.Error("legacy skip must not audit") + } + if n := strings.Count(logBuf.String(), "legacy whole-directory symlink"); n != 1 { + t.Errorf("legacy warning logged %d times, want once", n) + } +} + +// TestRepairClaudeCredentialBridge_NothingToRepair: a healthy link, an absent +// link and a directory are not this failure. +func TestRepairClaudeCredentialBridge_NothingToRepair(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("healthy link: repair = %+v", rep) + } + if err := os.Remove(link); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("absent link: repair = %+v", rep) + } + if err := os.Mkdir(link, 0o755); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Errorf("directory: repair = %+v", rep) + } + if sink.count() != 0 { + t.Error("nothing to repair must not audit") + } + // No per-agent home at all: no paths, no repair. + if _, _, ok := claudeCredentialBridgePaths("scanner", 0, "claude"); ok { + t.Error("uid 0 must have no bridge") + } + if _, _, ok := claudeCredentialBridgePaths("scanner", 1001, "vllm"); ok { + t.Error("inference backend must have no bridge") + } + t.Setenv("HIVE_SHARED_AGENT_HOME", "1") + if _, _, ok := claudeCredentialBridgePaths("scanner", 1001, "claude"); ok { + t.Error("forced shared home must have no bridge") + } +} + +// TestRepairClaudeCredentialBridge_StaleNamesNeverCollide: two repairs within +// the same second get distinct stale names on both sides. +func TestRepairClaudeCredentialBridge_StaleNamesNeverCollide(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + fixed := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) + oldNow := claudeBridgeNow + claudeBridgeNow = func() time.Time { return fixed } + t.Cleanup(func() { claudeBridgeNow = oldNow }) + if err := os.WriteFile(target, bridgeTestCreds(t, "s0", fixed.Add(time.Hour).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + for i := 1; i <= 2; i++ { + simulateClaudeRefresh(t, link, bridgeTestCreds(t, fmt.Sprintf("a%d", i), fixed.Add(time.Duration(i+1)*time.Hour).UnixMilli())) + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep == nil || rep.kept != "agent" { + t.Fatalf("repair %d = %+v", i, rep) + } + } + if got := staleFilesBeside(t, link); len(got) != 2 { + t.Errorf("agent stale files = %v, want 2 distinct", got) + } + got := staleFilesBeside(t, target) + if len(got) != 2 { + t.Fatalf("shared stale files = %v, want 2 distinct", got) + } + if !strings.HasSuffix(got[1], "-1") { + t.Errorf("second stale name = %q, want -1 suffix", got[1]) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-a2-") { + t.Errorf("shared token = %q, want the latest refresh", tok.AccessToken) + } +} + +func TestPreferAgentClaudeCredential(t *testing.T) { + now := time.Now() + live := func(tag string, d time.Duration) *claude.OAuthTokens { + return &claude.OAuthTokens{AccessToken: "a" + tag, RefreshToken: "r" + tag, ExpiresAt: now.Add(d).UnixMilli()} + } + cases := []struct { + name string + agent, shared *claude.OAuthTokens + want bool + }{ + {"nil agent", nil, live("s", time.Hour), false}, + {"agent newer", live("a", 2*time.Hour), live("s", time.Hour), true}, + {"agent older", live("a", time.Hour), live("s", 2*time.Hour), false}, + {"tie keeps shared", live("a", time.Hour), live("a", time.Hour), false}, + {"shared unusable", live("a", time.Hour), &claude.OAuthTokens{AccessToken: "x", ExpiresAt: now.Add(-time.Hour).UnixMilli()}, true}, + {"shared absent", live("a", time.Hour), nil, true}, + {"agent no refresh token", &claude.OAuthTokens{AccessToken: "a", ExpiresAt: now.Add(time.Hour).UnixMilli()}, nil, false}, + {"agent no expiry", &claude.OAuthTokens{AccessToken: "a", RefreshToken: "r"}, nil, false}, + {"agent expired, no refresh expiry", &claude.OAuthTokens{AccessToken: "a", RefreshToken: "r", ExpiresAt: now.Add(-time.Hour).UnixMilli()}, nil, true}, + {"agent empty", &claude.OAuthTokens{}, nil, false}, + } + for _, tc := range cases { + if got := preferAgentClaudeCredential(tc.agent, tc.shared, now); got != tc.want { + t.Errorf("%s: prefer agent = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestPromoteClaudeCredential_RefusesUnusableBytes(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, ".credentials.json") + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + for _, bad := range [][]byte{[]byte("not json"), []byte(`{}`), []byte(`{"claudeAiOauth":{"accessToken":"","refreshToken":"","expiresAt":0}}`)} { + stale, err := promoteClaudeCredential(bad, target, time.Now()) + if !errors.Is(err, errClaudeBridgeNotPromotable) || stale != "" { + t.Errorf("promote(%q) = (%q, %v), want errClaudeBridgeNotPromotable", bad, stale, err) + } + } + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared credential changed") + } + if got := staleFilesBeside(t, target); len(got) != 0 { + t.Errorf("preserved copies made for refused promotions: %v", got) + } + // Promotion with no shared file: nothing to preserve, chown seam called + // only when an owner was read. + fresh := filepath.Join(dir, "fresh", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(fresh), 0o755); err != nil { + t.Fatal(err) + } + stale, err := promoteClaudeCredential(sharedBytes, fresh, time.Now()) + if err != nil || stale != "" { + t.Fatalf("promote to absent target = (%q, %v)", stale, err) + } + if info, err := os.Stat(fresh); err != nil || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("promoted mode = %v err=%v", info.Mode(), err) + } +} + +func TestPromoteClaudeCredential_WritesOnlyTheTokenBlock(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, ".credentials.json") + if err := os.WriteFile(target, bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + exp := time.Now().Add(8 * time.Hour).UnixMilli() + // A credential with extra material around and inside the token block. + raw := fmt.Sprintf(`{"claudeAiOauth":{"accessToken":"acc-%s","refreshToken":"ref-%s","expiresAt":%d,"scopes":["user:inference"],"extraField":{"deep":"EXTRA-MATERIAL"}},"otherTopLevel":"EXTRA-MATERIAL"}`, + bridgeTestTokenSentinel, bridgeTestTokenSentinel, exp) + stale, err := promoteClaudeCredential([]byte(raw), target, time.Now()) + if err != nil || stale == "" { + t.Fatalf("promote = (%q, %v)", stale, err) + } + got := mustRead(t, target) + if bytes.Contains(got, []byte("EXTRA-MATERIAL")) { + t.Errorf("shared file carries bytes outside the typed token block: %s", got) + } + want, _ := json.Marshal(claude.Credentials{ClaudeAIOAuth: &claude.OAuthTokens{ + AccessToken: "acc-" + bridgeTestTokenSentinel, RefreshToken: "ref-" + bridgeTestTokenSentinel, + ExpiresAt: exp, Scopes: []string{"user:inference"}, + }}) + if !bytes.Equal(got, want) { + t.Errorf("shared file = %s, want canonical %s", got, want) + } + if info, err := os.Lstat(target); err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("shared file mode = %v err=%v", info.Mode(), err) + } + if info, err := os.Stat(stale); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("preserved shared copy mode = %v err=%v, want 0600", info.Mode(), err) + } + if _, err := os.Lstat(target + ".tmp-rebridge"); !errors.Is(err, fs.ErrNotExist) { + t.Error("a predictable temporary name was used") + } + if left, _ := filepath.Glob(target + ".*.tmp"); len(left) != 0 { + t.Errorf("temporary files left: %v", left) + } + if _, _, err := canonicalClaudeCredential(bytes.Repeat([]byte("x"), claudeBridgeMaxCredentialBytes+1), time.Now()); !errors.Is(err, errClaudeBridgeNotPromotable) { + t.Errorf("oversized input: err = %v", err) + } +} + +// TestRepairClaudeCredentialBridge_SharedTargetSymlinkIsRefused: the shared +// credential path holds a symlink. Nothing is written through it, the agent +// keeps its private copy in place, and the refusal is logged once. +func TestRepairClaudeCredentialBridge_SharedTargetSymlinkIsRefused(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + var logBuf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, target); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "fresh", time.Now().Add(8*time.Hour).UnixMilli())) + + for i := 0; i < 2; i++ { + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want refusal", rep) + } + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Errorf("victim rewritten: %q", got) + } + if info, _ := os.Lstat(victim); info.Mode().Perm() != 0o600 { + t.Errorf("victim mode changed: %v", info.Mode()) + } + if info, err := os.Lstat(target); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("symlink at the shared path was replaced: %v %v", info, err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("agent's private file not left in place: %v %v", info, err) + } + if got := staleFilesBeside(t, link); len(got) != 0 { + t.Errorf("private copy left aside: %v", got) + } + if sink.count() != 0 { + t.Error("a refusal must not audit") + } + if n := strings.Count(logBuf.String(), "location was refused"); n != 1 { + t.Errorf("refusal logged %d times over two cycles, want once:\n%s", n, logBuf.String()) + } + if strings.Contains(logBuf.String(), bridgeTestTokenSentinel) { + t.Error("log carries token material") + } +} + +// TestRepairClaudeCredentialBridge_SharedDirSymlinkIsRefused: the shared +// .claude directory itself is a symlink. Nothing is written into what it +// points at. +func TestRepairClaudeCredentialBridge_SharedDirSymlinkIsRefused(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + sharedBytes := bridgeTestCreds(t, "shared", time.Now().Add(time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + dir := filepath.Dir(target) + elsewhere := filepath.Join(t.TempDir(), "elsewhere") + if err := os.Rename(dir, elsewhere); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, dir); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "fresh", time.Now().Add(8*time.Hour).UnixMilli())) + + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("repair = %+v, want refusal", rep) + } + if !bytes.Equal(mustRead(t, filepath.Join(elsewhere, ".credentials.json")), sharedBytes) { + t.Error("wrote through the directory symlink") + } + entries, _ := os.ReadDir(elsewhere) + for _, e := range entries { + if e.Name() != ".credentials.json" { + t.Errorf("unexpected entry behind the directory symlink: %s", e.Name()) + } + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Errorf("agent's private file not left in place: %v %v", info, err) + } +} + +// TestMoveAsideClaudeCredential_NonRegularSourceIsRefused: a symlink where the +// private file should be is neither moved nor read. +func TestMoveAsideClaudeCredential_NonRegularSourceIsRefused(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim") + if err := os.WriteFile(victim, bridgeTestCreds(t, "v", time.Now().Add(time.Hour).UnixMilli()), 0o600); err != nil { + t.Fatal(err) + } + src := filepath.Join(dir, ".credentials.json") + if err := os.Symlink(victim, src); err != nil { + t.Fatal(err) + } + if _, err := moveAsideClaudeCredential(src, time.Now()); !errors.Is(err, errClaudeBridgeSourceChanged) { + t.Errorf("err = %v, want errClaudeBridgeSourceChanged", err) + } + if info, err := os.Lstat(src); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("symlink moved: %v %v", info, err) + } + if got := staleFilesBeside(t, src); len(got) != 0 { + t.Errorf("stale files created: %v", got) + } + // A directory is refused the same way. + d := filepath.Join(dir, "d") + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + if _, err := moveAsideClaudeCredential(d, time.Now()); !errors.Is(err, errClaudeBridgeSourceChanged) { + t.Errorf("directory: err = %v", err) + } + // The owner-read path refuses a symlink too, even one to a readable file. + if _, err := readClaudeCredentialAsOwner(src); !errors.Is(err, sharedfile.ErrNotRegular) { + t.Errorf("read through symlink: err = %v, want ErrNotRegular", err) + } +} + +// TestRepairClaudeCredentialBridge_UnreadableIsRelinkedOnlyAfterSignIn: an +// unreadable private file stays put on the watchdog, but a dashboard sign-in +// (a shared login newer than any private copy) moves it aside unread and +// relinks the agent. +func TestRepairClaudeCredentialBridge_UnreadableIsRelinkedOnlyAfterSignIn(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can read a mode-000 file; the permission fallback cannot be exercised") + } + m, sink, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "private", time.Now().Add(8*time.Hour).UnixMilli())) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + old := runAsAgentUser + runAsAgentUser = func(string, []byte, ...string) ([]byte, error) { return nil, errors.New("no su-exec") } + t.Cleanup(func() { runAsAgentUser = old }) + + // No shared login yet: even the sign-in trigger moves nothing. + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerSignIn); rep != nil { + t.Fatalf("sign-in without a shared login: repair = %+v", rep) + } + sharedBytes := bridgeTestCreds(t, "signin", time.Now().Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerWatchdog); rep != nil { + t.Fatalf("watchdog: repair = %+v, want none", rep) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("watchdog moved the unreadable file: %v %v", info, err) + } + + m.ReloadClaudeToken() + assertLinkRestored(t, link, target) + stale := staleFilesBeside(t, link) + if len(stale) != 1 { + t.Fatalf("stale files = %v, want the unreadable copy moved aside", stale) + } + if info, err := os.Lstat(stale[0]); err != nil || info.Mode().Perm() != 0 { + t.Errorf("moved-aside copy mode = %v err=%v, want unchanged (000)", info.Mode(), err) + } + _ = os.Chmod(stale[0], 0o600) + if !bytes.Equal(mustRead(t, target), sharedBytes) { + t.Error("shared login changed") + } + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil || rec.Fields["kept"] != "shared" || rec.Fields["trigger"] != claudeBridgeTriggerSignIn { + t.Errorf("audit = %+v", rec) + } +} + +// TestCredentialWatchdog_RebridgesWithinOneCycle: the bridge poll repairs a +// replaced link without waiting for the 5-minute probe. +func TestCredentialWatchdog_RebridgesWithinOneCycle(t *testing.T) { + t.Setenv(ClaudeBridgeIntervalEnv, "20ms") + t.Setenv(CredentialWatchdogIntervalEnv, "1h") + m, _, _, link, target := bridgeTestFixture(t) + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "old", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + fresh := bridgeTestCreds(t, "fresh", now.Add(8*time.Hour).UnixMilli()) + simulateClaudeRefresh(t, link, fresh) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { m.StartCredentialWatchdog(ctx); close(done) }() + + testutil.Eventually(t, 5*time.Second, func() bool { + got, err := os.Readlink(link) + return err == nil && got == target + }, "watchdog did not re-bridge the credential within 5s") + cancel() + <-done + if !bytes.Equal(mustRead(t, target), fresh) { + t.Error("watchdog did not publish the refresh to the shared login") + } +} + +func TestClaudeBridgeInterval(t *testing.T) { + t.Setenv(ClaudeBridgeIntervalEnv, "") + if got := claudeBridgeInterval(); got != defaultClaudeBridgeInterval { + t.Errorf("default = %v", got) + } + t.Setenv(ClaudeBridgeIntervalEnv, "250ms") + if got := claudeBridgeInterval(); got != 250*time.Millisecond { + t.Errorf("override = %v", got) + } + for _, bad := range []string{"nope", "-1s", "0"} { + t.Setenv(ClaudeBridgeIntervalEnv, bad) + if got := claudeBridgeInterval(); got != defaultClaudeBridgeInterval { + t.Errorf("%q: = %v, want default", bad, got) + } + } +} + +// TestReloadClaudeToken_RebridgesAfterSignIn: a dashboard sign-in reaches an +// agent whose link a refresh had replaced, at once rather than on the next +// poll; a logout (nothing usable) moves nothing. +func TestReloadClaudeToken_RebridgesAfterSignIn(t *testing.T) { + m, sink, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", time.Now().Add(-time.Hour).UnixMilli())) + + // Logout: no shared file. Nothing moves. + m.ReloadClaudeToken() + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("logout must not touch the private file: info=%v err=%v", info, err) + } + if sink.count() != 0 { + t.Fatal("logout audited a rebridge") + } + + sharedBytes := bridgeTestCreds(t, "signin", time.Now().Add(8*time.Hour).UnixMilli()) + if err := os.WriteFile(target, sharedBytes, 0o660); err != nil { + t.Fatal(err) + } + m.ReloadClaudeToken() + assertLinkRestored(t, link, target) + rec := sink.find(AuditClaudeCredentialRebridged) + if rec == nil || rec.Fields["trigger"] != claudeBridgeTriggerSignIn || rec.Fields["kept"] != "shared" { + t.Errorf("audit = %+v", rec) + } +} + +// TestRepairClaudeCredentialBridge_MidRenameIsNextCycle: the link vanishes +// between the Lstat and the read (the CLI is mid-rename). Nothing to do. +func TestRepairClaudeCredentialBridge_MidRenameIsNextCycle(t *testing.T) { + m, _, _, link, target := bridgeTestFixture(t) + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "agent", time.Now().Add(time.Hour).UnixMilli())) + if err := os.Chmod(link, 0); err != nil { + t.Fatal(err) + } + old := runAsAgentUser + runAsAgentUser = func(spec string, stdin []byte, argv ...string) ([]byte, error) { + _ = os.Remove(link) + return nil, &fs.PathError{Op: "open", Path: link, Err: fs.ErrNotExist} + } + t.Cleanup(func() { runAsAgentUser = old }) + if os.Getuid() == 0 { + // Root reads the file directly; remove it from under the Lstat another way. + runAsAgentUser = old + t.Skip("root cannot exercise the owner-read path") + } + rep, err := m.repairClaudeCredentialBridge("scanner", link, target, claudeBridgeTriggerWatchdog) + if rep != nil || err != nil { + t.Errorf("mid-rename: (%+v, %v), want nothing", rep, err) + } +} diff --git a/src/crux/agent/copilottoken/copilottoken.go b/src/crux/agent/copilottoken/copilottoken.go index 480036a3d3..de66bcbfb7 100644 --- a/src/crux/agent/copilottoken/copilottoken.go +++ b/src/crux/agent/copilottoken/copilottoken.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/sharedfile" ) // ConfigHasTokens returns true if either the Copilot config or Claude @@ -134,19 +135,18 @@ func ReadConfig(path string) (map[string]interface{}, error) { } // WriteConfig marshals cfg back to config.json with the CLI's // header -// preserved and the CLI-expected mode. Written via a temp file + rename so a -// concurrent CLI read never sees a half-written file. +// preserved and the CLI-expected mode. Written through sharedfile.Replace (a +// random temporary beside path, renamed into place) so a concurrent CLI read +// never sees a half-written file, and so a non-regular file at path, or a +// directory that is not this process's own, is refused rather than written +// through: config.json lives in a directory every agent uid can write to. func WriteConfig(path string, cfg map[string]interface{}, mode os.FileMode) error { out, err := json.MarshalIndent(cfg, "", " ") if err != nil { return err } content := ConfigHeader + string(out) - tmp := path + ".tmp" - if err := os.WriteFile(tmp, []byte(content), mode); err != nil { - return err - } - return os.Rename(tmp, path) + return sharedfile.Replace(path, []byte(content), mode, sharedfile.Options{PreserveOwner: true}) } // ClearExpiredTokens removes stored copilot tokens from config.json. @@ -361,9 +361,5 @@ func WriteDurableToken(durablePath, token string) error { if token == "" { return nil } - tmp := durablePath + ".tmp" - if err := os.WriteFile(tmp, []byte(token), 0o600); err != nil { - return err - } - return os.Rename(tmp, durablePath) + return sharedfile.Replace(durablePath, []byte(token), 0o600, sharedfile.Options{}) } diff --git a/src/crux/agent/coverage_boost_test.go b/src/crux/agent/coverage_boost_test.go index ec674b8f83..e5ee3839ed 100644 --- a/src/crux/agent/coverage_boost_test.go +++ b/src/crux/agent/coverage_boost_test.go @@ -1844,6 +1844,32 @@ func TestClearExpiredTokens_MissingFile(t *testing.T) { // fixSharedConfigPerms — with actual path // --------------------------------------------------------------------------- +// A symlink at the shared config path is left alone: the file it points at +// keeps its mode. +func TestFixSharedConfigPerms_SymlinkIsNotFollowed(t *testing.T) { + cleanup := configTestHelper(t) + defer cleanup() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, sharedCopilotConfigPath); err != nil { + t.Fatal(err) + } + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "claude"}, + }, discardLogger(), ProjectContext{}) + m.mu.RLock() + agent := m.agents["scanner"] + m.mu.RUnlock() + + m.fixSharedConfigPerms(agent) + + if info, _ := os.Stat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode = %v, want 0600 untouched", info.Mode()) + } +} + func TestFixSharedConfigPerms_FixesPerms(t *testing.T) { cleanup := configTestHelper(t) defer cleanup() diff --git a/src/crux/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go index f64c76eca0..5a3c02d85e 100644 --- a/src/crux/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -25,6 +25,19 @@ const ( // only for overrides) — disabling is intentionally not offered so the // safety net cannot be silently turned off. CredentialWatchdogIntervalEnv = "HIVE_CREDENTIAL_WATCHDOG_INTERVAL" + + // defaultClaudeBridgeInterval is how often the watchdog checks each + // per-agent Claude credential link for a token refresh that replaced it + // (claude_credential_bridge.go). Short on purpose: every agent holds the + // SAME access token, so they all reach expiry together and refresh within + // a minute or two of each other; the first refresh rotates the grant, and + // each agent that tries before that refresh is published fails and signs + // itself out. The check is one Lstat per claude agent, so 5s costs + // nothing; the su-exec read happens only when a link has been replaced. + defaultClaudeBridgeInterval = 5 * time.Second + // ClaudeBridgeIntervalEnv overrides the bridge poll interval with a Go + // duration string. + ClaudeBridgeIntervalEnv = "HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL" ) // credentialWatchdogInterval resolves the watchdog interval from @@ -38,6 +51,17 @@ func credentialWatchdogInterval() time.Duration { return defaultCredentialWatchdogInterval } +// claudeBridgeInterval resolves the bridge poll interval from +// ClaudeBridgeIntervalEnv, falling back to the default. +func claudeBridgeInterval() time.Duration { + if v := os.Getenv(ClaudeBridgeIntervalEnv); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultClaudeBridgeInterval +} + // credentialWatch describes one CLI backend whose usable credential lives in a // durable file on the hive PVC that the hive relies on but does NOT itself keep // alive. Only backends of this shape belong here: bob resolves its key at @@ -199,6 +223,8 @@ func credentialWatches() []credentialWatch { func (m *Manager) StartCredentialWatchdog(ctx context.Context) { ticker := time.NewTicker(credentialWatchdogInterval()) defer ticker.Stop() + bridgeTicker := time.NewTicker(claudeBridgeInterval()) + defer bridgeTicker.Stop() // Per-backend transition tracker so we log/audit on TRANSITIONS (usable -> // unusable and back) rather than every tick — a standing condition would // otherwise flood the Audit Log at the tick rate. @@ -207,7 +233,16 @@ func (m *Manager) StartCredentialWatchdog(ctx context.Context) { select { case <-ctx.Done(): return + case <-bridgeTicker.C: + if m.backendInUse("claude") { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog) + } case <-ticker.C: + // Re-bridge before probing, so a just-published refresh is what + // the shared-credential probe sees. + if m.backendInUse("claude") { + m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog) + } for _, w := range credentialWatches() { m.evalCredentialWatch(w, lastUnusable) } diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 114e404d1e..4b9798400d 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -425,6 +425,12 @@ func (m *Manager) setupPerAgentCLIHome(agentName, home string, uid int, spec per for _, name := range spec.bridges { link := filepath.Join(dir, name) target := filepath.Join(shared, name) + if spec.dir == claudeHomeDirName && name == claudeCredentialsFileName { + // A token refresh since the last launch replaced the link with a + // private file; publish it to the shared login and relink BEFORE + // bridgeHomeEntry, which refuses to clobber a regular file. + m.rebridgeClaudeCredential(agentName, link, target, claudeBridgeTriggerLaunch) + } if _, err := os.Stat(target); err != nil && isPerAgentCLIBridgeDir(spec.dir, name) { // Absent target: a dangling FILE link goes live on first login; a // dangling DIRECTORY link would make the CLI's own mkdir -p fail, diff --git a/src/crux/agent/manager_credentials.go b/src/crux/agent/manager_credentials.go index b681e0a9c2..232ba58d45 100644 --- a/src/crux/agent/manager_credentials.go +++ b/src/crux/agent/manager_credentials.go @@ -2,14 +2,16 @@ package agent import ( "context" + "errors" "fmt" - "os" + "io/fs" "strings" "time" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/claude" "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/sharedfile" ) // sharedCopilotConfigPath and sharedClaudeCredentialPath are vars (not consts) @@ -304,19 +306,22 @@ func compactPaneTail(output string, maxLines int) string { // fixSharedConfigPerms ensures /data/home/.copilot/config.json is group-readable // before launching an agent. Copilot CLI rewrites this file with 600 perms on // token refresh, locking out other agent UIDs that share the same HOME. +// +// The chmod goes through an open descriptor (sharedfile.Chmod), so anything +// other than a regular file at that path is left alone and logged. func (m *Manager) fixSharedConfigPerms(agent *AgentProcess) { - info, err := os.Stat(sharedCopilotConfigPath) - if err != nil { + prev, changed, err := sharedfile.Chmod(sharedCopilotConfigPath, sharedConfigDesiredMode) + switch { + case errors.Is(err, fs.ErrNotExist): return - } - if info.Mode().Perm() == sharedConfigDesiredMode { + case err != nil: + m.logger.Warn("failed to fix config.json perms", "agent", agent.Name, "error", err) + return + case !changed: return } - m.logger.Warn("fixing shared config.json perms before launch", + m.logger.Warn("fixed shared config.json perms before launch", "agent", agent.Name, - "was", fmt.Sprintf("%04o", info.Mode().Perm()), + "was", fmt.Sprintf("%04o", prev), "fix", fmt.Sprintf("%04o", sharedConfigDesiredMode)) - if err := os.Chmod(sharedCopilotConfigPath, sharedConfigDesiredMode); err != nil { - m.logger.Warn("failed to fix config.json perms", "error", err) - } } diff --git a/src/crux/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go index d6d5a23444..44cc4933dc 100644 --- a/src/crux/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -21,18 +21,18 @@ type ( AgentMintIssuer = crux.AgentMintIssuer ) -// ReloadClaudeToken re-reads the Claude credentials file and updates the -// cached token. Called by the dashboard after a successful OAuth login. +// ReloadClaudeToken re-reads the Claude credentials file, updates the cached +// token and (after m.mu is released) re-bridges agents a refresh unlinked. func (m *Manager) ReloadClaudeToken() { + defer m.rebridgeClaudeCredentialsAfterSignIn() // LIFO: runs after Unlock m.mu.Lock() defer m.mu.Unlock() m.claudeAuthToken = claude.ReadAccessToken(claude.CredentialsPath) } // Copilot token sources, as reported by CopilotTokenSource. Operator-facing -// phrases: they are spliced verbatim into the model-picker notice and the -// "rejected by upstream" log line, so each one has to read as an answer to -// "which credential did GitHub just reject?". +// phrases, spliced verbatim into the model-picker notice and the "rejected by +// upstream" log line: each must answer "which credential did GitHub reject?". const ( // Defined in the crux root package; see crux.CopilotTokenSourceEnv. CopilotTokenSourceEnv = crux.CopilotTokenSourceEnv diff --git a/src/crux/agent/permissions_watcher.go b/src/crux/agent/permissions_watcher.go index d8b5fbb394..648b60fb3d 100644 --- a/src/crux/agent/permissions_watcher.go +++ b/src/crux/agent/permissions_watcher.go @@ -620,7 +620,9 @@ func fixEntry(path string, fi os.FileInfo, logger *slog.Logger) { if fi.IsDir() { // Directories need u+rwx to be usable. if mode.Perm()&DirPerms != DirPerms { - newMode := mode.Perm() | DirPerms + // Keep setgid and sticky: the entrypoint sets both on the shared + // dot-dirs, and a repair of the group bits must not drop them. + newMode := mode&(os.ModePerm|os.ModeSetgid|os.ModeSticky) | DirPerms if err := chmod(newMode); err != nil { warnDeduped(logger, "chmod dir", "permissions watcher: chmod dir failed", path, err) } else { diff --git a/src/crux/audit.go b/src/crux/audit.go index 350cfa09e4..367c026240 100644 --- a/src/crux/audit.go +++ b/src/crux/audit.go @@ -40,6 +40,15 @@ const ( // failure shape and same operator-login recovery as the Copilot case; the // detail's outcome= distinguishes "missing" from "invalid or expired". AuditClaudeTokenMissing = "claude_token_missing" + + // AuditClaudeCredentialRebridged is recorded when an agent's per-agent + // ~/.claude/.credentials.json symlink to the shared Claude login had been + // replaced by a regular file (Claude Code saves a token refresh by renaming + // over the path) and hive re-bridged it: either promoting the agent's + // fresher token into the shared login or re-linking the agent to the + // shared one. The detail's kept= says which; it never carries token + // material. + AuditClaudeCredentialRebridged = "claude_credential_rebridged" ) // AuditSink receives agent lifecycle events for durable, queryable recording. diff --git a/src/crux/claude/oauth.go b/src/crux/claude/oauth.go index 47ef515a51..6777c081a8 100644 --- a/src/crux/claude/oauth.go +++ b/src/crux/claude/oauth.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -13,6 +14,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/hivecommons/hive/crux/sharedfile" ) const ( @@ -187,8 +190,27 @@ func ExchangeCode(code, codeVerifier, redirectURI string) (*OAuthTokens, error) }, nil } -// WriteCredentials atomically writes tokens to the credentials file. +// ErrNoAccessToken is returned by WriteCredentials for a token set with no +// access token. Writing one would replace whatever login is on file with a +// signed-out credential. +var ErrNoAccessToken = errors.New("refusing to write a Claude credential with no access token") + +// CredentialsFileMode is the mode of the shared credentials file: owner and +// group read/write, so every agent uid reaches it through the shared group. +const CredentialsFileMode = 0o660 + +// WriteCredentials atomically writes tokens to the credentials file. It +// refuses a nil token set or one with an empty access token +// (ErrNoAccessToken), so no hive caller can replace the shared login with a +// signed-out credential. The write is sharedfile.Replace: the credentials +// directory is shared with every agent uid, so the temporary name is random, +// metadata is set by descriptor, and a non-regular file at the destination or +// a directory that is not this process's own is refused rather than written +// through. func WriteCredentials(tokens *OAuthTokens, path string) error { + if tokens == nil || tokens.AccessToken == "" { + return ErrNoAccessToken + } dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o2775); err != nil { return fmt.Errorf("create credentials dir: %w", err) @@ -199,13 +221,8 @@ func WriteCredentials(tokens *OAuthTokens, path string) error { if err != nil { return fmt.Errorf("marshal credentials: %w", err) } - - tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, data, 0o660); err != nil { - return fmt.Errorf("write temp credentials: %w", err) - } - if err := os.Rename(tmpPath, path); err != nil { - return fmt.Errorf("rename credentials: %w", err) + if err := sharedfile.Replace(path, data, CredentialsFileMode, sharedfile.Options{PreserveOwner: true}); err != nil { + return fmt.Errorf("write credentials: %w", err) } return nil } @@ -214,10 +231,29 @@ func WriteCredentials(tokens *OAuthTokens, path string) error { // nil when the file is absent, malformed, or carries no token block. It never // applies an expiry rule — callers decide what a stale token means to them. func LoadTokens(path string) *OAuthTokens { - data, err := os.ReadFile(path) + tokens, err := ReadTokens(path) if err != nil { return nil } + return tokens +} + +// ReadTokens is LoadTokens with the read error kept, for callers that must +// tell "this process cannot read the file" (fs.ErrNotExist, or +// fs.ErrPermission on another uid's 0600 credential) apart from "the file +// holds no usable token". A file that was read but is malformed, or carries +// no token block, returns (nil, nil). The error never carries file content. +func ReadTokens(path string) (*OAuthTokens, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseTokens(data), nil +} + +// ParseTokens decodes a credentials file already in memory. Nil for malformed +// bytes or a file with no token block, like LoadTokens. +func ParseTokens(data []byte) *OAuthTokens { var creds Credentials if err := json.Unmarshal(data, &creds); err != nil { return nil @@ -281,11 +317,16 @@ func HasValidToken(path string) bool { // Positive evidence only: an unreadable or absent file returns false, leaving // callers exactly where they were before they asked. func HasUsableToken(path string) bool { - tokens := LoadTokens(path) + return TokensUsable(LoadTokens(path), time.Now()) +} + +// TokensUsable is the HasUsableToken rule applied to an already-loaded token +// set at a given instant: a live access token, or a refresh grant that has not +// expired. A nil token set is never usable. +func TokensUsable(tokens *OAuthTokens, now time.Time) bool { if tokens == nil { return false } - now := time.Now() if tokens.AccessToken != "" && !expired(tokens.ExpiresAt, now) { return true } diff --git a/src/crux/claude/oauth_write_error_branches_test.go b/src/crux/claude/oauth_write_error_branches_test.go index b8b9e8881b..97a1e9a2dc 100644 --- a/src/crux/claude/oauth_write_error_branches_test.go +++ b/src/crux/claude/oauth_write_error_branches_test.go @@ -1,10 +1,14 @@ package claude import ( + "errors" "os" "path/filepath" "strings" "testing" + "time" + + "github.com/hivecommons/hive/crux/sharedfile" ) // tokenEndpoint must fall back to the real Claude TokenURL when no test @@ -35,8 +39,11 @@ func TestWriteCredentials_TempWriteError(t *testing.T) { if err == nil { t.Fatalf("WriteCredentials into read-only dir: err = nil, want error") } - if !strings.Contains(err.Error(), "write temp credentials") { - t.Fatalf("error = %q, want it to mention write temp credentials", err) + if !strings.Contains(err.Error(), "write credentials") { + t.Fatalf("error = %q, want it to mention write credentials", err) + } + if left, _ := filepath.Glob(filepath.Join(dir, "creds.json.*")); len(left) != 0 { + t.Fatalf("temporary files left behind: %v", left) } } @@ -51,9 +58,54 @@ func TestWriteCredentials_RenameError(t *testing.T) { err := WriteCredentials(&OAuthTokens{AccessToken: "at"}, path) if err == nil { - t.Fatalf("WriteCredentials over non-empty dir: err = nil, want error") + t.Fatalf("WriteCredentials over a directory: err = nil, want error") + } + if !errors.Is(err, sharedfile.ErrUnsafeLocation) { + t.Fatalf("error = %q, want sharedfile.ErrUnsafeLocation", err) + } + if left, _ := filepath.Glob(path + ".*"); len(left) != 0 { + t.Fatalf("temporary files left behind: %v", left) + } +} + +// A symlink at the credentials path is refused, and the file it points at is +// left exactly as it was. +func TestWriteCredentials_SymlinkAtPathIsRefused(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, ".credentials.json") + if err := os.Symlink(victim, path); err != nil { + t.Fatal(err) + } + err := WriteCredentials(&OAuthTokens{AccessToken: "at"}, path) + if !errors.Is(err, sharedfile.ErrUnsafeLocation) { + t.Fatalf("error = %v, want sharedfile.ErrUnsafeLocation", err) + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Fatalf("victim rewritten: %q", got) + } + if info, _ := os.Lstat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode changed: %v", info.Mode()) + } +} + +// TestWriteCredentialsRefusesSignedOutTokens: no hive writer may put a +// credential with no access token over the shared login. +func TestWriteCredentialsRefusesSignedOutTokens(t *testing.T) { + path := filepath.Join(t.TempDir(), ".credentials.json") + if err := WriteCredentials(&OAuthTokens{AccessToken: "live", ExpiresAt: time.Now().Add(time.Hour).UnixMilli()}, path); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(path) + for _, tok := range []*OAuthTokens{nil, {}, {RefreshToken: "r", ExpiresAt: 1}} { + if err := WriteCredentials(tok, path); !errors.Is(err, ErrNoAccessToken) { + t.Errorf("WriteCredentials(%+v) = %v, want ErrNoAccessToken", tok, err) + } } - if !strings.Contains(err.Error(), "rename credentials") { - t.Fatalf("error = %q, want it to mention rename credentials", err) + if after, _ := os.ReadFile(path); string(after) != string(before) { + t.Error("the shared login was replaced by a signed-out credential") } } diff --git a/src/crux/claude/usable_token_test.go b/src/crux/claude/usable_token_test.go index bde265b4cc..720c474a5b 100644 --- a/src/crux/claude/usable_token_test.go +++ b/src/crux/claude/usable_token_test.go @@ -2,6 +2,8 @@ package claude import ( "encoding/json" + "errors" + "io/fs" "os" "path/filepath" "testing" @@ -132,3 +134,66 @@ func TestRefreshTokenExpiresAtRoundTrips(t *testing.T) { t.Fatalf("refreshTokenExpiresAt lost on round trip: got %+v", got) } } + +// TestReadTokens_KeepsTheReadError: the credential bridge must tell "this +// process may not open another uid's 0600 file" apart from "no token here"; +// LoadTokens flattens both to nil, ReadTokens does not. +func TestReadTokens_KeepsTheReadError(t *testing.T) { + if _, err := ReadTokens(filepath.Join(t.TempDir(), "missing")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("missing file: err = %v, want ErrNotExist", err) + } + bad := filepath.Join(t.TempDir(), ".credentials.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if tok, err := ReadTokens(bad); err != nil || tok != nil { + t.Errorf("malformed: (%v, %v), want nil, nil (readable, nothing usable)", tok, err) + } + if tok := LoadTokens(bad); tok != nil { + t.Errorf("LoadTokens malformed = %v, want nil", tok) + } + empty := filepath.Join(t.TempDir(), ".credentials.json") + if err := os.WriteFile(empty, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if tok, err := ReadTokens(empty); err != nil || tok != nil { + t.Errorf("no token block: (%v, %v), want nil, nil", tok, err) + } + if os.Getuid() != 0 { + locked := writeCredFile(t, &OAuthTokens{AccessToken: "a", ExpiresAt: ms(time.Hour)}) + if err := os.Chmod(locked, 0); err != nil { + t.Fatal(err) + } + if _, err := ReadTokens(locked); !errors.Is(err, fs.ErrPermission) { + t.Errorf("unreadable: err = %v, want ErrPermission", err) + } + } +} + +func TestParseTokens(t *testing.T) { + if tok := ParseTokens([]byte("nope")); tok != nil { + t.Errorf("malformed = %v", tok) + } + if tok := ParseTokens([]byte(`{}`)); tok != nil { + t.Errorf("no block = %v", tok) + } + tok := ParseTokens([]byte(`{"claudeAiOauth":{"accessToken":"a","refreshToken":"r","expiresAt":42}}`)) + if tok == nil || tok.AccessToken != "a" || tok.RefreshToken != "r" || tok.ExpiresAt != 42 { + t.Errorf("parsed = %+v", tok) + } +} + +func TestTokensUsable_AtAnInstant(t *testing.T) { + now := time.Now() + later := now.Add(2 * time.Hour) + tok := &OAuthTokens{AccessToken: "a", ExpiresAt: now.Add(time.Hour).UnixMilli()} + if !TokensUsable(tok, now) { + t.Error("live access token at now should be usable") + } + if TokensUsable(tok, later) { + t.Error("the same token judged after its expiry should be unusable") + } + if TokensUsable(nil, now) { + t.Error("nil is never usable") + } +} diff --git a/src/crux/sharedfile/sharedfile.go b/src/crux/sharedfile/sharedfile.go new file mode 100644 index 0000000000..9d4fd05ce0 --- /dev/null +++ b/src/crux/sharedfile/sharedfile.go @@ -0,0 +1,278 @@ +// Package sharedfile replaces and reads files that live in directories other +// uids can write to: the shared CLI credential and config files under +// /data/home that the hive maintains for a fleet of per-UID agents. +// +// A plain "write .tmp, chmod, rename" in such a directory acts on +// whatever is at those names when each call runs. Here every step acts on the +// inode this process created, by descriptor, and the name of that inode is +// unpredictable; the directory and the destination are checked to be what the +// caller expects (a directory this process owns, a regular file or nothing) +// right before the rename, and a symlink at any of those places is a refusal, +// never something to write through. +package sharedfile + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" +) + +// ErrUnsafeLocation is wrapped by Replace when the destination directory or +// the destination itself is not in a state this package will write through: +// the directory is missing, a symlink, world-writable, or not owned by this +// process, or the destination exists and is not a regular file. Nothing is +// written when it is returned. Callers that poll should log it once, not per +// cycle. +var ErrUnsafeLocation = errors.New("refusing to replace a file at an unsafe location") + +// ErrTooLarge is returned by ReadNoFollow for a file over the caller's cap. +var ErrTooLarge = errors.New("file is larger than the read cap") + +// ErrNotRegular is returned by ReadNoFollow when the path is a symlink or any +// other non-regular file. +var ErrNotRegular = errors.New("not a regular file") + +// createAttempts bounds the search for a free temporary name. A collision on +// a 64-bit random suffix is not expected; the retries make an EEXIST from a +// deliberately pre-created name a failure rather than a write-through. +const createAttempts = 8 + +// chownFile applies an owner to the inode behind an open descriptor. A seam: +// tests run as one uid and cannot change an owner for real. +var chownFile = func(f *os.File, uid, gid int) error { return f.Chown(uid, gid) } + +// randRead fills the temporary-name suffix; a seam so tests can force a +// collision or a failure. +var randRead = rand.Read + +// syncFile flushes the written temporary; a seam so tests can fail it. +var syncFile = func(f *os.File) error { return f.Sync() } + +// Options tune Replace. +type Options struct { + // PreserveOwner re-applies the previous destination's owner to the new + // file, best-effort: an unprivileged process cannot chown, and the file it + // wrote is already readable to the group by mode. + PreserveOwner bool +} + +// Replace atomically replaces target with data at mode. The data is written +// to a new file with a random name beside target (O_EXCL|O_NOFOLLOW, 0600), +// chmodded, optionally chowned, fsynced and verified by descriptor, and then +// renamed over target; the directory is fsynced afterwards. On any failure the +// temporary file is removed and target is untouched. +// +// Before the rename, the directory must be a real directory (not a symlink) +// owned by this process and not world-writable, and target must be absent or +// a regular file (not a symlink); otherwise ErrUnsafeLocation. +func Replace(target string, data []byte, mode fs.FileMode, opts Options) error { + dir := filepath.Dir(target) + prev, err := checkLocation(dir, target) + if err != nil { + return err + } + f, tmp, err := createTemp(target) + if err != nil { + return err + } + err = fill(f, data, mode, prev, opts) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(tmp) + return err + } + // The location can have changed while the file was being written. + if _, err := checkLocation(dir, target); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, target); err != nil { + _ = os.Remove(tmp) + return err + } + syncDir(dir) + return nil +} + +// CheckLocation verifies target's directory and target as documented on +// Replace, and returns the current target's Lstat (nil when absent). Callers +// that do preparatory work beside target (a backup copy, say) use it to refuse +// before that work, not only at the final write. +func CheckLocation(target string) (os.FileInfo, error) { + return checkLocation(filepath.Dir(target), target) +} + +// checkLocation verifies dir and target as documented on Replace and returns +// the current target's stat (nil when absent). +func checkLocation(dir, target string) (os.FileInfo, error) { + dinfo, err := os.Lstat(dir) + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrUnsafeLocation, dir, err) + } + if !dinfo.IsDir() { + return nil, fmt.Errorf("%w: %s is not a directory (mode %s)", ErrUnsafeLocation, dir, dinfo.Mode()) + } + if dinfo.Mode().Perm()&0o002 != 0 { + return nil, fmt.Errorf("%w: %s is world-writable (mode %s)", ErrUnsafeLocation, dir, dinfo.Mode()) + } + if st, ok := dinfo.Sys().(*syscall.Stat_t); ok { + if euid := os.Geteuid(); euid != 0 && int(st.Uid) != euid { + return nil, fmt.Errorf("%w: %s is owned by uid %d, this process is uid %d", ErrUnsafeLocation, dir, st.Uid, euid) + } + } + tinfo, err := os.Lstat(target) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrUnsafeLocation, target, err) + } + if !tinfo.Mode().IsRegular() { + return nil, fmt.Errorf("%w: %s is not a regular file (mode %s)", ErrUnsafeLocation, target, tinfo.Mode()) + } + return tinfo, nil +} + +// createTemp creates a new 0600 file with a random name beside target. The +// name is ".<16 hex>.tmp", so sweeps that match ".*.tmp" still +// recognise a leftover. +func createTemp(target string) (*os.File, string, error) { + for i := 0; i < createAttempts; i++ { + var b [8]byte + if _, err := randRead(b[:]); err != nil { + return nil, "", err + } + tmp := target + "." + hex.EncodeToString(b[:]) + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return nil, "", err + } + return f, tmp, nil + } + return nil, "", fmt.Errorf("no free temporary name beside %s after %d attempts", target, createAttempts) +} + +// fill writes data to f and sets its metadata, all through the descriptor, +// then verifies by fstat that the inode is a regular file with exactly the +// mode (and, when a chown succeeded, the owner) asked for. +func fill(f *os.File, data []byte, mode fs.FileMode, prev os.FileInfo, opts Options) error { + if _, err := f.Write(data); err != nil { + return err + } + if err := f.Chmod(mode); err != nil { + return err + } + wantUID, wantGID := -1, -1 + if opts.PreserveOwner && prev != nil { + if st, ok := prev.Sys().(*syscall.Stat_t); ok { + if err := chownFile(f, int(st.Uid), int(st.Gid)); err == nil { + wantUID, wantGID = int(st.Uid), int(st.Gid) + } + } + } + if err := syncFile(f); err != nil { + return err + } + info, err := f.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode().Perm() != mode.Perm() { + return fmt.Errorf("%s: written file is mode %s, want regular %s", f.Name(), info.Mode(), mode.Perm()) + } + if st, ok := info.Sys().(*syscall.Stat_t); ok { + if st.Nlink != 1 { + return fmt.Errorf("%s: written file has %d links, want 1", f.Name(), st.Nlink) + } + if wantUID >= 0 && (int(st.Uid) != wantUID || int(st.Gid) != wantGID) { + return fmt.Errorf("%s: written file is owned by %d:%d, want %d:%d", f.Name(), st.Uid, st.Gid, wantUID, wantGID) + } + } + return nil +} + +// syncDir makes the rename durable; best-effort, like every writer before it. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} + +// openRegularNoFollow opens path without following a symlink and confirms the +// opened file is regular. ErrNotRegular for a symlink or any other kind. +func openRegularNoFollow(path string, flag int) (*os.File, os.FileInfo, error) { + f, err := os.OpenFile(path, flag|syscall.O_NOFOLLOW, 0) + if err != nil { + var pe *fs.PathError + if errors.As(err, &pe) && (errors.Is(pe.Err, syscall.ELOOP) || errors.Is(pe.Err, syscall.EMLINK)) { + return nil, nil, fmt.Errorf("%w: %s is a symlink", ErrNotRegular, path) + } + return nil, nil, err + } + info, err := f.Stat() + if err == nil && !info.Mode().IsRegular() { + err = fmt.Errorf("%w: %s (mode %s)", ErrNotRegular, path, info.Mode()) + } + if err != nil { + f.Close() + return nil, nil, err + } + return f, info, nil +} + +// Chmod sets mode on the regular file at path through an open descriptor, so +// a symlink planted at path is refused (ErrNotRegular) instead of followed. +// Returns the mode the file had before, and changed=false when it already +// matched (nothing is done in that case). +func Chmod(path string, mode fs.FileMode) (prev fs.FileMode, changed bool, err error) { + f, info, err := openRegularNoFollow(path, os.O_RDONLY) + if err != nil { + return 0, false, err + } + defer f.Close() + prev = info.Mode().Perm() + if prev == mode.Perm() { + return prev, false, nil + } + if err := f.Chmod(mode.Perm()); err != nil { + return prev, false, err + } + return prev, true, nil +} + +// ReadNoFollow reads a regular file of at most max bytes without following a +// symlink at path. ErrNotRegular for a symlink or other non-regular file, +// ErrTooLarge over the cap; the open error (fs.ErrNotExist, fs.ErrPermission) +// otherwise. +func ReadNoFollow(path string, max int64) ([]byte, error) { + f, info, err := openRegularNoFollow(path, os.O_RDONLY) + if err != nil { + return nil, err + } + defer f.Close() + if info.Size() > max { + return nil, fmt.Errorf("%w: %s is %d bytes, cap %d", ErrTooLarge, path, info.Size(), max) + } + data, err := io.ReadAll(io.LimitReader(f, max+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > max { + return nil, fmt.Errorf("%w: %s grew past %d bytes while being read", ErrTooLarge, path, max) + } + return data, nil +} diff --git a/src/crux/sharedfile/sharedfile_test.go b/src/crux/sharedfile/sharedfile_test.go new file mode 100644 index 0000000000..d8307e37bc --- /dev/null +++ b/src/crux/sharedfile/sharedfile_test.go @@ -0,0 +1,480 @@ +package sharedfile + +import ( + "bytes" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func leftovers(t *testing.T, target string) []string { + t.Helper() + m, err := filepath.Glob(target + ".*.tmp") + if err != nil { + t.Fatal(err) + } + return m +} + +func modeOf(t *testing.T, path string) fs.FileMode { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + return info.Mode() +} + +func TestReplace_WritesAtModeAndLeavesNoTemp(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "config.json") + if err := Replace(target, []byte("one"), 0o660, Options{}); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "one" { + t.Errorf("content = %q", got) + } + if m := modeOf(t, target); !m.IsRegular() || m.Perm() != 0o660 { + t.Errorf("mode = %v, want regular 0660", m) + } + // Replacing an existing regular file works and keeps the new mode. + if err := Replace(target, []byte("two"), 0o640, Options{PreserveOwner: true}); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "two" { + t.Errorf("content after replace = %q", got) + } + if m := modeOf(t, target); m.Perm() != 0o640 { + t.Errorf("mode after replace = %v", m) + } + if l := leftovers(t, target); len(l) != 0 { + t.Errorf("temporary files left: %v", l) + } +} + +// Two writes never share a temporary name, and the name is not derivable +// from the target alone. +func TestReplace_TempNameIsUnpredictable(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + var names []string + old := chownFile + chownFile = func(f *os.File, uid, gid int) error { + names = append(names, filepath.Base(f.Name())) + return nil + } + t.Cleanup(func() { chownFile = old }) + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := Replace(target, []byte("x"), 0o600, Options{PreserveOwner: true}); err != nil { + t.Fatal(err) + } + } + if len(names) != 2 || names[0] == names[1] { + t.Fatalf("temp names = %v, want two distinct", names) + } + for _, n := range names { + if !strings.HasPrefix(n, "f.") || !strings.HasSuffix(n, ".tmp") || len(n) != len("f.")+16+len(".tmp") { + t.Errorf("temp name %q is not .<16 hex>.tmp", n) + } + } + // The sentinel a fixed-name writer would use is never created. + if _, err := os.Lstat(target + ".tmp"); !errors.Is(err, fs.ErrNotExist) { + t.Error("a predictable .tmp exists") + } +} + +func TestReplace_RefusesSymlinkAtTarget(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "cred") + if err := os.Symlink(victim, target); err != nil { + t.Fatal(err) + } + err := Replace(target, []byte("evil"), 0o660, Options{PreserveOwner: true}) + if !errors.Is(err, ErrUnsafeLocation) { + t.Fatalf("err = %v, want ErrUnsafeLocation", err) + } + if got, _ := os.ReadFile(victim); string(got) != "keep" { + t.Errorf("victim rewritten: %q", got) + } + if m := modeOf(t, victim); m.Perm() != 0o600 { + t.Errorf("victim mode changed: %v", m) + } + if m := modeOf(t, target); m&fs.ModeSymlink == 0 { + t.Error("symlink at target was replaced") + } + if l := leftovers(t, target); len(l) != 0 { + t.Errorf("temporary files left: %v", l) + } +} + +func TestReplace_RefusesUnsafeDirectory(t *testing.T) { + real := t.TempDir() + // A symlink where the directory should be. + linkDir := filepath.Join(t.TempDir(), "home") + if err := os.Symlink(real, linkDir); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(linkDir, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("symlinked dir: err = %v, want ErrUnsafeLocation", err) + } + if entries, _ := os.ReadDir(real); len(entries) != 0 { + t.Errorf("wrote through the directory symlink: %v", entries) + } + // A missing directory. + if err := Replace(filepath.Join(real, "nope", "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("missing dir: err = %v", err) + } + // A world-writable directory. + ww := filepath.Join(t.TempDir(), "ww") + if err := os.Mkdir(ww, 0o777); err != nil { + t.Fatal(err) + } + if err := os.Chmod(ww, 0o1777); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(ww, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("world-writable dir: err = %v", err) + } + // A file where the directory should be. + plain := filepath.Join(t.TempDir(), "plain") + if err := os.WriteFile(plain, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(plain, "f"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("file as dir: err = %v", err) + } + // A directory where the target should be. + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "d"), 0o700); err != nil { + t.Fatal(err) + } + if err := Replace(filepath.Join(dir, "d"), []byte("x"), 0o600, Options{}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("dir as target: err = %v", err) + } +} + +func TestReplace_ChownFailureIsBestEffortAndVerified(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + old := chownFile + chownFile = func(*os.File, int, int) error { return errors.New("EPERM") } + t.Cleanup(func() { chownFile = old }) + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err != nil { + t.Fatalf("a failed chown must not fail the write: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "new" { + t.Errorf("content = %q", got) + } + // A chown that "succeeds" but leaves the wrong owner is caught by fstat. + chownFile = func(*os.File, int, int) error { return nil } + victimOwner := filepath.Join(dir, "g") + if err := os.WriteFile(victimOwner, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + // Pretend the previous file belonged to someone else by faking prev. + prev := fakeInfo{FileInfo: must(os.Lstat(victimOwner)), uid: 12345, gid: 12345} + f, tmp, err := createTemp(victimOwner) + if err != nil { + t.Fatal(err) + } + err = fill(f, []byte("x"), 0o660, prev, Options{PreserveOwner: true}) + _ = f.Close() + _ = os.Remove(tmp) + if err == nil || !strings.Contains(err.Error(), "owned by") { + t.Errorf("owner mismatch not detected: %v", err) + } +} + +type fakeInfo struct { + os.FileInfo + uid, gid uint32 +} + +func (f fakeInfo) Sys() any { + st, _ := f.FileInfo.Sys().(*syscall.Stat_t) + cp := *st + cp.Uid, cp.Gid = f.uid, f.gid + return &cp +} + +func must(info os.FileInfo, err error) os.FileInfo { + if err != nil { + panic(err) + } + return info +} + +func TestCreateTemp_SkipsTakenNames(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + f, tmp, err := createTemp(target) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + if m := modeOf(t, tmp); m.Perm() != 0o600 { + t.Errorf("temp mode = %v, want 0600", m) + } + // A second create beside the first gets another name. + g, tmp2, err := createTemp(target) + if err != nil { + t.Fatal(err) + } + _ = g.Close() + if tmp == tmp2 { + t.Error("same temp name twice") + } +} + +func TestReadNoFollow(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "f") + if err := os.WriteFile(p, []byte("hello"), 0o600); err != nil { + t.Fatal(err) + } + got, err := ReadNoFollow(p, 64) + if err != nil || !bytes.Equal(got, []byte("hello")) { + t.Errorf("read = %q, %v", got, err) + } + if _, err := ReadNoFollow(p, 4); !errors.Is(err, ErrTooLarge) { + t.Errorf("over cap: err = %v", err) + } + link := filepath.Join(dir, "l") + if err := os.Symlink(p, link); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(link, 64); !errors.Is(err, ErrNotRegular) { + t.Errorf("symlink: err = %v, want ErrNotRegular", err) + } + if _, err := ReadNoFollow(dir, 64); !errors.Is(err, ErrNotRegular) { + t.Errorf("directory: err = %v, want ErrNotRegular", err) + } + if _, err := ReadNoFollow(filepath.Join(dir, "missing"), 64); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("missing: err = %v", err) + } + if os.Getuid() != 0 { + if err := os.Chmod(p, 0); err != nil { + t.Fatal(err) + } + if _, err := ReadNoFollow(p, 64); !errors.Is(err, fs.ErrPermission) { + t.Errorf("unreadable: err = %v, want ErrPermission", err) + } + } +} + +func TestChmod(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + prev, changed, err := Chmod(path, 0o660) + if err != nil || !changed || prev != 0o600 { + t.Fatalf("Chmod = (%o, %v, %v)", prev, changed, err) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o660 { + t.Fatalf("mode = %v", info.Mode()) + } + if _, changed, err := Chmod(path, 0o660); err != nil || changed { + t.Fatalf("already matching: changed=%v err=%v", changed, err) + } + if _, _, err := Chmod(filepath.Join(dir, "absent"), 0o660); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("absent: %v", err) + } + + victim := filepath.Join(dir, "victim") + if err := os.WriteFile(victim, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link") + if err := os.Symlink(victim, link); err != nil { + t.Fatal(err) + } + if _, _, err := Chmod(link, 0o666); !errors.Is(err, ErrNotRegular) { + t.Fatalf("symlink: %v, want ErrNotRegular", err) + } + if info, _ := os.Stat(victim); info.Mode().Perm() != 0o600 { + t.Fatalf("victim mode changed through the symlink: %v", info.Mode()) + } + if _, _, err := Chmod(dir, 0o700); !errors.Is(err, ErrNotRegular) { + t.Fatalf("directory: %v, want ErrNotRegular", err) + } +} + +func TestCheckLocation(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + if info, err := CheckLocation(target); err != nil || info != nil { + t.Fatalf("absent: (%v, %v)", info, err) + } + if err := os.WriteFile(target, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if info, err := CheckLocation(target); err != nil || info == nil || !info.Mode().IsRegular() { + t.Fatalf("regular: (%v, %v)", info, err) + } + if os.Geteuid() != 0 { + // A directory that cannot be searched: the target's Lstat fails with + // something other than "not exist", and that is refused too. + closed := filepath.Join(dir, "closed") + if err := os.Mkdir(closed, 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(closed, 0o700) }) + if _, err := CheckLocation(filepath.Join(closed, "f")); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("unsearchable dir: %v", err) + } + } +} + +func TestCreateTemp_RandomFailures(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "f") + old := randRead + t.Cleanup(func() { randRead = old }) + + randRead = func([]byte) (int, error) { return 0, errors.New("no entropy") } + if _, _, err := createTemp(target); err == nil || !strings.Contains(err.Error(), "no entropy") { + t.Errorf("entropy failure: %v", err) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers: %v", left) + } + + // Every name the generator can produce is already taken: refused after + // createAttempts, never written through the existing file. + randRead = func(b []byte) (int, error) { + for i := range b { + b[i] = 0xAB + } + return len(b), nil + } + taken := target + "." + strings.Repeat("ab", 8) + ".tmp" + if err := os.WriteFile(taken, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := Replace(target, []byte("new"), 0o600, Options{}); err == nil || !strings.Contains(err.Error(), "no free temporary name") { + t.Errorf("all names taken: %v", err) + } + if got, _ := os.ReadFile(taken); string(got) != "keep" { + t.Errorf("the taken name was written through: %q", got) + } + if _, err := os.Lstat(target); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("target appeared: %v", err) + } +} + +func TestReplace_FailuresLeaveNoTemp(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root is not stopped by directory modes") + } + old := chownFile + t.Cleanup(func() { chownFile = old }) + + // Cannot create the temporary: the directory is read-only. + ro := t.TempDir() + if err := os.Chmod(ro, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(ro, 0o700) }) + if err := Replace(filepath.Join(ro, "f"), []byte("x"), 0o600, Options{}); err == nil || errors.Is(err, ErrUnsafeLocation) { + t.Errorf("read-only dir: err = %v, want a create error", err) + } + + // The written file fails verification (mode changed under us). + dir := t.TempDir() + target := filepath.Join(dir, "f") + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + chownFile = func(f *os.File, _, _ int) error { return f.Chmod(0o644) } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err == nil || !strings.Contains(err.Error(), "written file is mode") { + t.Errorf("mode drift: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite failed verification: %q", got) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after failed fill: %v", left) + } + + // The flush fails: no rename, no leftover. + oldSync := syncFile + syncFile = func(*os.File) error { return errors.New("EIO") } + err := Replace(target, []byte("new"), 0o660, Options{}) + syncFile = oldSync + if err == nil || !strings.Contains(err.Error(), "EIO") { + t.Errorf("sync failure: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite failed sync: %q", got) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after failed sync: %v", left) + } + + // The temporary gained a second name while being written. + chownFile = func(f *os.File, _, _ int) error { + hl := f.Name() + ".hl" + t.Cleanup(func() { _ = os.Remove(hl) }) + return os.Link(f.Name(), hl) + } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); err == nil || !strings.Contains(err.Error(), "links, want 1") { + t.Errorf("extra link: %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target replaced despite extra link: %q", got) + } + + // The location changes while the temporary is being written. + chownFile = func(*os.File, int, int) error { + _ = os.Remove(target) + return os.Symlink("/nonexistent", target) + } + if err := Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}); !errors.Is(err, ErrUnsafeLocation) { + t.Errorf("location changed mid-write: %v", err) + } + if info, err := os.Lstat(target); err != nil || info.Mode()&fs.ModeSymlink == 0 { + t.Errorf("the planted link was replaced: %v %v", info, err) + } + if left := leftovers(t, target); len(left) != 0 { + t.Errorf("leftovers after refused rename: %v", left) + } + _ = os.Remove(target) + if err := os.WriteFile(target, []byte("seed"), 0o600); err != nil { + t.Fatal(err) + } + + // The rename itself fails: the directory went read-only after the + // temporary was created. + chownFile = func(*os.File, int, int) error { return os.Chmod(dir, 0o500) } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + err = Replace(target, []byte("new"), 0o660, Options{PreserveOwner: true}) + _ = os.Chmod(dir, 0o700) + if err == nil || errors.Is(err, ErrUnsafeLocation) { + t.Errorf("rename failure: err = %v", err) + } + if got, _ := os.ReadFile(target); string(got) != "seed" { + t.Errorf("target changed on failed rename: %q", got) + } + // The temporary could not be removed from the read-only directory at + // the time; it is the one leftover this path can produce, and it is + // removable now. + for _, l := range leftovers(t, target) { + _ = os.Remove(l) + } +} diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index b2f04031b4..66e68db376 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -150,7 +150,9 @@ hive_run_uid_isolation_migration() { else echo "[entrypoint] UID isolation: repairing ${_agent_name} (uid ${_agent_uid}) in background" hive_reown_agent_paths_if_needed "$_agent_name" "$_agent_uid" - chmod g+rwX "$_agent_dir" 2>/dev/null || true + if [ -d "$_agent_dir" ] && ! [ -L "$_agent_dir" ]; then + chmod g+rwX "$_agent_dir" 2>/dev/null || true + fi hive_publish_uid_isolation_marker "$_agent_marker" "$_agent_expected" || true echo "[entrypoint] UID isolation: ${_agent_name} pass complete" fi @@ -685,6 +687,9 @@ hive_sweep_root_phase_paths() { _sweep_failed="" for _p in $HIVE_DATA_ROOT_PHASE_PATHS; do [ -e "$_p" ] || continue + # Never through a link: chown without -h acts on what the link names, and + # /data/home is writable by every agent uid. + [ -L "$_p" ] && continue # Already owned by the runtime user — the steady state on almost every # boot. Skip the syscall entirely so the common path costs one stat. [ "$(stat -c '%u' "$_p" 2>/dev/null || echo)" = "1001" ] && continue @@ -1421,6 +1426,9 @@ if [ "$(id -u)" = "0" ]; then fi for _seeded_dir in /data/agents/*/; do [ -d "$_seeded_dir" ] || continue + # The trailing-slash glob also matches a link to a directory; a + # recursive chown/chmod must never start from one. + [ -L "${_seeded_dir%/}" ] && continue if [ "$(stat -c '%u' "$_seeded_dir" 2>/dev/null || echo 0)" = "0" ]; then chown -R dev:node "$_seeded_dir" 2>/dev/null || true chmod -R g+rwX "$_seeded_dir" 2>/dev/null || true @@ -1489,6 +1497,27 @@ if [ "$(id -u)" = "0" ]; then chown dev:node /data/home/.copilot 2>/dev/null || true chmod 2775 /data/home/.claude /data/home/.claude/session-env 2>/dev/null || true chown dev:node /data/home/.claude /data/home/.claude/session-env 2>/dev/null || true + # hive_shared_dir_sticky DIR... — add the sticky bit to the shared CLI + # directories whose files the root-run perm guard and the in-process Go + # writers rename over: in a group-writable directory only the sticky bit + # keeps a dev-owned entry from being removed or replaced by another uid. + # Under the per-agent home layout every CLI rewrites its own copy inside + # /data/home/agents/, so nothing legitimate renames over a dev-owned + # file here. The legacy whole-directory layout (HIVE_SHARED_AGENT_HOME=1) + # does — each agent uid refreshes the shared file in place — so the bit is + # left off there. Group bits are untouched; the existing 2770/2775 stays. + # Only directories that are real directories get it: a chmod through a + # symlink would land on whatever the link names. + hive_shared_dir_sticky() { + [ "${HIVE_SHARED_AGENT_HOME:-0}" = "1" ] && return 0 + for _d; do + if [ -d "$_d" ] && ! [ -L "$_d" ]; then + chmod +t "$_d" 2>/dev/null || true + fi + done + return 0 + } + hive_shared_dir_sticky /data/home /data/home/.claude /data/home/.copilot # Codex (newer method) writes a local sqlite state db under $HOME/.codex — # pre-create it group-writable + setgid so every agent UID (node group) can # init it, matching .claude/.copilot. Without this, switching an agent to the @@ -1536,6 +1565,9 @@ if [ "$(id -u)" = "0" ]; then # process touches .bob first. 2770: no world access, these hold auth settings. chmod 2770 /data/home/.bob /data/home/.bob/settings 2>/dev/null || true chown dev:node /data/home/.bob /data/home/.bob/settings 2>/dev/null || true + # Not antigravity-cli/ or its cache/: those are bridged into each agent home + # as directories, so agy renames its token there under the agent's own uid. + hive_shared_dir_sticky /data/home/.gemini /data/home/.bob ln -sfn /data/config/github-copilot /home/dev/.config/github-copilot ln -sfn /data/config/github-copilot /data/home/.config/github-copilot ln -sfn /data/home/.copilot /home/dev/.copilot @@ -1625,18 +1657,52 @@ if [ "$(id -u)" = "0" ]; then # this is an OAuth token. The chown restores dev ownership as well, because # that is the invariant the in-process Go permissions watcher depends on — it # runs as dev after the privilege drop and can only chmod what dev owns. + # + # Both single-file repairs go through hive_fix_shared_file, which acts only + # on a regular file inside a regular directory: chmod and chown (without -h) + # act on whatever a path resolves to, and these directories are writable by + # every agent uid. The invariant is that a root-run repair never changes the + # mode or owner of anything an agent uid could have pointed the path at. hive_fix_shared_credential() { - [ -f "$1" ] || return 0 - chmod g+r "$1" 2>/dev/null || true - chown dev:node "$1" 2>/dev/null || true + hive_fix_shared_file "$1" g+r return 0 } # hive_fix_copilot_config — copilot rewrites config.json 0600 on refresh. hive_fix_copilot_config() { - [ -f /data/home/.copilot/config.json ] || return 0 - chmod 660 /data/home/.copilot/config.json 2>/dev/null || true - chown dev:node /data/home/.copilot/config.json 2>/dev/null || true + hive_fix_shared_file /data/home/.copilot/config.json 660 + return 0 + } + + # hive_fix_shared_file PATH MODE — chmod MODE and chown dev:node the regular + # file at PATH. Nothing is done when PATH is absent. When PATH or its + # directory is anything other than a regular file in a regular directory, + # nothing is done either, and a WARN is logged once per path per guard + # process (HIVE_PERM_GUARD_REFUSED is that process's memory of it). + # + # The `[ -L ]` tests are the decision; the find runs the chmod and chown + # from the same enumeration (`-type f` never matches a link) and `chown -h` + # never follows, so the time between the check and each action is as short + # as the shell can make it. The sticky bit set on the shared dot-dirs at + # boot is what keeps a dev-owned entry from being swapped out from under it. + hive_fix_shared_file() { + _p="$1"; _m="$2" + _dir="${_p%/*}"; _base="${_p##*/}" + if [ -d "$_dir" ] && ! [ -L "$_dir" ] && [ -f "$_p" ] && ! [ -L "$_p" ]; then + find "$_dir" -maxdepth 1 -name "$_base" -type f ! -type l -exec chmod "$_m" {} + -exec chown -h dev:node {} + 2>/dev/null || true + return 0 + fi + # Absent (and not a dangling link): nothing to repair, nothing to say. + if ! [ -e "$_p" ] && ! [ -L "$_p" ]; then + return 0 + fi + case " ${HIVE_PERM_GUARD_REFUSED:-} " in + *" $_p "*) ;; + *) + HIVE_PERM_GUARD_REFUSED="${HIVE_PERM_GUARD_REFUSED:-} $_p" + echo "[entrypoint] WARN: perm guard: $_p is not a regular file in a regular directory; left untouched" >&2 + ;; + esac return 0 } @@ -1747,7 +1813,9 @@ if [ "$(id -u)" = "0" ]; then # sweep still happens, on the 5-minute cycle. hive_fix_claude_instant() { hive_fix_shared_credential /data/home/.claude/.credentials.json - chmod g+rwx /data/home/.claude 2>/dev/null || true + if [ -d /data/home/.claude ] && ! [ -L /data/home/.claude ]; then + chmod g+rwx /data/home/.claude 2>/dev/null || true + fi return 0 } @@ -1765,7 +1833,11 @@ if [ "$(id -u)" = "0" ]; then hive_fix_gemini_instant() { hive_fix_shared_credential /data/home/.gemini/antigravity-cli/antigravity-oauth-token hive_fix_shared_credential /data/home/.gemini/antigravity-cli/cache/onboarding.json - chmod g+rwx /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache 2>/dev/null || true + for _d in /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache; do + if [ -d "$_d" ] && ! [ -L "$_d" ]; then + chmod g+rwx "$_d" 2>/dev/null || true + fi + done return 0 } @@ -1898,6 +1970,11 @@ if [ "$(id -u)" = "0" ]; then # launch. Falling back to the proxy CA alone if the combined bundle isn't there # yet keeps proxied HTTPS working; the combined bundle is strictly safer because # it also trusts real public roots for any non-proxied endpoint (advisory mode). + # Root writes this file into a directory every agent uid can write to, so + # it is only ever written when the path is absent or already a regular file. + if [ -L /data/home/.bashrc ]; then + echo "[entrypoint] WARN: /data/home/.bashrc is not a regular file; left untouched" + else cat > /data/home/.bashrc <<'BASHRC' 2>/dev/null || true # Hive agent shell environment if [ -f /data/proxy-ca-bundle.pem ]; then @@ -1914,15 +1991,16 @@ BASHRC # Written by root via `cat >` above, so it is created root:root. Hand it to # dev at the point of creation (#5369): the recursive /data chown is guarded # off on every normal boot and cannot be relied on to fix it later. - chown dev:node /data/home/.bashrc 2>/dev/null || true + chown -h dev:node /data/home/.bashrc 2>/dev/null || true + fi # Login shells (tmux default-command) read ~/.profile, not ~/.bashrc — chain # them so both shell flavors get the same environment. - if [ ! -f /data/home/.profile ]; then + if [ ! -f /data/home/.profile ] && ! [ -L /data/home/.profile ]; then # shellcheck disable=SC2016 printf '[ -f "$HOME/.bashrc" ] && . "$HOME/.bashrc"\n' > /data/home/.profile 2>/dev/null || true chmod 644 /data/home/.profile 2>/dev/null || true # Same as .bashrc above — created by root, so hand it over here (#5369). - chown dev:node /data/home/.profile 2>/dev/null || true + chown -h dev:node /data/home/.profile 2>/dev/null || true fi # ── #5369: targeted post-phase ownership sweep ──────────────────────── @@ -2020,8 +2098,10 @@ BASHRC fi # Ensure agent dirs are group-writable so the dev user (same node group) # can clean up stale workspaces at runtime without root privileges. - chmod g+rwX "/data/agents/${agent_name}" 2>/dev/null || true - if [ -d "/data/beads/${agent_name}" ]; then + if [ -d "/data/agents/${agent_name}" ] && ! [ -L "/data/agents/${agent_name}" ]; then + chmod g+rwX "/data/agents/${agent_name}" 2>/dev/null || true + fi + if [ -d "/data/beads/${agent_name}" ] && ! [ -L "/data/beads/${agent_name}" ]; then BEAD_DIR_OWNER=$(stat -c '%u' "/data/beads/${agent_name}" 2>/dev/null || echo "0") if [ "$BEAD_DIR_OWNER" != "$AGENT_UID" ]; then chown -R "hive-${agent_name}:node" "/data/beads/${agent_name}" 2>/dev/null || true diff --git a/src/deploy/test_entrypoint_perm_guard.sh b/src/deploy/test_entrypoint_perm_guard.sh index 857dfce8dd..9b34f8ca07 100755 --- a/src/deploy/test_entrypoint_perm_guard.sh +++ b/src/deploy/test_entrypoint_perm_guard.sh @@ -594,6 +594,90 @@ for repair in hive_fix_tree_recent hive_fix_tree; do done rm -f "$SETTINGS" "$SIBLING" +# ── 14. The single-file repairs act only on a regular file in a regular dir ── +# chmod and chown (without -h) act on whatever a path resolves to. The shared +# dot-dirs are writable by every agent uid, so the file-level repairs must do +# nothing at all when the path, or the directory it is in, is not what the +# guard expects — and say so once, not once per 5-second poll. +mode_of() { stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1"; } +PLANT_HOME="$WORK/plant-home" +PLANT_OUT="$WORK/plant-outside" +mkdir -p "$PLANT_HOME/.claude" "$PLANT_HOME/.copilot" +sed "s#/data/home#${PLANT_HOME}#g" "$GUARDS" > "$WORK/guards.plant.sh" +printf 'outside\n' > "$PLANT_OUT" +chmod 0600 "$PLANT_OUT" + +# A link where the credential should be. +ln -s "$PLANT_OUT" "$PLANT_HOME/.claude/.credentials.json" +ln -s "$PLANT_OUT" "$PLANT_HOME/.copilot/config.json" +PLANT_LOG="$WORK/plant.log" +sh -c 'set -e; . "$1"; hive_fix_credentials_fast; hive_fix_credentials_fast; hive_fix_claude_instant' \ + sh "$WORK/guards.plant.sh" >/dev/null 2>"$PLANT_LOG" +if [ "$(mode_of "$PLANT_OUT")" = "600" ]; then + ok "a link planted at the credential path is not acted on (target stays 0600)" +else + bad "a link planted at the credential path is not acted on" \ + "target went to mode $(mode_of "$PLANT_OUT")" + chmod 0600 "$PLANT_OUT" +fi +if [ -L "$PLANT_HOME/.claude/.credentials.json" ] && [ -L "$PLANT_HOME/.copilot/config.json" ]; then + ok "the planted links themselves are left in place" +else + bad "the planted links themselves are left in place" +fi +warn_count="$(grep -c 'WARN: perm guard' "$PLANT_LOG" || true)" +if [ "$warn_count" = "2" ]; then + ok "a refused path is reported once per guard process (2 paths, 3 passes, 2 WARNs)" +else + bad "a refused path is reported once per guard process" \ + "expected 2 WARN lines, got $warn_count: $(cat "$PLANT_LOG")" +fi +rm -f "$PLANT_HOME/.claude/.credentials.json" "$PLANT_HOME/.copilot/config.json" + +# A link where the directory should be, with a real file behind it. +rmdir "$PLANT_HOME/.copilot" +mkdir -p "$WORK/plant-elsewhere" +printf '{}\n' > "$WORK/plant-elsewhere/config.json" +chmod 0600 "$WORK/plant-elsewhere/config.json" +ln -s "$WORK/plant-elsewhere" "$PLANT_HOME/.copilot" +sh -c 'set -e; . "$1"; hive_fix_copilot_config' sh "$WORK/guards.plant.sh" >/dev/null 2>&1 +if [ "$(mode_of "$WORK/plant-elsewhere/config.json")" = "600" ]; then + ok "a link planted at the directory is not acted through (file behind it stays 0600)" +else + bad "a link planted at the directory is not acted through" \ + "file behind the link went to mode $(mode_of "$WORK/plant-elsewhere/config.json")" +fi +rm -f "$PLANT_HOME/.copilot" +mkdir -p "$PLANT_HOME/.copilot" + +# A regular file in a regular directory is still repaired, and an absent one +# is silent. +printf '{}\n' > "$PLANT_HOME/.copilot/config.json" +chmod 0600 "$PLANT_HOME/.copilot/config.json" +sh -c 'set -e; . "$1"; hive_fix_copilot_config' sh "$WORK/guards.plant.sh" >/dev/null 2>"$PLANT_LOG" +if group_rw "$(mode_of "$PLANT_HOME/.copilot/config.json")"; then + ok "a regular config.json in a regular .copilot is still reopened (mode $(mode_of "$PLANT_HOME/.copilot/config.json"))" +else + bad "a regular config.json in a regular .copilot is still reopened" \ + "mode $(mode_of "$PLANT_HOME/.copilot/config.json")" +fi +rm -f "$PLANT_HOME/.copilot/config.json" +sh -c 'set -e; . "$1"; hive_fix_credentials_fast' sh "$WORK/guards.plant.sh" >/dev/null 2>>"$PLANT_LOG" +if [ -s "$PLANT_LOG" ]; then + bad "an absent shared file is silent" "$(cat "$PLANT_LOG")" +else + ok "an absent shared file is silent" +fi + +# The guard that survives a planted link also survives the next loop: the +# refusal returns 0 under set -e. +if sh -c 'set -e; . "$1"; ln -s "$2" "$3/.claude/.credentials.json"; hive_fix_shared_credential "$3/.claude/.credentials.json"; echo alive' \ + sh "$WORK/guards.plant.sh" "$PLANT_OUT" "$PLANT_HOME" 2>/dev/null | grep -q alive; then + ok "a refusal does not end the guard under set -e" +else + bad "a refusal does not end the guard under set -e" +fi + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1 diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 02f8baaaf1..e270605f8b 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -633,6 +633,20 @@ Each per-UID agent runs with its own durable `HOME` under `/data/home/agents//memory/MEMORY.md`, per-agent by path) was writable by every other agent and persisted on the PVC forever. Provisioning now retires those whole-directory bridges on the next launch, keeps only the login/settings entries above, and never deletes or moves anything under `/data/home`: a pre-existing shared memory store is left in place and logged once (`shared CLI home still holds a persistent memory store …`) so you can review or drain it ([#17033](https://github.com/hivecommons/hive/issues/17033)) before removing it by hand. The dashboard's token/cost scanners (`data.claude_sessions_dir`, `data.bob_sessions_dir`, `data.copilot_sessions_dir`) walk the shared path **and** every `/data/home/agents/*/…` sibling, attributing each per-agent tree to its owner, so cost accounting does not regress. `HIVE_SHARED_AGENT_HOME=1` restores the legacy single shared `HOME` for every agent (and with it the shared memory). +#### How the Claude login stays shared when the CLI replaces the link + +Claude Code saves a refreshed OAuth token (about every 8 hours) by writing a temp file in `~/.claude` and renaming it over `.credentials.json`. A rename replaces a symlink rather than writing through it, so after the first refresh the agent is running on a private `0600` copy of the token, the refresh has rotated the grant the shared file still holds, and every other agent fails its own refresh. On per-UID hives this logged the whole fleet out about every 8 hours ([#17039](https://github.com/hivecommons/hive/pull/17039) follow-up). + +Hive now treats the login as the one piece of `.claude` state that *is* shared, and publishes it back: at every agent launch, every `HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL` (default `5s`) from the credential watchdog, and right after a dashboard sign-in, it checks each claude agent's `~/.claude/.credentials.json`. When a regular file has replaced the link, hive reads it — directly, or as the agent's own uid through the same `su-exec` seam that adopts `.claude.json` sessions, so an agent-owned `0600` file is no obstacle — and: + +- keeps whichever credential is newer **and** usable in the shared `/data/home/.claude/.credentials.json`: the agent's copy wins only when it is complete (access token, refresh token, recorded `expiresAt`), usable, and expires later than the shared one, or the shared one is unusable. A failed refresh (empty tokens, `expiresAt` 0) never wins; +- moves the private file aside to `.credentials.json.stale-` in the agent's `.claude` (owner and mode unchanged) and, when the agent's copy was promoted, preserves the old shared file beside the shared credential the same way. Nothing is deleted; remove the `.stale-*` files by hand once you no longer want them; +- restores the symlink and records a `claude_credential_rebridged` audit entry (`kept=agent|shared`, never token material). + +When the file cannot be read either way (a deployment without `su-exec`), nothing is moved or relinked, the agent keeps the token it already had, and one `ERROR` per agent and file version names the path. The one exception is a dashboard sign-in: a fresh shared login is newer than anything an unreadable private copy could hold, so that trigger moves the unreadable file aside unread and relinks the agent. Memory, transcripts and todos stay per agent throughout; only the credential file is published. + +Every hive-side write to a shared login or CLI config (`.claude/.credentials.json`, `.copilot/config.json`, the durable Copilot token) goes through `crux/sharedfile`: a randomly named temporary beside the target, mode and owner set on the open descriptor, verified, then renamed into place, and only when the target is absent or a regular file inside a regular directory the server owns. Anything else at those paths is refused and logged once. The entrypoint's root permission guard applies the same rule to the files it repairs, and the shared dot-dirs carry the sticky bit so a dev-owned entry cannot be replaced by another uid. + ### CLI auto-memory (`agents.cli_memory`) Subscription CLIs keep their own long-lived "memory" next to the transcripts: Claude Code's auto-memory (`~/.claude/projects/-data-agents-/memory/MEMORY.md`), Codex memories (`$CODEX_HOME/memories/`), Copilot CLI's `memory` setting, and Gemini's `~/.gemini/GEMINI.md`. Hive agents already get their durable context from beads, the knowledge store, and the kick prompt, so a hidden per-CLI notebook mostly accumulates stale one-off inferences ([#17032](https://github.com/hivecommons/hive/issues/17032)). The `agents.cli_memory` block controls it fleet-wide, and each agent may override it with its own `cli_memory:` block: diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index c97f5fc7dd..fd410f8a76 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -311,6 +311,7 @@ and [network/credential controls](security-model.md#5-network-and-credentials). | `CODEX_API_KEY` | No | none | API key read by `pkg/agent/authprobe.go` for the Codex CLI backend; either this or `OPENAI_API_KEY` makes Codex API-key mode count as configured. | | `HIVE_AGENT_TOKEN_REFRESH_INTERVAL` | No | `40m` | Go duration overriding the per-agent token refresh interval. Invalid or non-positive values fall back to the default. | | `HIVE_CREDENTIAL_WATCHDOG_INTERVAL` | No | `5m` | Go duration overriding how often the credential watchdog verifies each in-use backend credential file. `0` does NOT disable the watchdog — disabling is intentionally not offered. | +| `HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL` | No | `5s` | Go duration overriding how often the credential watchdog checks each per-UID claude agent's `~/.claude/.credentials.json` link for a token refresh that replaced it with a private file, and publishes that refresh to the shared login (see [Per-agent homes](agent-configuration.md)). One `lstat` per claude agent per tick. | | `HIVE_PROVIDER_ERROR_BACKOFF_BASE` | No | `2m` | Go duration for the first inference-provider error backoff before another kick is allowed. Invalid or non-positive values fall back to the default. | | `HIVE_PROVIDER_ERROR_BACKOFF_MAX` | No | `30m` | Go duration cap for exponential inference-provider error backoff. Invalid or non-positive values fall back to the default. | | `HIVE_START_FAILURE_BLOCK_THRESHOLD` | No | `3` | Consecutive identical start failures (positive integer) before an agent is blocked from automatic relaunch (`pkg/agent/start_failure.go`). Backoff is applied from the first failure regardless; invalid values fall back to the default. | diff --git a/src/pkg/dashboard/audit.go b/src/pkg/dashboard/audit.go index af149dfa61..3e4a1e1a52 100644 --- a/src/pkg/dashboard/audit.go +++ b/src/pkg/dashboard/audit.go @@ -56,6 +56,7 @@ var auditSensitiveActions = map[string]bool{ "claude_auth_complete": true, "claude_auth_logout": true, "claude_auth_start": true, + "claude_credential_rebridged": true, "claude_token_missing": true, "config_agent_cadences": true, "config_agent_channels": true, From ef6c5406f3d331afe1eea38731e65e4071ecb1c6 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 00:22:14 +0000 Subject: [PATCH 031/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.224.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.224.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 15 +++++++++++++++ changelog.d/added-benchmarks-export-cost.md | 1 - changelog.d/added-marketplace-curated-techdocs.md | 1 - changelog.d/fixed-marketplace-stream-tarball.md | 1 - .../security-17730-upstreamproxy-url-redact.md | 1 - 5 files changed, 15 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/added-benchmarks-export-cost.md delete mode 100644 changelog.d/added-marketplace-curated-techdocs.md delete mode 100644 changelog.d/fixed-marketplace-stream-tarball.md delete mode 100644 changelog.d/security-17730-upstreamproxy-url-redact.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 541d4ef742..9e157c27e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,21 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.224.0) + +### Added + +- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. +- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). + +### Fixed + +- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. + +### Security + +- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). + ## 2026-11-11 (v5.223.0) ### Added diff --git a/changelog.d/added-benchmarks-export-cost.md b/changelog.d/added-benchmarks-export-cost.md deleted file mode 100644 index 62f95173f7..0000000000 --- a/changelog.d/added-benchmarks-export-cost.md +++ /dev/null @@ -1 +0,0 @@ -- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. diff --git a/changelog.d/added-marketplace-curated-techdocs.md b/changelog.d/added-marketplace-curated-techdocs.md deleted file mode 100644 index 4a22b467ee..0000000000 --- a/changelog.d/added-marketplace-curated-techdocs.md +++ /dev/null @@ -1 +0,0 @@ -- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). diff --git a/changelog.d/fixed-marketplace-stream-tarball.md b/changelog.d/fixed-marketplace-stream-tarball.md deleted file mode 100644 index fd6a16ff29..0000000000 --- a/changelog.d/fixed-marketplace-stream-tarball.md +++ /dev/null @@ -1 +0,0 @@ -- Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. diff --git a/changelog.d/security-17730-upstreamproxy-url-redact.md b/changelog.d/security-17730-upstreamproxy-url-redact.md deleted file mode 100644 index 432e2a58ef..0000000000 --- a/changelog.d/security-17730-upstreamproxy-url-redact.md +++ /dev/null @@ -1 +0,0 @@ -- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). From edd341e27b9cede8bedc3449aeb641c789fb4822 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 17:23:38 -0700 Subject: [PATCH 032/166] =?UTF-8?q?=E2=9C=A8=20hub:=20per-cluster=20host?= =?UTF-8?q?=5Faliases=20for=20hosted=20hives=20(#17805)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: per-cluster host_aliases for hosted hives Adds ClusterConfig.host_aliases ([{ip, hostnames[]}]) to /data/saas/clusters.json, validated at load (IP parses, hostnames non-empty, lowercased; invalid blocks are dropped with a warning, never the cluster). Rendered as Pod hostAliases in the hosted-hive Deployment template next to the upstream_proxy env injection, and a throttled, rate-limited reconcile sweep (3 patches / 15 min) converges existing hives while preserving unrelated hand-applied aliases. Needed for GHE hosts reachable only through a TLS reverse proxy on a fixed IP (no CONNECT proxy), where upstream_proxy cannot apply. Fixes #17801 Signed-off-by: Andrew Anderson * 📖 api-reference: resync hub-server.go citations after host_aliases field Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-hub-cluster-host-aliases.md | 1 + src/docs/api-reference/hub-server.md | 58 ++-- src/docs/hub-deployment.md | 2 + src/docs/network-requirements.md | 30 +++ src/pkg/hub/cluster_host_aliases.go | 247 ++++++++++++++++++ src/pkg/hub/cluster_host_aliases_test.go | 150 +++++++++++ src/pkg/hub/clusters_registry.go | 10 + src/pkg/hub/saas_provision.go | 20 ++ src/pkg/hub/saas_sha_poller.go | 6 + src/pkg/hub/server.go | 3 + 10 files changed, 498 insertions(+), 29 deletions(-) create mode 100644 changelog.d/added-hub-cluster-host-aliases.md create mode 100644 src/pkg/hub/cluster_host_aliases.go create mode 100644 src/pkg/hub/cluster_host_aliases_test.go diff --git a/changelog.d/added-hub-cluster-host-aliases.md b/changelog.d/added-hub-cluster-host-aliases.md new file mode 100644 index 0000000000..ebc6e54a5c --- /dev/null +++ b/changelog.d/added-hub-cluster-host-aliases.md @@ -0,0 +1 @@ +- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 0c4f7404d3..c8891dd864 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1789` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1790` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1792` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1793` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1721` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1722` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1723` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1724` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1725` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1736` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1737` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1738` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1739` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1724` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1725` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1726` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1727` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1728` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1739` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1740` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1741` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1742` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1747` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1757` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1758` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1759` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1760` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1761` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1762` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1763` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1770` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1771` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1778` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1779` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1780` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1802` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1804` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1750` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1760` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1761` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1762` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1763` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1764` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1765` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1766` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1773` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1783` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1800` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1805` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1807` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index f617e25ad2..fd842d5b06 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -101,6 +101,8 @@ For push-reported clusters where the hub cannot run `kubectl`, follow the manife ### Push-reported cluster entries +`/data/saas/clusters.json` also accepts per-cluster network overrides for hives it provisions: `upstream_proxy` (`{url, hosts[], no_proxy[]}`, a CONNECT forward proxy, injected as `HIVE_UPSTREAM_PROXY_*` env) and `host_aliases` (`[{ip, hostnames[]}]`, rendered as Pod `hostAliases` for a GHE reachable only via a TLS reverse proxy on a fixed IP; existing hives are converged by a rate-limited reconcile sweep). See [network-requirements.md](network-requirements.md). + `/data/saas/clusters.json` keeps the historical compatibility key `pull_only`. Set `pull_only: true` when the hub cannot reach that cluster's Kubernetes API and the spokes must dial out to the hub instead. Operator-facing surfaces call these clusters **push-reported** because their node stats, dashboard token proofs, and liveness arrive in spoke heartbeats pushed from the cluster. A push-reported entry still needs its stable identity (for example `id`, `name`, `domain`, app/forge fields), but it does not need a usable `kubeconfig_path` for hub-side reads. If node stats are absent, upgrade the spokes to a build that sends heartbeat cluster health and then check the error reason shown in the Clusters panel. diff --git a/src/docs/network-requirements.md b/src/docs/network-requirements.md index 198496a7ed..ddb1af9e3e 100644 --- a/src/docs/network-requirements.md +++ b/src/docs/network-requirements.md @@ -97,6 +97,36 @@ slots on that cluster are provisioned with the env vars above: Existing hive Deployments are not reconciled; patch them once with `kubectl -n set env deploy/hive -c hive HIVE_UPSTREAM_PROXY_URL=... HIVE_UPSTREAM_PROXY_HOSTS=...`. +### GitHub Enterprise behind a TLS reverse proxy (hostAliases) + +Some networks expose an enterprise GitHub not through a CONNECT forward proxy +but through a **TLS reverse proxy** pinned to a fixed IP: the proxy serves a +certificate valid for the GHE hostnames on :443 and forwards to GHE, while +`CONNECT` is rejected (e.g. openresty answering `405 Not Allowed`) and direct +egress to GHE times out. `upstream_proxy` cannot express that; the fix is a +Pod `hostAliases` entry that resolves the GHE hostnames to the proxy IP so TLS +SNI/verification and every path (API, git smart-HTTP, MITM) keep working. + +Hosted hubs declare it per cluster in `/data/saas/clusters.json`: + +```json +{ + "id": "corp-cluster", + "host_aliases": [ + { "ip": "129.34.104.21", "hostnames": ["github.ibm.com", "api.github.ibm.com"] } + ] +} +``` + +Entries are validated at load (the IP must parse, every entry needs at least +one hostname; hostnames are lowercased). An invalid `host_aliases` block is +logged and ignored — it never drops the cluster. New pool slots on the cluster +are provisioned with the matching `spec.template.spec.hostAliases`, and a +throttled reconcile sweep (at most 3 patches per 15 minutes, since each one +rolls that hive's pod) converges existing hives. Unrelated hand-applied +aliases for other IPs are preserved. Non-hub (standalone) deployments add the +same `hostAliases` block to their own pod spec. + ## Firewall recommendations - Public edge: expose only the dashboard/hub front door (3001 for Docker/hub, or your HTTPS Ingress/Route for Kubernetes). Keep 3002 and 7681 cluster/private unless you have an explicit operator access path. diff --git a/src/pkg/hub/cluster_host_aliases.go b/src/pkg/hub/cluster_host_aliases.go new file mode 100644 index 0000000000..d3940710a3 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases.go @@ -0,0 +1,247 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sort" + "strings" + "time" +) + +const ( + // hostAliasesReconcileInterval throttles the sweep. Like NET_ADMIN and the + // per-hive env lane this is static remediation: a hive either carries the + // cluster's hostAliases or it does not, and a converged hive stays + // converged until clusters.json changes. + hostAliasesReconcileInterval = 15 * time.Minute + + // hostAliasesKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + hostAliasesKubectlTimeout = 15 * time.Second + + // hostAliasesMaxPatchesPerCycle caps PATCHES per sweep. A hostAliases change + // is a podspec change and rolls the pod; the 2026-11-11 vllm-d rollout + // showed 28 simultaneous rollouts saturating CRI-O on one worker (5.8 GB + // image) and liveness-killing neighbours, so convergence is deliberately + // slow: 3 per 15 min. + hostAliasesMaxPatchesPerCycle = 3 + + // podHostAliasesPath is the JSON-Patch path to the pod-level hostAliases + // list in the hive Deployment. + podHostAliasesPath = "/spec/template/spec/hostAliases" +) + +// ClusterHostAlias is one Pod hostAliases entry the hosted-hive provisioner +// renders into every hive Deployment on the cluster (and the reconcile sweep +// converges onto existing ones). It exists for GitHub Enterprise hosts that +// are reachable only through a TLS REVERSE proxy pinned to a fixed IP — not a +// CONNECT forward proxy, so upstream_proxy cannot express it (#17801). +type ClusterHostAlias struct { + IP string `json:"ip" yaml:"ip"` + Hostnames []string `json:"hostnames" yaml:"hostnames"` +} + +// normalizeClusterHostAliases validates and canonicalises a cluster's +// host_aliases: the IP must parse, every entry needs at least one hostname, +// hostnames are trimmed and lowercased, blanks dropped. Returns nil for an +// empty input so callers can test `len(out) == 0`. +func normalizeClusterHostAliases(in []ClusterHostAlias) ([]ClusterHostAlias, error) { + if len(in) == 0 { + return nil, nil + } + out := make([]ClusterHostAlias, 0, len(in)) + for i, a := range in { + ip := strings.TrimSpace(a.IP) + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("host_aliases[%d]: ip %q does not parse", i, a.IP) + } + names := make([]string, 0, len(a.Hostnames)) + for _, h := range a.Hostnames { + h = strings.ToLower(strings.TrimSpace(h)) + if h == "" { + continue + } + if strings.ContainsAny(h, " \t/:") { + return nil, fmt.Errorf("host_aliases[%d]: hostname %q is not a bare host", i, h) + } + names = append(names, h) + } + if len(names) == 0 { + return nil, fmt.Errorf("host_aliases[%d]: ip %s has no hostnames", i, ip) + } + out = append(out, ClusterHostAlias{IP: ip, Hostnames: names}) + } + return out, nil +} + +// hostAliasesPatchJSON builds the JSON-Patch that sets the pod-level +// hostAliases to exactly want. "add" on an existing path replaces it, so one +// op serves both the missing and the drifted case. +func hostAliasesPatchJSON(want []ClusterHostAlias) (string, error) { + val, err := json.Marshal(want) + if err != nil { + return "", err + } + patch, err := json.Marshal([]map[string]any{{ + "op": "add", "path": podHostAliasesPath, "value": json.RawMessage(val), + }}) + if err != nil { + return "", err + } + return string(patch), nil +} + +// hostAliasesConverged reports whether the live hostAliases (raw jsonpath +// output, possibly empty) already carry every entry in want. Order and +// hostname case are ignored; EXTRA live entries are tolerated so an operator's +// hand-applied alias for an unrelated host is never clobbered by the sweep. +func hostAliasesConverged(liveRaw []byte, want []ClusterHostAlias) bool { + if len(want) == 0 { + return true + } + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + if err := json.Unmarshal(liveRaw, &live); err != nil { + return false + } + } + liveSet := map[string]bool{} + for _, a := range live { + for _, h := range a.Hostnames { + liveSet[strings.TrimSpace(a.IP)+"|"+strings.ToLower(strings.TrimSpace(h))] = true + } + } + for _, a := range want { + for _, h := range a.Hostnames { + if !liveSet[a.IP+"|"+h] { + return false + } + } + } + return true +} + +// mergeHostAliases returns want merged over live: live entries for IPs not in +// want are kept (so unrelated hand-applied aliases survive), entries for IPs +// in want are replaced by the desired hostnames. Output is sorted by IP so the +// patch is deterministic. +func mergeHostAliases(liveRaw []byte, want []ClusterHostAlias) []ClusterHostAlias { + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + _ = json.Unmarshal(liveRaw, &live) + } + wantIPs := map[string]bool{} + for _, a := range want { + wantIPs[a.IP] = true + } + out := make([]ClusterHostAlias, 0, len(live)+len(want)) + for _, a := range live { + if !wantIPs[strings.TrimSpace(a.IP)] { + out = append(out, a) + } + } + out = append(out, want...) + sort.SliceStable(out, func(i, j int) bool { return out[i].IP < out[j].IP }) + return out +} + +// reconcileHostAliasesIfDue runs the sweep at most once per +// hostAliasesReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileNetAdminIfDue (poller-loop-only state). +func (s *HubServer) reconcileHostAliasesIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastHostAliasesReconcile.IsZero() || + time.Since(s.lastHostAliasesReconcile) >= hostAliasesReconcileInterval + if due { + s.lastHostAliasesReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileHostAliases() +} + +// reconcileHostAliases sweeps every hosted hive whose cluster declares +// host_aliases and patches the Deployment's pod hostAliases where the desired +// entries are absent — at most hostAliasesMaxPatchesPerCycle per cycle, since +// every patch rolls that hive's pod. Clusters WITHOUT host_aliases are never +// touched, so the lane cannot strip anything an operator applied by hand. +func (s *HubServer) reconcileHostAliases() { + wantByCluster := map[string][]ClusterHostAlias{} + for id, c := range s.clusters { + if len(c.HostAliases) > 0 { + wantByCluster[id] = c.HostAliases + } + } + if len(wantByCluster) == 0 { + return + } + + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil { + continue + } + want, ok := wantByCluster[cluster.ID] + if !ok || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "jsonpath={"+strings.ReplaceAll(podHostAliasesPath, "/", ".")+"}") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("host_aliases reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + if hostAliasesConverged(out, want) { + continue + } + if patched >= hostAliasesMaxPatchesPerCycle { + deferred++ + continue + } + patch, perr := hostAliasesPatchJSON(mergeHostAliases(out, want)) + if perr != nil { + s.logger.Warn("host_aliases reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("host_aliases reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled cluster host_aliases onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("host_aliases reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", hostAliasesMaxPatchesPerCycle, + "next_cycle_in", hostAliasesReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/cluster_host_aliases_test.go b/src/pkg/hub/cluster_host_aliases_test.go new file mode 100644 index 0000000000..cbb502f462 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases_test.go @@ -0,0 +1,150 @@ +package hub + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + "text/template" +) + +func renderManifestWithHostAliases(t *testing.T, aliases []ClusterHostAlias) string { + t.Helper() + tmpl, err := template.New("manifests").Parse(k8sManifestTemplate) + if err != nil { + t.Fatalf("template does not parse: %v", err) + } + data := map[string]any{ + "ID": "test-hive", "ImageTag": "v2-latest", "ImagePullPolicy": "Always", + "InitContainerUID": "1001", "InitContainerGID": "1000", + "TerminalPort": 7681, "DashboardPort": 8080, + "CPURequest": "500m", "MemRequest": "1Gi", "CPULimit": "2", "MemLimit": "4Gi", + "Namespace": "hive-test", "StorageSize": "10Gi", "StorageClass": "standard", + "HeartbeatKey": "hb", "SessionKey": "sk", "SSOPublicKey": "pk", "TerminalKey": "tk", + "AuthorizedUsers": "alice:owner", "Host": "test-hive.example", "Owner": "alice", + "HubPublicURL": "https://hub.example", "ProjectName": "test", "PrimaryRepo": "org/repo", + "AppKeys": []provisionAppKey{}, + "HostAliases": aliases, + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + t.Fatalf("template does not execute: %v", err) + } + return buf.String() +} + +func TestClusterHostAliasesRenderedIntoPodSpec(t *testing.T) { + m := renderManifestWithHostAliases(t, []ClusterHostAlias{ + {IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}, + }) + want := " spec:\n hostAliases:\n - ip: \"129.34.104.21\"\n hostnames:\n - \"github.ibm.com\"\n - \"api.github.ibm.com\"\n" + // The template carries a comment line between spec: and hostAliases:. + stripped := strings.Join(func() []string { + var out []string + for _, l := range strings.Split(m, "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "#") { + continue + } + out = append(out, l) + } + return out + }(), "\n") + if !strings.Contains(stripped, want) { + t.Fatalf("rendered manifest missing pod hostAliases block:\n%s\n--- got ---\n%s", want, m) + } +} + +func TestClusterHostAliasesOmittedWhenUnset(t *testing.T) { + for name, aliases := range map[string][]ClusterHostAlias{"nil": nil, "empty": {}} { + if m := renderManifestWithHostAliases(t, aliases); strings.Contains(m, "hostAliases") { + t.Errorf("%s: hostAliases must not be rendered when no aliases configured", name) + } + } +} + +func TestNormalizeClusterHostAliases(t *testing.T) { + got, err := normalizeClusterHostAliases([]ClusterHostAlias{ + {IP: " 129.34.104.21 ", Hostnames: []string{" GitHub.IBM.com ", "", "api.github.ibm.com"}}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 || got[0].IP != "129.34.104.21" || + strings.Join(got[0].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("unexpected normalization: %+v", got) + } + if out, err := normalizeClusterHostAliases(nil); err != nil || out != nil { + t.Fatalf("nil input should yield nil,nil; got %v,%v", out, err) + } + for name, in := range map[string][]ClusterHostAlias{ + "bad ip": {{IP: "not-an-ip", Hostnames: []string{"a.example"}}}, + "no hostnames": {{IP: "10.0.0.1", Hostnames: []string{" ", ""}}}, + "url not host": {{IP: "10.0.0.1", Hostnames: []string{"https://a.example"}}}, + } { + if _, err := normalizeClusterHostAliases(in); err == nil { + t.Errorf("%s: expected error", name) + } + } +} + +func TestValidateClusterEntriesKeepsClusterOnBadHostAliases(t *testing.T) { + logger := slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)) + got := validateClusterEntries([]ClusterConfig{ + {ID: "bad", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "nope", Hostnames: []string{"a"}}}}, + {ID: "good", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "10.1.2.3", Hostnames: []string{"A.Example"}}}}, + }, logger) + bad, ok := got["bad"] + if !ok { + t.Fatal("cluster with invalid host_aliases must still be admitted") + } + if bad.HostAliases != nil { + t.Fatalf("invalid host_aliases must be dropped, got %+v", bad.HostAliases) + } + if g := got["good"].HostAliases; len(g) != 1 || g[0].Hostnames[0] != "a.example" { + t.Fatalf("valid host_aliases must be normalized in registry, got %+v", g) + } +} + +func TestHostAliasesConvergedAndMerge(t *testing.T) { + want := []ClusterHostAlias{{IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}} + + if hostAliasesConverged(nil, want) { + t.Error("empty live must not be converged") + } + live := []byte(`[{"ip":"129.34.104.21","hostnames":["API.github.ibm.com","github.ibm.com"]},{"ip":"10.0.0.9","hostnames":["other.example"]}]`) + if !hostAliasesConverged(live, want) { + t.Error("live superset (any order/case) must be converged") + } + partial := []byte(`[{"ip":"129.34.104.21","hostnames":["github.ibm.com"]}]`) + if hostAliasesConverged(partial, want) { + t.Error("missing hostname must not be converged") + } + if !hostAliasesConverged([]byte(`[]`), nil) { + t.Error("no desired aliases is always converged") + } + + merged := mergeHostAliases([]byte(`[{"ip":"10.0.0.9","hostnames":["other.example"]},{"ip":"129.34.104.21","hostnames":["stale.example"]}]`), want) + if len(merged) != 2 || merged[0].IP != "10.0.0.9" || merged[1].IP != "129.34.104.21" || + strings.Join(merged[1].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("merge must keep unrelated IPs and replace desired IPs: %+v", merged) + } + + patch, err := hostAliasesPatchJSON(merged) + if err != nil { + t.Fatal(err) + } + var ops []struct { + Op string `json:"op"` + Path string `json:"path"` + Value []ClusterHostAlias `json:"value"` + } + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != podHostAliasesPath || len(ops[0].Value) != 2 { + t.Fatalf("unexpected patch: %s", patch) + } +} diff --git a/src/pkg/hub/clusters_registry.go b/src/pkg/hub/clusters_registry.go index e07495bc6e..d8fb409077 100644 --- a/src/pkg/hub/clusters_registry.go +++ b/src/pkg/hub/clusters_registry.go @@ -207,6 +207,16 @@ func validateClusterEntries(configs []ClusterConfig, logger *slog.Logger) map[st logger.Warn("skipping cluster with no domain", "cluster", c.ID) continue } + // host_aliases is validated but NOT an admission rule: a typo there + // must not drop the cluster (and strand its hives); the aliases are + // dropped instead and the operator told why. + aliases, err := normalizeClusterHostAliases(c.HostAliases) + if err != nil { + logger.Warn("ignoring invalid host_aliases for cluster — hives will NOT get them", + "cluster", c.ID, "error", err) + aliases = nil + } + c.HostAliases = aliases clusters[c.ID] = c } return clusters diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index f1652cd756..8ef3b3db00 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -245,6 +245,13 @@ type ClusterConfig struct { // reachable only via the proxy). Injected into new pool slots as // HIVE_UPSTREAM_PROXY_URL / HIVE_UPSTREAM_PROXY_HOSTS / HIVE_NO_PROXY. UpstreamProxy *ClusterUpstreamProxy `json:"upstream_proxy,omitempty" yaml:"upstream_proxy,omitempty"` + // HostAliases are Pod hostAliases entries rendered into every hive + // Deployment on THIS cluster, for GitHub Enterprise hosts reachable only + // through a TLS reverse proxy pinned to a fixed IP (no CONNECT proxy, so + // upstream_proxy cannot express it). Validated at load by + // normalizeClusterHostAliases; existing hives are converged by the + // host_aliases reconcile sweep (cluster_host_aliases.go, #17801). + HostAliases []ClusterHostAlias `json:"host_aliases,omitempty" yaml:"host_aliases,omitempty"` // GitHubAppSlug is the GitHub App's URL slug on THIS cluster's GitHub // host. A GitHub Enterprise instance hosts its own App registration, // which is rarely named "kubestellar-hive" — without this the spoke's @@ -2660,6 +2667,7 @@ func provisionHive(h *SaaSHive, req *CreateHiveRequest, cluster *ClusterConfig, "UpstreamProxyURL": cluster.UpstreamProxy.url(), "UpstreamProxyHosts": cluster.UpstreamProxy.hosts(), "UpstreamProxyNoProxy": cluster.UpstreamProxy.noProxy(), + "HostAliases": cluster.HostAliases, "IsPublic": h.IsPublic, "HiveType": "hosted", "HasGHE": effectiveGitHubBaseURL(h, cluster) != "", @@ -3404,6 +3412,18 @@ spec: app: hive hive-id: {{.ID}} spec: +{{- if .HostAliases}} + # Per-cluster host_aliases (clusters.json): pins GHE hostnames to a TLS + # reverse-proxy IP where no CONNECT proxy exists. See #17801. + hostAliases: +{{- range .HostAliases}} + - ip: "{{.IP}}" + hostnames: +{{- range .Hostnames}} + - "{{.}}" +{{- end}} +{{- end}} +{{- end}} {{- if .RequiresSCC}} serviceAccountName: hive-sa {{- else}} diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index 6c04fc0e9f..a7997a6a39 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -715,6 +715,11 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // rate-limited to perHiveEnvMaxPatchesPerCycle patches per cycle, because // each patch rolls that hive's pod. See perhive_env_reconcile.go. s.reconcilePerHiveEnvIfDue() + // Converge per-cluster host_aliases (clusters.json) onto existing hive + // Deployments so GHE-behind-reverse-proxy clusters never need a manual + // kubectl patch again (#17801). Throttled and rate-limited like the lane + // above because each patch rolls a pod. + s.reconcileHostAliasesIfDue() // Put #7457's auth-url / auth-response-headers onto the hive-contribute // Ingress of every hosted spoke provisioned before that fix, so // /api/contribute/me learns who is calling there too (#7517). The @@ -789,6 +794,7 @@ func (s *HubServer) pollLatestSHAsTick(ctx context.Context, now time.Time) { s.reconcileNetAdminIfDue() s.reconcileNodeHealthRBACIfDue() s.reconcilePerHiveEnvIfDue() + s.reconcileHostAliasesIfDue() s.reapOrphanedPodsIfDue() s.retireExpiredGenerationsIfDue() s.sweepExpiredAccessIfDue() diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 07b3e0232b..b024fc42e5 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1210,6 +1210,9 @@ type HubServer struct { // Same rationale and same guarding mutex as lastNetAdminReconcile above: // both are poller-loop-only state. lastPerHiveEnvReconcile time.Time + // lastHostAliasesReconcile throttles the per-cluster host_aliases + // reconcile (cluster_host_aliases.go). Same mutex, same rationale. + lastHostAliasesReconcile time.Time // lastContributeIngressReconcile throttles the hive-contribute Ingress // auth-url reconcile (contribute_ingress_reconcile.go), which puts the // #7457 annotations onto spokes provisioned before that fix (#7517). From ceee26d6a224675d99e3c61a2d18d9f35b5cfeba Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:24:46 +0000 Subject: [PATCH 033/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20count=20PR=20auth?= =?UTF-8?q?or's=20own=20sign-off=20after=20a=20hive=20rebase;=20no=20decis?= =?UTF-8?q?ion=20gate=20on=20DCO=20asks=20(#17795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-hive-rebase-signoff-attribution.md | 1 + .../stage_tracker_human_gate_test.go | 7 ++++ src/pkg/dashboard/stage_tracker_rework.go | 15 ++++++++ .../dashboard/stage_tracker_rework_test.go | 8 +++++ .../static-src/js/tracker-human-gate.js | 6 +++- .../dashboard/static-src/js/tracker-words.js | 3 ++ src/pkg/dashboard/static/index.html | 9 ++++- .../github/automerge/automerge_stage_dco.go | 30 +++++++++++++++- .../automerge/automerge_stage_labels_test.go | 36 +++++++++++++++++++ 9 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-hive-rebase-signoff-attribution.md diff --git a/changelog.d/fixed-hive-rebase-signoff-attribution.md b/changelog.d/fixed-hive-rebase-signoff-attribution.md new file mode 100644 index 0000000000..6463d9f05f --- /dev/null +++ b/changelog.d/fixed-hive-rebase-signoff-attribution.md @@ -0,0 +1 @@ +- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. diff --git a/src/pkg/dashboard/stage_tracker_human_gate_test.go b/src/pkg/dashboard/stage_tracker_human_gate_test.go index 53cec8f3a2..e21dc8e0a6 100644 --- a/src/pkg/dashboard/stage_tracker_human_gate_test.go +++ b/src/pkg/dashboard/stage_tracker_human_gate_test.go @@ -202,6 +202,13 @@ assert.deepStrictEqual(ids(['hold'], { kind: 'issue', stage: 10 }), ['Release ho assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row140, { sentinel_notice: true })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17480, { modifiers: ['needs-decision', 'draft'] })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons({ kind: 'pr', stage: 110, modifiers: ['merged', 'hold'] }), []); +// hivecommons/hive#17789: a 071 row flagged needs_human on a DCO sign-off +// ask is the author's move, not a decision; another 071 reason still is. +const row17780 = { repo: 'hivecommons/hive', number: 17780, kind: 'pr', stage: 71, head: '0e5ccbe', author: 'clubanderson', author_kind: 'human', needs_human: true, reason: 'hive-signoff-attribution' }; +assert.deepStrictEqual(stageTrackerHumanGateReasons(row17780), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'dco-missing' })), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'label:needs-rebase' })).map(r => r.key), ['decision']); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { modifiers: ['needs-decision'] })).map(r => r.key), ['decision'], 'an explicit needs-decision still gates'); assert.ok(!stageTrackerRowHtml({ repo: 'acme/console', number: 1, kind: 'pr', stage: 50 }, stages, ctx).includes('ham-human-gate'), 'a clean row has no gate'); // (d) secondary rows #17463 / #17476 of the shared PR #17480 (primary row diff --git a/src/pkg/dashboard/stage_tracker_rework.go b/src/pkg/dashboard/stage_tracker_rework.go index fcd7511ea4..4dbb341738 100644 --- a/src/pkg/dashboard/stage_tracker_rework.go +++ b/src/pkg/dashboard/stage_tracker_rework.go @@ -112,9 +112,24 @@ func stageTrackerReworkOwedBy(it StageTrackerItem) string { if strings.Contains(strings.ToLower(it.Reason), "conflict") && it.Rebase == nil { return reworkOwedByAuthor } + // A DCO ask parks the head on the author's sign-off; no re-review is + // queued for it (hivecommons/hive#17789). + if stageTrackerAuthorDCOReason(it.Reason) { + return reworkOwedByAuthor + } return reworkOwedByHiveReviewer } +// stageTrackerAuthorDCOReason reports a lane reason that asks the author to +// fix their commits' sign-off. +func stageTrackerAuthorDCOReason(reason string) bool { + switch strings.TrimSpace(reason) { + case stagelabels.LaneReasonDCOMissing, stagelabels.LaneReasonHiveSignoffAttribution: + return true + } + return false +} + // StageTrackerFixWork is the hub's fix-queue standing for one PR and one // contributor (ContributeWSHub.FixWorkFor). type StageTrackerFixWork struct { diff --git a/src/pkg/dashboard/stage_tracker_rework_test.go b/src/pkg/dashboard/stage_tracker_rework_test.go index ce47bbd4cb..f6e85b1e1d 100644 --- a/src/pkg/dashboard/stage_tracker_rework_test.go +++ b/src/pkg/dashboard/stage_tracker_rework_test.go @@ -247,6 +247,10 @@ assert.strictEqual(stageTrackerPresenceChipText(working), 'waiting on @Danathar // A pushed head with nothing against it waits on the hive's reviewer (hivecommons/hive#16615). assert.strictEqual(stageTrackerPresenceChipText(pushed), 'waiting on 🐝 hive reviewer · re-review of bbb'); assert.strictEqual(stageTrackerPresenceChipText(w(Object.assign({}, pushed.rework, { owed_by: 'author' }))), 'waiting on @Danathar · re-review pending', 'the server says the author still owes it'); +// hivecommons/hive#17789: an older payload without owed_by reads a DCO ask as the author's move too. +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'hive-signoff-attribution' })), false); +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'dco-missing' })), false); +assert.strictEqual(stageTrackerPresenceChipText(Object.assign({}, pushed, { reason: 'hive-signoff-attribution', rework: Object.assign({}, pushed.rework, { owed_by: 'author' }) })), 'waiting on @Danathar · sign-off owed', 'no re-review is queued for a DCO ask'); assert.match(stageTrackerReworkTip(requested, NOW), /^rework requested 1 hour ago \(\S+ .*\) · no response from @Danathar yet$/); assert.match(stageTrackerReworkTip(queued, NOW), /^rework queued for @Danathar's clanker · 2 ahead · queued 10 min ago \(/); assert.match(stageTrackerReworkTip(acked, NOW), /^@Danathar acknowledged · activity on the PR 20 min ago \(/); @@ -305,6 +309,10 @@ func TestStageTrackerReworkOwedBy16615(t *testing.T) { "red CI on the pushed head": func(it *StageTrackerItem) { it.CIActivity.State = "failing" }, "findings on the pushed head": func(it *StageTrackerItem) { it.ReviewActivity.State = "changes_requested" }, "conflict nobody in the hive rebases": func(it *StageTrackerItem) { it.Reason = "conflicting" }, + // hivecommons/hive#17789: a DCO ask is the author's move; no + // re-review of the pushed head is queued for it. + "hive-signoff-attribution": func(it *StageTrackerItem) { it.Reason = "hive-signoff-attribution" }, + "dco-missing": func(it *StageTrackerItem) { it.Reason = "dco-missing" }, } { it := base() mutate(&it) diff --git a/src/pkg/dashboard/static-src/js/tracker-human-gate.js b/src/pkg/dashboard/static-src/js/tracker-human-gate.js index 6bab762a8a..04ba8f70c8 100644 --- a/src/pkg/dashboard/static-src/js/tracker-human-gate.js +++ b/src/pkg/dashboard/static-src/js/tracker-human-gate.js @@ -83,7 +83,11 @@ const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. diff --git a/src/pkg/dashboard/static-src/js/tracker-words.js b/src/pkg/dashboard/static-src/js/tracker-words.js index b0e95b991e..df5063fcd7 100644 --- a/src/pkg/dashboard/static-src/js/tracker-words.js +++ b/src/pkg/dashboard/static-src/js/tracker-words.js @@ -343,6 +343,8 @@ const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -360,6 +362,7 @@ const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 68c2d1d5ce..bba58a4f63 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -18060,6 +18060,8 @@

Where to get help

const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -18077,6 +18079,7 @@

Where to get help

const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. @@ -26787,7 +26790,11 @@

All labels

const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. diff --git a/src/pkg/github/automerge/automerge_stage_dco.go b/src/pkg/github/automerge/automerge_stage_dco.go index 5237934be7..a8af6c9969 100644 --- a/src/pkg/github/automerge/automerge_stage_dco.go +++ b/src/pkg/github/automerge/automerge_stage_dco.go @@ -84,7 +84,7 @@ func dcoCommitStatsForHive(commits []*gh.RepositoryCommit, appLogin, prAuthor st } else { stats.unsignedOther++ } - } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !messageHasPRAuthorSignoff(msg, prAuthor) { + } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !commitHasPRAuthorSignoff(rc, prAuthor) { stats.squashUnsafeHive++ } } @@ -191,6 +191,34 @@ func botSquashAuthor(login string) bool { return strings.HasSuffix(strings.ToLower(strings.TrimSpace(login)), "[bot]") } +// commitHasPRAuthorSignoff mirrors signoff_is_pr_author in +// check-squash-signoff-attribution.sh: a sign-off certifies the PR author +// when it is that account's GitHub noreply, or when GitHub links the commit +// AUTHOR to the PR author and the sign-off is that author's own git email +// (or noreply). The committer is not consulted, so a hive rebase that makes +// the App the committer of the author's own signed commit does not file it +// as the hive's to re-sign (hivecommons/hive#17789). +func commitHasPRAuthorSignoff(commit *gh.RepositoryCommit, prAuthor string) bool { + msg := commit.GetCommit().GetMessage() + if messageHasPRAuthorSignoff(msg, prAuthor) { + return true + } + want := normalizeBotLogin(prAuthor) + if want == "" || normalizeBotLogin(commit.GetAuthor().GetLogin()) != want { + return false + } + authorEmail := strings.ToLower(strings.TrimSpace(commit.GetCommit().GetAuthor().GetEmail())) + for _, line := range strings.Split(msg, "\n") { + if !strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "signed-off-by:") { + continue + } + if email := signedOffEmail(line); email != "" && email == authorEmail { + return true + } + } + return false +} + func messageHasPRAuthorSignoff(message, prAuthor string) bool { login := strings.ToLower(strings.TrimSpace(prAuthor)) if login == "" { diff --git a/src/pkg/github/automerge/automerge_stage_labels_test.go b/src/pkg/github/automerge/automerge_stage_labels_test.go index 01a73647ff..3ed0e74cbb 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_test.go +++ b/src/pkg/github/automerge/automerge_stage_labels_test.go @@ -2465,6 +2465,42 @@ func TestDCOCommitStatsForHiveSquashUnsafeCountsHiveIdentities(t *testing.T) { } } +// hivecommons/hive#17789: a hive rebase + force-push makes the App bot the +// committer of the PR author's own commit. A sign-off with the commit +// author's own email, on a commit GitHub links to the PR author, certifies +// the PR author (the squash attribution gate agrees), so it is never +// squash-unsafe; a foreign sign-off under the same committer still is. +func TestDCOCommitStatsForHiveAuthorSignedAfterHiveRebase(t *testing.T) { + rebased := func(authorLogin, authorEmail, signoff string) *gh.RepositoryCommit { + return &gh.RepositoryCommit{ + Author: &gh.User{Login: gh.Ptr(authorLogin)}, + Committer: &gh.User{Login: gh.Ptr(testHiveAppBotLogin)}, + Commit: &gh.Commit{ + Author: &gh.CommitAuthor{Email: gh.Ptr(authorEmail)}, + Message: gh.Ptr("fix: openshell sandbox names\n\nSigned-off-by: " + signoff), + }, + Parents: []*gh.Commit{{}}, + } + } + for _, tc := range []struct { + name string + commit *gh.RepositoryCommit + want int + }{ + {"author's real email", rebased("clubanderson", "andy@clubanderson.com", "Andrew Anderson "), 0}, + {"author email case differs", rebased("ClubAnderson", "Andy@ClubAnderson.com", "Andrew Anderson "), 0}, + {"author's noreply", rebased("clubanderson", "andy@clubanderson.com", "A <123+clubanderson@users.noreply.github.com>"), 0}, + {"someone else's sign-off", rebased("clubanderson", "andy@clubanderson.com", "Other "), 1}, + {"commit authored by someone else", rebased("collaborator", "andy@clubanderson.com", "Andrew Anderson "), 1}, + {"author login unlinked", rebased("", "andy@clubanderson.com", "Andrew Anderson "), 1}, + } { + stats := dcoCommitStatsForHive([]*gh.RepositoryCommit{tc.commit}, testHiveAppBotLogin, "clubanderson", false, nil) + if stats.unsigned != 0 || stats.squashUnsafeHive != tc.want { + t.Errorf("%s: %+v, want squashUnsafeHive=%d", tc.name, stats, tc.want) + } + } +} + // hivecommons/hive#16701: a split cached before hive identities counted filed // a project.ai_author commit as "other"; stageDCO reads that head's commits // once more so the hive's own commit can be re-signed, then trusts the From 38f4d9a324a09bdd76b241eb0764319d6be28735 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:26:06 +0000 Subject: [PATCH 034/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20render=20Subst?= =?UTF-8?q?rate=20preflight=20warnings=20on=20the=20agent=20card=20(#17796?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat: render Substrate preflight warnings on the agent card Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 test: seed proc.Config so sandbox-warnings status test matches the builder's input Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...added-substrate-preflight-card-warnings.md | 1 + src/cmd/hive/main.go | 4 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++---- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/config/agent_sandbox_report.go | 45 ++++++++++++++ src/pkg/config/agent_sandbox_report_test.go | 27 +++++++++ src/pkg/config/config.go | 19 ++++-- .../dashboard/agent_sandbox_warnings.test.js | 29 ++++++++++ .../dashboard/agent_sandbox_warnings_test.go | 58 +++++++++++++++++++ src/pkg/dashboard/server.go | 16 +++-- .../dashboard/static-src/js/agent-cards.js | 7 +++ src/pkg/dashboard/static/index.html | 7 +++ src/pkg/dashboard/status_builder.go | 1 + 15 files changed, 218 insertions(+), 28 deletions(-) create mode 100644 changelog.d/added-substrate-preflight-card-warnings.md create mode 100644 src/pkg/config/agent_sandbox_report.go create mode 100644 src/pkg/config/agent_sandbox_report_test.go create mode 100644 src/pkg/dashboard/agent_sandbox_warnings.test.js create mode 100644 src/pkg/dashboard/agent_sandbox_warnings_test.go diff --git a/changelog.d/added-substrate-preflight-card-warnings.md b/changelog.d/added-substrate-preflight-card-warnings.md new file mode 100644 index 0000000000..e3bbc51847 --- /dev/null +++ b/changelog.d/added-substrate-preflight-card-warnings.md @@ -0,0 +1 @@ +- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 0c660bc8cb..bd17e6e5b7 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -7144,7 +7144,9 @@ func parseColorInt(color string) int { // operator who flips the Security tab's sandbox toggle never restarts, so a // boot-only check would never reach the person who most needs it. func logAgentSandboxPosture(logger *slog.Logger, cfg *config.Config) { - for _, warning := range config.AgentSandboxGateWarnings(cfg) { + report := config.AgentSandboxGateReport(cfg) + config.StoreSandboxGateReport(report) + for _, warning := range report.Warnings { logger.Warn("agent sandbox posture", "warning", warning) } } diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 08dba0af9e..8c5b76d503 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1683` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1684` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1685` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1687` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1688` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1689` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index c8255286a9..aec304411e 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1690` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1694` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index e41ee91d74..a0737fea7e 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1663` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1664` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1665` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1671` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1677` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1680` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1667` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1668` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1669` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1675` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1680` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1683` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1686` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index b8e71aaf96..96caa5bfa3 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1753` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1757` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/config/agent_sandbox_report.go b/src/pkg/config/agent_sandbox_report.go new file mode 100644 index 0000000000..b21a42c1b0 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report.go @@ -0,0 +1,45 @@ +package config + +import "sync" + +// SandboxGateReport is the result of AgentSandboxGateReport. OpenShell and +// Substrate are the preflight findings for agents on that runtime; both are +// also included in Warnings. +type SandboxGateReport struct { + Warnings []string + OpenShell []string + Substrate []string +} + +var ( + sandboxGateReportMu sync.RWMutex + sandboxGateReport SandboxGateReport +) + +// StoreSandboxGateReport records the report computed at boot/reload so +// readers such as the status builder never run a preflight themselves. +func StoreSandboxGateReport(r SandboxGateReport) { + sandboxGateReportMu.Lock() + defer sandboxGateReportMu.Unlock() + sandboxGateReport = r +} + +// LastSandboxGateReport returns the report last stored by StoreSandboxGateReport. +func LastSandboxGateReport() SandboxGateReport { + sandboxGateReportMu.RLock() + defer sandboxGateReportMu.RUnlock() + return sandboxGateReport +} + +// SandboxPreflightWarningsFor returns the stored preflight warnings that apply +// to an agent on the given sandbox runtime. +func SandboxPreflightWarningsFor(runtime string) []string { + r := LastSandboxGateReport() + switch runtime { + case SandboxRuntimeOpenShell: + return r.OpenShell + case SandboxRuntimeSubstrate: + return r.Substrate + } + return nil +} diff --git a/src/pkg/config/agent_sandbox_report_test.go b/src/pkg/config/agent_sandbox_report_test.go new file mode 100644 index 0000000000..f1e0115515 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxPreflightWarningsFor(t *testing.T) { + t.Cleanup(func() { StoreSandboxGateReport(SandboxGateReport{}) }) + StoreSandboxGateReport(SandboxGateReport{ + OpenShell: []string{"o"}, + Substrate: []string{"s"}, + }) + for _, tc := range []struct { + runtime string + want []string + }{ + {SandboxRuntimeOpenShell, []string{"o"}}, + {SandboxRuntimeSubstrate, []string{"s"}}, + {SandboxRuntimeJob, nil}, + {"", nil}, + } { + if got := SandboxPreflightWarningsFor(tc.runtime); !reflect.DeepEqual(got, tc.want) { + t.Errorf("SandboxPreflightWarningsFor(%q) = %v, want %v", tc.runtime, got, tc.want) + } + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 92917c4108..6c28e87550 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1586,10 +1586,17 @@ func SplitCadenceTargetKey(key string) (agent, repo string) { // legal config, and refusing to boot on it would turn a diagnostic into an // outage. func AgentSandboxGateWarnings(cfg *Config) []string { + return AgentSandboxGateReport(cfg).Warnings +} + +// AgentSandboxGateReport is AgentSandboxGateWarnings plus the preflight +// findings kept apart per runtime, so callers can attribute them to the agents +// on that runtime without parsing strings. Warnings is the same flat list. +func AgentSandboxGateReport(cfg *Config) SandboxGateReport { if cfg == nil || !cfg.AgentSandbox.Enabled { // The sandbox is off globally. That is the documented default and the // posture the docs describe; it is not a misconfiguration. - return nil + return SandboxGateReport{} } var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string @@ -1642,7 +1649,7 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "agent_sandbox.enabled is true but NO agent is opted in, so the sandbox is inert and all %d agent(s) still run unconfined on the tmux path — "+ "the per-agent gate is separate: set `sandbox: {enabled: true}` on each agent that should be sandboxed (#4918)", len(cfg.Agents))) - return out + return SandboxGateReport{Warnings: out} } if len(noImage) > 0 { out = append(out, fmt.Sprintf( @@ -1681,9 +1688,11 @@ func AgentSandboxGateWarnings(cfg *Config) []string { } // Last, because it may reach the network (bounded and cached): the static // warnings above must not wait on a gateway. - out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) - out = append(out, cruxconfig.SubstratePreflightWarnings(substrateBlocks)...) - return out + openshellWarnings := cruxconfig.OpenShellPreflightWarnings(openshellBlocks) + substrateWarnings := cruxconfig.SubstratePreflightWarnings(substrateBlocks) + out = append(out, openshellWarnings...) + out = append(out, substrateWarnings...) + return SandboxGateReport{Warnings: out, OpenShell: openshellWarnings, Substrate: substrateWarnings} } // CadenceIsOperatorOwned reports whether an operator explicitly set the diff --git a/src/pkg/dashboard/agent_sandbox_warnings.test.js b/src/pkg/dashboard/agent_sandbox_warnings.test.js new file mode 100644 index 0000000000..4c5b8dc18d --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings.test.js @@ -0,0 +1,29 @@ +// Agent card renders the per-agent sandbox_warnings preflight findings. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const src = fs.readFileSync(path.join(__dirname, 'static-src', 'js', 'agent-cards.js'), 'utf8'); +const m = src.match(/function agentSandboxWarningsHtml\(a\) \{[\s\S]*?\n \}\n/); +assert.ok(m, 'agentSandboxWarningsHtml not found'); +const esc = s => String(s).replace(/&/g, '&').replace(/ { + const html = agentSandboxWarningsHtml({ sandbox_warnings: ['operator missing', 'no '] }); + assert.strictEqual((html.match(/class="agent-error"/g) || []).length, 2); + assert.ok(html.includes('⚠ operator missing')); + assert.ok(html.includes('no <gvisor>')); +}); + +test('renders nothing without warnings', () => { + assert.strictEqual(agentSandboxWarningsHtml({}), ''); + assert.strictEqual(agentSandboxWarningsHtml({ sandbox_warnings: [] }), ''); + assert.strictEqual(agentSandboxWarningsHtml(null), ''); +}); + +test('shipped index.html carries the card hook', () => { + const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); + assert.ok(html.includes('${agentSandboxWarningsHtml(a)}')); +}); diff --git a/src/pkg/dashboard/agent_sandbox_warnings_test.go b/src/pkg/dashboard/agent_sandbox_warnings_test.go new file mode 100644 index 0000000000..2bb774ca7e --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings_test.go @@ -0,0 +1,58 @@ +package dashboard + +import ( + "reflect" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +func TestAgentStatusSandboxWarningsByRuntime(t *testing.T) { + config.StoreSandboxGateReport(config.SandboxGateReport{ + OpenShell: []string{"openshell gateway unreachable"}, + Substrate: []string{"substrate operator not found"}, + }) + t.Cleanup(func() { config.StoreSandboxGateReport(config.SandboxGateReport{}) }) + + enabled := true + agent := func(runtime string, sandboxed bool) config.AgentConfig { + a := config.AgentConfig{Backend: "claude", Enabled: true} + if sandboxed { + a.Sandbox = &config.AgentSandboxOverride{Enabled: &enabled, Runtime: runtime} + } + return a + } + cfg := &config.Config{ + AgentRuntimeConfig: config.AgentRuntimeConfig{ + AgentSandbox: config.AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]config.AgentConfig{ + "sub": agent("substrate", true), + "osh": agent("openshell", true), + "job": agent("job", true), + "none": agent("", false), + }, + }, + Governor: config.GovernorConfig{Modes: map[string]config.ModeConfig{}}, + } + statuses := map[string]*crux.Process{} + for n, a := range cfg.Agents { + statuses[n] = &crux.Process{Config: a} + } + payload := BuildAgentOnlyStatus(governor.State{Mode: "IDLE"}, statuses, cfg) + + want := map[string][]string{ + "sub": {"substrate operator not found"}, + "osh": {"openshell gateway unreachable"}, + "job": nil, + "none": nil, + } + got := map[string][]string{} + for _, a := range payload.Agents { + got[a.Name] = a.SandboxWarnings + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("sandbox warnings by agent = %#v, want %#v", got, want) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 7a72a42eb2..e1db31882f 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -955,12 +955,16 @@ type FrontendAgent struct { // SandboxRuntime is the effective sandbox.runtime for a sandboxed agent // ("podman" or "job"); the SPA routes "▶ terminal" to the Job pod log // viewer for "job" (hivecommons/hive#14798). Empty when not sandboxed. - SandboxRuntime string `json:"sandboxRuntime,omitempty"` - LastError string `json:"lastError,omitempty"` - StallNudges int `json:"stallNudges,omitempty"` - StallRecoveries int `json:"stallRecoveries,omitempty"` - ActionNudges int `json:"actionNudges,omitempty"` - TransientNudges int `json:"transientNudges,omitempty"` + SandboxRuntime string `json:"sandboxRuntime,omitempty"` + // SandboxWarnings are the boot/reload preflight findings for this agent's + // sandbox runtime (openshell or substrate), from the stored + // config.AgentSandboxGateReport; no preflight runs on the status path. + SandboxWarnings []string `json:"sandbox_warnings,omitempty"` + LastError string `json:"lastError,omitempty"` + StallNudges int `json:"stallNudges,omitempty"` + StallRecoveries int `json:"stallRecoveries,omitempty"` + ActionNudges int `json:"actionNudges,omitempty"` + TransientNudges int `json:"transientNudges,omitempty"` // SessionRssMb is the resident memory of the agent's CLI process tree // (MiB) and SessionAge how long the current CLI session has been up, // both sampled on the health tick (#14842). SessionRecycles counts diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 1fdf34d621..db730d57dd 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -747,6 +747,12 @@ tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `
⚠ ${esc(w)}
`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -985,6 +991,7 @@ const prefix = retrying ? 'retrying: ' : ''; return `
⚠ ${esc(prefix)}${esc(a.lastError)}
`; })()} + ${agentSandboxWarningsHtml(a)}
${isInferenceBackendName(a.cli) ? 'method' : 'cli'}${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` `}
model${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` `}
diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index bba58a4f63..fd73ff80e2 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -30945,6 +30945,12 @@

All labels

tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `
⚠ ${esc(w)}
`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -31183,6 +31189,7 @@

All labels

const prefix = retrying ? 'retrying: ' : ''; return `
⚠ ${esc(prefix)}${esc(a.lastError)}
`; })()} + ${agentSandboxWarningsHtml(a)}
${isInferenceBackendName(a.cli) ? 'method' : 'cli'}${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` `}
model${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` `}
diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index ff2e6c23c2..e85b6beac4 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1156,6 +1156,7 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config Sandboxed: agentCfg.SandboxEnabled(cfg.AgentSandbox), SandboxRun: sandboxRun, SandboxRuntime: effectiveSandboxRuntime(agentCfg, cfg), + SandboxWarnings: config.SandboxPreflightWarningsFor(effectiveSandboxRuntime(agentCfg, cfg)), Session: name, State: string(proc.State), Starting: proc.Starting && proc.State != crux.StateRunning, From f2152198831373385b5ed5c79e18fe7b1ed9ca56 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:36 +0000 Subject: [PATCH 035/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20dedupe=20repeated?= =?UTF-8?q?=20undelivered=20review=20kick=20warning=20(#17755)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-review-kick-undelivered-dedupe.md | 1 + src/cmd/hive/review_kick_order.go | 46 +++++++++++++++++++ src/cmd/hive/review_kick_order_test.go | 22 +++++++++ 3 files changed, 69 insertions(+) create mode 100644 changelog.d/fixed-review-kick-undelivered-dedupe.md diff --git a/changelog.d/fixed-review-kick-undelivered-dedupe.md b/changelog.d/fixed-review-kick-undelivered-dedupe.md new file mode 100644 index 0000000000..974cf9e77e --- /dev/null +++ b/changelog.d/fixed-review-kick-undelivered-dedupe.md @@ -0,0 +1 @@ +- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. diff --git a/src/cmd/hive/review_kick_order.go b/src/cmd/hive/review_kick_order.go index ba5aa40aa0..a949df169b 100644 --- a/src/cmd/hive/review_kick_order.go +++ b/src/cmd/hive/review_kick_order.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "sync" + "time" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/scheduler" @@ -99,10 +100,20 @@ func orderReviewFixKicksFirst(messages []scheduler.KickMessage, reviewKickByMess // particular PR is being starved. The planner drops the undelivered kick // from the dispatch state (review.ConfirmDelivered) and re-plans it on the // next cycle, so the line says so. +// +// The same kick for the same PR head fails with the same cause every cycle +// while an agent's pane is not ready, so repeats are logged at debug (and +// not re-audited) until the cause changes or undeliveredKickRewarnAfter +// passes. func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k review.DispatchKick, err error) { if logger == nil { logger = slog.Default() } + if !undeliveredKicks.shouldReport(k, err, time.Now()) { + logger.Debug("review swarm kick still not delivered", + "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) + return + } logger.Warn("review swarm kick not delivered; re-planned next cycle", "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) if auditor == nil { @@ -112,6 +123,41 @@ func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k r fmt.Sprintf("kind=%s pr=%s head=%s agent=%s error=%v", k.Kind, k.PRRef, k.HeadSHA, k.Agent, err), k.Agent) } +const ( + undeliveredKickRewarnAfter = 30 * time.Minute + undeliveredKickMaxTracked = 1024 +) + +type undeliveredKickReport struct { + cause string + at time.Time +} + +type undeliveredKickTracker struct { + mu sync.Mutex + seen map[string]undeliveredKickReport +} + +var undeliveredKicks = &undeliveredKickTracker{seen: map[string]undeliveredKickReport{}} + +func (t *undeliveredKickTracker) shouldReport(k review.DispatchKick, err error, now time.Time) bool { + key := k.Kind + "\x00" + k.Agent + "\x00" + k.PRRef + "\x00" + k.HeadSHA + cause := "" + if err != nil { + cause = err.Error() + } + t.mu.Lock() + defer t.mu.Unlock() + if prev, ok := t.seen[key]; ok && prev.cause == cause && now.Sub(prev.at) < undeliveredKickRewarnAfter { + return false + } + if len(t.seen) >= undeliveredKickMaxTracked { + t.seen = map[string]undeliveredKickReport{} + } + t.seen[key] = undeliveredKickReport{cause: cause, at: now} + return true +} + // governorKickDisplacementLimit is how many consecutive cycles an agent's // governor kick may be deferred behind a delivered fix kick before the // governor kick goes first for one cycle. Two: a single displacement is the diff --git a/src/cmd/hive/review_kick_order_test.go b/src/cmd/hive/review_kick_order_test.go index 6822ee190b..8a45547f40 100644 --- a/src/cmd/hive/review_kick_order_test.go +++ b/src/cmd/hive/review_kick_order_test.go @@ -87,6 +87,28 @@ func TestLogUndeliveredReviewKickNamesPR(t *testing.T) { logUndeliveredReviewKick(logger, nil, k, errors.New("x")) } +// The same undelivered kick (same PR head, same cause) repeats every cycle +// while the pane is not ready; only the first occurrence may warn/audit. +func TestLogUndeliveredReviewKickDedupesRepeats(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + auditor := &recordingFixAuditor{} + k := review.DispatchKick{Kind: "fix", Agent: "dedupe-agent", PRRef: "hivecommons/hive#17724", HeadSHA: "abc1234"} + for i := 0; i < 5; i++ { + logUndeliveredReviewKick(logger, auditor, k, errors.New("pane not ready")) + } + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 1 { + t.Fatalf("warn count = %d, want 1: %s", n, buf.String()) + } + if len(auditor.entries) != 1 { + t.Fatalf("audit entries = %d, want 1", len(auditor.entries)) + } + logUndeliveredReviewKick(logger, auditor, k, errors.New("different cause")) + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 2 { + t.Fatalf("warn count after cause change = %d, want 2", n) + } +} + // hivecommons/hive#15166: a fix kick delivered first leaves the lane busy, so // the governor kick behind it is deferred; an adopted PR under // changes_requested is re-planned as a fix kick every cycle, so the scanner's From 31b93219c1e3130b20bbd6daafb0665a9784d188 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:56 +0000 Subject: [PATCH 036/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20cache=20app=5Fkey?= =?UTF-8?q?s=5Fheld=20key=20fingerprints=20so=20heartbeat=20collect=20stay?= =?UTF-8?q?s=20in=20budget=20(#17806)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-app-keys-held-fingerprint-cache.md | 1 + src/pkg/appkey/appkey.go | 37 +++++++++++++++++-- src/pkg/appkey/write_error_test.go | 33 +++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-app-keys-held-fingerprint-cache.md diff --git a/changelog.d/fixed-app-keys-held-fingerprint-cache.md b/changelog.d/fixed-app-keys-held-fingerprint-cache.md new file mode 100644 index 0000000000..82e292b784 --- /dev/null +++ b/changelog.d/fixed-app-keys-held-fingerprint-cache.md @@ -0,0 +1 @@ +- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. diff --git a/src/pkg/appkey/appkey.go b/src/pkg/appkey/appkey.go index 1e8772937f..9a57a80fc0 100644 --- a/src/pkg/appkey/appkey.go +++ b/src/pkg/appkey/appkey.go @@ -20,6 +20,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/hivecommons/hive/pkg/config" ) @@ -62,6 +64,17 @@ type Resolver struct { ProvisionedKeyPath string // FileMode is applied to keys this Resolver writes. FileMode os.FileMode + + fpMu sync.Mutex + fpCache map[string]heldFingerprint +} + +// heldFingerprint memoizes a key file's fingerprint against the size and mtime +// it was computed from, so an unchanged file is never re-read. +type heldFingerprint struct { + size int64 + mtime time.Time + fp string } type tempKeyFile interface { @@ -77,8 +90,9 @@ var ( createTempKey = func(dir, pattern string) (tempKeyFile, error) { return os.CreateTemp(dir, pattern) } - removeFile = os.Remove - renameFile = os.Rename + fingerprintFromFile = config.AppKeyFingerprintFromFile + removeFile = os.Remove + renameFile = os.Rename ) // Default returns a Resolver using the production spoke layout. @@ -125,6 +139,10 @@ func (r *Resolver) HeldFingerprints() map[string]string { return nil } var held map[string]string + r.fpMu.Lock() + defer r.fpMu.Unlock() + seen := make(map[string]heldFingerprint) + defer func() { r.fpCache = seen }() for _, e := range entries { if e.IsDir() { continue @@ -138,10 +156,21 @@ func (r *Resolver) HeldFingerprints() map[string]string { if convErr != nil || id <= 0 { continue } - fp, fpErr := config.AppKeyFingerprintFromFile(filepath.Join(r.DataDir, name)) - if fpErr != nil || fp == "" { + var fp string + info, infoErr := e.Info() + if infoErr != nil { continue } + if c, ok := r.fpCache[name]; ok && c.size == info.Size() && c.mtime.Equal(info.ModTime()) { + fp = c.fp + } else { + var fpErr error + fp, fpErr = fingerprintFromFile(filepath.Join(r.DataDir, name)) + if fpErr != nil || fp == "" { + continue + } + } + seen[name] = heldFingerprint{size: info.Size(), mtime: info.ModTime(), fp: fp} if held == nil { held = make(map[string]string) } diff --git a/src/pkg/appkey/write_error_test.go b/src/pkg/appkey/write_error_test.go index 084a2ac422..0e7c91a1fa 100644 --- a/src/pkg/appkey/write_error_test.go +++ b/src/pkg/appkey/write_error_test.go @@ -338,3 +338,36 @@ func TestWriteKeyFileAtomicReplacesUnwritableFile(t *testing.T) { t.Fatal("rejected writes must leave the key untouched") } } + +func TestHeldFingerprintsDoesNotRereadUnchangedKeys(t *testing.T) { + r := newTestResolver(t) + if err := os.MkdirAll(r.DataDir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(r.DataDir, "gh-app-key-42.pem") + writeTestAppKey(t, path) + + reads := 0 + orig := fingerprintFromFile + fingerprintFromFile = func(p string) (string, error) { + reads++ + return orig(p) + } + t.Cleanup(func() { fingerprintFromFile = orig }) + + first := r.HeldFingerprints() + second := r.HeldFingerprints() + if first["42"] == "" || second["42"] != first["42"] { + t.Fatalf("fingerprints = %v then %v", first, second) + } + if reads != 1 { + t.Fatalf("key file read %d times across two scans, want 1", reads) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if got := r.HeldFingerprints(); got != nil { + t.Fatalf("removed key still reported: %v", got) + } +} From 73991e204ac78f586ea38adc214e1a954ed8db4c Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 00:55:22 +0000 Subject: [PATCH 037/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.225.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.225.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 20 +++++++++++++++++++ changelog.d/added-hub-cluster-host-aliases.md | 1 - .../added-settings-substrate-leaves.md | 1 - .../fixed-agent-lifecycle-repeated-class.md | 1 - .../fixed-automerge-orphaned-queued-runs.md | 1 - .../fixed-claude-shared-credentials.md | 1 - .../fixed-hive-rebase-signoff-attribution.md | 1 - .../fixed-hub-digest-stale-tracker-read.md | 1 - .../fixed-issue-claims-retry-warn-dedupe.md | 1 - .../security-marketplace-model-provenance.md | 1 - 10 files changed, 20 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/added-hub-cluster-host-aliases.md delete mode 100644 changelog.d/added-settings-substrate-leaves.md delete mode 100644 changelog.d/fixed-agent-lifecycle-repeated-class.md delete mode 100644 changelog.d/fixed-automerge-orphaned-queued-runs.md delete mode 100644 changelog.d/fixed-claude-shared-credentials.md delete mode 100644 changelog.d/fixed-hive-rebase-signoff-attribution.md delete mode 100644 changelog.d/fixed-hub-digest-stale-tracker-read.md delete mode 100644 changelog.d/fixed-issue-claims-retry-warn-dedupe.md delete mode 100644 changelog.d/security-marketplace-model-provenance.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e157c27e3..8cb83973d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,26 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.225.0) + +### Added + +- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). +- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. + +### Fixed + +- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. +- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). +- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. +- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. +- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. +- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). + +### Security + +- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. + ## 2026-11-12 (v5.224.0) ### Added diff --git a/changelog.d/added-hub-cluster-host-aliases.md b/changelog.d/added-hub-cluster-host-aliases.md deleted file mode 100644 index ebc6e54a5c..0000000000 --- a/changelog.d/added-hub-cluster-host-aliases.md +++ /dev/null @@ -1 +0,0 @@ -- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). diff --git a/changelog.d/added-settings-substrate-leaves.md b/changelog.d/added-settings-substrate-leaves.md deleted file mode 100644 index 59d543165c..0000000000 --- a/changelog.d/added-settings-substrate-leaves.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. diff --git a/changelog.d/fixed-agent-lifecycle-repeated-class.md b/changelog.d/fixed-agent-lifecycle-repeated-class.md deleted file mode 100644 index fda13f7ad7..0000000000 --- a/changelog.d/fixed-agent-lifecycle-repeated-class.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. diff --git a/changelog.d/fixed-automerge-orphaned-queued-runs.md b/changelog.d/fixed-automerge-orphaned-queued-runs.md deleted file mode 100644 index 6e7f72b821..0000000000 --- a/changelog.d/fixed-automerge-orphaned-queued-runs.md +++ /dev/null @@ -1 +0,0 @@ -- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). diff --git a/changelog.d/fixed-claude-shared-credentials.md b/changelog.d/fixed-claude-shared-credentials.md deleted file mode 100644 index db3fa495ed..0000000000 --- a/changelog.d/fixed-claude-shared-credentials.md +++ /dev/null @@ -1 +0,0 @@ -- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. diff --git a/changelog.d/fixed-hive-rebase-signoff-attribution.md b/changelog.d/fixed-hive-rebase-signoff-attribution.md deleted file mode 100644 index 6463d9f05f..0000000000 --- a/changelog.d/fixed-hive-rebase-signoff-attribution.md +++ /dev/null @@ -1 +0,0 @@ -- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. diff --git a/changelog.d/fixed-hub-digest-stale-tracker-read.md b/changelog.d/fixed-hub-digest-stale-tracker-read.md deleted file mode 100644 index 12611376c5..0000000000 --- a/changelog.d/fixed-hub-digest-stale-tracker-read.md +++ /dev/null @@ -1 +0,0 @@ -- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. diff --git a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md b/changelog.d/fixed-issue-claims-retry-warn-dedupe.md deleted file mode 100644 index 0942f81c03..0000000000 --- a/changelog.d/fixed-issue-claims-retry-warn-dedupe.md +++ /dev/null @@ -1 +0,0 @@ -- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). diff --git a/changelog.d/security-marketplace-model-provenance.md b/changelog.d/security-marketplace-model-provenance.md deleted file mode 100644 index 2f6054f7a6..0000000000 --- a/changelog.d/security-marketplace-model-provenance.md +++ /dev/null @@ -1 +0,0 @@ -- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. From 0a7ae0402d8c912b3f1ca2234606be3d26877000 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:05:05 +0000 Subject: [PATCH 038/166] =?UTF-8?q?=F0=9F=90=9B=20fix(marketplace):=20rest?= =?UTF-8?q?rict=20spoke-triggered=20=3Frepo=3D=20refresh=20to=20tracked=20?= =?UTF-8?q?repositories=20(#17800)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(marketplace): restrict spoke-triggered ?repo= refresh to tracked repositories A registered spoke could pass any owner/name to POST /api/marketplace/refresh and the hub would fetch it with its own GitHub identity and publish the ingested model under the hub's name. Non-admin callers must now name a corpus or HIVE_HUB_MARKETPLACE_EXTRA_REPOS entry; admins are unchanged. Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: refresh head to re-report required checks Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-marketplace-refresh-corpus-only.md | 1 + src/pkg/hub/marketplace.go | 10 +++++++++- src/pkg/hub/marketplace_refresh.go | 12 ++++++++++++ src/pkg/hub/marketplace_test.go | 17 +++++++++++++++-- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 changelog.d/security-marketplace-refresh-corpus-only.md diff --git a/changelog.d/security-marketplace-refresh-corpus-only.md b/changelog.d/security-marketplace-refresh-corpus-only.md new file mode 100644 index 0000000000..9c888537a7 --- /dev/null +++ b/changelog.d/security-marketplace-refresh-corpus-only.md @@ -0,0 +1 @@ +- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 0388838bb4..c886fcac26 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -222,7 +222,11 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ // catalogue" button works through the spoke. Spokes are rate-limited (one // full run per marketplaceFullRefreshInterval across all spokes, // marketplaceRepoRefreshBurst single-repo runs per minute) → 429 with -// Retry-After; admins are not. +// Retry-After; admins are not. A spoke's ?repo= must also name a repository +// the hub already tracks (the corpus or HIVE_HUB_MARKETPLACE_EXTRA_REPOS), +// otherwise 403: the ingest runs with the hub's GitHub identity and +// publishes under the hub's name, so only an admin may widen what it +// fetches. func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return @@ -241,6 +245,10 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ return } if !who.admin { + if !marketplaceTracksRepo(repo) { + marketplaceJSONError(w, http.StatusForbidden, "repo is not in the marketplace corpus; only a hub admin may refresh other repositories") + return + } if ok, wait := rf.allowRepoRefresh(now); !ok { marketplaceTooMany(w, wait, "single-repository refresh") return diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index baf5d208a7..f55dd82fc0 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -192,6 +192,18 @@ func marketplaceRepos() []string { return out } +// marketplaceTracksRepo reports whether repo (already lower-cased and +// trimmed) is one the hub ingests on its own: a corpus entry or an +// HIVE_HUB_MARKETPLACE_EXTRA_REPOS entry. +func marketplaceTracksRepo(repo string) bool { + for _, r := range marketplaceRepos() { + if r == repo { + return true + } + } + return false +} + // marketplaceDisabled reports HIVE_HUB_MARKETPLACE_DISABLED. func marketplaceDisabled() bool { v := strings.ToLower(strings.TrimSpace(os.Getenv(hubMarketplaceDisableEnv))) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index d62f22c3f0..e28ba73458 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -796,7 +796,7 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { s := marketplaceTestHub(t, fetch) clock := s.refresher().now() s.refresher().now = func() time.Time { return clock } - t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo") + t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo,ok/repo") spoke := func(target string) *http.Request { req := httptest.NewRequest(http.MethodPost, target, nil) @@ -831,8 +831,21 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { t.Fatalf("bad bearer: %d", rec.Code) } + // Spoke single repo outside the corpus: refused before any fetch or + // rate-limit accounting; the same request as admin is allowed. + rec := mktDo(s, spoke("/api/marketplace/refresh?repo=outside/corpus")) + if rec.Code != 403 || !strings.Contains(rec.Body.String(), "not in the marketplace corpus") { + t.Fatalf("spoke outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh?repo=outside/corpus")); rec.Code != 200 { + t.Fatalf("admin outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if _, ok := s.marketplace().refreshRecord("outside/corpus"); !ok { + t.Fatal("admin refresh of an outside-corpus repo should have run") + } + // Spoke single repo: ok, then burst-limited. - rec := mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) + rec = mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) if rec.Code != 200 || !strings.Contains(rec.Body.String(), "404 Not Found") { t.Fatalf("spoke single: %d %s", rec.Code, rec.Body.String()) } From 2e16f7c1dab1b5c4e01535118856464eeae2cc01 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 01:27:27 +0000 Subject: [PATCH 039/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.226.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.226.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 11 +++++++++++ .../added-substrate-preflight-card-warnings.md | 1 - changelog.d/fixed-app-keys-held-fingerprint-cache.md | 1 - changelog.d/fixed-review-kick-undelivered-dedupe.md | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/added-substrate-preflight-card-warnings.md delete mode 100644 changelog.d/fixed-app-keys-held-fingerprint-cache.md delete mode 100644 changelog.d/fixed-review-kick-undelivered-dedupe.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb83973d8..4da2380372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.226.0) + +### Added + +- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). + +### Fixed + +- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. +- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. + ## 2026-11-12 (v5.225.0) ### Added diff --git a/changelog.d/added-substrate-preflight-card-warnings.md b/changelog.d/added-substrate-preflight-card-warnings.md deleted file mode 100644 index e3bbc51847..0000000000 --- a/changelog.d/added-substrate-preflight-card-warnings.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). diff --git a/changelog.d/fixed-app-keys-held-fingerprint-cache.md b/changelog.d/fixed-app-keys-held-fingerprint-cache.md deleted file mode 100644 index 82e292b784..0000000000 --- a/changelog.d/fixed-app-keys-held-fingerprint-cache.md +++ /dev/null @@ -1 +0,0 @@ -- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. diff --git a/changelog.d/fixed-review-kick-undelivered-dedupe.md b/changelog.d/fixed-review-kick-undelivered-dedupe.md deleted file mode 100644 index 974cf9e77e..0000000000 --- a/changelog.d/fixed-review-kick-undelivered-dedupe.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. From 9333f350640a61c18809c305809430d0e58207c2 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:30:22 +0000 Subject: [PATCH 040/166] =?UTF-8?q?=F0=9F=94=92=20substratebridge:=20refus?= =?UTF-8?q?e=20tar=20entries=20that=20write=20through=20symlinks=20on=20Un?= =?UTF-8?q?pack=20(#17813)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔒 substrate: unpack pulled workspaces through os.Root so tar symlinks cannot escape Fixes #17769 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: re-trigger required checks after base moved Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-substrate-unpack-symlink.md | 1 + src/crux/sandbox/substratebridge/bridge.go | 35 ++++++++++++------- .../sandbox/substratebridge/bridge_test.go | 30 ++++++++++++++++ 3 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 changelog.d/security-substrate-unpack-symlink.md diff --git a/changelog.d/security-substrate-unpack-symlink.md b/changelog.d/security-substrate-unpack-symlink.md new file mode 100644 index 0000000000..28e56900e6 --- /dev/null +++ b/changelog.d/security-substrate-unpack-symlink.md @@ -0,0 +1 @@ +- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 653616887e..6804a7ba3b 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -267,7 +267,10 @@ func Pack(dir string, w io.Writer) error { return gz.Close() } -// Unpack reads a Pack stream into dir, refusing entries that would escape it. +// Unpack reads a Pack stream into dir, refusing entries that escape it +// lexically (absolute or ".." names) or through symlinks: every write goes +// through an os.Root, so a symlink entry may be created but nothing is ever +// written through one that leaves dir. func Unpack(r io.Reader, dir string) error { gz, err := gzip.NewReader(r) if err != nil { @@ -277,6 +280,11 @@ func Unpack(r io.Reader, dir string) error { if err := os.MkdirAll(dir, 0o755); err != nil { return err } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() tr := tar.NewReader(gz) for { hdr, err := tr.Next() @@ -286,34 +294,34 @@ func Unpack(r io.Reader, dir string) error { if err != nil { return fmt.Errorf("tar: %w", err) } - target, err := safeJoin(dir, hdr.Name) + rel, err := safeRel(hdr.Name) if err != nil { return err } - if err := unpackEntry(tr, hdr, target); err != nil { - return err + if err := unpackEntry(tr, hdr, root, rel); err != nil { + return fmt.Errorf("tar: entry %q: %w", hdr.Name, err) } } } -func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { +func unpackEntry(tr *tar.Reader, hdr *tar.Header, root *os.Root, rel string) error { switch hdr.Typeflag { case tar.TypeDir: - return os.MkdirAll(target, os.FileMode(hdr.Mode)&os.ModePerm|0o700) + return root.MkdirAll(rel, os.FileMode(hdr.Mode)&os.ModePerm|0o700) case tar.TypeSymlink: - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { return err } - _ = os.Remove(target) - return os.Symlink(hdr.Linkname, target) + _ = root.Remove(rel) + return root.Symlink(hdr.Linkname, rel) case tar.TypeReg: if hdr.Size > maxTarEntry { return fmt.Errorf("tar: entry %q is too large (%d bytes)", hdr.Name, hdr.Size) } - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { return err } - f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) + f, err := root.OpenFile(rel, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) if err != nil { return err } @@ -328,12 +336,13 @@ func unpackEntry(tr *tar.Reader, hdr *tar.Header, target string) error { } } -func safeJoin(dir, name string) (string, error) { +// safeRel is the lexical first filter; os.Root enforces the rest. +func safeRel(name string) (string, error) { clean := filepath.Clean(filepath.FromSlash(name)) if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("tar: entry %q escapes the workspace", name) } - return filepath.Join(dir, clean), nil + return clean, nil } // Serve runs the bridge on ln until ctx ends. The actor's command is diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index bc18a77a56..5483abc2a9 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -154,6 +154,36 @@ func TestUnpackSymlinkReplacesExistingAndNestedEntries(t *testing.T) { } } +func TestUnpackRefusesWriteThroughSymlink(t *testing.T) { + for name, linkname := range map[string]func(outside string) string{ + "absolute": func(outside string) string { return outside }, + "relative": func(string) string { return "../outside-rel" }, + } { + t.Run(name, func(t *testing.T) { + parent := t.TempDir() + dst := filepath.Join(parent, "ws") + outside := t.TempDir() + if name == "relative" { + outside = filepath.Join(parent, "outside-rel") + if err := os.Mkdir(outside, 0o755); err != nil { + t.Fatal(err) + } + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: linkname(outside), Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "link/pwned", Typeflag: tar.TypeReg, Size: 2, Mode: 0o644}) + _, _ = tw.Write([]byte("no")) + }) + if err := Unpack(stream, dst); err == nil { + t.Fatal("write through escaping symlink succeeded") + } + if _, err := os.Stat(filepath.Join(outside, "pwned")); !os.IsNotExist(err) { + t.Errorf("file written outside the workspace: %v", err) + } + }) + } +} + func TestUnpackEntryErrorsWhenParentIsAFile(t *testing.T) { dst := t.TempDir() writeFile(t, filepath.Join(dst, "blocker"), "x") From 1290ce8b176d5464382f808065fb0211290066b3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:30:43 +0000 Subject: [PATCH 041/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20show=20substra?= =?UTF-8?q?te=20kicks=20and=20stream=20bridge=20output=20in=20the=20termin?= =?UTF-8?q?al=20pane=20(#17809)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: show substrate kicks and stream bridge output in the terminal pane Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substratebridge: create the workspace dir before the streaming run test newBridgeServer intentionally leaves the workspace path uncreated so the PUT/GET workspace tests can exercise the missing-directory paths, but TestHandlerStreamsRunOutput ran a command in it and failed with chdir: no such file or directory before any frame was streamed. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substrate kicks: stream only transitions after the viewer connected; cover new sandbox paths Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../added-terminal-pane-substrate-kicks.md | 1 + src/crux/sandbox/substrate.go | 41 ++- src/crux/sandbox/substrate_bridge.go | 50 ++- src/crux/sandbox/substrate_client_test.go | 10 +- src/crux/sandbox/substrate_kicks.go | 243 +++++++++++++ src/crux/sandbox/substrate_kicks_test.go | 331 ++++++++++++++++++ src/crux/sandbox/substrate_test.go | 17 +- src/crux/sandbox/substratebridge/bridge.go | 85 ++++- .../sandbox/substratebridge/bridge_test.go | 54 +++ src/docs/sandbox-isolation.md | 2 +- src/pkg/dashboard/job_terminal.go | 130 ++++++- .../dashboard/job_terminal_substrate_test.go | 250 +++++++++++++ .../dashboard/static-src/js/agent-cards.js | 11 +- src/pkg/dashboard/static/index.html | 11 +- 14 files changed, 1200 insertions(+), 36 deletions(-) create mode 100644 changelog.d/added-terminal-pane-substrate-kicks.md create mode 100644 src/crux/sandbox/substrate_kicks.go create mode 100644 src/crux/sandbox/substrate_kicks_test.go create mode 100644 src/pkg/dashboard/job_terminal_substrate_test.go diff --git a/changelog.d/added-terminal-pane-substrate-kicks.md b/changelog.d/added-terminal-pane-substrate-kicks.md new file mode 100644 index 0000000000..1514ab649f --- /dev/null +++ b/changelog.d/added-terminal-pane-substrate-kicks.md @@ -0,0 +1 @@ +- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index 9ad2d81f2d..e79f4a87d7 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "time" @@ -174,7 +175,8 @@ type substrateControl interface { type substrateBridge interface { WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error - Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) + // Run copies the kick's output to live (nil allowed) as it arrives. + Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error } @@ -192,6 +194,16 @@ type SubstrateLauncher struct { newClient func(SubstrateOptions) (substrateClient, error) // now overrides the clock in tests. now func() time.Time + // kicks is where each kick's actor states and output are recorded for + // the dashboard's terminal pane; nil means DefaultSubstrateKicks. + kicks *SubstrateKicks +} + +func (l *SubstrateLauncher) kickRecord() *SubstrateKicks { + if l.kicks != nil { + return l.kicks + } + return DefaultSubstrateKicks } func (l *SubstrateLauncher) client() (substrateClient, error) { @@ -277,14 +289,18 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e } cleanupTemplate := func() { _ = control.DeleteActorTemplate(live(), ref) } - if _, err := control.CreateActor(ctx, &substrateapi.Actor{ + created, err := control.CreateActor(ctx, &substrateapi.Actor{ Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: name}, ActorTemplate: ref, WorkerSelector: substrateSelector(l.Options.WorkerSelector), - }); err != nil { + }) + if err != nil { cleanupTemplate() return res, fmt.Errorf("substrate: creating actor %s: %w", name, err) } + kicks := l.kickRecord() + kick := kicks.begin(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace, State: SubstrateStateCreated}) + kicks.observe(kick, created) // Delete is the default: a kick is a one-shot and keeping a snapshot // around costs object storage. snapshot_on_idle keeps a resumable // snapshot instead, so the next kick's warm state (caches, builds) is @@ -292,11 +308,15 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e cleanupActor := func() { if l.Options.SnapshotOnIdle && ctx.Err() == nil { if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { + kicks.update(kick, func(i *SubstrateActorInfo) { + i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true + }) cleanupTemplate() return } } _ = control.DeleteActor(live(), ref) + kicks.update(kick, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) cleanupTemplate() } @@ -314,7 +334,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e cleanupActor() return res, fmt.Errorf("substrate: resuming actor %s: %w", name, err) } - if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll); err != nil { + kicks.update(kick, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + observe := func(a *substrateapi.Actor) { kicks.observe(kick, a) } + if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll, observe); err != nil { cleanupActor() if ctx.Err() != nil { return res, ctx.Err() @@ -336,9 +358,10 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e if workDir == "" { workDir = SubstrateWorkspaceMount } - out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}) + out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}, kicks.output(kick)) if runErr == nil { res.Stdout, res.Stderr, res.ExitCode = out.Stdout, out.Stderr, out.ExitCode + kicks.update(kick, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, out.ExitCode }) if out.Error != "" { runErr = errors.New(out.Error) } @@ -364,8 +387,9 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e // substrateWaitRunning polls GetActor until the actor is RUNNING, or fails // when it crashes. A deleted or suspended actor during the wait is a failure -// too: nothing hive did asked for it. -func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration) error { +// too: nothing hive did asked for it. observe (nil allowed) sees every +// actor GetActor returns. +func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration, observe func(*substrateapi.Actor)) error { ticker := time.NewTicker(poll) defer ticker.Stop() for { @@ -373,6 +397,9 @@ func substrateWaitRunning(ctx context.Context, control substrateControl, ref *su if err != nil { return err } + if observe != nil { + observe(actor) + } switch state := actor.GetStatus().GetState(); state { case substrateapi.ActorState_ACTOR_STATE_RUNNING: return nil diff --git a/src/crux/sandbox/substrate_bridge.go b/src/crux/sandbox/substrate_bridge.go index 77f3ca4a16..5762b39bc8 100644 --- a/src/crux/sandbox/substrate_bridge.go +++ b/src/crux/sandbox/substrate_bridge.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -52,9 +53,14 @@ func (b *routerBridge) Push(ctx context.Context, ref *substrateapi.ObjectRef, wo return resp.Body.Close() } -// Run executes the kick command in the actor and waits for it to exit. -func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { +// Run executes the kick command in the actor and waits for it to exit. It +// asks the bridge to stream (RunRequest.Stream) and copies stdout and +// stderr to live as they arrive, so the dashboard's terminal pane can follow +// the kick; the returned RunResponse still carries the whole (capped) +// output for the transcript. live may be nil. +func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { var out substratebridge.RunResponse + req.Stream = true body, err := json.Marshal(req) if err != nil { return out, err @@ -64,10 +70,44 @@ func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req return out, err } defer resp.Body.Close() - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return out, fmt.Errorf("decoding run response: %w", err) + return substrateDecodeRun(resp.Body, live) +} + +// substrateDecodeRun reads a /run answer: Partial frames from a streaming +// bridge, then the final frame with the exit code. A bridge that does not +// stream answers with one complete, non-Partial RunResponse, which is read +// the same way. +func substrateDecodeRun(r io.Reader, live io.Writer) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + var stdout, stderr bytes.Buffer + dec := json.NewDecoder(r) + for { + var frame substratebridge.RunResponse + if err := dec.Decode(&frame); err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return out, fmt.Errorf("decoding run response: %w", err) + } + substrateAppendCapped(&stdout, frame.Stdout) + substrateAppendCapped(&stderr, frame.Stderr) + if live != nil { + _, _ = io.WriteString(live, frame.Stdout) + _, _ = io.WriteString(live, frame.Stderr) + } + if !frame.Partial { + out.Stdout, out.Stderr = stdout.String(), stderr.String() + out.ExitCode, out.Error = frame.ExitCode, frame.Error + return out, nil + } + } +} + +func substrateAppendCapped(buf *bytes.Buffer, s string) { + if room := substratebridge.MaxOutput - buf.Len(); len(s) > room { + s = s[:max(room, 0)] } - return out, nil + buf.WriteString(s) } // Pull replaces the local workspace with the actor's, so the commits the diff --git a/src/crux/sandbox/substrate_client_test.go b/src/crux/sandbox/substrate_client_test.go index fa9f0c8ea4..5a5638de27 100644 --- a/src/crux/sandbox/substrate_client_test.go +++ b/src/crux/sandbox/substrate_client_test.go @@ -394,10 +394,14 @@ func TestRouterBridgeDrivesTheBridgeProtocol(t *testing.T) { if got, _ := os.ReadFile(filepath.Join(actorWS, "sub", "file.txt")); string(got) != "before" { t.Fatalf("actor workspace = %q", got) } - out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}) + var live strings.Builder + out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}, &live) if err != nil || out.ExitCode != 4 || !strings.Contains(out.Stdout, "ran") { t.Fatalf("Run: %+v %v", out, err) } + if live.String() != "ran\n" { + t.Errorf("streamed output = %q", live.String()) + } if err := b.Pull(ctx, ref, local); err != nil { t.Fatalf("Pull: %v", err) } @@ -427,7 +431,7 @@ func TestRouterBridgeErrors(t *testing.T) { if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "unpack failed") { t.Errorf("non-2xx: %v", err) } - if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil { + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil { t.Error("Run swallowed a non-2xx") } if err := b.Pull(ctx, ref, t.TempDir()); err == nil { @@ -438,7 +442,7 @@ func TestRouterBridgeErrors(t *testing.T) { _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("{not json")) })) - if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}); err == nil || !strings.Contains(err.Error(), "decoding run response") { + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { t.Errorf("bad json: %v", err) } ws := filepath.Join(t.TempDir(), "ws") diff --git a/src/crux/sandbox/substrate_kicks.go b/src/crux/sandbox/substrate_kicks.go new file mode 100644 index 0000000000..1dffb978de --- /dev/null +++ b/src/crux/sandbox/substrate_kicks.go @@ -0,0 +1,243 @@ +package sandbox + +import ( + "context" + "io" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Substrate kick reader (hivecommons/hive#17777), the substrate counterpart +// of the OpenShell log reader in openshell_logs.go. +// +// A substrate-runtime agent has no tmux session, so the dashboard's +// read-only kick viewer needs the agent's live actor and its output. The +// Control API has no ListActors and Substrate has no logs RPC, so neither +// can come from the control plane: the launcher, which runs in the same +// hive process as the dashboard, records each kick here instead — the +// actor's name, every state it is seen in (RESUMING, RUNNING, then +// SUSPENDED or DELETING), the worker's SPIFFE id, and the kick's stdout +// and stderr as the bridge streams /run. Unlike the OpenShell gateway log, +// this is the kick's own output and the exit code is the kick's. + +// SubstrateActorInfo is what the viewer needs to know about a kick's actor. +type SubstrateActorInfo struct { + Name string + Atespace string + // State is the actor's ActorState without its ACTOR_STATE_ prefix, or + // CREATED before the control plane has reported one. + State string + // SpiffeID is the identity of the worker the actor was scheduled on + // (WorkerAssignment.worker_spiffe_id), once known. + SpiffeID string + // Exited is true once the bridge's /run reported the kick's exit code. + Exited bool + ExitCode int + // Suspended is true when the launcher kept a snapshot of the actor + // (snapshot_on_idle) instead of deleting it. + Suspended bool + // Done is true once the launcher has finished with the actor. + Done bool +} + +// Finished reports whether the kick will produce no more output. +func (i SubstrateActorInfo) Finished() bool { return i.Done } + +// SubstrateLogReader is the dashboard's view of the substrate kicks this +// process runs. *SubstrateKicks implements it. +type SubstrateLogReader interface { + // LatestActor returns the newest kick the launcher recorded for + // launchName (see LaunchName). A finished kick is still returned so the + // viewer can say how its actor ended; found is false when the agent has + // not run a substrate kick since hive started. + LatestActor(ctx context.Context, launchName string) (info SubstrateActorInfo, found bool, err error) + // StreamRun writes the kick's output to w, calling onState for every + // state the actor reaches after the call (the caller has already shown + // the one LatestActor reported), polling every poll + // until the launcher is done with the actor or ctx ends, and returns + // the last state it observed. + StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) +} + +// DefaultSubstrateKicks is the process-wide record the launcher writes and +// the dashboard reads. +var DefaultSubstrateKicks = &SubstrateKicks{} + +var _ SubstrateLogReader = (*SubstrateKicks)(nil) + +// SubstrateKicks records the latest substrate kick per launch name. +type SubstrateKicks struct { + mu sync.Mutex + byLaunch map[string]*substrateKick +} + +// substrateKick is one recorded kick: every state snapshot it went through +// and its output so far, capped like the bridge's. +type substrateKick struct { + history []SubstrateActorInfo + out []byte +} + +func (k *substrateKick) latest() SubstrateActorInfo { return k.history[len(k.history)-1] } + +// begin records a new kick for launchName, replacing the previous one. +func (s *SubstrateKicks) begin(launchName string, info SubstrateActorInfo) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + if s.byLaunch == nil { + s.byLaunch = map[string]*substrateKick{} + } + k := &substrateKick{history: []SubstrateActorInfo{info}} + s.byLaunch[launchName] = k + return k +} + +// update applies fn to the kick's latest state and records the result as +// a new snapshot when anything changed. +func (s *SubstrateKicks) update(k *substrateKick, fn func(*SubstrateActorInfo)) { + if k == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + next := k.latest() + fn(&next) + if next != k.latest() { + k.history = append(k.history, next) + } +} + +// observe records what GetActor reported: state and worker SPIFFE id. +func (s *SubstrateKicks) observe(k *substrateKick, actor *substrateapi.Actor) { + s.update(k, func(i *SubstrateActorInfo) { + if st := substrateStateName(actor.GetStatus().GetState()); st != "" { + i.State = st + } + if id := actor.GetStatus().GetWorkerAssignment().GetWorkerSpiffeId(); id != "" { + i.SpiffeID = id + } + }) +} + +// output returns a writer that appends to the kick's recorded output. +func (s *SubstrateKicks) output(k *substrateKick) io.Writer { + if k == nil { + return nil + } + return substrateKickWriter{s: s, k: k} +} + +type substrateKickWriter struct { + s *SubstrateKicks + k *substrateKick +} + +func (w substrateKickWriter) Write(p []byte) (int, error) { + w.s.mu.Lock() + defer w.s.mu.Unlock() + chunk := p + if room := substratebridge.MaxOutput - len(w.k.out); len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + w.k.out = append(w.k.out, chunk...) + return len(p), nil +} + +// LatestActor implements SubstrateLogReader. +func (s *SubstrateKicks) LatestActor(_ context.Context, launchName string) (SubstrateActorInfo, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + k, ok := s.byLaunch[launchName] + if !ok { + return SubstrateActorInfo{}, false, nil + } + return k.latest(), true, nil +} + +func (s *SubstrateKicks) byActor(name string) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + for _, k := range s.byLaunch { + if k.history[0].Name == name { + return k + } + } + return nil +} + +// StreamRun implements SubstrateLogReader. +func (s *SubstrateKicks) StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) { + k := s.byActor(actorName) + if k == nil { + // Replaced by a newer kick before the viewer got here: this one is over. + return SubstrateActorInfo{Name: actorName, Done: true}, nil + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + // The caller has just rendered LatestActor's snapshot, the newest one: + // replaying older history would show the actor going backwards. + s.mu.Lock() + seenStates, seenOut := len(k.history), 0 + s.mu.Unlock() + for { + s.mu.Lock() + states := append([]SubstrateActorInfo(nil), k.history[seenStates:]...) + out := append([]byte(nil), k.out[seenOut:]...) + seenStates, seenOut = len(k.history), len(k.out) + latest := k.latest() + s.mu.Unlock() + // States reached before the kick exited come before this poll's + // output, the rest after it, so the pane reads in order. + split := len(states) + for i, st := range states { + if st.Exited || st.Done { + split = i + break + } + } + emit := func(sts []SubstrateActorInfo) { + if onState != nil { + for _, st := range sts { + onState(st) + } + } + } + emit(states[:split]) + if len(out) > 0 { + if _, err := w.Write(out); err != nil { + return latest, err + } + } + emit(states[split:]) + if latest.Done { + return latest, nil + } + select { + case <-ctx.Done(): + return latest, ctx.Err() + case <-ticker.C: + } + } +} + +// substrateStateName is an ActorState as the viewer shows it: RUNNING, not +// ACTOR_STATE_RUNNING; empty for UNSPECIFIED. +func substrateStateName(st substrateapi.ActorState) string { + if st == substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED { + return "" + } + return strings.TrimPrefix(st.String(), "ACTOR_STATE_") +} + +// Substrate actor states as SubstrateActorInfo.State carries them. +const ( + SubstrateStateCreated = "CREATED" + SubstrateStateResuming = "RESUMING" + SubstrateStateRunning = "RUNNING" + SubstrateStateSuspended = "SUSPENDED" + SubstrateStateDeleting = "DELETING" +) diff --git a/src/crux/sandbox/substrate_kicks_test.go b/src/crux/sandbox/substrate_kicks_test.go new file mode 100644 index 0000000000..50857ad705 --- /dev/null +++ b/src/crux/sandbox/substrate_kicks_test.go @@ -0,0 +1,331 @@ +package sandbox + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +func substrateStates(t *testing.T, kicks *SubstrateKicks, launchName string) ([]SubstrateActorInfo, string) { + t.Helper() + kicks.mu.Lock() + defer kicks.mu.Unlock() + k, ok := kicks.byLaunch[launchName] + if !ok { + t.Fatalf("no kick recorded for %q", launchName) + } + return append([]SubstrateActorInfo(nil), k.history...), string(k.out) +} + +func TestSubstrateLauncherRecordsKickForTerminal(t *testing.T) { + fake := &fakeSubstrate{ + spiffeID: "spiffe://ate.local/ns/ate-workers/sa/worker", + runResp: substratebridge.RunResponse{Stdout: "step 1\n", Stderr: "warn\n", ExitCode: 5}, + } + l := newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, out := substrateStates(t, l.kicks, "deps-sandbox") + if out != "step 1\nwarn\n" { + t.Errorf("recorded output = %q", out) + } + var states []string + for _, h := range history { + states = append(states, h.State) + } + if got := strings.Join(states, ","); got != "CREATED,RESUMING,RUNNING,RUNNING,DELETING" { + t.Errorf("states = %s", got) + } + final := history[len(history)-1] + if final.Name != "hive-kick-deps-1700000000" || final.Atespace != "hive" || final.SpiffeID != fake.spiffeID || + !final.Exited || final.ExitCode != 5 || !final.Done || final.Suspended { + t.Errorf("final = %+v", final) + } + if info, found, err := l.kicks.LatestActor(context.Background(), "deps-sandbox"); err != nil || !found || info != final { + t.Errorf("LatestActor = %+v %v %v", info, found, err) + } + if _, found, _ := l.kicks.LatestActor(context.Background(), "other-sandbox"); found { + t.Error("an agent without a kick must not be found") + } +} + +func TestSubstrateLauncherRecordsSuspendAndFailure(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, _ := substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateSuspended || !final.Suspended || !final.Done { + t.Errorf("snapshot_on_idle final = %+v", final) + } + + fake = &fakeSubstrate{waitReadyErr: errors.New("no bridge")} + l = newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil { + t.Fatal("expected bridge error") + } + history, _ = substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateDeleting || !final.Done || final.Exited { + t.Errorf("failed kick final = %+v", final) + } + + // A kick that fails before the actor exists records nothing. + fake = &fakeSubstrate{actorErr: errors.New("quota")} + l = newTestSubstrate(fake) + _, _ = l.Run(context.Background(), substrateTestSpec()) + if _, found, _ := l.kicks.LatestActor(context.Background(), "deps-sandbox"); found { + t.Error("no actor was created; nothing must be recorded") + } +} + +// substrateStreamWriter records what StreamRun writes and closes started on +// the first write, so a test knows the stream has begun polling before it +// records further transitions. +type substrateStreamWriter struct { + strings.Builder + started chan struct{} + once sync.Once +} + +func newSubstrateStreamWriter() *substrateStreamWriter { + return &substrateStreamWriter{started: make(chan struct{})} +} + +func (w *substrateStreamWriter) Write(p []byte) (int, error) { + defer w.once.Do(func() { close(w.started) }) + return w.Builder.Write(p) +} + +func substrateStateNames(infos []SubstrateActorInfo) string { + var states []string + for _, i := range infos { + states = append(states, i.State) + } + return strings.Join(states, ",") +} + +func TestSubstrateKicksStreamRunFollowsStatesAndOutput(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + w := kicks.output(k) + _, _ = w.Write([]byte("line 0\n")) + + type event struct{ state, out string } + var events []event + buf := newSubstrateStreamWriter() + done := make(chan SubstrateActorInfo) + go func() { + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, buf, func(i SubstrateActorInfo) { + events = append(events, event{state: i.State, out: buf.String()}) + }) + if err != nil { + t.Errorf("StreamRun: %v", err) + } + done <- final + }() + <-buf.started + kicks.observe(k, &substrateapi.Actor{Status: &substrateapi.ActorStatus{ + State: substrateapi.ActorState_ACTOR_STATE_RUNNING, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: "spiffe://x/w"}, + }}) + _, _ = w.Write([]byte("line 1\n")) + _, _ = w.Write([]byte("line 2\n")) + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 0 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true }) + + final := <-done + if buf.String() != "line 0\nline 1\nline 2\n" { + t.Fatalf("streamed = %q", buf.String()) + } + if final.State != SubstrateStateSuspended || !final.Done || final.SpiffeID != "spiffe://x/w" { + t.Fatalf("final = %+v", final) + } + // RESUMING is what LatestActor reported before the stream started, so + // only the transitions recorded afterwards are reported. + var states []string + for _, e := range events { + states = append(states, e.state) + } + if got := strings.Join(states, ","); got != "RUNNING,RUNNING,SUSPENDED" { + t.Fatalf("states = %s", got) + } + if last := events[len(events)-1]; last.out != "line 0\nline 1\nline 2\n" { + t.Fatalf("SUSPENDED must be reported after the output, saw output %q", last.out) + } +} + +func TestSubstrateKicksStreamRunMidKickDoesNotReplayOlderStates(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.SpiffeID = SubstrateStateRunning, "spiffe://x/w" }) + w := kicks.output(k) + _, _ = w.Write([]byte("booting\n")) + + // The viewer connects while the kick is already RUNNING. + info, found, err := kicks.LatestActor(context.Background(), "deps-sandbox") + if err != nil || !found || info.State != SubstrateStateRunning { + t.Fatalf("LatestActor = %+v %v %v", info, found, err) + } + var seen []SubstrateActorInfo + buf := newSubstrateStreamWriter() + done := make(chan error) + go func() { + _, err := kicks.StreamRun(context.Background(), info.Name, time.Millisecond, buf, func(i SubstrateActorInfo) { + seen = append(seen, i) + }) + done <- err + }() + <-buf.started + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 3 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) + if err := <-done; err != nil { + t.Fatalf("StreamRun: %v", err) + } + for _, st := range seen { + if st.State == SubstrateStateCreated || st.State == SubstrateStateResuming { + t.Fatalf("a state older than RUNNING was replayed: %s", substrateStateNames(seen)) + } + } + if got := substrateStateNames(seen); got != "RUNNING,DELETING" { + t.Fatalf("states = %s", got) + } + if !seen[0].Exited || seen[0].ExitCode != 3 || !seen[1].Done { + t.Fatalf("transitions = %+v", seen) + } + if buf.String() != "booting\n" { + t.Fatalf("streamed = %q", buf.String()) + } +} + +func TestSubstrateKicksStreamRunWriteError(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + _, _ = kicks.output(k).Write([]byte("out")) + boom := errors.New("viewer gone") + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, failingWriter{err: boom}, nil) + if !errors.Is(err, boom) || final.State != SubstrateStateRunning || final.Done { + t.Fatalf("StreamRun = %+v %v", final, err) + } +} + +func TestSubstrateKicksStreamRunReplacedKick(t *testing.T) { + kicks := &SubstrateKicks{} + kicks.begin("other-sandbox", SubstrateActorInfo{Name: "b1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a2", State: SubstrateStateCreated}) + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, &strings.Builder{}, func(SubstrateActorInfo) { + t.Fatal("a replaced kick has no transitions to report") + }) + if err != nil || !final.Done || final.Name != "a1" { + t.Fatalf("replaced kick: %+v %v", final, err) + } +} + +func TestSubstrateLauncherDefaultsToProcessKicks(t *testing.T) { + if got := (&SubstrateLauncher{}).kickRecord(); got != DefaultSubstrateKicks { + t.Fatalf("kickRecord = %p, want DefaultSubstrateKicks", got) + } + own := &SubstrateKicks{} + if got := (&SubstrateLauncher{kicks: own}).kickRecord(); got != own { + t.Fatalf("kickRecord = %p, want %p", got, own) + } +} + +func TestSubstrateStateName(t *testing.T) { + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED); got != "" { + t.Errorf("UNSPECIFIED = %q", got) + } + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_RUNNING); got != SubstrateStateRunning { + t.Errorf("RUNNING = %q", got) + } +} + +func TestSubstrateKicksStreamRunUnknownActorAndCancel(t *testing.T) { + kicks := &SubstrateKicks{} + final, err := kicks.StreamRun(context.Background(), "gone", time.Millisecond, &strings.Builder{}, nil) + if err != nil || !final.Done || final.Name != "gone" { + t.Fatalf("unknown actor: %+v %v", final, err) + } + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := kicks.StreamRun(ctx, "a1", time.Millisecond, &strings.Builder{}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled stream: %v", err) + } +} + +func TestSubstrateKickOutputIsCapped(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("x", SubstrateActorInfo{Name: "a"}) + w := kicks.output(k) + big := make([]byte, substratebridge.MaxOutput+10) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + if n, _ := w.Write([]byte("more")); n != 4 { + t.Fatal("a full buffer must still accept writes") + } + if len(k.out) != substratebridge.MaxOutput { + t.Fatalf("recorded %d bytes", len(k.out)) + } + if kicks.output(nil) != nil { + t.Fatal("no kick, no writer") + } + kicks.update(nil, func(*SubstrateActorInfo) { t.Fatal("must not run") }) +} + +func TestSubstrateDecodeRunStreamedAndLegacy(t *testing.T) { + var live strings.Builder + stream := `{"stdout":"a","stderr":"","exit_code":0,"partial":true} +{"stdout":"","stderr":"b","exit_code":0,"partial":true} +{"stdout":"","stderr":"","exit_code":2,"error":"boom"} +` + out, err := substrateDecodeRun(strings.NewReader(stream), &live) + if err != nil || out.Stdout != "a" || out.Stderr != "b" || out.ExitCode != 2 || out.Error != "boom" || live.String() != "ab" { + t.Fatalf("streamed: %+v %v live=%q", out, err, live.String()) + } + // A bridge that predates streaming answers with one complete response. + live.Reset() + out, err = substrateDecodeRun(strings.NewReader(`{"stdout":"all","stderr":"err","exit_code":1}`), &live) + if err != nil || out.Stdout != "all" || out.Stderr != "err" || out.ExitCode != 1 || live.String() != "allerr" { + t.Fatalf("legacy: %+v %v live=%q", out, err, live.String()) + } + // A stream cut before the final frame is an error, not exit code 0. + if _, err := substrateDecodeRun(strings.NewReader(`{"stdout":"a","partial":true}`), nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Fatalf("truncated: %v", err) + } +} + +func TestSubstrateDecodeRunCapsOutput(t *testing.T) { + big := strings.Repeat("o", substratebridge.MaxOutput) + var stream strings.Builder + enc := json.NewEncoder(&stream) + for _, frame := range []substratebridge.RunResponse{ + {Stdout: big, Partial: true}, + {Stdout: "more", Stderr: "e", Partial: true}, + {ExitCode: 0}, + } { + if err := enc.Encode(frame); err != nil { + t.Fatal(err) + } + } + out, err := substrateDecodeRun(strings.NewReader(stream.String()), nil) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(out.Stdout) != substratebridge.MaxOutput || strings.Contains(out.Stdout, "more") || out.Stderr != "e" { + t.Fatalf("stdout %d bytes, stderr %q", len(out.Stdout), out.Stderr) + } +} diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go index f306ab9d39..1bdb458a75 100644 --- a/src/crux/sandbox/substrate_test.go +++ b/src/crux/sandbox/substrate_test.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "errors" + "io" "strings" "sync" "testing" @@ -32,6 +33,7 @@ type fakeSubstrate struct { getErr error states []substrateapi.ActorState crashMsg string + spiffeID string getCancels context.CancelFunc waitReadyErr error pushErr error @@ -98,7 +100,10 @@ func (f *fakeSubstrate) GetActor(ctx context.Context, ref *substrateapi.ObjectRe } return &substrateapi.Actor{ Metadata: &substrateapi.ResourceMetadata{Name: ref.GetName()}, - Status: &substrateapi.ActorStatus{State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}}, + Status: &substrateapi.ActorStatus{ + State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: f.spiffeID}, + }, }, nil } @@ -143,9 +148,12 @@ func (f *fakeSubstrate) Push(ctx context.Context, _ *substrateapi.ObjectRef, ws return f.pushErr } -func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest) (substratebridge.RunResponse, error) { +func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { f.note("Run", ctx) f.runReq = req + if live != nil { + _, _ = io.WriteString(live, f.runResp.Stdout+f.runResp.Stderr) + } if f.runCancels != nil { f.runCancels() } @@ -181,6 +189,7 @@ func newTestSubstrate(fake *fakeSubstrate) *SubstrateLauncher { }} l.newClient = func(SubstrateOptions) (substrateClient, error) { return fake, nil } l.now = func() time.Time { return time.Unix(1_700_000_000, 0) } + l.kicks = &SubstrateKicks{} return l } @@ -499,14 +508,14 @@ func TestSubstrateWaitRunningPollsUntilRunning(t *testing.T) { substrateapi.ActorState_ACTOR_STATE_RUNNING, }} ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} - if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err != nil { + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err != nil { t.Fatalf("err = %v", err) } if n := len(fake.calls); n != 3 { t.Errorf("GetActor calls = %d, want 3", n) } fake = &fakeSubstrate{states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_SUSPENDING}} - if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { t.Errorf("err = %v", err) } } diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go index 6804a7ba3b..498b1aaa62 100644 --- a/src/crux/sandbox/substratebridge/bridge.go +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -24,6 +24,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" ) @@ -57,14 +58,24 @@ type RunRequest struct { Command []string `json:"command"` Env map[string]string `json:"env,omitempty"` WorkDir string `json:"workdir,omitempty"` + // Stream asks for the output as it is produced: the response is then a + // sequence of JSON RunResponse frames, every one but the last marked + // Partial and carrying only new output, the last carrying the exit + // code. A bridge that predates the field ignores it and answers with a + // single complete RunResponse, which a streaming client reads as a + // final frame, so either side can be upgraded first. + Stream bool `json:"stream,omitempty"` } -// RunResponse is the body POST /run answers with once the command exits. +// RunResponse is the body POST /run answers with once the command exits, +// or one frame of a streamed answer (see RunRequest.Stream). type RunResponse struct { Stdout string `json:"stdout"` Stderr string `json:"stderr"` ExitCode int `json:"exit_code"` Error string `json:"error,omitempty"` + // Partial marks a streamed output frame; its ExitCode means nothing. + Partial bool `json:"partial,omitempty"` } // Handler serves the bridge protocol for one workspace directory. @@ -159,16 +170,79 @@ func (h *Handler) run(w http.ResponseWriter, r *http.Request) { ctx, cancel = context.WithTimeout(ctx, h.RunTimeout) defer cancel() } - resp := Run(ctx, req, h.workspace()) w.Header().Set("Content-Type", contentTypeJSON) + if !req.Stream { + resp := Run(ctx, req, h.workspace()) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) + return + } w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(resp) + frames := &frameEncoder{enc: json.NewEncoder(w)} + if f, ok := w.(http.Flusher); ok { + frames.flush = f.Flush + } + resp := run(ctx, req, h.workspace(), &frameWriter{frames: frames, stderr: false}, &frameWriter{frames: frames, stderr: true}) + frames.send(RunResponse{ExitCode: resp.ExitCode, Error: resp.Error}) +} + +// frameEncoder serializes the streamed frames of one POST /run; stdout and +// stderr are written from separate goroutines by os/exec. +type frameEncoder struct { + mu sync.Mutex + enc *json.Encoder + flush func() +} + +func (f *frameEncoder) send(frame RunResponse) { + f.mu.Lock() + defer f.mu.Unlock() + _ = f.enc.Encode(frame) + if f.flush != nil { + f.flush() + } +} + +// frameWriter turns one of the command's output streams into Partial +// frames, capped at MaxOutput like the buffered answer. It never fails the +// command: a client that went away only loses the rest of the output. +type frameWriter struct { + frames *frameEncoder + stderr bool + written int +} + +func (w *frameWriter) Write(p []byte) (int, error) { + chunk := p + if room := MaxOutput - w.written; len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + if len(chunk) > 0 { + w.written += len(chunk) + frame := RunResponse{Partial: true} + if w.stderr { + frame.Stderr = string(chunk) + } else { + frame.Stdout = string(chunk) + } + w.frames.send(frame) + } + return len(p), nil } // Run executes one RunRequest with the request env replacing the process // env — the launcher already filtered it through hive's credential-free // allowlist — and reports the exit code the way the Podman launcher does. func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { + var stdout, stderr bytes.Buffer + resp := run(ctx, req, workspace, &stdout, &stderr) + resp.Stdout, resp.Stderr = capString(stdout.Bytes()), capString(stderr.Bytes()) + return resp +} + +// run executes req with its output going to stdout and stderr and reports +// the exit code; the output fields of the result are left empty. +func run(ctx context.Context, req RunRequest, workspace string, stdout, stderr io.Writer) RunResponse { cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) cmd.Dir = req.WorkDir if cmd.Dir == "" { @@ -187,10 +261,9 @@ func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { env = append(env, "PATH="+os.Getenv("PATH")) } cmd.Env = env - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr + cmd.Stdout, cmd.Stderr = stdout, stderr err := cmd.Run() - resp := RunResponse{Stdout: capString(stdout.Bytes()), Stderr: capString(stderr.Bytes())} + var resp RunResponse var exitErr *exec.ExitError switch { case err == nil: diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go index 5483abc2a9..c7e40bde59 100644 --- a/src/crux/sandbox/substratebridge/bridge_test.go +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -469,3 +469,57 @@ func TestPutWorkspaceClearsInPlace(t *testing.T) { t.Errorf("clearDir of a missing dir = %v, want nil", err) } } + +func TestHandlerStreamsRunOutput(t *testing.T) { + srv, ws := newBridgeServer(t) + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo one; echo two >&2; exit 2"}, Stream: true}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var stdout, stderr strings.Builder + var frames []RunResponse + for { + var f RunResponse + if err := dec.Decode(&f); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + frames = append(frames, f) + stdout.WriteString(f.Stdout) + stderr.WriteString(f.Stderr) + } + if len(frames) < 2 { + t.Fatalf("frames = %+v", frames) + } + for _, f := range frames[:len(frames)-1] { + if !f.Partial { + t.Fatalf("only the last frame may be final: %+v", frames) + } + } + if last := frames[len(frames)-1]; last.Partial || last.ExitCode != 2 || last.Stdout != "" || last.Stderr != "" { + t.Fatalf("final frame = %+v", last) + } + if stdout.String() != "one\n" || stderr.String() != "two\n" { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestFrameWriterCapsOutput(t *testing.T) { + var buf bytes.Buffer + w := &frameWriter{frames: &frameEncoder{enc: json.NewEncoder(&buf)}} + big := make([]byte, MaxOutput+5) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + buf.Reset() + if n, err := w.Write([]byte("more")); err != nil || n != 4 || buf.Len() != 0 { + t.Fatalf("past the cap: n=%d err=%v frames=%q", n, err, buf.String()) + } +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 13199d6fee..4b9cf5a4fc 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -325,7 +325,7 @@ agents: **What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). -**The terminal pane.** A substrate-runtime agent has no tmux session and, until [#17699](https://github.com/hivecommons/hive/issues/17699), no live pane: the ▶ terminal shows the last persisted transcript like the job viewer between kicks. +**The terminal pane.** A substrate-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer behind the same handoff and role check ([#17777](https://github.com/hivecommons/hive/issues/17777)). The Control API has no `ListActors` and Substrate has no logs RPC, so the launcher — which runs in the same hive process as the dashboard — records each kick for the pane: the actor's `/`, every state it is seen in (`RESUMING`, `RUNNING`, then `SUSPENDED` with `snapshot_on_idle` or `DELETING`), and the worker's SPIFFE id from the actor's worker assignment. The bridge's `POST /run` streams the kick's stdout and stderr when asked (`"stream": true`, newline-delimited `RunResponse` frames; an older bridge answers with one complete response and still works), and the pane follows that output live, scrubbed of token-shaped values like the pod log, with each actor state change as a status line. Unlike OpenShell this is the kick's own output, and the pane ends with the kick's exit code and whether the actor was suspended or deleted. Between kicks it names how the last actor ended above the last persisted transcript; after a hive restart, before the agent's next kick, it shows the transcript alone. **Limitations (phase 2).** Workspace copied, not shared; `crux substrate-bridge` entrypoint and a published sandbox image land with the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); no resume of suspended actors; `api.github.com` not read-only at the policy layer; latency and image size not yet measured. diff --git a/src/pkg/dashboard/job_terminal.go b/src/pkg/dashboard/job_terminal.go index 032f93eff8..43800908c9 100644 --- a/src/pkg/dashboard/job_terminal.go +++ b/src/pkg/dashboard/job_terminal.go @@ -66,8 +66,20 @@ import ( // So while a kick runs the pane shows what the sandbox did around it // (network and filesystem decisions, supervisor state) and then points the // operator at the transcript, which is where the kick's own output and -// exit code land once the launcher's Exec().Run returns. Only the job pod -// log shows the kick's stdout live. +// exit code land once the launcher's Exec().Run returns. The job pod log +// and the substrate stream below show the kick's stdout live. +// +// Substrate runtime (hivecommons/hive#17777). An agent with +// sandbox.runtime: substrate runs each kick in a Substrate actor, again with +// no tmux session, so it uses the same page, route and three states. The +// Control API has no ListActors and Substrate no logs RPC, so the live +// source is sandbox.SubstrateLogReader: the launcher, in this same process, +// records the actor (name, atespace, every state it passes through, the +// worker's SPIFFE id) and tees the kick's stdout and stderr from the +// bridge's streamed /run into it. Unlike OpenShell this is the kick's own +// output and exit code. When the launcher is done the actor is deleted or, +// with snapshot_on_idle, suspended; the pane names which, and between kicks +// shows how the last actor ended above the persisted transcript. const ( // Both live under terminalPathPrefix so the terminal handoff code, @@ -99,6 +111,7 @@ type jobPodLogReader interface { type jobTerminalDeps struct { reader jobPodLogReader openshell sandbox.OpenShellLogReader + substrate sandbox.SubstrateLogReader workspaceRoot string pollInterval time.Duration } @@ -139,6 +152,15 @@ func (s *Server) openshellLogReader(name string) sandbox.OpenShellLogReader { return &sandbox.OpenShellLauncher{Options: opts} } +// substrateLogReader is the substrate counterpart: the process-wide record +// the substrate launcher writes each kick into. +func (s *Server) substrateLogReader() sandbox.SubstrateLogReader { + if s.jobTerminal.substrate != nil { + return s.jobTerminal.substrate + } + return sandbox.DefaultSubstrateKicks +} + func (s *Server) jobTerminalWorkspaceRoot() string { if s.jobTerminal.workspaceRoot != "" { return s.jobTerminal.workspaceRoot @@ -163,11 +185,11 @@ func (s *Server) jobTerminalPollInterval() time.Duration { // runtime runs its kicks outside the hive pod, so it has no tmux session // and its terminal is this read-only viewer. func sandboxRuntimeWithoutTmux(rt string) bool { - return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell + return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell || rt == config.SandboxRuntimeSubstrate } // jobTerminalAgent resolves the path's agent and checks that its effective -// sandbox runtime has no tmux session (job or openshell), returning that +// sandbox runtime has no tmux session (job, openshell or substrate), returning that // runtime. The status code says which precondition failed: 404 for an // unknown agent, 409 for one that has a tmux terminal instead. func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) { @@ -185,7 +207,7 @@ func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) } rt := agentCfg.SandboxRuntime(cfg.AgentSandbox) if !sandboxRuntimeWithoutTmux(rt) { - return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job or OpenShell sandbox; there is no sandbox log to show", name, rt) + return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job, OpenShell sandbox or Substrate actor; there is no sandbox log to show", name, rt) } return name, rt, http.StatusOK, "" } @@ -291,6 +313,10 @@ func (s *Server) handleJobTerminalStream(w http.ResponseWriter, r *http.Request) s.streamOpenShellSandbox(ctx, out, name, s.openshellLogReader(name)) return } + if rt == config.SandboxRuntimeSubstrate { + s.streamSubstrateActor(ctx, out, name, s.substrateLogReader()) + return + } reader := s.jobPodLogReader() pod, found, err := reader.LatestPod(ctx, sandbox.LaunchName(name)) @@ -407,6 +433,89 @@ func (s *Server) streamOpenShellSandbox(ctx context.Context, out *jobTerminalWri } } +// streamSubstrateActor is the substrate-runtime counterpart: name the +// agent's newest actor and its state and SPIFFE id, follow the kick's +// output as the bridge streams it, scrubbed line by line, with each actor +// state transition as a status line, then report the kick's exit code and +// whether the actor was suspended or deleted. A finished kick's actor is +// still named, above the persisted transcript. +func (s *Server) streamSubstrateActor(ctx context.Context, out *jobTerminalWriter, name string, reader sandbox.SubstrateLogReader) { + info, found, err := reader.LatestActor(ctx, sandbox.LaunchName(name)) + if err != nil { + out.status("cannot read Substrate actors for agent %q: %v", name, err) + out.status("showing the last persisted transcript instead") + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + if !found { + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("substrate actor %s", describeSubstrateActor(info)) + if info.Finished() { + out.status("%s", substrateActorEnd(info)) + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("%s", substrateLiveLogNote) + last := info + scrub := &kubejob.LineFilterWriter{Dst: out, Fn: scrubLogLine} + final, err := reader.StreamRun(ctx, info.Name, s.jobTerminalPollInterval(), scrub, func(next sandbox.SubstrateActorInfo) { + if next.State == last.State && next.SpiffeID == last.SpiffeID { + return + } + last = next + out.status("actor %s", describeSubstrateActor(next)) + }) + _ = scrub.Close() + if err != nil { + if ctx.Err() != nil { + return + } + out.status("output stream ended with an error: %v", err) + } + if final.Name == "" { + final.Name = info.Name + } + if final.Exited { + out.status("kick exited with code %d", final.ExitCode) + } + out.status("%s", substrateActorEnd(final)) +} + +// describeSubstrateActor is " · · ", leaving out +// what the launcher has not learned yet. +func describeSubstrateActor(i sandbox.SubstrateActorInfo) string { + parts := []string{i.Name, orUnknown(i.State)} + if i.Atespace != "" { + parts[0] = i.Atespace + "/" + i.Name + } + if i.SpiffeID != "" { + parts = append(parts, i.SpiffeID) + } + return strings.Join(parts, " · ") +} + +// substrateActorEnd says what became of the actor once the launcher was done +// with it, or that it has not finished. +func substrateActorEnd(i sandbox.SubstrateActorInfo) string { + switch { + case i.Suspended: + return fmt.Sprintf("actor %s was suspended with a snapshot (snapshot_on_idle): the kick has finished. %s", i.Name, substrateKickResultNote) + case i.Done: + return fmt.Sprintf("actor %s was deleted by the launcher: the kick has finished. %s", i.Name, substrateKickResultNote) + default: + return fmt.Sprintf("actor %s · %s (exit code not reported). %s", i.Name, orUnknown(i.State), substrateKickResultNote) + } +} + +const ( + // substrateLiveLogNote opens every live substrate stream. + substrateLiveLogNote = "this is the kick's stdout and stderr as the actor's bridge streams /run, scrubbed; actor state changes are shown as they happen" + // substrateKickResultNote closes it. + substrateKickResultNote = "the full output and exit code are in the persisted transcript; reopen this pane to see them" +) + const ( // openshellLiveLogNote opens every live OpenShell stream so an operator // never mistakes the sandbox's event log for the kick's output. @@ -440,11 +549,20 @@ var ( live: "OpenShell sandbox", where: "on the gateway", gone: "the launcher deletes a sandbox when its kick finishes", } + substrateTerms = sandboxTerms{ + title: "substrate actor log", + about: "the kick runs in a Substrate actor, so this is the kick's output as the actor's bridge streams it while it runs and the kick's transcript afterwards", + live: "Substrate actor", where: "started by this hive process", + gone: "the launcher deletes, or with snapshot_on_idle suspends, an actor when its kick finishes", + } ) func sandboxTermsFor(rt string) sandboxTerms { - if rt == config.SandboxRuntimeOpenShell { + switch rt { + case config.SandboxRuntimeOpenShell: return openshellTerms + case config.SandboxRuntimeSubstrate: + return substrateTerms } return jobTerms } diff --git a/src/pkg/dashboard/job_terminal_substrate_test.go b/src/pkg/dashboard/job_terminal_substrate_test.go new file mode 100644 index 0000000000..9db874f5f2 --- /dev/null +++ b/src/pkg/dashboard/job_terminal_substrate_test.go @@ -0,0 +1,250 @@ +package dashboard + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/pkg/config" +) + +// fakeSubstrateLogReader scripts sandbox.SubstrateLogReader: an optional +// recorded actor, the states it passes through, the output its bridge +// streams and the state it ends in. +type fakeSubstrateLogReader struct { + mu sync.Mutex + info sandbox.SubstrateActorInfo + found bool + latestErr error + states []sandbox.SubstrateActorInfo + lines []string + final sandbox.SubstrateActorInfo + streamErr error + launchNames []string + streamed []string + polls []time.Duration +} + +func (f *fakeSubstrateLogReader) LatestActor(_ context.Context, launchName string) (sandbox.SubstrateActorInfo, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.launchNames = append(f.launchNames, launchName) + return f.info, f.found, f.latestErr +} + +func (f *fakeSubstrateLogReader) StreamRun(_ context.Context, name string, poll time.Duration, w io.Writer, onState func(sandbox.SubstrateActorInfo)) (sandbox.SubstrateActorInfo, error) { + f.mu.Lock() + f.streamed = append(f.streamed, name) + f.polls = append(f.polls, poll) + states, lines, final, err := f.states, f.lines, f.final, f.streamErr + f.mu.Unlock() + for _, st := range states { + onState(st) + } + for _, l := range lines { + if _, werr := io.WriteString(w, l); werr != nil { + return final, werr + } + } + return final, err +} + +// substrateTerminalServer adds a substrate-runtime agent "actor" to the +// job-terminal fixture (which already has job "deps", podman "lint" and the +// tmux agent "scanner"). +func substrateTerminalServer(t *testing.T, reader *fakeSubstrateLogReader) (*Server, string) { + t.Helper() + s, root := jobTerminalServer(t, &fakeJobPodLogReader{}) + on := true + s.deps.Config.Agents["actor"] = config.AgentConfig{Backend: "claude", Enabled: true, Sandbox: &config.AgentSandboxOverride{Enabled: &on, Runtime: config.SandboxRuntimeSubstrate}} + s.jobTerminal.substrate = reader + return s, root +} + +const testSpiffeID = "spiffe://ate.local/ns/ate-workers/sa/worker" + +func TestJobTerminalGateAdmitsSubstrateRuntime(t *testing.T) { + if !sandboxRuntimeWithoutTmux(config.SandboxRuntimeSubstrate) { + t.Fatal("substrate agents have no tmux session; the viewer must admit them") + } + s, _ := substrateTerminalServer(t, &fakeSubstrateLogReader{}) + rec := doGet(s, "/terminal/job/actor") + if rec.Code != http.StatusOK { + t.Fatalf("substrate agent page: status = %d (%s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{"actor — substrate actor log", "Substrate actor", "'./actor/stream'"} { + if !strings.Contains(body, want) { + t.Errorf("substrate page missing %q", want) + } + } + if strings.Contains(body, "Kubernetes Job") || strings.Contains(body, "OpenShell") { + t.Fatal("substrate page must not describe the kick as a Job or OpenShell sandbox") + } + if rec := doGet(s, "/terminal/job/lint/stream"); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "Substrate actor") { + t.Fatalf("podman agent stream: status = %d (%s)", rec.Code, rec.Body.String()) + } +} + +func TestSubstrateTerminalStreamsKickWithActorTransitionsScrubbed(t *testing.T) { + token := "ghp_" + strings.Repeat("Z", 30) + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateResuming}, + found: true, + states: []sandbox.SubstrateActorInfo{ + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID}, + // Only the exit code changed: no new status line. + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4}, + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + }, + lines: []string{"step 1\n", "auth " + token + " ok\n", "partial"}, + final: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + } + s, _ := substrateTerminalServer(t, reader) + rec := doGet(s, "/terminal/job/actor/stream") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · RESUMING", + "this is the kick's stdout and stderr as the actor's bridge streams /run", + "actor hive/hive-kick-actor-1 · RUNNING · " + testSpiffeID, + "actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "step 1\n", + "auth ghp_", + "partial", + // Unlike OpenShell, the bridge's exit code is the kick's own. + "kick exited with code 4", + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle): the kick has finished", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if n := strings.Count(body, "· RUNNING ·"); n != 1 { + t.Errorf("RUNNING reported %d times; a state is reported once per transition:\n%s", n, body) + } + if strings.Contains(body, token) { + t.Fatalf("live kick output reached the browser unscrubbed:\n%s", body) + } + reader.mu.Lock() + defer reader.mu.Unlock() + if len(reader.launchNames) != 1 || reader.launchNames[0] != sandbox.LaunchName("actor") { + t.Fatalf("actor must be looked up by the executor's launch name, got %v", reader.launchNames) + } + if len(reader.streamed) != 1 || reader.streamed[0] != "hive-kick-actor-1" || reader.polls[0] != time.Millisecond { + t.Fatalf("streamed %v at %v", reader.streamed, reader.polls) + } +} + +func TestSubstrateTerminalStreamReportsErrorAndDeletedActor(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateDeleting, Done: true}, + streamErr: errors.New("router refused connect"), + } + s, _ := substrateTerminalServer(t, reader) + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"output stream ended with an error: router refused connect", "actor a was deleted by the launcher: the kick has finished. the full output and exit code are in the persisted transcript"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if strings.Contains(body, "kick exited with code") { + t.Fatalf("no exit code was reported; none must be shown:\n%s", body) + } + + reader = &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, + } + s, _ = substrateTerminalServer(t, reader) + if body := doGet(s, "/terminal/job/actor/stream").Body.String(); !strings.Contains(body, "actor b · RUNNING (exit code not reported)") { + t.Fatalf("unfinished stream:\n%s", body) + } +} + +// Between kicks the recorded actor is finished: the pane names it and how +// it ended (suspended with snapshot_on_idle), then shows the transcript. +func TestSubstrateTerminalFinishedActorShowsTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, Suspended: true, Done: true}, + found: true, + } + s, root := substrateTerminalServer(t, reader) + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle)", + path, "from the last kick", "last run exited with code 0", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + reader.mu.Lock() + streamed := reader.streamed + reader.mu.Unlock() + if len(streamed) != 0 { + t.Fatalf("finished actor: nothing must be streamed, got %v", streamed) + } +} + +func TestSubstrateTerminalFallsBackToTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{} + s, root := substrateTerminalServer(t, reader) + + body := doGet(s, "/terminal/job/actor/stream").Body.String() + if !strings.Contains(body, `no sandbox run yet for agent "actor": no Substrate actor started by this hive process`) || !strings.Contains(body, root) { + t.Fatalf("no-run state:\n%s", body) + } + + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{`no Substrate actor for agent "actor" right now`, path, "from the last kick", "last run exited with code 0"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + + reader.latestErr = errors.New("registry unavailable") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"cannot read Substrate actors", "registry unavailable", "from the last kick"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } +} + +func TestSubstrateTerminalDefaultReaderIsLauncherRecord(t *testing.T) { + s, _ := apiServer(t) + if r, ok := s.substrateLogReader().(*sandbox.SubstrateKicks); !ok || r != sandbox.DefaultSubstrateKicks { + t.Fatal("default reader must be the record the substrate launcher writes") + } +} + +func TestStaticTerminalLinkRoutesSubstrateRuntimeToViewer(t *testing.T) { + body, err := os.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + html := string(body) + for _, want := range []string{ + "const SUBSTRATE_SANDBOX_RUNTIME = 'substrate';", + "|| sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME;", + "if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) {", + } { + if !strings.Contains(html, want) { + t.Fatalf("static dashboard substrate-terminal wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index db730d57dd..5253df7de5 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -73,11 +73,15 @@ by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -93,6 +97,9 @@ } function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index fd73ff80e2..21292a873d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -30271,11 +30271,15 @@

All labels

by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -30291,6 +30295,9 @@

All labels

} function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; From 9468b5e0c9c4117a9e2604d2eb6f60764a5447a3 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 18:33:35 -0700 Subject: [PATCH 042/166] =?UTF-8?q?=F0=9F=90=9B=20Mark=20hive's=20in-clust?= =?UTF-8?q?er=20k8s=20API=20dials=20so=20the=20egress=20redirect=20never?= =?UTF-8?q?=20MITMs=20them=20(#17818)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Mark hive's in-cluster k8s API dials so the egress redirect never MITMs them On nodes without the xt_owner netfilter module (IBM Cloud IKS/ROKS) the HIVE_PROXY chain has only the packet-mark self-exemption, so every :443 dial the hive makes for itself must carry SO_MARK. The proxy's upstream dials did; the spoke's own Kubernetes API clients (self-upgrade, startup probe, node metrics, ingress probes, kubejob) did not, so they were REDIRECTed into the MITM proxy, hit the private-range forward guard and got EOF — every wec2 hive failed self-upgrade with 'reading startup probe before upgrade … EOF'. Move the SO_MARK dialer into a new crux/egressmark package (crux/proxy imports crux/agent which imports kubejob, so kubejob cannot import proxy) and use it for all five in-cluster API transports. Signed-off-by: Andrew Anderson * 🌱 Allow crux/egressmark in the dashboard import ratchet Signed-off-by: Andrew Anderson * 🌱 Add crux/egressmark tests for the coverage gate Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../fixed-k8s-api-client-egress-mark.md | 1 + src/crux/egressmark/egressmark.go | 82 +++++++++++++++ src/crux/egressmark/egressmark_test.go | 84 ++++++++++++++++ src/crux/egressmark/somark_linux.go | 11 +++ src/crux/egressmark/somark_other.go | 9 ++ src/crux/kubejob/kubejob.go | 3 +- src/crux/proxy/github_proxy.go | 99 +++---------------- src/crux/proxy/somark_linux.go | 20 ---- src/crux/proxy/somark_other.go | 11 --- src/docs/net-admin-requirement.md | 2 + .../testutil/dashboard_import_allowlist.txt | 2 + src/pkg/dashboard/deployment_upgrade.go | 3 +- src/pkg/hub/spoke/cluster_metrics.go | 4 +- src/pkg/hub/spoke/heartbeat.go | 2 + src/pkg/hub/spoke/self_upgrade.go | 3 +- 15 files changed, 217 insertions(+), 119 deletions(-) create mode 100644 changelog.d/fixed-k8s-api-client-egress-mark.md create mode 100644 src/crux/egressmark/egressmark.go create mode 100644 src/crux/egressmark/egressmark_test.go create mode 100644 src/crux/egressmark/somark_linux.go create mode 100644 src/crux/egressmark/somark_other.go delete mode 100644 src/crux/proxy/somark_linux.go delete mode 100644 src/crux/proxy/somark_other.go diff --git a/changelog.d/fixed-k8s-api-client-egress-mark.md b/changelog.d/fixed-k8s-api-client-egress-mark.md new file mode 100644 index 0000000000..b33246cd5f --- /dev/null +++ b/changelog.d/fixed-k8s-api-client-egress-mark.md @@ -0,0 +1 @@ +- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). diff --git a/src/crux/egressmark/egressmark.go b/src/crux/egressmark/egressmark.go new file mode 100644 index 0000000000..0d70e8fb1d --- /dev/null +++ b/src/crux/egressmark/egressmark.go @@ -0,0 +1,82 @@ +// Package egressmark stamps the hive's OWN outbound sockets with the fwmark +// (SO_MARK) that the entrypoint's forced-egress iptables rule exempts +// (`-m mark --mark -j RETURN`), so control-plane traffic the hive +// itself originates is never redirected into its own MITM proxy. +// +// It is shared by the proxy's upstream dials and by every in-cluster +// Kubernetes API client (self-upgrade, startup-probe backfill, node metrics, +// ingress probes, kubejob). The entrypoint also exempts the hive's uid with +// `-m owner`, but that match needs the xt_owner kernel module, which IBM Cloud +// IKS/ROKS and other OVN nodes do not load: there the mark is the ONLY +// exemption, and an unmarked dial to kubernetes.default.svc:443 lands in the +// proxy, whose private-range forward guard closes it ("EOF" / "unexpected eof +// while reading"), failing every spoke self-upgrade on that cluster. +package egressmark + +import ( + "log/slog" + "net" + "os" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + // DefaultMark is the fwmark the entrypoint exempts when + // HIVE_PROXY_EGRESS_MARK is unset. entrypoint.sh is the single source of + // truth and exports the same value into the hive process. + DefaultMark = 0x1112 + + // Env is the variable entrypoint.sh exports so the Go side and the + // iptables rule stay in lockstep. + Env = "HIVE_PROXY_EGRESS_MARK" +) + +// Mark returns the fwmark to stamp on the hive's own dials, read from +// HIVE_PROXY_EGRESS_MARK (hex with 0x prefix or decimal). It falls back to +// DefaultMark when unset or unparseable: that default matches the entrypoint +// default, so accidental drift stays on the agreed value rather than silently +// disabling the exemption. +func Mark() int { + raw := strings.TrimSpace(os.Getenv(Env)) + if raw == "" { + return DefaultMark + } + if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { + return int(v) + } + return DefaultMark +} + +// Dialer returns a *net.Dialer whose Control hook stamps Mark() on the socket +// before connect. The optional timeout is applied when non-zero. +// +// The mark is BEST-EFFORT: setsockopt(SO_MARK) needs CAP_NET_ADMIN in the +// effective set, and a hive running without it must still dial (on OKE the +// owner-uid exemption covers it). A mark failure is therefore logged once and +// the dial proceeds unmarked instead of aborting. +func Dialer(timeout time.Duration) *net.Dialer { + mark := Mark() + return &net.Dialer{ + Timeout: timeout, + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + if err := setSockMark(fd, mark); err != nil { + warnOnce(mark, err) + } + }) + }, + } +} + +var warnSync sync.Once + +func warnOnce(mark int, err error) { + warnSync.Do(func() { + slog.Warn("egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies where xt_owner is loaded); expected when the hive runs without CAP_NET_ADMIN in its effective set", + "mark", mark, "error", err) + }) +} diff --git a/src/crux/egressmark/egressmark_test.go b/src/crux/egressmark/egressmark_test.go new file mode 100644 index 0000000000..9c3ed7040d --- /dev/null +++ b/src/crux/egressmark/egressmark_test.go @@ -0,0 +1,84 @@ +package egressmark + +import ( + "errors" + "net" + "syscall" + "testing" + "time" +) + +// TestMarkDefaultAndEnvOverride verifies the mark resolves to the entrypoint +// default when unset, honours hex and decimal overrides, and fails safe back +// to the default on garbage or zero rather than disabling the exemption. +func TestMarkDefaultAndEnvOverride(t *testing.T) { + cases := []struct { + name string + env string + want int + }{ + {"unset", "", DefaultMark}, + {"hex", "0x2223", 0x2223}, + {"decimal", "8739", 8739}, + {"garbage falls back", "not-a-number", DefaultMark}, + {"zero falls back", "0", DefaultMark}, + {"negative falls back", "-5", DefaultMark}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(Env, tc.env) + if got := Mark(); got != tc.want { + t.Fatalf("Mark() with %s=%q = %#x, want %#x", Env, tc.env, got, tc.want) + } + }) + } +} + +// TestDialerControlDoesNotBreakDial dials a loopback listener through the +// marked dialer. The Control hook must run and attempt SO_MARK without failing +// the dial: the hive runs without CAP_NET_ADMIN in its effective set in many +// deployments, so setsockopt(SO_MARK) returns EPERM there and a propagated +// error would break every self-upgrade / startup-probe / kubejob API call. +func TestDialerControlDoesNotBreakDial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + }() + + d := Dialer(2 * time.Second) + if d.Timeout != 2*time.Second { + t.Fatalf("Dialer timeout = %v, want 2s", d.Timeout) + } + conn, err := d.Dial("tcp", ln.Addr().String()) + if err != nil { + if errors.Is(err, syscall.EPERM) { + t.Fatalf("Dialer aborted the dial on SO_MARK EPERM — mark must be best-effort: %v", err) + } + t.Fatalf("Dialer Dial failed (Control hook must not break the dial): %v", err) + } + conn.Close() +} + +// TestDialerZeroTimeoutLeavesDefault confirms a zero timeout is passed through +// unchanged so callers that rely on the http.Transport's own deadlines are not +// given a surprise dial timeout. +func TestDialerZeroTimeoutLeavesDefault(t *testing.T) { + if d := Dialer(0); d.Timeout != 0 || d.Control == nil { + t.Fatalf("Dialer(0) = timeout %v, control nil=%v; want 0 and a Control hook", d.Timeout, d.Control == nil) + } +} + +// TestWarnOnceIsIdempotent exercises the once-only warning path directly so a +// mark failure on every dial does not spam the log. +func TestWarnOnceIsIdempotent(t *testing.T) { + warnOnce(DefaultMark, errors.New("first")) + warnOnce(DefaultMark, errors.New("second")) +} diff --git a/src/crux/egressmark/somark_linux.go b/src/crux/egressmark/somark_linux.go new file mode 100644 index 0000000000..0e7ca6e503 --- /dev/null +++ b/src/crux/egressmark/somark_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package egressmark + +import "syscall" + +// setSockMark sets SO_MARK on the raw socket so the kernel stamps every packet +// it emits with the fwmark the entrypoint's nat/OUTPUT RETURN rule matches. +func setSockMark(fd uintptr, mark int) error { + return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) +} diff --git a/src/crux/egressmark/somark_other.go b/src/crux/egressmark/somark_other.go new file mode 100644 index 0000000000..9cb20460a3 --- /dev/null +++ b/src/crux/egressmark/somark_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package egressmark + +// setSockMark is a no-op off Linux: SO_MARK is Linux-only, but the package must +// still compile on the CI matrix and developer machines. +func setSockMark(fd uintptr, mark int) error { + return nil +} diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index 5a7001bc59..bc24f16b9b 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -41,6 +41,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/sandbox" ) @@ -779,6 +780,6 @@ func (l *Launcher) client() (*http.Client, error) { if !pool.AppendCertsFromPEM(pem) { return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates") } - l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}} return l.HTTPClient, nil } diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 35a1ad0844..c47abfedf8 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -11,15 +11,14 @@ import ( "net" "net/http" "os" - "strconv" "strings" "sync" - "syscall" "time" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/inferencehealth" "github.com/hivecommons/hive/crux/upstreamproxy" ) @@ -38,94 +37,26 @@ const ( maxViolationLog = 1000 maxGitHubWriteBodyScan = 2 << 20 - // defaultProxyEgressMark is the fwmark (SO_MARK) the proxy stamps onto its - // OWN upstream :443 dials so the forced-egress iptables redirect can exempt - // them (via `-m mark --mark -j RETURN`) and avoid looping the proxy's - // traffic back into itself. - // - // This replaces the `-m owner --uid-owner` self-exemption, which needs the - // xt_owner kernel module — absent on OpenShift/OVN nodes. A packet mark works - // on both OKE and OpenShift. The value is arbitrary but MUST match the mark - // the entrypoint exempts; entrypoint.sh is the single source of truth and - // exports HIVE_PROXY_EGRESS_MARK (defaulting to this same value) into the - // proxy process, which reads it via proxyEgressMark(). - defaultProxyEgressMark = 0x1112 - - // proxyEgressMarkEnv is the env var the entrypoint exports to keep the proxy - // and the iptables rule in sync from one source of truth. - proxyEgressMarkEnv = "HIVE_PROXY_EGRESS_MARK" + // defaultProxyEgressMark and proxyEgressMarkEnv mirror crux/egressmark, + // the shared home of the SO_MARK self-exemption (also used by the hive's + // in-cluster Kubernetes API clients). entrypoint.sh remains the single + // source of truth for the value. + defaultProxyEgressMark = egressmark.DefaultMark + proxyEgressMarkEnv = egressmark.Env ) -// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials, -// read from HIVE_PROXY_EGRESS_MARK (set by entrypoint.sh) so the Go proxy and -// the iptables exemption stay in lockstep. Accepts hex (0x1112) or decimal. -// Falls back to defaultProxyEgressMark when unset or unparseable — that default -// matches the entrypoint default, so an accidental drift fails safe by staying -// on the agreed value rather than silently disabling the exemption. +// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials; +// see egressmark.Mark. func proxyEgressMark() int { - raw := strings.TrimSpace(os.Getenv(proxyEgressMarkEnv)) - if raw == "" { - return defaultProxyEgressMark - } - // ParseInt with base 0 honours a 0x prefix (hex) and plain decimal. - if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { - return int(v) - } - return defaultProxyEgressMark + return egressmark.Mark() } -// markDialer returns a *net.Dialer whose Control hook stamps the proxy egress -// fwmark (SO_MARK) on the outbound socket before connect. On non-Linux builds -// setSockMark is a no-op, so the dialer is portable. The optional timeout is -// applied when non-zero. -// -// SO_MARK is a BEST-EFFORT egress hint, never a hard requirement: it exists so -// the forced-egress iptables redirect can exempt the proxy's own upstream dials -// via `-m mark`. But setsockopt(SO_MARK) needs CAP_NET_ADMIN in the process's -// EFFECTIVE set, and the hive Go process runs as a non-root user (entrypoint -// drops from root to `dev` via gosu, which strips all capabilities). So the mark -// call fails with EPERM ("operation not permitted") on exactly the deployments -// that run non-root — which is all of them. Returning that error from the -// Control hook ABORTS the dial, so every proxy upstream dial fails and agents -// can't reach GitHub at all. That is a regression: on OKE the `-m owner -// --uid-owner` exemption already covers the proxy's uid dials, so an unmarked -// dial succeeds; on OpenShift (no xt_owner) the mark is the only exemption, but -// there too, failing the dial helps nothing. So we log the mark failure ONCE and -// proceed unmarked rather than killing the connection. +// markDialer returns a *net.Dialer whose sockets carry the proxy egress fwmark +// so the forced-egress REDIRECT exempts them. The mark is best-effort: without +// CAP_NET_ADMIN the dial proceeds unmarked (owner-UID exemption on OKE); see +// egressmark.Dialer for the full rationale. func markDialer(timeout time.Duration) *net.Dialer { - mark := proxyEgressMark() - return &net.Dialer{ - Timeout: timeout, - Control: func(_, _ string, c syscall.RawConn) error { - // A failure to enter the raw-conn control context is a real dial - // problem (the fd is unusable) — propagate it. - if err := c.Control(func(fd uintptr) { - if markErr := setSockMark(fd, mark); markErr != nil { - warnSockMarkOnce(mark, markErr) - } - }); err != nil { - return err - } - // Deliberately do NOT propagate a setSockMark failure: the mark is a - // best-effort exemption hint, and the owner-UID iptables RETURN - // already exempts the proxy's own uid on OKE, so an unmarked dial - // still reaches GitHub. - return nil - }, - } -} - -// warnSockMarkOnce logs the first SO_MARK failure at WARN and every subsequent -// one at DEBUG, so a non-CAP_NET_ADMIN deployment (the common non-root case) -// does not flood the log with one line per dial while still surfacing the -// condition once. -var sockMarkWarnOnce sync.Once - -func warnSockMarkOnce(mark int, err error) { - sockMarkWarnOnce.Do(func() { - slog.Warn("proxy egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies on OKE); this is expected when the hive runs non-root without CAP_NET_ADMIN in its effective set", - "mark", mark, "error", err) - }) + return egressmark.Dialer(timeout) } // CACertPath and caKeyPath are the PVC locations of the persisted MITM CA. diff --git a/src/crux/proxy/somark_linux.go b/src/crux/proxy/somark_linux.go deleted file mode 100644 index b7b0158b68..0000000000 --- a/src/crux/proxy/somark_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build linux - -package proxy - -import ( - "syscall" -) - -// setSockMark sets SO_MARK on the raw socket file descriptor. The kernel then -// stamps every packet the socket emits with this fwmark, which the entrypoint's -// iptables nat/OUTPUT rule matches (-m mark --mark -j RETURN) to exempt -// the proxy's OWN upstream traffic from the :443 -> :18443 redirect, so it does -// not loop back into itself. -// -// Unlike `-m owner --uid-owner`, packet marks do NOT require the xt_owner kernel -// module, which is absent on OpenShift/OVN nodes. The mark exemption is the -// single self-exemption mechanism that works on both OKE and OpenShift. -func setSockMark(fd uintptr, mark int) error { - return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) -} diff --git a/src/crux/proxy/somark_other.go b/src/crux/proxy/somark_other.go deleted file mode 100644 index e5fee60dc7..0000000000 --- a/src/crux/proxy/somark_other.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !linux - -package proxy - -// setSockMark is a no-op on non-Linux platforms: SO_MARK is a Linux-only socket -// option. The proxy only runs inside the Linux container, but the package must -// still compile on the CI build matrix (and on developer darwin machines), so -// this stub keeps the mark-setting dialer portable. -func setSockMark(fd uintptr, mark int) error { - return nil -} diff --git a/src/docs/net-admin-requirement.md b/src/docs/net-admin-requirement.md index d0a9e06078..aea764ad62 100644 --- a/src/docs/net-admin-requirement.md +++ b/src/docs/net-admin-requirement.md @@ -127,6 +127,8 @@ match no traffic even mid-probe) and is unconditionally torn down: | `xt_REDIRECT` | the `REDIRECT` target that forces `:443` through the proxy | **FATAL, exit 77** | | `xt_owner` | owner-UID self-exemption (`-m owner`) | WARN only — optional by design; OpenShift/OVN runs without it and the mark exemption carries the proxy's egress | +Without `xt_owner` the mark is the only exemption, so every outbound `:443` dial the hive makes for itself must carry it. The proxy's upstream dials and the in-cluster Kubernetes API clients (self-upgrade, startup-probe backfill, node metrics, ingress probes, kubejob) share `crux/egressmark.Dialer`; an unmarked dial to `kubernetes.default.svc` would be redirected into the proxy and closed by its private-range forward guard (`EOF`), which is how IBM Cloud IKS spokes failed every self-upgrade with `reading startup probe before upgrade … EOF`. + When a required module is missing, the FATAL names it explicitly: ``` diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 43b8a642cf..63c7035f19 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -8,9 +8,11 @@ # seam or be added here with PR justification; removed imports must be # deleted so the ratchet tightens. Entries are sorted. # workflowmodel: /api/workflows handlers compile lifecycle models (Compile, Profile diagnostics) and the tracker emits per-repo workflow columns; the store and bindings already route through Dependencies (hivecommons/hive#17620 P2). +# egressmark: deployment_upgrade.go dials the in-cluster Kubernetes API for the hive's own rollout; the dial must carry the forced-egress SO_MARK or nodes without xt_owner redirect it into the MITM proxy (hivecommons/hive#17817). github.com/hivecommons/hive/crux github.com/hivecommons/hive/crux/agentparse github.com/hivecommons/hive/crux/claude +github.com/hivecommons/hive/crux/egressmark github.com/hivecommons/hive/crux/kubejob github.com/hivecommons/hive/crux/logscrub github.com/hivecommons/hive/crux/paneclass diff --git a/src/pkg/dashboard/deployment_upgrade.go b/src/pkg/dashboard/deployment_upgrade.go index 1c15e90f3b..61f21bbec3 100644 --- a/src/pkg/dashboard/deployment_upgrade.go +++ b/src/pkg/dashboard/deployment_upgrade.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" ) @@ -356,7 +357,7 @@ func kubernetesSelfUpgradeRBACPrecheck() error { } client := &http.Client{ Timeout: selfUpgradePrecheckTimeout, - Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}}, + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}, } path := fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/%s", namespace, selfUpgradeDeploymentName) if err := kubernetesSelfUpgradeRequest(client, http.MethodGet, path, nil, token); err != nil { diff --git a/src/pkg/hub/spoke/cluster_metrics.go b/src/pkg/hub/spoke/cluster_metrics.go index aef70da225..031bc03a0c 100644 --- a/src/pkg/hub/spoke/cluster_metrics.go +++ b/src/pkg/hub/spoke/cluster_metrics.go @@ -14,6 +14,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/crux/egressmark" ) const ( @@ -547,7 +549,7 @@ func k8sAPIGetWithContext(ctx context.Context, path string) ([]byte, error) { client := &http.Client{ Timeout: metricsCollectionTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(ctx, metricsCollectionTimeout) diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 705efdbaa7..5ab3c0990e 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -23,6 +23,7 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" "github.com/hivecommons/hive/pkg/hivestate" @@ -2279,6 +2280,7 @@ func inClusterHTTPClient(cfg *inClusterConfig) (*http.Client, error) { Timeout: routeExistenceProbeTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + DialContext: egressmark.Dialer(0).DialContext, }, }, nil } diff --git a/src/pkg/hub/spoke/self_upgrade.go b/src/pkg/hub/spoke/self_upgrade.go index d699fa2bec..d21d0d23b1 100644 --- a/src/pkg/hub/spoke/self_upgrade.go +++ b/src/pkg/hub/spoke/self_upgrade.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/imageref" ) @@ -686,7 +687,7 @@ func k8sAPIPatchOnce(path string, body []byte) error { client := &http.Client{ Timeout: selfUpgradePatchTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(context.Background(), selfUpgradePatchTimeout) From 16d910e595acac2e9e2d6713b56d31c9bed02caa Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 18:36:45 -0700 Subject: [PATCH 043/166] =?UTF-8?q?=F0=9F=90=9B=20hub:=20stop=20flagging?= =?UTF-8?q?=20off-cadence=20and=20scheduled=20agents=20as=20idle-with-work?= =?UTF-8?q?=20(#17825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit classifyInactiveAgent ignored AgentSummary.ExpectedActive, so an agent with no cadence in the governor's current mode (e.g. redteam while quiet) was alerted as idle whenever the hive had queued work. Spokes also reported KickIntervalSec=0 for times/cron cadences, dropping the hub to the 45m rule between slots. Skip the idle rule when the spoke says the agent is not expected active, and report the gap from the last fired slot to the next as the kick interval for scheduled cadences. Signed-off-by: Andrew Anderson --- ...xed-hub-idle-off-cadence-false-positive.md | 1 + src/pkg/hub/agent_inactivity.go | 7 +++++ src/pkg/hub/agent_inactivity_test.go | 21 ++++++++++++++ src/pkg/hub/spoke/heartbeat_activity.go | 29 ++++++++++++++++++- src/pkg/hub/spoke/heartbeat_activity_test.go | 25 ++++++++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-hub-idle-off-cadence-false-positive.md diff --git a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md new file mode 100644 index 0000000000..dfd16fe914 --- /dev/null +++ b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md @@ -0,0 +1 @@ +- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. diff --git a/src/pkg/hub/agent_inactivity.go b/src/pkg/hub/agent_inactivity.go index 0cd7fb41e4..3531af4a61 100644 --- a/src/pkg/hub/agent_inactivity.go +++ b/src/pkg/hub/agent_inactivity.go @@ -153,6 +153,13 @@ func classifyInactiveAgent(a AgentSummary, queuedWork int, now time.Time) agentI // turns a healthy fresh digest into "idle with work queued". return agentInactiveNone } + if !legacyAgent(a) && !a.ExpectedActive { + // A new-protocol spoke that says the agent is not expected active in + // the governor's current mode (no cadence, or a paused one — e.g. + // redteam in quiet mode) will never kick it; sitting idle is the + // configured behaviour, not a fault. + return agentInactiveNone + } lastActivity, activityOK := parseAgentTime(a.LastActivityAt) if !activityOK { // The spoke has never observed a pane change for this agent. That is diff --git a/src/pkg/hub/agent_inactivity_test.go b/src/pkg/hub/agent_inactivity_test.go index 39aec10838..a1fbba6c37 100644 --- a/src/pkg/hub/agent_inactivity_test.go +++ b/src/pkg/hub/agent_inactivity_test.go @@ -188,6 +188,27 @@ func TestIdleRuleRespectsKickCadence(t *testing.T) { } } +func TestIdleRuleSkipsOffCadenceAgents(t *testing.T) { + now := time.Now() + a := AgentSummary{ + Name: "redteam", State: "running", Backend: "copilot", + Enabled: true, CanOpenIssue: true, CanOpenPR: true, + StartedAt: settled(now), LastActivityAt: activeAt(now, 6*time.Hour), + } + + // Off-cadence in the governor's current mode (projectbluefin redteam in + // quiet mode): the governor never kicks it, so idleness is by design. + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveNone { + t.Fatalf("off-cadence agent classified %v; want none", got.label()) + } + + // Same agent once the mode gives it a cadence is held to the idle rule. + a.ExpectedActive = true + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveIdleWithWork { + t.Fatalf("expected-active idle agent classified %v; want idle-with-work", got.label()) + } +} + func TestIdleRuleSkipsWriteIncapableAdvisoryAgents(t *testing.T) { now := time.Now() a := AgentSummary{ diff --git a/src/pkg/hub/spoke/heartbeat_activity.go b/src/pkg/hub/spoke/heartbeat_activity.go index 566c4fd5fc..bdcec1e543 100644 --- a/src/pkg/hub/spoke/heartbeat_activity.go +++ b/src/pkg/hub/spoke/heartbeat_activity.go @@ -122,12 +122,39 @@ func HeartbeatKickInterval(govState governor.State, name string, proc *crux.Proc return 0 } cadence, ok := govState.Cadences[name] - if !ok || cadence.Paused || cadence.Interval <= 0 { + if !ok || cadence.Paused { + return 0 + } + if cadence.Schedule != "" && cadence.Schedule.Mode() != config.CadenceModeInterval { + return scheduledKickGap(cadence, govState.LastKick[name], time.Now()) + } + if cadence.Interval <= 0 { return 0 } return cadence.Interval } +// scheduledKickGap is the cadence evidence for a times/cron agent: the gap +// from the slot it last fired at to the slot it fires at next. The hub's +// idle rule compares pane quiet against this (plus slack), so without it a +// "09:00,12:00,15:00 weekdays" agent reads as idle-with-work every evening +// and all weekend — exactly when it is supposed to be quiet. With no fired +// slot yet the gap runs from now to the first slot. Zero means no evidence. +func scheduledKickGap(cadence governor.AgentCadence, lastKick, now time.Time) time.Duration { + base := cadence.LastOccurrence + if base.IsZero() { + base = lastKick + } + if base.IsZero() || base.After(now) { + base = now + } + next, ok := cadence.Schedule.NextAfter(base) + if !ok || !next.After(base) { + return 0 + } + return next.Sub(base) +} + func QuotaExhaustedAgentCount(agents []AgentSummary) int { count := 0 for _, a := range agents { diff --git a/src/pkg/hub/spoke/heartbeat_activity_test.go b/src/pkg/hub/spoke/heartbeat_activity_test.go index fd0e6053cc..eb8100c9a9 100644 --- a/src/pkg/hub/spoke/heartbeat_activity_test.go +++ b/src/pkg/hub/spoke/heartbeat_activity_test.go @@ -102,6 +102,31 @@ func TestHeartbeatKickInterval(t *testing.T) { } } +func TestScheduledKickGapSpansSlots(t *testing.T) { + // Weekday 09:00/12:00/15:00 UTC. Last fired Friday 15:00 → next slot is + // Monday 09:00, so the gap must cover the whole weekend, not 3h. + cad := governor.AgentCadence{Schedule: config.Cadence(`tod:{"times":["09:00","12:00","15:00"],"tz":"UTC","days":["mon","tue","wed","thu","fri"]}`)} + if err := cad.Schedule.Validate(); err != nil { + t.Fatalf("fixture cadence invalid: %v", err) + } + fri := time.Date(2026, 11, 13, 15, 0, 0, 0, time.UTC) + cad.LastOccurrence = fri + if got := scheduledKickGap(cad, fri, fri.Add(5*time.Hour)); got != 66*time.Hour { + t.Errorf("weekend gap = %v, want 66h", got) + } + // Mid-day slot: 09:00 → 12:00. + cad.LastOccurrence = time.Date(2026, 11, 11, 9, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, cad.LastOccurrence.Add(time.Hour)); got != 3*time.Hour { + t.Errorf("intra-day gap = %v, want 3h", got) + } + // Never fired: gap runs from now to the first slot. + cad.LastOccurrence = time.Time{} + now := time.Date(2026, 11, 11, 7, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, now); got != 2*time.Hour { + t.Errorf("unfired gap = %v, want 2h", got) + } +} + func TestQuotaExhaustedCountsAndReasons(t *testing.T) { statuses := map[string]*crux.Process{ "counted": {State: crux.StateRunning, QuotaExhausted: true}, From 8bb5332ea895a2d786f1353c4c388480ee268e3c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:50:48 +0000 Subject: [PATCH 044/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20use=20unique=20te?= =?UTF-8?q?mp=20files=20when=20persisting=20hive=20and=20reach=20state=20(?= =?UTF-8?q?#17740)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use unique temp files when persisting hive and reach state Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: re-run PR verifier after cancelled concurrent run Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: refresh head to clear a Copilot PR Automation run stuck in queued Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-state-persist-rename-race.md | 1 + src/pkg/snapshot/state.go | 20 +++++++++++-- src/pkg/snapshot/state_test.go | 29 +++++++++++++++++++ src/pkg/tracing/reach.go | 20 +++++++++++-- src/pkg/tracing/reach_test.go | 28 ++++++++++++++++++ 5 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-state-persist-rename-race.md diff --git a/changelog.d/fixed-state-persist-rename-race.md b/changelog.d/fixed-state-persist-rename-race.md new file mode 100644 index 0000000000..cbe357487b --- /dev/null +++ b/changelog.d/fixed-state-persist-rename-race.md @@ -0,0 +1 @@ +- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). diff --git a/src/pkg/snapshot/state.go b/src/pkg/snapshot/state.go index fdc7230281..419489f0bc 100644 --- a/src/pkg/snapshot/state.go +++ b/src/pkg/snapshot/state.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "time" "github.com/hivecommons/hive/pkg/config" @@ -183,11 +184,26 @@ func SaveState(path string, state *PersistedState, logger *slog.Logger) error { return fmt.Errorf("marshaling state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // A unique temp name per call: concurrent savers sharing path+".tmp" + // would rename each other's file away and fail with ENOENT (#17726). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("writing temp state file: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("writing temp state file: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("renaming state file: %w", err) } diff --git a/src/pkg/snapshot/state_test.go b/src/pkg/snapshot/state_test.go index 2d9905f7e2..61b6de9a78 100644 --- a/src/pkg/snapshot/state_test.go +++ b/src/pkg/snapshot/state_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -239,3 +240,31 @@ func TestLoadState_ReadErrorNotMissing(t *testing.T) { t.Error("state should be nil on read error") } } + +func TestSaveState_ConcurrentWritersDoNotRace(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveState(path, &PersistedState{Agents: map[string]AgentState{}}, testLogger()) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveState: %v", err) + } + } + if _, err := LoadState(path, testLogger()); err != nil { + t.Fatalf("LoadState after concurrent saves: %v", err) + } + leftovers, _ := filepath.Glob(path + "*.tmp") + if len(leftovers) != 0 { + t.Fatalf("leftover temp files: %v", leftovers) + } +} diff --git a/src/pkg/tracing/reach.go b/src/pkg/tracing/reach.go index 78caf7461e..7bdb660b81 100644 --- a/src/pkg/tracing/reach.go +++ b/src/pkg/tracing/reach.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "sort" "sync" "sync/atomic" @@ -388,11 +389,26 @@ func SaveReachState(path string) error { if err != nil { return fmt.Errorf("marshal reach state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // Unique temp name per call so concurrent savers cannot rename each + // other's file away (ENOENT, #17728). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("write reach state: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("write reach state: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("rename reach state: %w", err) } return nil diff --git a/src/pkg/tracing/reach_test.go b/src/pkg/tracing/reach_test.go index bb5feb4a31..8a8da61d02 100644 --- a/src/pkg/tracing/reach_test.go +++ b/src/pkg/tracing/reach_test.go @@ -311,3 +311,31 @@ func TestReach_WindowRollover(t *testing.T) { c.winStart.Load(), c.winTotal.Load(), c.winErrors.Load()) } } + +func TestSaveReachState_ConcurrentWritersDoNotRace(t *testing.T) { + resetReach(t) + _, span := StartSpan(context.Background(), "governor.eval_cycle") + span.End() + path := filepath.Join(t.TempDir(), "reach-state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveReachState(path) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveReachState: %v", err) + } + } + entries, _ := filepath.Glob(path + "*.tmp") + if len(entries) != 0 { + t.Fatalf("leftover temp files: %v", entries) + } +} From 607441fe04b73dbc015ef47ad32e1c28d7107c45 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:53:24 +0000 Subject: [PATCH 045/166] =?UTF-8?q?=F0=9F=90=9B=20hub=20heartbeat:=20back?= =?UTF-8?q?=20off=20and=20classify=20a=20permanent=20403=20rejection=20(#1?= =?UTF-8?q?7823)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hub heartbeat: back off and classify a permanent 403 rejection instead of retrying every tick Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 fleetreport: add threshold case for hub_heartbeat_rejected_unprovisioned TestEveryRuntimeClassTriggerAndNonTriggerThresholds requires every class in Classes to have a trigger/non-trigger case; the new class was missing one. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-hub-heartbeat-403-backoff.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/fleetreport/runtime.go | 2 + src/pkg/fleetreport/runtime_test.go | 1 + src/pkg/hub/spoke/heartbeat.go | 34 ++++++++++ src/pkg/hub/spoke/heartbeat_testhooks.go | 1 + .../hub/spoke/heartbeat_unprovisioned_test.go | 63 +++++++++++++++++++ 7 files changed, 103 insertions(+) create mode 100644 changelog.d/fixed-hub-heartbeat-403-backoff.md create mode 100644 src/pkg/hub/spoke/heartbeat_unprovisioned_test.go diff --git a/changelog.d/fixed-hub-heartbeat-403-backoff.md b/changelog.d/fixed-hub-heartbeat-403-backoff.md new file mode 100644 index 0000000000..cf59938b9a --- /dev/null +++ b/changelog.d/fixed-hub-heartbeat-403-backoff.md @@ -0,0 +1 @@ +- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index bd96ad2143..ad7f77ae2a 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -145,6 +145,7 @@ title is `[selfreport] : `. | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index c0b7a6779d..9e9b3c3c48 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -77,6 +77,7 @@ const ( ClassBeadFactIngestFailed = "bead_fact_ingest_failed" ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" ClassInferenceUpstreamError = "inference_upstream_error" @@ -197,6 +198,7 @@ var Classes = map[string]ClassSpec{ ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, + ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 1593918264..da3b933e4a 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -453,6 +453,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, + {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 5ab3c0990e..86c71807b5 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1528,11 +1528,38 @@ func StartHeartbeat(ctx context.Context, hubURL string, collect StatusCollector, logger.Info("hub heartbeat stopped") return case <-ticker.C: + if heartbeatUnprovisionedBackoffActive(time.Now()) { + // Keep the attempt clock advancing so /api/livez does not + // read the deliberate pause as a stalled loop. + recordHeartbeatAttempt() + continue + } processHeartbeatResponse(sendHeartbeat(ctx, hubURL, collect, logger, freshCollect)) } } } +// heartbeatUnprovisionedBackoff is how long the loop pauses after the hub +// answers 403 "hosted/saas hive IDs can only be created via provisioning". +// The hub never auto-creates hosted IDs, so retrying every tick cannot +// succeed until an operator provisions (or the hive is re-IDed); a slow probe +// still notices when that happens. A var so tests can shorten it. +var heartbeatUnprovisionedBackoff = 10 * time.Minute + +// heartbeatUnprovisionedUntil is the UnixNano deadline of that pause (0 = none). +var heartbeatUnprovisionedUntil atomic.Int64 + +func heartbeatUnprovisionedBackoffActive(now time.Time) bool { + until := heartbeatUnprovisionedUntil.Load() + return until != 0 && now.UnixNano() < until +} + +// hubHeartbeatRejectsUnprovisioned reports whether the hub refused the beat +// because it does not know this hosted/saas hive ID (a permanent condition). +func hubHeartbeatRejectsUnprovisioned(status int, body string) bool { + return status == http.StatusForbidden && strings.Contains(body, "can only be created via provisioning") +} + // waitForReady's knobs are vars (not consts) for the same reason as // taskPushInterval: tests need a StartHeartbeat that reaches its send loop in // milliseconds, not after a 3-minute readiness wait. Production keeps the @@ -1843,6 +1870,12 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa if resp.StatusCode >= 300 { respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + if hubHeartbeatRejectsUnprovisioned(resp.StatusCode, string(respBody)) { + heartbeatUnprovisionedUntil.Store(time.Now().Add(heartbeatUnprovisionedBackoff).UnixNano()) + logger.Warn("hub heartbeat rejected: hive id is not provisioned on the hub — backing off", + "status", resp.StatusCode, "body", string(respBody), "retry_in", heartbeatUnprovisionedBackoff.String()) + return nil + } logger.Warn("hub heartbeat rejected", "status", resp.StatusCode, "body", string(respBody)) if hubHeartbeatStatusIsDefect(resp.StatusCode) { fleetreport.Observe(fleetreport.ClassUnexpectedHub5xx, "heartbeat", fmt.Sprintf("hub heartbeat returned HTTP %d", resp.StatusCode), string(respBody)) @@ -1850,6 +1883,7 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa return nil } fleetreport.Clear(fleetreport.ClassUnexpectedHub5xx, "heartbeat") + heartbeatUnprovisionedUntil.Store(0) // The hub accepted the beat (status < 300) — this is what /api/livez // treats as "the heartbeat goroutine is alive and doing its job", so diff --git a/src/pkg/hub/spoke/heartbeat_testhooks.go b/src/pkg/hub/spoke/heartbeat_testhooks.go index 87216c3f17..48743a0619 100644 --- a/src/pkg/hub/spoke/heartbeat_testhooks.go +++ b/src/pkg/hub/spoke/heartbeat_testhooks.go @@ -29,6 +29,7 @@ func SetHeartbeatStateForTest(enabled bool, lastAttempt, lastSuccess time.Time) // still had fresh stats to fall back on. func ResetHeartbeatStateForTest() { SetHeartbeatStateForTest(false, time.Time{}, time.Time{}) + heartbeatUnprovisionedUntil.Store(0) lastGoodPayload.Lock() // Invalidate any collect goroutine still running from an earlier test // (same lock as its late store) so it cannot repopulate the cache. diff --git a/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go new file mode 100644 index 0000000000..4aa9227c4d --- /dev/null +++ b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go @@ -0,0 +1,63 @@ +package spoke + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/fleetreport" +) + +func TestPostHeartbeat_UnprovisionedForbiddenBacksOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "hosted/saas hive IDs can only be created via provisioning", http.StatusForbidden) + })) + defer srv.Close() + + if resp := postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()); resp != nil { + t.Fatalf("rejected beat returned a response: %+v", resp) + } + if !heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("expected backoff after unprovisioned 403") + } + if heartbeatUnprovisionedBackoffActive(time.Now().Add(heartbeatUnprovisionedBackoff + time.Second)) { + t.Fatal("backoff must expire") + } +} + +func TestPostHeartbeat_OtherForbiddenDoesNotBackOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "bad secret", http.StatusForbidden) + })) + defer srv.Close() + + postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()) + if heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("a 403 without the provisioning body must not back off") + } +} + +func TestHubHeartbeatRejectsUnprovisioned(t *testing.T) { + body := "hosted/saas hive IDs can only be created via provisioning" + if !hubHeartbeatRejectsUnprovisioned(http.StatusForbidden, body) { + t.Error("403 + body should match") + } + if hubHeartbeatRejectsUnprovisioned(http.StatusUnauthorized, body) { + t.Error("401 must not match") + } +} + +func TestUnprovisionedWarnIsLogFedClass(t *testing.T) { + msg := "hub heartbeat rejected: hive id is not provisioned on the hub — backing off" + if got := fleetreport.LogFedClass(msg); got != fleetreport.ClassHubHeartbeatUnprovisioned { + t.Fatalf("LogFedClass = %q", got) + } +} From 0616380793a672a7b47e633412a0ff78d13a5d5b Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 02:05:31 +0000 Subject: [PATCH 046/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.227.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.227.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 16 ++++++++++++++++ .../added-terminal-pane-substrate-kicks.md | 1 - .../fixed-hub-idle-off-cadence-false-positive.md | 1 - changelog.d/fixed-k8s-api-client-egress-mark.md | 1 - .../security-marketplace-refresh-corpus-only.md | 1 - changelog.d/security-substrate-unpack-symlink.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-terminal-pane-substrate-kicks.md delete mode 100644 changelog.d/fixed-hub-idle-off-cadence-false-positive.md delete mode 100644 changelog.d/fixed-k8s-api-client-egress-mark.md delete mode 100644 changelog.d/security-marketplace-refresh-corpus-only.md delete mode 100644 changelog.d/security-substrate-unpack-symlink.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4da2380372..1481511f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.227.0) + +### Added + +- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. + +### Fixed + +- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. +- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). + +### Security + +- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. +- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. + ## 2026-11-12 (v5.226.0) ### Added diff --git a/changelog.d/added-terminal-pane-substrate-kicks.md b/changelog.d/added-terminal-pane-substrate-kicks.md deleted file mode 100644 index 1514ab649f..0000000000 --- a/changelog.d/added-terminal-pane-substrate-kicks.md +++ /dev/null @@ -1 +0,0 @@ -- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. diff --git a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md b/changelog.d/fixed-hub-idle-off-cadence-false-positive.md deleted file mode 100644 index dfd16fe914..0000000000 --- a/changelog.d/fixed-hub-idle-off-cadence-false-positive.md +++ /dev/null @@ -1 +0,0 @@ -- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. diff --git a/changelog.d/fixed-k8s-api-client-egress-mark.md b/changelog.d/fixed-k8s-api-client-egress-mark.md deleted file mode 100644 index b33246cd5f..0000000000 --- a/changelog.d/fixed-k8s-api-client-egress-mark.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). diff --git a/changelog.d/security-marketplace-refresh-corpus-only.md b/changelog.d/security-marketplace-refresh-corpus-only.md deleted file mode 100644 index 9c888537a7..0000000000 --- a/changelog.d/security-marketplace-refresh-corpus-only.md +++ /dev/null @@ -1 +0,0 @@ -- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. diff --git a/changelog.d/security-substrate-unpack-symlink.md b/changelog.d/security-substrate-unpack-symlink.md deleted file mode 100644 index 28e56900e6..0000000000 --- a/changelog.d/security-substrate-unpack-symlink.md +++ /dev/null @@ -1 +0,0 @@ -- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. From 6deac859a40a808d82f6e7d0bc50d1d8d08c15e1 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 19:19:13 -0700 Subject: [PATCH 047/166] =?UTF-8?q?=E2=9C=A8=20knowledge:=20Plaud=20voice?= =?UTF-8?q?=20notes=20as=20a=20knowledge=20source=20(#17821)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: Plaud voice notes as a knowledge source Add a plaud knowledge connector that reads Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server over stdio, with a minimal JSON-RPC MCP client, Connect/Disconnect sign-in on the Knowledge Integrations card, docs and tests. Fixes #17820 Signed-off-by: Andrew Anderson * 🌱 knowledge/plaud: port dashboard edits to static-src, document auth routes in openapi.json, drop unused field Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-knowledge-plaud-source.md | 1 + dashboard/openapi.json | 373 ++++++++++++++++ src/docs/api-reference/knowledge.md | 3 + src/docs/api-reference/packs-and-acmm.md | 2 +- src/docs/knowledge-connectors.md | 89 +++- src/pkg/dashboard/api.go | 1 + .../dashboard/api_knowledge_connector_auth.go | 114 +++++ .../api_knowledge_connector_auth_test.go | 81 ++++ .../knowledge_integrations_modal_test.go | 42 ++ .../dashboard/static-src/js/config-state.js | 1 + .../dashboard/static-src/js/config-tabs.js | 5 +- src/pkg/dashboard/static-src/js/knowledge.js | 89 ++++ src/pkg/dashboard/static/index.html | 95 +++- src/pkg/knowledge/connector/connector.go | 3 +- src/pkg/knowledge/connector/connector_test.go | 2 +- src/pkg/knowledge/connector/mcpstdio.go | 379 ++++++++++++++++ src/pkg/knowledge/connector/mcpstdio_test.go | 182 ++++++++ src/pkg/knowledge/connector/plaud.go | 421 ++++++++++++++++++ src/pkg/knowledge/connector/plaud_auth.go | 374 ++++++++++++++++ src/pkg/knowledge/connector/plaud_test.go | 229 ++++++++++ src/pkg/knowledge/connector/syncer.go | 10 + 21 files changed, 2489 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added-knowledge-plaud-source.md create mode 100644 src/pkg/dashboard/api_knowledge_connector_auth.go create mode 100644 src/pkg/dashboard/api_knowledge_connector_auth_test.go create mode 100644 src/pkg/knowledge/connector/mcpstdio.go create mode 100644 src/pkg/knowledge/connector/mcpstdio_test.go create mode 100644 src/pkg/knowledge/connector/plaud.go create mode 100644 src/pkg/knowledge/connector/plaud_auth.go create mode 100644 src/pkg/knowledge/connector/plaud_test.go diff --git a/changelog.d/added-knowledge-plaud-source.md b/changelog.d/added-knowledge-plaud-source.md new file mode 100644 index 0000000000..c7cf5b02d6 --- /dev/null +++ b/changelog.d/added-knowledge-plaud-source.md @@ -0,0 +1 @@ +- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 72944b0d0f..cc87eca843 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -12676,6 +12676,379 @@ } } }, + "/api/knowledge/connectors/{name}/auth": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Get a knowledge connector's sign-in state", + "description": "Returns the cached sign-in state of a connector that authenticates interactively (e.g. `plaud` via the Plaud MCP OAuth flow). Pass `refresh=1` to re-query the upstream account. Tokens are never returned; they stay in the connector's own credential store.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + }, + { + "name": "refresh", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "1", + "true" + ] + }, + "description": "Re-query the upstream account instead of returning the cached state" + } + ], + "responses": { + "200": { + "description": "Sign-in state", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/connect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Start a knowledge connector's interactive sign-in", + "description": "Owner-only. Starts the connector's OAuth login (for `plaud`, the MCP server's `login` tool) and returns the pending state including `auth_url` for the user to open in a browser. The login session is kept alive for the callback; poll `GET /api/knowledge/connectors/{name}/auth?refresh=1` for the outcome.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "202": { + "description": "Login started (or already connected)", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to start the login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/disconnect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Sign a knowledge connector out", + "description": "Owner-only. Signs the connector out upstream (for `plaud`, the MCP server's `logout` tool), which removes the token from the connector's own credential store, and returns the new state.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "200": { + "description": "Signed out", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to sign out", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/api/knowledge/connectors/{name}/sync": { "post": { "tags": [ diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index ac9bc8f73a..1228e273fb 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -62,3 +62,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | +| `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | +| `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | +| `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index d0067822c6..f01162b768 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:555`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:556`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 69f7c3b801..d9328667f9 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -1,6 +1,6 @@ # Knowledge connectors -External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents and GitHub wikis) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. +External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis and Plaud voice notes) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. For the shared configuration fields, the fact format and how connectors relate to `git_sources` and `documents`, see [knowledge-curator.md → Connectors](knowledge-curator.md#connectors-knowledgeconnectors). @@ -10,7 +10,7 @@ For the shared configuration fields, the fact format and how connectors relate t knowledge: connectors: - name: team-wiki # lowercase letters, digits, dashes - type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint + type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud enabled: true interval: 15m # minimum 1m; blank = 15m layer: project # personal | project | org | community @@ -218,6 +218,91 @@ knowledge: ends the content when any were dropped. See [Context7 content cap](knowledge-curator.md#context7-content-cap). +## Plaud Note Pro (voice notes, `type: plaud`) + +The Plaud connector turns the recordings on a [Plaud Note / Note Pro](https://plaud.ai) +into knowledge facts: one fact per recording with the recording time, +duration, Plaud's AI summary and the full speaker-labelled transcript. Agents +search and cite them like any other fact, so a voice memo recorded on the way +to work can inform the issue or PR an agent picks up an hour later. + +It talks to the **official, free Plaud MCP server** (`npx -y @plaud-ai/mcp`) +over stdio as an MCP client — there is no Zapier/Make/IFTTT hop and nothing to +pay beyond your existing Plaud plan. Plaud lists the MCP server as a Preview +feature on every tier including Starter; Plaud's docs note that preview +features may later move behind paid plans. + +```yaml +knowledge: + connectors: + - name: plaud + type: plaud + layer: personal + interval: 15m + scope: + command: npx -y @plaud-ai/mcp@latest # default; any stdio MCP command + since_days: "30" # how far back the first sync reaches + max_files: "200" # recordings fetched per sync + include_audio: "false" # add Plaud's 24h presigned MP3 link + query: "" # optional name filter (list_files query) + tags: "voice, ideas" # extra tags written as attr_tags + # home: /data/knowledge/connector-state/plaud/home # token store; default shown, or "inherit" +``` + +No `auth` block: Plaud signs you in with its own OAuth flow and keeps the token +in the MCP server's store (`$HOME/.plaud/tokens-mcp.json`). hive never sees, +stores or logs the token. By default the connector points the MCP server's +`HOME` at its own state directory under `/data/knowledge/connector-state//home` +so the token survives restarts; set `home: inherit` to use the hive process's +HOME instead. + +### Setup + +1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run + the Plaud MCP server. +2. Add the connector from **Knowledge → 🔌 Integrations → Plaud voice notes** + (or the YAML above) and restart the hive so the syncer picks it up. +3. On the Plaud card click **Connect Plaud**. The card shows the Plaud + authorization URL; open it, sign in and click **Authorize**. The OAuth + callback lands on `localhost:8199` of the machine running hive, so for a + remote hive forward that port first (`ssh -L 8199:localhost:8199 user@host`) + and open the URL from the forwarding machine. The login session stays open + for 10 minutes; click **Check** to confirm "connected as …". + - Alternative when port-forwarding is impractical: authorize once on your + laptop with `npx -y @plaud-ai/mcp@latest install`, then copy + `~/.plaud/tokens-mcp.json` into the connector's `home` directory + (`…/connector-state//home/.plaud/`). The file is Plaud's own + token store, not a hive secret. +4. Click **Sync now**. New recordings are picked up every `interval`. + +### Behaviour + +- Each sync calls `list_files` (from the newest recording seen minus a 24 h + overlap, or `since_days` back on the first run), then `get_note` and + `get_transcript` for each recording. A recording whose transcript and + summary are both still processing is skipped and retried next sync; the + status reports it as a partial sync. +- Facts carry `source: plaud`, `attr_plaud_file_id`, `attr_recorded_at`, + `attr_duration_ms` and `attr_device_serial`. Audio is never downloaded; + with `include_audio: true` the fact links Plaud's 24-hour presigned URL. +- The listing is incremental, so recordings deleted in Plaud are not + tombstoned automatically; remove the fact file or the connector. +- Sign-in state (`connected as`, pending authorization, last error) comes from + `GET /api/knowledge/connectors/{name}/auth`; see [API](#api). + +### Privacy and cost + +- Transcripts and summaries are produced in **Plaud's cloud** and fetched from + there; the hive stores them in the vault for the layer you choose (use + `personal` unless the whole hive should see them). Do not record material + that may not leave your organisation. +- **Local-only alternative:** Plaud's app can auto-sync notes to an Obsidian + vault or Drive/Dropbox folder, and exports TXT/Markdown/DOCX by hand. Point + the existing `git` connector at a Git-synced copy of that folder (see the + Obsidian entry in 🔌 Integrations), or import single files with a + `document` connector (`file_path`). No code, still $0, and no MCP sign-in. +- Everything in this section costs **$0** beyond your Plaud subscription. + ## Notion (`type: notion`) The Notion connector syncs pages and database rows through the public Notion diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index 52c63d7fa0..af8380cc4a 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -548,6 +548,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.mux.HandleFunc("GET /api/hotspots", s.handleHotspots) // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() + s.registerKnowledgeConnectorAuthRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) // Per-agent Variables tab (#11458, api_variables.go). s.registerAgentVariableRoutes() diff --git a/src/pkg/dashboard/api_knowledge_connector_auth.go b/src/pkg/dashboard/api_knowledge_connector_auth.go new file mode 100644 index 0000000000..a5bd882be1 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth.go @@ -0,0 +1,114 @@ +package dashboard + +import ( + "context" + "net/http" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// knowledgeConnectorLookup is implemented by the connector syncer so the +// dashboard can reach a connector's optional interfaces (Authenticator). +type knowledgeConnectorLookup interface { + Connector(name string) (connector.Connector, bool) +} + +// knowledgeAuthTimeout bounds one auth status/connect/disconnect request, +// which may spawn the connector's MCP server. +const knowledgeAuthTimeout = 90 * time.Second + +// Interactive connector sign-in (Plaud OAuth through its MCP server): status, +// connect (returns the authorization URL) and disconnect. Tokens stay in the +// vendor tool's own store and are never returned. +func (s *Server) registerKnowledgeConnectorAuthRoutes() { + s.mux.HandleFunc("GET /api/knowledge/connectors/{name}/auth", s.handleKnowledgeConnectorAuthStatus) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/connect", s.handleKnowledgeConnectorAuthConnect) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/disconnect", s.handleKnowledgeConnectorAuthDisconnect) +} + +// knowledgeAuthenticator resolves the named connector's Authenticator, +// writing the matching error response when it is unavailable. +func (s *Server) knowledgeAuthenticator(w http.ResponseWriter, name string) (connector.Authenticator, bool) { + rt := s.knowledgeConnectors() + if rt == nil { + jsonError(w, "no knowledge connectors are configured", http.StatusNotFound) + return nil, false + } + lookup, ok := rt.(knowledgeConnectorLookup) + if !ok { + jsonError(w, "connector sign-in is not available on this runtime", http.StatusServiceUnavailable) + return nil, false + } + c, ok := lookup.Connector(name) + if !ok { + jsonError(w, "unknown knowledge connector", http.StatusNotFound) + return nil, false + } + auth, ok := c.(connector.Authenticator) + if !ok { + jsonError(w, "this connector type does not use interactive sign-in", http.StatusBadRequest) + return nil, false + } + return auth, true +} + +func knowledgeAuthContext(r *http.Request) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(r.Context()), knowledgeAuthTimeout) +} + +// handleKnowledgeConnectorAuthStatus returns the cached sign-in state, or a +// fresh one when ?refresh=1 (which may start the MCP server briefly). +func (s *Server) handleKnowledgeConnectorAuthStatus(w http.ResponseWriter, r *http.Request) { + auth, ok := s.knowledgeAuthenticator(w, r.PathValue("name")) + if !ok { + return + } + refresh := r.URL.Query().Get("refresh") == "1" + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + jsonResponse(w, auth.AuthStatus(ctx, refresh)) +} + +// handleKnowledgeConnectorAuthConnect (owner) starts the sign-in flow and +// answers 202 with the state; auth_url is the page the owner must open. +func (s *Server) handleKnowledgeConnectorAuthConnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Connect(ctx) + if err != nil { + jsonError(w, "connect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_connect", auditDetail("connector", name), "") + jsonStatusResponse(w, http.StatusAccepted, st) +} + +// handleKnowledgeConnectorAuthDisconnect (owner) signs out upstream. +func (s *Server) handleKnowledgeConnectorAuthDisconnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Disconnect(ctx) + if err != nil { + jsonError(w, "disconnect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_disconnect", auditDetail("connector", name), "") + jsonResponse(w, st) +} diff --git a/src/pkg/dashboard/api_knowledge_connector_auth_test.go b/src/pkg/dashboard/api_knowledge_connector_auth_test.go new file mode 100644 index 0000000000..7a7b7239f2 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// authConn is a fake connector implementing connector.Authenticator. +type authConn struct { + blockingConn + state connector.AuthState +} + +func (c *authConn) AuthStatus(context.Context, bool) connector.AuthState { return c.state } +func (c *authConn) Connect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Pending: true, AuthURL: "https://auth.example/authorize"} + return c.state, nil +} +func (c *authConn) Disconnect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Message: "signed out"} + return c.state, nil +} + +func decodeAuthState(t *testing.T, body []byte) connector.AuthState { + t.Helper() + var st connector.AuthState + if err := json.Unmarshal(body, &st); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return st +} + +func TestKnowledgeConnectorAuthEndpoints(t *testing.T) { + s, deps := apiServer(t) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("auth without syncer = %d, want 404", rec.Code) + } + + conn := &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + + rec := doGet(s, "/api/knowledge/connectors/wiki/auth") + if rec.Code != http.StatusOK { + t.Fatalf("auth status = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("state = %+v", st) + } + if rec := doGet(s, "/api/knowledge/connectors/nope/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("unknown connector = %d, want 404", rec.Code) + } + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/connect", nil); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner connect = %d, want 403", rec.Code) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/connect", nil) + if rec.Code != http.StatusAccepted { + t.Fatalf("connect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Pending || st.AuthURL != "https://auth.example/authorize" { + t.Fatalf("connect state = %+v", st) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/disconnect", nil) + if rec.Code != http.StatusOK { + t.Fatalf("disconnect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); st.Connected || st.Message != "signed out" { + t.Fatalf("disconnect state = %+v", st) + } +} + +func TestKnowledgeConnectorAuthNotInteractive(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &blockingConn{}) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusBadRequest { + t.Fatalf("non-authenticator connector = %d, want 400: %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 574593bc45..0e79b35a80 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -142,3 +142,45 @@ func TestKnowledgeIntegrationsModalContext7Card(t *testing.T) { } } } + +// TestKnowledgeIntegrationsModalPlaudCard guards the Plaud entry: the card +// exposes Connect/Disconnect/Check wired to the connector auth endpoints, +// the Knowledge pane links to it, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'plaud': { label: 'Plaud voice notes'`, + `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, + `if (type === 'plaud') html += kbPlaudAuthLine(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, + `data-action="kbPlaudConnect"`, + `data-action="kbPlaudDisconnect"`, + `data-action="kbPlaudRefreshAuth"`, + `data-action="kbOpenPlaudIntegration"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect'`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect'`, + `async function kbPlaudConnect(name)`, + `async function kbPlaudDisconnect(name)`, + `async function kbPlaudRefreshAuth(name)`, + `function kbOpenPlaudIntegration()`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Plaud card snippet %q", want) + } + } + start := strings.Index(html, "const _kbPlaudAuth = {};") + end := strings.Index(html, "function kbOpenPlaudIntegration()") + if start < 0 || end < start { + t.Fatal("Plaud sign-in block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Plaud sign-in block uses a browser-native dialog %q", native) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 6bb1f4d866..459c8a898a 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -127,6 +127,7 @@ { tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index c16d315d5b..fbd7bfa2ef 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -289,7 +289,7 @@ } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -298,6 +298,7 @@ 'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } }; @@ -313,6 +314,7 @@ // Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -328,6 +330,7 @@ 'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] }; diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 94a3fdc231..eb322b6c55 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -1975,6 +1975,12 @@ html += ''; html += '
'; + html += '
🎙️ Plaud voice notes'; + html += '
'; + html += '
Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -2421,9 +2427,11 @@ html += '
' + escapeHtml(c.name) + ' ' + escapeHtml(status) + '
'; html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name); html += '
'; if (owner) { html += '
'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += ''; html += ''; html += ''; @@ -2460,6 +2468,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; html += '
'; @@ -2543,3 +2552,83 @@ await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '
Plaud: checking sign-in…
'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — open the Plaud authorization page ↗ (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as ' + escapeHtml(a.account) + '' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '
' + text + '
'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += ''; + else html += ''; + html += ''; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 21292a873d..1511394525 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -45491,6 +45491,12 @@

Import'; html += '

'; + html += '
🎙️ Plaud voice notes'; + html += '
'; + html += '
Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -45937,9 +45943,11 @@

How Agents Use Knowledge (Priming)

html += '
' + escapeHtml(c.name) + ' ' + escapeHtml(status) + '
'; html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name); html += '
'; if (owner) { html += '
'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += ''; html += ''; html += ''; @@ -45976,6 +45984,7 @@

How Agents Use Knowledge (Priming)

html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; html += '
'; @@ -46059,6 +46068,86 @@

How Agents Use Knowledge (Priming)

await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '
Plaud: checking sign-in…
'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — open the Plaud authorization page ↗ (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as ' + escapeHtml(a.account) + '' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '
' + text + '
'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += ''; + else html += ''; + html += ''; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } // ── Vault Management ── function kbOpenVaults() { @@ -54905,6 +54994,7 @@

Configure LiteLLM

{ tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; @@ -56185,7 +56275,7 @@

Configure LiteLLM

} // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -56194,6 +56284,7 @@

Configure LiteLLM

'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } }; @@ -56209,6 +56300,7 @@

Configure LiteLLM

// Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -56224,6 +56316,7 @@

Configure LiteLLM

'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] }; diff --git a/src/pkg/knowledge/connector/connector.go b/src/pkg/knowledge/connector/connector.go index f66542b913..777984bb2b 100644 --- a/src/pkg/knowledge/connector/connector.go +++ b/src/pkg/knowledge/connector/connector.go @@ -216,7 +216,7 @@ func NewRegistry() *Registry { // DefaultRegistry returns a registry with the built-in connector types // (`git`, `document`, `github-wiki`, `repo-wiki`, `confluence`, `notion`, -// `sharepoint`, `google-drive` and `context7`) registered. +// `sharepoint`, `google-drive`, `context7` and `plaud`) registered. func DefaultRegistry() *Registry { r := NewRegistry() _ = r.Register(TypeGit, newGitConnector) @@ -228,6 +228,7 @@ func DefaultRegistry() *Registry { _ = r.Register(TypeSharePoint, newSharePointConnector) _ = r.Register(TypeRepoWiki, newRepoWikiConnector) _ = r.Register(TypeContext7, newContext7Connector) + _ = r.Register(TypePlaud, newPlaudConnector) return r } diff --git a/src/pkg/knowledge/connector/connector_test.go b/src/pkg/knowledge/connector/connector_test.go index 076c37a641..33e132d104 100644 --- a/src/pkg/knowledge/connector/connector_test.go +++ b/src/pkg/knowledge/connector/connector_test.go @@ -95,7 +95,7 @@ func TestRegistryRegister(t *testing.T) { if got := r.Types(); !reflect.DeepEqual(got, []string{"another", "fake"}) { t.Fatalf("Types() = %v", got) } - if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypeRepoWiki, TypeSharePoint}) { + if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint}) { t.Fatalf("DefaultRegistry().Types() = %v", got) } } diff --git a/src/pkg/knowledge/connector/mcpstdio.go b/src/pkg/knowledge/connector/mcpstdio.go new file mode 100644 index 0000000000..6cc36e2f15 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio.go @@ -0,0 +1,379 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" +) + +// mcpProtocolVersion is the MCP revision hive speaks as a stdio client. +const mcpProtocolVersion = "2024-11-05" + +// mcpMaxLineBytes caps one JSON-RPC line read from an MCP server. +const mcpMaxLineBytes = 32 << 20 + +// mcpClient is one connected MCP session. Call invokes a tool and returns +// the concatenated text content of the result. +type mcpClient interface { + Call(ctx context.Context, tool string, args map[string]any) (string, error) + // Notices returns side-channel text seen so far (logging notifications + // and stderr), used to surface OAuth URLs printed by the server. + Notices() string + Close() error +} + +// mcpDialer opens a fresh MCP session. +type mcpDialer func(ctx context.Context) (mcpClient, error) + +// mcpSession speaks newline-delimited JSON-RPC 2.0 over a reader/writer +// pair. Requests are serialized; notifications from the server are kept as +// notices. Close terminates the underlying process when there is one. +type mcpSession struct { + w io.Writer + lines chan []byte + done chan struct{} + nextID atomic.Int64 + mu sync.Mutex + notice strings.Builder + stderr *lockedBuffer + closer func() error + reqMu sync.Mutex +} + +type rpcMessage struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// newMCPSession wires a session over r/w and performs the MCP handshake. +func newMCPSession(ctx context.Context, r io.Reader, w io.Writer, stderr *lockedBuffer, closer func() error) (*mcpSession, error) { + s := &mcpSession{w: w, lines: make(chan []byte, 64), done: make(chan struct{}), stderr: stderr, closer: closer} + go s.readLoop(r) + init := map[string]any{ + "protocolVersion": mcpProtocolVersion, + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "hive-knowledge-connector", "version": "1"}, + } + if _, err := s.request(ctx, "initialize", init); err != nil { + _ = s.Close() + return nil, err + } + _ = s.notify("notifications/initialized", nil) + return s, nil +} + +func (s *mcpSession) readLoop(r io.Reader) { + defer close(s.done) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + s.lines <- []byte(line) + } +} + +func (s *mcpSession) send(v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.w.Write(append(data, '\n')) + return err +} + +func (s *mcpSession) notify(method string, params any) error { + msg := map[string]any{"jsonrpc": "2.0", "method": method} + if params != nil { + msg["params"] = params + } + return s.send(msg) +} + +// request sends one request and waits for its reply, recording any +// notifications that arrive in between. +func (s *mcpSession) request(ctx context.Context, method string, params any) (json.RawMessage, error) { + s.reqMu.Lock() + defer s.reqMu.Unlock() + id := s.nextID.Add(1) + if err := s.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + return nil, fmt.Errorf("mcp: sending %s: %w", method, err) + } + want := fmt.Sprintf("%d", id) + for { + select { + case <-ctx.Done(): + return nil, s.withStderr(fmt.Errorf("mcp: %s: %w", method, ctx.Err())) + case <-s.done: + // Drain replies that arrived just before the server hung up. + for { + select { + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + continue + default: + } + return nil, s.withStderr(fmt.Errorf("mcp: %s: server closed the connection", method)) + } + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + } + } +} + +// match decodes one line; ok is true when it is the reply to request id want. +func (s *mcpSession) match(line []byte, want, method string) (json.RawMessage, bool, error) { + var msg rpcMessage + if err := json.Unmarshal(line, &msg); err != nil { + return nil, false, nil + } + if len(msg.ID) == 0 || msg.Method != "" { + s.recordNotification(msg) + return nil, false, nil + } + if strings.Trim(string(msg.ID), `"`) != want { + return nil, false, nil + } + if msg.Error != nil { + return nil, true, fmt.Errorf("mcp: %s: %s (code %d)", method, msg.Error.Message, msg.Error.Code) + } + return msg.Result, true, nil +} + +func (s *mcpSession) recordNotification(msg rpcMessage) { + if len(msg.Params) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.notice.Len() < 64<<10 { + s.notice.Write(msg.Params) + s.notice.WriteByte('\n') + } +} + +func (s *mcpSession) withStderr(err error) error { + if s.stderr == nil { + return err + } + if tail := strings.TrimSpace(s.stderr.Tail(2000)); tail != "" { + return fmt.Errorf("%w (stderr: %s)", err, tail) + } + return err +} + +// Notices implements mcpClient. +func (s *mcpSession) Notices() string { + s.mu.Lock() + out := s.notice.String() + s.mu.Unlock() + if s.stderr != nil { + out += s.stderr.Tail(16 << 10) + } + return out +} + +// Call implements mcpClient: tools/call with the text content joined. +func (s *mcpSession) Call(ctx context.Context, tool string, args map[string]any) (string, error) { + if args == nil { + args = map[string]any{} + } + raw, err := s.request(ctx, "tools/call", map[string]any{"name": tool, "arguments": args}) + if err != nil { + return "", err + } + var res struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(raw, &res); err != nil { + return "", fmt.Errorf("mcp: %s: decoding result: %w", tool, err) + } + var parts []string + for _, c := range res.Content { + if c.Type == "text" && c.Text != "" { + parts = append(parts, c.Text) + } + } + text := strings.Join(parts, "\n") + if res.IsError { + if text == "" { + text = "tool reported an error" + } + return text, fmt.Errorf("mcp: %s: %s", tool, firstLine(text)) + } + return text, nil +} + +// Close implements mcpClient. +func (s *mcpSession) Close() error { + if s.closer != nil { + return s.closer() + } + return nil +} + +// lockedBuffer is a goroutine-safe byte buffer used for child stderr. +type lockedBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.buf.Len() > 256<<10 { + return len(p), nil + } + return b.buf.Write(p) +} + +// Tail returns the last n bytes written. +func (b *lockedBuffer) Tail(n int) string { + b.mu.Lock() + defer b.mu.Unlock() + s := b.buf.String() + if len(s) > n { + return s[len(s)-n:] + } + return s +} + +// stdioMCPDialer starts command as a child process per session. home, when +// non-empty, overrides HOME so the server keeps its own token store in a +// persistent directory. +func stdioMCPDialer(command, home string) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + argv, err := splitCommandLine(command) + if err != nil { + return nil, err + } + if len(argv) == 0 { + return nil, errors.New("mcp: command is empty") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = os.Environ() + if home != "" { + if err := os.MkdirAll(home, 0o700); err != nil { + return nil, fmt.Errorf("mcp: creating home %s: %w", home, err) + } + cmd.Env = append(cmd.Env, "HOME="+home) + cmd.Dir = home + } + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr := &lockedBuffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("mcp: starting %q: %w", argv[0], err) + } + closer := sync.OnceValue(func() error { + _ = stdin.Close() + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + return cmd.Wait() + }) + return newMCPSession(ctx, stdout, stdin, stderr, closer) + } +} + +// splitCommandLine splits a shell-like command line on whitespace, honouring +// single and double quotes. No expansion is performed. +func splitCommandLine(s string) ([]string, error) { + var out []string + var cur strings.Builder + inWord := false + var quote rune + for _, r := range s { + switch { + case quote != 0: + if r == quote { + quote = 0 + } else { + cur.WriteRune(r) + } + case r == '\'' || r == '"': + quote = r + inWord = true + case r == ' ' || r == '\t' || r == '\n': + if inWord { + out = append(out, cur.String()) + cur.Reset() + inWord = false + } + default: + cur.WriteRune(r) + inWord = true + } + } + if quote != 0 { + return nil, fmt.Errorf("mcp: unterminated quote in command %q", s) + } + if inWord { + out = append(out, cur.String()) + } + return out, nil +} + +var httpsURLRe = regexp.MustCompile(`https://[^\s"'<>\\)\]]+`) + +// firstHTTPSURL returns the first https:// URL found in s, or "". +func firstHTTPSURL(s string) string { + return httpsURLRe.FindString(s) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 300 { + s = s[:300] + "…" + } + return s +} + +// ensureDir creates dir (and parents) and returns it. +func ensureDir(dir string) (string, error) { + if dir == "" { + return "", nil + } + if err := os.MkdirAll(filepath.Clean(dir), 0o700); err != nil { + return "", err + } + return dir, nil +} diff --git a/src/pkg/knowledge/connector/mcpstdio_test.go b/src/pkg/knowledge/connector/mcpstdio_test.go new file mode 100644 index 0000000000..97a63e0df1 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio_test.go @@ -0,0 +1,182 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fakeMCPServer answers JSON-RPC over pipes like a stdio MCP server. tools +// maps a tool name to a handler returning (text, isError). +type fakeMCPServer struct { + tools map[string]func(args map[string]any) (string, bool) + mu sync.Mutex + calls []string + initReq map[string]any +} + +func (f *fakeMCPServer) serve(r io.Reader, w io.Writer) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments"` + Proto string `json:"protocolVersion"` + } `json:"params"` + } + if err := json.Unmarshal(sc.Bytes(), &req); err != nil { + continue + } + if len(req.ID) == 0 { + continue // notification + } + var result any + var rpcErr map[string]any + switch req.Method { + case "initialize": + f.mu.Lock() + f.initReq = map[string]any{"protocolVersion": req.Params.Proto} + f.mu.Unlock() + // Interleave a logging notification to prove the client skips it. + fmt.Fprintf(w, `{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"booting https://plaud.example/notice"}}`+"\n") + result = map[string]any{"protocolVersion": mcpProtocolVersion, "capabilities": map[string]any{}} + case "tools/call": + f.mu.Lock() + f.calls = append(f.calls, req.Params.Name) + h := f.tools[req.Params.Name] + f.mu.Unlock() + if h == nil { + rpcErr = map[string]any{"code": -32601, "message": "unknown tool " + req.Params.Name} + break + } + text, isErr := h(req.Params.Arguments) + result = map[string]any{"content": []map[string]any{{"type": "text", "text": text}}, "isError": isErr} + default: + rpcErr = map[string]any{"code": -32601, "message": "method not found"} + } + resp := map[string]any{"jsonrpc": "2.0", "id": json.RawMessage(req.ID)} + if rpcErr != nil { + resp["error"] = rpcErr + } else { + resp["result"] = result + } + b, _ := json.Marshal(resp) + fmt.Fprintf(w, "%s\n", b) + } +} + +func (f *fakeMCPServer) callNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +// pipeDialer connects an mcpSession to srv over in-memory pipes. +func pipeDialer(srv *fakeMCPServer) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + cr, sw := io.Pipe() // server → client + sr, cw := io.Pipe() // client → server + go srv.serve(sr, sw) + closer := func() error { _ = cw.Close(); _ = sw.Close(); return nil } + return newMCPSession(ctx, cr, cw, nil, closer) + } +} + +func TestMCPSessionHandshakeCallAndErrors(t *testing.T) { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){ + "echo": func(a map[string]any) (string, bool) { return fmt.Sprint(a["x"]), false }, + "boom": func(map[string]any) (string, bool) { return "nope: not authenticated", true }, + }} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := pipeDialer(srv)(ctx) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if got, err := c.Call(ctx, "echo", map[string]any{"x": "hi"}); err != nil || got != "hi" { + t.Fatalf("echo = %q, %v", got, err) + } + if _, err := c.Call(ctx, "boom", nil); err == nil || !strings.Contains(err.Error(), "not authenticated") { + t.Fatalf("isError must surface as error, got %v", err) + } + if _, err := c.Call(ctx, "missing", nil); err == nil || !strings.Contains(err.Error(), "unknown tool") { + t.Fatalf("rpc error = %v", err) + } + if !strings.Contains(c.Notices(), "plaud.example/notice") { + t.Fatalf("notification not recorded: %q", c.Notices()) + } + if srv.initReq["protocolVersion"] != mcpProtocolVersion { + t.Fatalf("initialize params = %v", srv.initReq) + } +} + +func TestMCPSessionServerExit(t *testing.T) { + cr, sw := io.Pipe() + sr, cw := io.Pipe() + go func() { _, _ = io.Copy(io.Discard, sr) }() + go func() { + // Reply to initialize, then hang up before any tool call is answered. + fmt.Fprintln(sw, `{"jsonrpc":"2.0","id":1,"result":{}}`) + _ = sw.Close() + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + s, err := newMCPSession(ctx, cr, cw, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := s.Call(ctx, "x", nil); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("want closed-connection error, got %v", err) + } +} + +func TestSplitCommandLine(t *testing.T) { + got, err := splitCommandLine(`npx -y "@plaud-ai/mcp@latest" --flag 'a b'`) + if err != nil { + t.Fatal(err) + } + want := []string{"npx", "-y", "@plaud-ai/mcp@latest", "--flag", "a b"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("split = %q", got) + } + if _, err := splitCommandLine(`npx "unterminated`); err == nil { + t.Fatal("unterminated quote must fail") + } +} + +func TestStdioMCPDialerSpawnsProcess(t *testing.T) { + // A tiny shell MCP server: answers initialize and one tools/call. + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\necho '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"pong\"}]}}'\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + if got, err := c.Call(ctx, "ping", nil); err != nil || got != "pong" { + t.Fatalf("ping = %q, %v", got, err) + } + if _, err := os.Stat(filepath.Join(dir, "home")); err != nil { + t.Fatalf("HOME dir must be created: %v", err) + } +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go new file mode 100644 index 0000000000..de0c27e432 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud.go @@ -0,0 +1,421 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// TypePlaud is the connector type for Plaud voice-note recordings, read +// through the official (free) Plaud MCP server over stdio. +const TypePlaud = "plaud" + +const ( + // plaudDefaultCommand launches the official Plaud MCP server. + plaudDefaultCommand = "npx -y @plaud-ai/mcp@latest" + // plaudDefaultSinceDays bounds the first sync. + plaudDefaultSinceDays = 30 + // plaudDefaultMaxFiles caps recordings fetched per sync. + plaudDefaultMaxFiles = 200 + // plaudListPageSize is the list_files page size requested. + plaudListPageSize = 50 + // plaudRelistOverlap re-lists this far behind the cursor so late + // transcripts and edited summaries are picked up. + plaudRelistOverlap = 24 * time.Hour + // plaudLoginWindow is how long a login session stays alive to receive + // the OAuth callback. + plaudLoginWindow = 10 * time.Minute + // plaudLoginReplyWait bounds how long Connect waits for the login tool + // to answer before returning whatever URL has surfaced so far. + plaudLoginReplyWait = 8 * time.Second + // plaudHomeInherit keeps the hive process HOME for the MCP server. + plaudHomeInherit = "inherit" +) + +// plaudConnector syncs Plaud recordings: list_files → get_transcript + +// get_note → one page per recording. Scope keys: command, home, since_days, +// max_files, include_audio, query, tags. No auth.env/auth.file: Plaud's +// OAuth token lives in the MCP server's own store ($HOME/.plaud). +type plaudConnector struct { + cfg ConnectorConfig + logger *slog.Logger + dial mcpDialer + now func() time.Time + stateDir string + + mu sync.Mutex + login *plaudLogin + auth AuthState +} + +// plaudLogin is an in-flight OAuth login kept alive for the callback. +type plaudLogin struct { + client mcpClient + cancel context.CancelFunc + result string + done chan struct{} +} + +func newPlaudConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &plaudConnector{cfg: cfg, logger: deps.logger(), now: time.Now, stateDir: deps.StateDir} + home := cfg.ScopeValue("home") + switch { + case home == plaudHomeInherit: + home = "" + case home == "" && deps.StateDir != "": + home = filepath.Join(deps.StateDir, "home") + } + if _, err := ensureDir(home); err != nil { + return nil, fmt.Errorf("plaud: %w", err) + } + command := cfg.ScopeValue("command") + if command == "" { + command = plaudDefaultCommand + } + c.dial = stdioMCPDialer(command, home) + return c, nil +} + +func (c *plaudConnector) Type() string { return TypePlaud } + +// Validate checks the scope. Plaud never takes a hive-managed credential. +func (c *plaudConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("plaud: auth.env/auth.file are not used; Plaud signs in through its own OAuth flow (Connect in the dashboard)") + } + if cmd := cfg.ScopeValue("command"); cmd != "" { + if _, err := splitCommandLine(cmd); err != nil { + return err + } + } + if h := cfg.ScopeValue("home"); h != "" && h != plaudHomeInherit && !filepath.IsAbs(h) { + return fmt.Errorf("plaud: scope.home must be an absolute path or %q", plaudHomeInherit) + } + for _, key := range []string{"since_days", "max_files"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return fmt.Errorf("plaud: scope.%s must be a positive integer", key) + } + } + } + if v := cfg.ScopeValue("include_audio"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("plaud: scope.include_audio must be true or false") + } + } + return nil +} + +func (c *plaudConnector) scopeInt(key string, def int) int { + n, err := strconv.Atoi(c.cfg.ScopeValue(key)) + if err != nil || n <= 0 { + return def + } + return n +} + +func (c *plaudConnector) includeAudio() bool { + b, _ := strconv.ParseBool(c.cfg.ScopeValue("include_audio")) + return b +} + +// plaudCursor is the newest recording time seen, RFC3339. +func parsePlaudCursor(cur Cursor) time.Time { + t, err := time.Parse(time.RFC3339, string(cur)) + if err != nil { + return time.Time{} + } + return t +} + +// Sync lists recordings created since the cursor (minus an overlap window) +// and emits one page per recording. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *plaudConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + client, err := c.dial(ctx) + if err != nil { + return cur, err + } + defer func() { _ = client.Close() }() + + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-plaudRelistOverlap) + } + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return cur, err + } + newest := parsePlaudCursor(cur) + var skipped []string + for _, f := range files { + if err := ctx.Err(); err != nil { + return cur, err + } + page, err := c.fetchPage(ctx, client, f) + if err != nil { + skipped = append(skipped, fmt.Sprintf("%s: %v", f.ID, err)) + continue + } + if err := emit(page); err != nil { + return cur, err + } + if f.CreatedAt.After(newest) { + newest = f.CreatedAt + } + } + c.refreshAuth(ctx, client) + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if len(skipped) > 0 { + return next, &PartialError{Err: fmt.Errorf("plaud: %d recording(s) skipped: %s", len(skipped), strings.Join(skipped, "; "))} + } + return next, nil +} + +// listFiles pages through list_files until a page is empty, repeats, or the +// cap is hit. Recordings older than since are dropped client-side as well, +// in case the server ignores date_from. +func (c *plaudConnector) listFiles(ctx context.Context, client mcpClient, since time.Time, maxFiles int) ([]plaudFile, error) { + seen := map[string]bool{} + var out []plaudFile + for page := 1; len(out) < maxFiles; page++ { + args := map[string]any{"page": page, "page_size": plaudListPageSize} + if !since.IsZero() { + args["date_from"] = since.UTC().Format("2006-01-02") + } + if q := c.cfg.ScopeValue("query"); q != "" { + args["query"] = q + } + text, err := client.Call(ctx, "list_files", args) + if err != nil { + return nil, plaudAuthError(err) + } + files, err := decodePlaudFiles(text) + if err != nil { + return nil, err + } + added := 0 + for _, f := range files { + if f.ID == "" || seen[f.ID] { + continue + } + seen[f.ID] = true + if !since.IsZero() && !f.CreatedAt.IsZero() && f.CreatedAt.Before(since) { + continue + } + out = append(out, f) + added++ + if len(out) >= maxFiles { + break + } + } + if added == 0 || len(files) < plaudListPageSize { + break + } + } + return out, nil +} + +// fetchPage builds the knowledge page for one recording. +func (c *plaudConnector) fetchPage(ctx context.Context, client mcpClient, f plaudFile) (Page, error) { + noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": f.ID}) + transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": f.ID}) + if noteErr != nil && trErr != nil { + return Page{}, plaudAuthError(trErr) + } + if noteErr != nil { + noteText = "" + } + if trErr != nil { + transcriptText = "" + } + note := renderPlaudNote(noteText) + transcript := renderPlaudTranscript(transcriptText) + if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { + return Page{}, fmt.Errorf("no note or transcript available yet") + } + audioURL := "" + if c.includeAudio() { + if text, err := client.Call(ctx, "get_file", map[string]any{"file_id": f.ID}); err == nil { + audioURL = plaudPresignedURL(text) + } + } + title := strings.TrimSpace(f.Name) + if title == "" { + title = "Plaud recording " + f.ID + } + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + if !f.StartAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.StartAt.UTC().Format(time.RFC3339)) + } else if !f.CreatedAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.CreatedAt.UTC().Format(time.RFC3339)) + } + if f.DurationMS > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(f.DurationMS) * time.Millisecond).Round(time.Second)) + } + if audioURL != "" { + fmt.Fprintf(&md, "- **Audio (24h link):** %s\n", audioURL) + } + md.WriteString("\n") + if strings.TrimSpace(note) != "" { + md.WriteString("## Summary (Plaud)\n\n") + md.WriteString(strings.TrimSpace(note)) + md.WriteString("\n\n") + } + if strings.TrimSpace(transcript) != "" { + md.WriteString("## Transcript\n\n") + md.WriteString(strings.TrimSpace(transcript)) + md.WriteString("\n") + } + attrs := map[string]string{"plaud_file_id": f.ID} + if f.DurationMS > 0 { + attrs["duration_ms"] = strconv.FormatInt(f.DurationMS, 10) + } + if !f.StartAt.IsZero() { + attrs["recorded_at"] = f.StartAt.UTC().Format(time.RFC3339) + } + if tags := c.cfg.ScopeValue("tags"); tags != "" { + attrs["tags"] = tags + } + if f.Serial != "" { + attrs["device_serial"] = f.Serial + } + updated := f.CreatedAt + if f.StartAt.After(updated) { + updated = f.StartAt + } + return Page{ + ID: f.ID, + Title: title, + Markdown: md.String(), + UpdatedAt: updated, + Path: []string{"Plaud"}, + Attrs: attrs, + }, nil +} + +// plaudAuthError rewrites "not authenticated" tool errors into a hint that +// points at the dashboard Connect button. +func plaudAuthError(err error) error { + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "401") || strings.Contains(msg, "not authenticated") || strings.Contains(msg, "unauthorized") || strings.Contains(msg, "login") { + return fmt.Errorf("plaud: not connected — use Connect in the Knowledge → Integrations → Plaud card (%w)", err) + } + return err +} + +// Attrs returned by the MCP server for one recording. +type plaudFile struct { + ID string + Name string + CreatedAt time.Time + StartAt time.Time + DurationMS int64 + Serial string +} + +// decodePlaudFiles accepts a JSON array of recordings, or an object wrapping +// one under files/items/data/results/recordings. +func decodePlaudFiles(text string) ([]plaudFile, error) { + text = strings.TrimSpace(text) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + if i := strings.IndexAny(text, "[{"); i > 0 { + return decodePlaudFiles(text[i:]) + } + return nil, fmt.Errorf("plaud: list_files returned non-JSON output: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"files", "items", "data", "results", "recordings"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + if _, ok := v["id"]; ok { + items = []any{v} + } + } + } + out := make([]plaudFile, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + f := plaudFile{ + ID: plaudString(m, "id", "file_id"), + Name: plaudString(m, "name", "title", "filename"), + Serial: plaudString(m, "serial_number"), + CreatedAt: plaudTime(m, "created_at", "create_time"), + StartAt: plaudTime(m, "start_at", "start_time"), + } + if d, ok := m["duration"].(float64); ok { + f.DurationMS = int64(d) + } + if f.ID != "" { + out = append(out, f) + } + } + return out, nil +} + +func plaudString(m map[string]any, keys ...string) string { + for _, k := range keys { + switch v := m[k].(type) { + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + case float64: + return strconv.FormatInt(int64(v), 10) + } + } + return "" +} + +func plaudTime(m map[string]any, keys ...string) time.Time { + for _, k := range keys { + switch v := m[k].(type) { + case string: + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, strings.TrimSpace(v)); err == nil { + return t + } + } + case float64: + // Epoch seconds or milliseconds. + if v > 1e12 { + return time.UnixMilli(int64(v)).UTC() + } + if v > 0 { + return time.Unix(int64(v), 0).UTC() + } + } + } + return time.Time{} +} diff --git a/src/pkg/knowledge/connector/plaud_auth.go b/src/pkg/knowledge/connector/plaud_auth.go new file mode 100644 index 0000000000..710ea2d291 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth.go @@ -0,0 +1,374 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// AuthState describes a connector's external sign-in. Connectors that own +// their credential flow (OAuth through a vendor MCP server) report it here; +// the dashboard renders Connect/Disconnect from it. Tokens never appear. +type AuthState struct { + // Connected is true when the upstream reports a signed-in account. + Connected bool `json:"connected"` + // Account is the signed-in identity (email or display name), if known. + Account string `json:"account,omitempty"` + // Pending is true while a login awaits the user's browser authorization. + Pending bool `json:"pending"` + // AuthURL is the authorization URL the user must open, while pending. + AuthURL string `json:"auth_url,omitempty"` + // Message is the last human-readable login/logout outcome. + Message string `json:"message,omitempty"` + CheckedAt time.Time `json:"checked_at,omitempty"` +} + +// Authenticator is optionally implemented by connectors whose credential is +// obtained interactively rather than from auth.env/auth.file. +type Authenticator interface { + // AuthStatus reports the current sign-in state. It may contact the + // upstream when refresh is true; otherwise it returns the cached state. + AuthStatus(ctx context.Context, refresh bool) AuthState + // Connect begins the sign-in flow and returns the state, including the + // URL the user must open when the flow needs a browser. + Connect(ctx context.Context) (AuthState, error) + // Disconnect signs out and revokes the stored credential upstream. + Disconnect(ctx context.Context) (AuthState, error) +} + +var _ Authenticator = (*plaudConnector)(nil) + +// AuthStatus implements Authenticator. +func (c *plaudConnector) AuthStatus(ctx context.Context, refresh bool) AuthState { + c.mu.Lock() + st := c.auth + login := c.login + c.mu.Unlock() + if login != nil { + select { + case <-login.done: + st.Pending = false + if login.result != "" { + st.Message = firstLine(login.result) + } + default: + st.Pending = true + if st.AuthURL == "" { + st.AuthURL = firstHTTPSURL(login.client.Notices()) + } + } + } + if !refresh || st.Pending { + return st + } + client, err := c.dial(ctx) + if err != nil { + st.Message = err.Error() + st.CheckedAt = c.now().UTC() + c.setAuth(st) + return st + } + defer func() { _ = client.Close() }() + c.refreshAuth(ctx, client) + c.mu.Lock() + defer c.mu.Unlock() + return c.auth +} + +// refreshAuth asks get_current_user and caches the answer. +func (c *plaudConnector) refreshAuth(ctx context.Context, client mcpClient) { + st := AuthState{CheckedAt: c.now().UTC()} + text, err := client.Call(ctx, "get_current_user", nil) + switch { + case err != nil: + st.Message = firstLine(err.Error()) + default: + st.Connected = true + st.Account = plaudAccountName(text) + } + c.setAuth(st) +} + +func (c *plaudConnector) setAuth(st AuthState) { + c.mu.Lock() + c.auth = st + c.mu.Unlock() +} + +// Connect implements Authenticator: it starts a long-lived MCP session, +// calls the server's login tool and keeps the process alive for +// plaudLoginWindow so the OAuth callback (localhost:8199) can land. The +// authorization URL is taken from the tool reply or the server's log output. +func (c *plaudConnector) Connect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + select { + case <-l.done: + c.login = nil + default: + c.mu.Unlock() + return c.AuthStatus(ctx, false), nil + } + } + c.mu.Unlock() + + loginCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), plaudLoginWindow) + client, err := c.dial(loginCtx) + if err != nil { + cancel() + return c.AuthStatus(ctx, false), err + } + l := &plaudLogin{client: client, cancel: cancel, done: make(chan struct{})} + c.mu.Lock() + c.login = l + c.auth = AuthState{Pending: true, CheckedAt: c.now().UTC()} + c.mu.Unlock() + + go c.runLogin(loginCtx, l) + + timer := time.NewTimer(plaudLoginReplyWait) + defer timer.Stop() + select { + case <-l.done: + case <-timer.C: + case <-ctx.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +func (c *plaudConnector) runLogin(ctx context.Context, l *plaudLogin) { + defer close(l.done) + defer l.cancel() + text, err := l.client.Call(ctx, "login", nil) + if err != nil { + text = err.Error() + } + l.result = text + st := AuthState{CheckedAt: c.now().UTC(), Message: firstLine(text)} + if url := firstHTTPSURL(text); url != "" && err == nil { + // The tool answered with the URL only: the callback still has to + // arrive, so keep the session alive until the window closes. + c.mu.Lock() + c.auth.AuthURL = url + c.auth.Pending = true + c.auth.Message = "" + c.mu.Unlock() + if !c.waitForLogin(ctx, l.client) { + st.Message = "authorization not completed before the login window closed" + st.AuthURL = "" + } else { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + } else if err == nil { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + _ = l.client.Close() + c.mu.Lock() + c.auth = st + c.mu.Unlock() + c.clearLogin(l) +} + +// waitForLogin polls get_current_user until it succeeds or ctx expires. +func (c *plaudConnector) waitForLogin(ctx context.Context, client mcpClient) bool { + for { + if _, err := client.Call(ctx, "get_current_user", nil); err == nil { + return true + } + if sleepCtx(ctx, 3*time.Second) != nil { + return false + } + } +} + +func (c *plaudConnector) clearLogin(l *plaudLogin) { + c.mu.Lock() + if c.login == l { + c.login = nil + } + c.auth.Pending = false + c.mu.Unlock() +} + +// Disconnect implements Authenticator via the server's logout tool. +func (c *plaudConnector) Disconnect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.mu.Unlock() + client, err := c.dial(ctx) + if err != nil { + return c.AuthStatus(ctx, false), err + } + defer func() { _ = client.Close() }() + text, err := client.Call(ctx, "logout", nil) + st := AuthState{CheckedAt: c.now().UTC()} + if err != nil { + st.Message = firstLine(err.Error()) + c.setAuth(st) + return st, err + } + st.Message = firstLine(text) + if st.Message == "" { + st.Message = "signed out" + } + c.setAuth(st) + return st, nil +} + +// plaudAccountName extracts an identity from get_current_user output. +func plaudAccountName(text string) string { + text = strings.TrimSpace(text) + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err == nil { + for _, key := range []string{"email", "name", "nickname", "username", "id"} { + if s := plaudString(m, key); s != "" { + return s + } + } + if u, ok := m["user"].(map[string]any); ok { + return plaudAccountName(mustJSON(u)) + } + } + return firstLine(text) +} + +func mustJSON(v any) string { + b, _ := json.Marshal(v) + return string(b) +} + +// renderPlaudNote turns get_note output into markdown. JSON replies are +// flattened: note_list entries and content/summary/markdown fields. +func renderPlaudNote(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var parts []string + var walk func(v any) + walk = func(v any) { + switch t := v.(type) { + case string: + if s := strings.TrimSpace(t); s != "" { + parts = append(parts, s) + } + case []any: + for _, it := range t { + walk(it) + } + case map[string]any: + for _, key := range []string{"note_list", "notes", "markdown", "content", "summary", "text", "note"} { + if inner, ok := t[key]; ok { + walk(inner) + return + } + } + if s := plaudString(t, "title"); s != "" { + parts = append(parts, "### "+s) + } + } + } + walk(raw) + // Parsed JSON with no text means the note is not ready yet. + return strings.Join(parts, "\n\n") +} + +// renderPlaudTranscript turns get_transcript output into "[mm:ss] Speaker: +// text" lines when it is JSON; plain text passes through. +func renderPlaudTranscript(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var segs []any + switch t := raw.(type) { + case []any: + segs = t + case map[string]any: + for _, key := range []string{"source_list", "segments", "transcript", "items", "data"} { + if arr, ok := t[key].([]any); ok { + segs = arr + break + } + if s, ok := t[key].(string); ok { + return strings.TrimSpace(s) + } + } + } + var b strings.Builder + for _, s := range segs { + m, ok := s.(map[string]any) + if !ok { + if str, ok := s.(string); ok { + b.WriteString(str + "\n") + } + continue + } + content := plaudString(m, "text", "content", "sentence") + if content == "" { + continue + } + if ts := plaudTimestamp(m); ts != "" { + fmt.Fprintf(&b, "[%s] ", ts) + } + if sp := plaudString(m, "speaker", "speaker_name", "speaker_id"); sp != "" { + b.WriteString(sp + ": ") + } + b.WriteString(content + "\n") + } + // Parsed JSON with no segments means the transcript is not ready yet. + return b.String() +} + +// plaudTimestamp formats a segment start (ms or s) as mm:ss or h:mm:ss. +func plaudTimestamp(m map[string]any) string { + for _, key := range []string{"start_time", "start", "start_ms", "begin"} { + switch v := m[key].(type) { + case float64: + // Plaud reports segment offsets in milliseconds. + d := (time.Duration(v) * time.Millisecond).Round(time.Second) + h := int(d.Hours()) + mm := int(d.Minutes()) % 60 + ss := int(d.Seconds()) % 60 + if h > 0 { + return fmt.Sprintf("%d:%02d:%02d", h, mm, ss) + } + return fmt.Sprintf("%02d:%02d", mm, ss) + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + } + } + return "" +} + +// plaudPresignedURL pulls the presigned audio URL out of get_file output. +func plaudPresignedURL(text string) string { + var m map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(text)), &m); err == nil { + if u := plaudString(m, "presigned_url", "audio_url", "url"); u != "" { + return u + } + } + return firstHTTPSURL(text) +} diff --git a/src/pkg/knowledge/connector/plaud_test.go b/src/pkg/knowledge/connector/plaud_test.go new file mode 100644 index 0000000000..eada7e56b8 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_test.go @@ -0,0 +1,229 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func plaudTestFiles() []map[string]any { + return []map[string]any{ + {"id": "rec-1", "name": "Standup ideas", "created_at": "2026-11-10T09:00:00Z", "start_at": "2026-11-10T08:58:00Z", "duration": 125000.0, "serial_number": "PN123"}, + {"id": "rec-2", "name": "Drive home", "created_at": "2026-11-11T18:00:00Z", "duration": 61000.0}, + {"id": "rec-old", "name": "Ancient", "created_at": "2020-01-01T00:00:00Z"}, + } +} + +func newPlaudTestServer() *fakeMCPServer { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){}} + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + if p, _ := a["page"].(float64); p > 1 { + return "[]", false + } + b, _ := json.Marshal(map[string]any{"files": plaudTestFiles()}) + return string(b), false + } + srv.tools["get_note"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return "", true // still processing + } + b, _ := json.Marshal(map[string]any{"note_list": []string{"## Key points\n- ship plaud connector"}}) + return string(b), false + } + srv.tools["get_transcript"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return `{"source_list":[]}`, false + } + return `{"source_list":[{"start_time":1500,"speaker":"Andy","text":"Hello hive"},{"start_time":65000,"speaker":"Andy","text":"Second line"}]}`, false + } + srv.tools["get_current_user"] = func(map[string]any) (string, bool) { + return `{"email":"andy@example.com"}`, false + } + return srv +} + +func newTestPlaud(t *testing.T, srv *fakeMCPServer, scope map[string]string) *plaudConnector { + t.Helper() + cfg := ConnectorConfig{Name: "plaud", Type: TypePlaud, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + pc := c.(*plaudConnector) + pc.dial = pipeDialer(srv) + pc.now = func() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + return pc +} + +func TestPlaudSyncEmitsPagesAndAdvancesCursor(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, map[string]string{"tags": "voice, ideas"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) || !strings.Contains(err.Error(), "rec-2") { + t.Fatalf("want partial error naming rec-2, got %v", err) + } + if len(pages) != 1 || pages[0].ID != "rec-1" { + t.Fatalf("pages = %+v", pages) + } + p := pages[0] + for _, want := range []string{"# Standup ideas", "**Recorded:** 2026-11-10T08:58:00Z", "**Duration:** 2m5s", "## Summary (Plaud)", "ship plaud connector", "## Transcript", "[00:02] Andy: Hello hive", "[01:05] Andy: Second line"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["plaud_file_id"] != "rec-1" || p.Attrs["tags"] != "voice, ideas" || p.Attrs["device_serial"] != "PN123" || p.Attrs["duration_ms"] != "125000" { + t.Fatalf("attrs = %v", p.Attrs) + } + if next != Cursor("2026-11-10T09:00:00Z") { + t.Fatalf("cursor = %q", next) + } + // The sync refreshes the sign-in state for the dashboard card. + if st := pc.AuthStatus(ctx, false); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("auth after sync = %+v", st) + } + calls := strings.Join(srv.callNames(), ",") + if strings.Contains(calls, "get_file") { + t.Fatalf("audio must not be fetched by default: %s", calls) + } +} + +func TestPlaudSyncUsesCursorWindowAndAudio(t *testing.T) { + srv := newPlaudTestServer() + var dateFrom string + list := srv.tools["list_files"] + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + dateFrom, _ = a["date_from"].(string) + return list(a) + } + srv.tools["get_file"] = func(map[string]any) (string, bool) { + return `{"id":"rec-1","presigned_url":"https://cdn.plaud.example/rec-1.mp3?sig=1"}`, false + } + pc := newTestPlaud(t, srv, map[string]string{"include_audio": "true"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "2026-11-11T00:00:00Z", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) { + t.Fatalf("err = %v", err) + } + // Cursor minus the 24h overlap → 2026-11-10; rec-1 (Nov 10) is re-listed. + if dateFrom != "2026-11-10" { + t.Fatalf("date_from = %q", dateFrom) + } + if len(pages) != 1 || !strings.Contains(pages[0].Markdown, "cdn.plaud.example/rec-1.mp3") { + t.Fatalf("pages = %+v", pages) + } + // The cursor never moves backwards. + if next != Cursor("2026-11-11T00:00:00Z") { + t.Fatalf("cursor = %q", next) + } +} + +func TestPlaudSyncNotAuthenticated(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["list_files"] = func(map[string]any) (string, bool) { return "401 Not authenticated", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, err := pc.Sync(ctx, "", func(Page) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "Connect") { + t.Fatalf("want connect hint, got %v", err) + } +} + +func TestPlaudValidate(t *testing.T) { + base := ConnectorConfig{Name: "plaud", Type: TypePlaud, Layer: "personal"} + reg := DefaultRegistry() + deps := Deps{StateDir: t.TempDir()} + if _, err := reg.New(base, deps); err != nil { + t.Fatalf("default config must validate: %v", err) + } + bad := []ConnectorConfig{ + {Name: "p", Type: TypePlaud, Layer: "personal", Auth: Auth{Env: "PLAUD_TOKEN"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"since_days": "zero"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"include_audio": "maybe"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"home": "relative/dir"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"command": `npx "oops`}}, + } + for i, cfg := range bad { + if _, err := reg.New(cfg, deps); err == nil { + t.Errorf("config %d must be rejected: %+v", i, cfg.Scope) + } + } +} + +func TestPlaudConnectCompletesLogin(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { + return "Open https://auth.plaud.example/authorize?state=abc to sign in", false + } + srv.tools["logout"] = func(map[string]any) (string, bool) { return "Signed out", false } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + // get_current_user succeeds right away in the fake, so the login + // finishes inside the reply window and the state is already connected. + if !st.Connected || st.Account != "andy@example.com" || st.Pending { + t.Fatalf("state after connect = %+v", st) + } + st, err = pc.Disconnect(ctx) + if err != nil || st.Connected || st.Message != "Signed out" { + t.Fatalf("disconnect = %+v, %v", st, err) + } +} + +func TestPlaudConnectLoginError(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { return "browser could not be opened", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + if st.Connected || st.Pending || !strings.Contains(st.Message, "browser could not be opened") { + t.Fatalf("state = %+v", st) + } +} + +func TestPlaudRenderHelpers(t *testing.T) { + if got := renderPlaudNote("plain **markdown**"); got != "plain **markdown**" { + t.Fatalf("plain note = %q", got) + } + if got := renderPlaudNote(`{"note_list":[{"title":"Action items","content":"- do it"}]}`); got != "- do it" { + t.Fatalf("nested note = %q", got) + } + if got := renderPlaudTranscript("raw transcript"); got != "raw transcript" { + t.Fatalf("plain transcript = %q", got) + } + if got := renderPlaudTranscript(`{"transcript":"as string"}`); got != "as string" { + t.Fatalf("string transcript = %q", got) + } + if got := renderPlaudTranscript(`[{"start":3725000,"text":"late"}]`); !strings.HasPrefix(got, "[1:02:05] late") { + t.Fatalf("hour timestamp = %q", got) + } + files, err := decodePlaudFiles("Here are your files:\n" + `[{"id": 7, "name": "n", "created_at": 1762848000}]`) + if err != nil || len(files) != 1 || files[0].ID != "7" || files[0].CreatedAt.Year() != 2025 { + t.Fatalf("decode with prose prefix = %+v, %v", files, err) + } + if _, err := decodePlaudFiles("no json here"); err == nil { + t.Fatal("non-JSON listing must error") + } + if got := plaudAccountName(`{"user":{"name":"Andy"}}`); got != "Andy" { + t.Fatalf("account = %q", got) + } + if got := fmt.Sprint(plaudAuthError(fmt.Errorf("mcp: list_files: Unauthorized"))); !strings.Contains(got, "not connected") { + t.Fatalf("auth error = %q", got) + } +} diff --git a/src/pkg/knowledge/connector/syncer.go b/src/pkg/knowledge/connector/syncer.go index 160749e364..efe5efc6f7 100644 --- a/src/pkg/knowledge/connector/syncer.go +++ b/src/pkg/knowledge/connector/syncer.go @@ -159,6 +159,16 @@ func (s *Syncer) update(name string, fn func(*Status)) Status { return st } +// Connector returns the built connector for name; ok is false for an +// unknown name or one whose config failed to build. +func (s *Syncer) Connector(name string) (Connector, bool) { + e, ok := s.entries[name] + if !ok || e.buildErr != nil || e.conn == nil { + return nil, false + } + return e.conn, true +} + // SyncNow runs one sync of the named connector immediately, whether or not // it is enabled. It returns ErrSyncInProgress if a sync is already running. func (s *Syncer) SyncNow(ctx context.Context, name string) (Status, error) { From d3b0aeb89c85f4ef350e8234b1a53d609b21e701 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:35:14 +0000 Subject: [PATCH 048/166] =?UTF-8?q?=F0=9F=90=9B=20hive:=20do=20not=20gate?= =?UTF-8?q?=20the=20dashboard=20bind=20on=20the=20advisory-issue=20ensure?= =?UTF-8?q?=20(#17829)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hive: do not gate the dashboard bind on the advisory-issue ensure Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🐛 hive: add missing time import in boot_github_deps_test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...board-bind-not-gated-on-advisory-ensure.md | 1 + src/cmd/hive/boot_github_deps_test.go | 27 +++++++++++++++++++ src/cmd/hive/main.go | 16 +++++++++-- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md diff --git a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md new file mode 100644 index 0000000000..b7cfafe241 --- /dev/null +++ b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md @@ -0,0 +1 @@ +- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). diff --git a/src/cmd/hive/boot_github_deps_test.go b/src/cmd/hive/boot_github_deps_test.go index 438b34332c..410e612bfa 100644 --- a/src/cmd/hive/boot_github_deps_test.go +++ b/src/cmd/hive/boot_github_deps_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" @@ -281,3 +282,29 @@ func TestBootAdvisoryWithRateLimitDoesNotClassify(t *testing.T) { t.Fatalf("log:\n%s", log.String()) } } + +func TestBootAdvisoryWithBoundsBlockedEnsure(t *testing.T) { + old := advisoryEnsureBudget + advisoryEnsureBudget = 50 * time.Millisecond + defer func() { advisoryEnsureBudget = old }() + + cfg := bootAdvisoryConfig(t) + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + f.deps.ensureAdvisoryIssue = func(ctx context.Context, _ *github.Client, _ string) (int, error) { + <-ctx.Done() + return 0, ctx.Err() + } + + done := make(chan struct{}) + go func() { b.bootAdvisoryWith(f.deps); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("bootAdvisory must not block on a hung advisory-issue ensure") + } + if len(b.advisoryIssues) != 0 { + t.Fatalf("advisoryIssues = %v", b.advisoryIssues) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index bd17e6e5b7..a7fca4a85b 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1652,6 +1652,11 @@ func (b *boot) newKnowledgePrimer() *knowledge.Primer { return p } +// advisoryEnsureBudget caps the startup advisory-issue ensure (and the failure +// classification after it) so a slow or unreachable GitHub cannot hold the +// dashboard bind past the kubelet liveness budget (#17827). +var advisoryEnsureBudget = 15 * time.Second + // bootAdvisory builds the notifier, infers the ACMM level, seeds the // GitHub App banner state, opens the mutation-convergence ledger and journal, // finds or creates the pinned advisory issue and seeds the embedded default @@ -1733,7 +1738,12 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { primaryRepo = b.cfg.Project.Repos[0] } if primaryRepo != "" { - num, err := deps.ensureAdvisoryIssue(b.ctx, b.ghClient, primaryRepo) + // Bounded: every GitHub App token mint against an unreachable + // GHE burns ~8s, and this runs before the dashboard binds. An + // unresolved issue is retried per eval cycle (ensurePinnedAdvisoryIssue). + ensureCtx, cancelEnsure := context.WithTimeout(b.ctx, advisoryEnsureBudget) + num, err := deps.ensureAdvisoryIssue(ensureCtx, b.ghClient, primaryRepo) + cancelEnsure() if err != nil { b.logger.Error("failed to ensure advisory issue", "repo", primaryRepo, "error", err) // GitHub returns 403 for rate limiting too — a transient @@ -1750,7 +1760,9 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { // vanished on the first Re-check with nothing fixed. // classifyGitHubAppFailure is the same verdict Re-check // uses, and it declines to raise on AppStateUnknown. - raise, diag, state := deps.classifyAppFailure(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + classifyCtx, cancelClassify := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyAppFailure(classifyCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + cancelClassify() if raise { b.githubAppRequired = true b.githubAppDiag, b.githubAppState = diag, state From 7e9574be6e71868e545d7fc364f858d2a5185d4e Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 02:36:59 +0000 Subject: [PATCH 049/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.227.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.227.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-hub-heartbeat-403-backoff.md | 1 - changelog.d/fixed-state-persist-rename-race.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-hub-heartbeat-403-backoff.md delete mode 100644 changelog.d/fixed-state-persist-rename-race.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 1481511f99..102ac33d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.227.1) + +### Fixed + +- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). +- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). + ## 2026-11-12 (v5.227.0) ### Added diff --git a/changelog.d/fixed-hub-heartbeat-403-backoff.md b/changelog.d/fixed-hub-heartbeat-403-backoff.md deleted file mode 100644 index cf59938b9a..0000000000 --- a/changelog.d/fixed-hub-heartbeat-403-backoff.md +++ /dev/null @@ -1 +0,0 @@ -- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). diff --git a/changelog.d/fixed-state-persist-rename-race.md b/changelog.d/fixed-state-persist-rename-race.md deleted file mode 100644 index cbe357487b..0000000000 --- a/changelog.d/fixed-state-persist-rename-race.md +++ /dev/null @@ -1 +0,0 @@ -- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). From 10ce002d4c30cf10952f9ce556fcc5c33a44ca40 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:03:20 +0000 Subject: [PATCH 050/166] =?UTF-8?q?=F0=9F=90=9B=20substrate:=20garbage-col?= =?UTF-8?q?lect=20older=20suspended=20kick=20actors=20when=20snapshot=5Fon?= =?UTF-8?q?=5Fidle=20is=20on=20(#17830)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 substrate: garbage-collect older suspended kick actors when snapshot_on_idle is on Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 substrate: scope superseded-actor GC to the atespace it was created in and log via slog Review follow-up for #17830: - lastSuspended records {Name, Atespace}; the delete uses the stored atespace and is skipped (with a warning) when the agent's atespace changed between kicks, instead of targeting a name never created there. - Replace stdlib log.Printf with an injected *slog.Logger on SubstrateOptions (slog.Default() fallback), wired from the agent manager, with agent/actor/atespace/error attrs. - Document the one-snapshot-per-agent retention rule in sandbox-isolation.md, ADR 0026 and the dashboard snapshot_on_idle help. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-substrate-snapshot-on-idle-gc.md | 1 + src/crux/agent/manager_sandbox_substrate.go | 4 +- src/crux/config/agent_config_substrate.go | 8 +- src/crux/sandbox/substrate.go | 34 +++++- src/crux/sandbox/substrate_kicks.go | 18 +++ src/crux/sandbox/substrate_test.go | 108 +++++++++++++++++- .../adr/0026-sandbox-runtime-selection.md | 2 +- src/docs/sandbox-isolation.md | 2 +- .../static-src/js/config-watchdog-security.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- 10 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 changelog.d/fixed-substrate-snapshot-on-idle-gc.md diff --git a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md new file mode 100644 index 0000000000..97a4d8f193 --- /dev/null +++ b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md @@ -0,0 +1 @@ +- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). diff --git a/src/crux/agent/manager_sandbox_substrate.go b/src/crux/agent/manager_sandbox_substrate.go index 4d535bf238..861ec4936c 100644 --- a/src/crux/agent/manager_sandbox_substrate.go +++ b/src/crux/agent/manager_sandbox_substrate.go @@ -43,7 +43,9 @@ func (m *Manager) substrateLauncherLocked(cfg config.SandboxSubstrateConfig) san if m.sandboxSubstrateLauncherFactory != nil { return m.sandboxSubstrateLauncherFactory(cfg) } - return &sandbox.SubstrateLauncher{Options: SubstrateOptionsFromConfig(cfg)} + opts := SubstrateOptionsFromConfig(cfg) + opts.Logger = m.logger + return &sandbox.SubstrateLauncher{Options: opts} } // SubstrateOptionsFromConfig converts the operator's substrate block into diff --git a/src/crux/config/agent_config_substrate.go b/src/crux/config/agent_config_substrate.go index 327eb6959b..1a847d7d8e 100644 --- a/src/crux/config/agent_config_substrate.go +++ b/src/crux/config/agent_config_substrate.go @@ -106,8 +106,12 @@ type SandboxSubstrateConfig struct { // pool. WorkerSelector map[string]string `yaml:"worker_selector,omitempty" json:"worker_selector,omitempty"` // SnapshotOnIdle suspends the actor (asynchronous snapshot) after the - // kick instead of deleting it, so Substrate keeps a resumable image. - // Default false: a kick is one-shot and snapshots cost object storage. + // kick instead of deleting it, so Substrate keeps a snapshot. The + // snapshot is retained until the agent's next idle snapshot replaces it + // (at most one per agent per hive process); it is not yet resumed on the + // next kick, and actors left from before a hive restart are not + // reclaimed. Default false: a kick is one-shot and snapshots cost + // object storage. SnapshotOnIdle bool `yaml:"snapshot_on_idle,omitempty" json:"snapshot_on_idle,omitempty"` // AllowHosts are additional egress hosts (host or host:port, leading // *. wildcard allowed) the actor may reach, on top of the GitHub hosts diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go index e79f4a87d7..965a3e8471 100644 --- a/src/crux/sandbox/substrate.go +++ b/src/crux/sandbox/substrate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "strings" "time" @@ -105,6 +106,16 @@ type SubstrateOptions struct { SnapshotOnIdle bool AllowHosts []string PolicyProfile string + // Logger receives cleanup diagnostics that do not fail the kick; nil + // means slog.Default(). + Logger *slog.Logger +} + +func (o SubstrateOptions) logger() *slog.Logger { + if o.Logger != nil { + return o.Logger + } + return slog.Default() } // Validate reports the first configuration error a kick would hit. @@ -302,9 +313,11 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e kick := kicks.begin(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace, State: SubstrateStateCreated}) kicks.observe(kick, created) // Delete is the default: a kick is a one-shot and keeping a snapshot - // around costs object storage. snapshot_on_idle keeps a resumable - // snapshot instead, so the next kick's warm state (caches, builds) is - // a resume rather than a cold boot. + // around costs object storage. snapshot_on_idle keeps the actor + // suspended instead. Nothing resumes it yet (the next kick is a cold + // boot), so only the latest snapshot per agent is retained: the one it + // replaces is deleted. The Control API has no ListActors, so actors left + // by an earlier hive process are not reclaimed. cleanupActor := func() { if l.Options.SnapshotOnIdle && ctx.Err() == nil { if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { @@ -312,6 +325,21 @@ func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, e i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true }) cleanupTemplate() + prev := kicks.swapSuspended(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace}) + switch { + case prev.Name == "" || prev.Name == name: + case prev.Atespace != ref.Atespace: + // This launcher's client is for the current atespace; + // an actor left in another one is not ours to reach. + l.Options.logger().Warn("substrate: superseded suspended actor left in another atespace", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "current_atespace", ref.Atespace) + default: + prevRef := &substrateapi.ObjectRef{Atespace: prev.Atespace, Name: prev.Name} + if err := control.DeleteActor(live(), prevRef); err != nil { + l.Options.logger().Warn("substrate: deleting superseded suspended actor", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "error", err) + } + } return } } diff --git a/src/crux/sandbox/substrate_kicks.go b/src/crux/sandbox/substrate_kicks.go index 1dffb978de..cd7aba007b 100644 --- a/src/crux/sandbox/substrate_kicks.go +++ b/src/crux/sandbox/substrate_kicks.go @@ -73,6 +73,10 @@ var _ SubstrateLogReader = (*SubstrateKicks)(nil) type SubstrateKicks struct { mu sync.Mutex byLaunch map[string]*substrateKick + // lastSuspended is the newest actor each launch name left suspended + // (snapshot_on_idle), with the atespace it was created in; in memory + // only, so a hive restart forgets it. + lastSuspended map[string]SubstrateActorInfo } // substrateKick is one recorded kick: every state snapshot it went through @@ -96,6 +100,20 @@ func (s *SubstrateKicks) begin(launchName string, info SubstrateActorInfo) *subs return k } +// swapSuspended records actor (name and atespace) as launchName's retained +// suspended actor and returns the one it replaces, or a zero value when +// there was none. +func (s *SubstrateKicks) swapSuspended(launchName string, actor SubstrateActorInfo) SubstrateActorInfo { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastSuspended == nil { + s.lastSuspended = map[string]SubstrateActorInfo{} + } + prev := s.lastSuspended[launchName] + s.lastSuspended[launchName] = SubstrateActorInfo{Name: actor.Name, Atespace: actor.Atespace} + return prev +} + // update applies fn to the kick's latest state and records the result as // a new snapshot when anything changed. func (s *SubstrateKicks) update(k *substrateKick, fn func(*SubstrateActorInfo)) { diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go index 1bdb458a75..29bee045e4 100644 --- a/src/crux/sandbox/substrate_test.go +++ b/src/crux/sandbox/substrate_test.go @@ -1,9 +1,11 @@ package sandbox import ( + "bytes" "context" "errors" "io" + "log/slog" "strings" "sync" "testing" @@ -46,6 +48,7 @@ type fakeSubstrate struct { pulledWorkspace string workers []*substrateapi.Worker listErr error + deleted []string closed bool cleanupCtxDead bool } @@ -118,8 +121,9 @@ func (f *fakeSubstrate) SuspendActor(ctx context.Context, _ *substrateapi.Object return f.suspendErr } -func (f *fakeSubstrate) DeleteActor(ctx context.Context, _ *substrateapi.ObjectRef) error { +func (f *fakeSubstrate) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { f.note("DeleteActor", ctx) + f.deleted = append(f.deleted, ref.GetName()) return f.deleteErr } @@ -452,6 +456,108 @@ func TestSubstrateRunSnapshotOnIdle(t *testing.T) { } } +func TestSubstrateSnapshotOnIdleDeletesSupersededActor(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 0 { + t.Fatalf("first suspend deleted %v, want nothing", fake.deleted) + } + + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + second := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 1 || fake.deleted[0] != first || first == second { + t.Errorf("deleted = %v, want only the first actor %s (second %s survives)", fake.deleted, first, second) + } +} + +func TestSubstrateSnapshotOnIdleKeepsOtherAgentsActors(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + other := substrateTestSpec() + other.Name = "docs-sandbox" + if _, err := l.Run(context.Background(), other); err != nil { + t.Fatalf("Run other: %v", err) + } + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted %v: one agent's suspend must not delete another's actor", fake.deleted) + } +} + +func TestSubstrateSnapshotOnIdleSkipsSupersededActorInOtherAtespace(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + + // The same agent is reconfigured onto another atespace: the launcher's + // client is for the new one, so the old actor must not be deleted + // through it (the delete would target a name never created there). + l.Options.Atespace = "hive-staging" + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted = %v, want no delete across atespaces", fake.deleted) + } + if out := logs.String(); !strings.Contains(out, first) || !strings.Contains(out, "another atespace") || !strings.Contains(out, "agent="+spec.Name) { + t.Errorf("log = %q, want a warning naming %s and the agent", out, first) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorLogsStructured(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + } + out := logs.String() + if !strings.Contains(out, "level=WARN") || !strings.Contains(out, "agent=deps-sandbox") || !strings.Contains(out, "error=\"not found\"") { + t.Errorf("log = %q, want a WARN record with agent and error attrs", out) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorDoesNotFailRun(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v (a failed GC delete must not fail the kick)", err) + } + } + if len(fake.deleted) != 1 { + t.Errorf("deleted = %v, want one attempted delete", fake.deleted) + } +} + func TestSubstrateRunContextCancelledMidKick(t *testing.T) { // The kick's context dies while the command runs: Pull is skipped, the // actor is deleted on a fresh context, and the caller gets ctx.Err(). diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index b2d0a8c838..d5f27798c2 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -254,7 +254,7 @@ opt-in and inert unless selected. | Go client `go.ate.dev/substrate` | **not imported**: it needs Go 1.27 (hive is on 1.26) and a very large module graph. A trimmed, wire-compatible proto subset is generated in-repo (`crux/sandbox/substrateapi`, field numbers pinned by tests); zero new Go modules, +578 KB on the `crux` binary | | image by tag or digest | digest-pinned `sandbox.substrate.image` only, enforced at boot (`AgentSandboxGateWarnings`) and at kick | | no `policy_profile` concept | passed to the actor as `HIVE_POLICY_PROFILE`; Substrate has no annotations | -| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; resume-on-next-kick is not implemented (#17697) | +| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; only the latest suspended actor per agent per hive process is retained (the previous one is deleted on the next suspend; actors from before a restart are not reclaimed); resume-on-next-kick is not implemented (#17697) | | `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | ### Cluster prerequisites and which hive clusters qualify diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 4b9cf5a4fc..a1e1779719 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -321,7 +321,7 @@ agents: **The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. -**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. +**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). At most one suspended actor is retained per agent per hive process: the previous one is deleted after the next successful suspend (skipped with a warning if the agent's `atespace` changed in between), and actors left by an earlier hive process are not reclaimed because the Control API has no `ListActors`. `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. **What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 8ac65300be..f8cb3e1d36 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -543,7 +543,7 @@ { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, - { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, ]; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 1511394525..0b3097ddd5 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -66690,7 +66690,7 @@

Escalation Breaker

{ key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents..sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents..sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.', agent: 'agents..sandbox.substrate.worker_selector.', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, - { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Costs object storage.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents..sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents..sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents..sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, ]; From d4a38e70896330146f99fcef159592ed5fd4ff2c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:03:33 +0000 Subject: [PATCH 051/166] =?UTF-8?q?=F0=9F=90=9B=20hub:=20converge=20startu?= =?UTF-8?q?pProbe=20and=20init=20image=20onto=20existing=20hosted=20deploy?= =?UTF-8?q?ments=20(#17831)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 hub: converge startupProbe and init image onto existing hosted deployments Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * docs: resync hub-server api-reference citations after server.go field insertion Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .../fixed-hub-reconcile-startup-probe.md | 1 + src/docs/api-reference/hub-server.md | 58 ++-- src/pkg/hub/saas_sha_poller.go | 5 + src/pkg/hub/server.go | 3 + src/pkg/hub/startup_probe_reconcile.go | 258 ++++++++++++++++++ src/pkg/hub/startup_probe_reconcile_test.go | 136 +++++++++ 6 files changed, 432 insertions(+), 29 deletions(-) create mode 100644 changelog.d/fixed-hub-reconcile-startup-probe.md create mode 100644 src/pkg/hub/startup_probe_reconcile.go create mode 100644 src/pkg/hub/startup_probe_reconcile_test.go diff --git a/changelog.d/fixed-hub-reconcile-startup-probe.md b/changelog.d/fixed-hub-reconcile-startup-probe.md new file mode 100644 index 0000000000..71a7a69c72 --- /dev/null +++ b/changelog.d/fixed-hub-reconcile-startup-probe.md @@ -0,0 +1 @@ +- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index c8891dd864..9a605a3930 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1792` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1793` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1795` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1796` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1724` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1725` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1726` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1727` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1728` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1739` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1740` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1741` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1742` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1727` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1728` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1729` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1730` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1731` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1742` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1743` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1744` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1745` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1750` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1760` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1761` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1762` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1763` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1764` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1765` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1766` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1773` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1783` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1800` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1805` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1807` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1753` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1763` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1764` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1765` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1766` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1767` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1768` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1776` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1784` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1786` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1803` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1808` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1810` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:36` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:37` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:38` | diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index a7997a6a39..be74e30875 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -720,6 +720,10 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // kubectl patch again (#17801). Throttled and rate-limited like the lane // above because each patch rolls a pod. s.reconcileHostAliasesIfDue() + // Converge the template startupProbe (#9692) and current init image repo + // onto hive Deployments provisioned before them, so a slow boot is not a + // permanent liveness kill-loop (#17827). Throttled and rate-limited. + s.reconcileStartupProbeIfDue() // Put #7457's auth-url / auth-response-headers onto the hive-contribute // Ingress of every hosted spoke provisioned before that fix, so // /api/contribute/me learns who is calling there too (#7517). The @@ -795,6 +799,7 @@ func (s *HubServer) pollLatestSHAsTick(ctx context.Context, now time.Time) { s.reconcileNodeHealthRBACIfDue() s.reconcilePerHiveEnvIfDue() s.reconcileHostAliasesIfDue() + s.reconcileStartupProbeIfDue() s.reapOrphanedPodsIfDue() s.retireExpiredGenerationsIfDue() s.sweepExpiredAccessIfDue() diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index b024fc42e5..aedd7a7e1f 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1213,6 +1213,9 @@ type HubServer struct { // lastHostAliasesReconcile throttles the per-cluster host_aliases // reconcile (cluster_host_aliases.go). Same mutex, same rationale. lastHostAliasesReconcile time.Time + // lastStartupProbeReconcile throttles the template startupProbe / init + // image reconcile (startup_probe_reconcile.go). Same mutex, same rationale. + lastStartupProbeReconcile time.Time // lastContributeIngressReconcile throttles the hive-contribute Ingress // auth-url reconcile (contribute_ingress_reconcile.go), which puts the // #7457 annotations onto spokes provisioned before that fix (#7517). diff --git a/src/pkg/hub/startup_probe_reconcile.go b/src/pkg/hub/startup_probe_reconcile.go new file mode 100644 index 0000000000..6ef0007df4 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile.go @@ -0,0 +1,258 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +const ( + // startupProbeReconcileInterval throttles the sweep. Like the NET_ADMIN and + // host_aliases lanes the drift is STATIC: a Deployment either carries the + // template's startupProbe / current init image or it does not. + startupProbeReconcileInterval = 15 * time.Minute + + // startupProbeKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + startupProbeKubectlTimeout = 15 * time.Second + + // startupProbeMaxPatchesPerCycle caps PATCHES per sweep: every patch + // changes the pod template and rolls that hive's pod once. + startupProbeMaxPatchesPerCycle = 3 + + // legacyHiveImageRepo is the pre-rename image repository. Its tags are no + // longer published, so an init container still pinned to it fails to pull + // on the next reschedule (#17827). + legacyHiveImageRepo = "ghcr.io/kubestellar/hive" + + // templateHiveImageRepo is the repository the provisioning template renders + // for every hive container and init container. + templateHiveImageRepo = bulkHiveImageRepo + + // hiveContainerName is the main container's name in the template. + hiveContainerName = "hive" +) + +// templateStartupProbe is the startupProbe the provisioning template renders +// onto the hive container (#9692), as the JSON value a patch writes. +type templateStartupProbe struct { + HTTPGet struct { + Path string `json:"path"` + Port int `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +func wantHostedStartupProbe() templateStartupProbe { + var p templateStartupProbe + p.HTTPGet.Path = "/api/health" + p.HTTPGet.Port = dashboardPort + p.InitialDelaySeconds = hostedStartupProbeInitialDelaySeconds + p.PeriodSeconds = hostedStartupProbePeriodSeconds + p.FailureThreshold = hostedStartupProbeFailureThreshold + return p +} + +// liveProbe is the subset of a live probe the template controls. Fields the +// API server defaults (scheme, timeoutSeconds, successThreshold) are ignored +// so a converged Deployment never compares unequal. +type liveProbe struct { + HTTPGet *struct { + Path string `json:"path"` + Port json.RawMessage `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +type liveContainer struct { + Name string `json:"name"` + Image string `json:"image"` + StartupProbe *liveProbe `json:"startupProbe"` +} + +type liveHiveDeployment struct { + Spec struct { + Template struct { + Spec struct { + InitContainers []liveContainer `json:"initContainers"` + Containers []liveContainer `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` +} + +func startupProbeMatches(live *liveProbe, want templateStartupProbe) bool { + if live == nil || live.HTTPGet == nil { + return false + } + port := strings.Trim(string(live.HTTPGet.Port), `"`) + return live.HTTPGet.Path == want.HTTPGet.Path && + port == strconv.Itoa(want.HTTPGet.Port) && + live.InitialDelaySeconds == want.InitialDelaySeconds && + live.PeriodSeconds == want.PeriodSeconds && + live.FailureThreshold == want.FailureThreshold +} + +// rewriteLegacyHiveImage returns the image with a legacyHiveImageRepo +// repository swapped for templateHiveImageRepo, keeping the tag (or digest). +// ok is false when the image is not on the legacy repository. +func rewriteLegacyHiveImage(image string) (string, bool) { + image = strings.TrimSpace(image) + if !strings.HasPrefix(image, legacyHiveImageRepo) { + return "", false + } + rest := image[len(legacyHiveImageRepo):] + if rest != "" && rest[0] != ':' && rest[0] != '@' { + return "", false + } + return templateHiveImageRepo + rest, true +} + +// startupProbePatchJSON is the PURE decision function for the sweep. Given the +// live Deployment JSON (`kubectl get deployment hive -o json`), it returns the +// JSON-Patch that sets the hive container's startupProbe to the template's and +// rewrites any init-container image on the legacy repository, plus a short +// description of each change. An empty patch means already converged. +func startupProbePatchJSON(deployJSON []byte) (string, []string, error) { + var d liveHiveDeployment + if err := json.Unmarshal(deployJSON, &d); err != nil { + return "", nil, fmt.Errorf("parsing deployment: %w", err) + } + podSpec := d.Spec.Template.Spec + var ops []map[string]any + var changes []string + + idx := -1 + for i, c := range podSpec.Containers { + if c.Name == hiveContainerName { + idx = i + break + } + } + if idx >= 0 { + want := wantHostedStartupProbe() + if !startupProbeMatches(podSpec.Containers[idx].StartupProbe, want) { + ops = append(ops, map[string]any{ + "op": "add", + "path": fmt.Sprintf("/spec/template/spec/containers/%d/startupProbe", idx), + "value": want, + }) + changes = append(changes, "startupProbe") + } + } + + for i, c := range podSpec.InitContainers { + img, ok := rewriteLegacyHiveImage(c.Image) + if !ok { + continue + } + ops = append(ops, map[string]any{ + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/initContainers/%d/image", i), + "value": img, + }) + changes = append(changes, "initContainers["+c.Name+"].image="+img) + } + + if len(ops) == 0 { + return "", nil, nil + } + b, err := json.Marshal(ops) + if err != nil { + return "", nil, err + } + return string(b), changes, nil +} + +// reconcileStartupProbeIfDue runs the sweep at most once per +// startupProbeReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileHostAliasesIfDue (poller-loop-only state). +func (s *HubServer) reconcileStartupProbeIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastStartupProbeReconcile.IsZero() || + time.Since(s.lastStartupProbeReconcile) >= startupProbeReconcileInterval + if due { + s.lastStartupProbeReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileStartupProbe() +} + +// reconcileStartupProbe converges the template's startupProbe (#9692) and the +// current init-container image repository onto hosted hive Deployments +// provisioned before those landed (#17827). Without the startupProbe a slow +// boot is killed by the livenessProbe at ~90s and never recovers. At most +// startupProbeMaxPatchesPerCycle patches per cycle, since each rolls a pod. +func (s *HubServer) reconcileStartupProbe() { + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "json") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("startupProbe reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + patch, changes, perr := startupProbePatchJSON(out) + if perr != nil { + s.logger.Warn("startupProbe reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + if patch == "" { + continue + } + if patched >= startupProbeMaxPatchesPerCycle { + deferred++ + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("startupProbe reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled template startupProbe/init image onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, + "changes", strings.Join(changes, ","), "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("startupProbe reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", startupProbeMaxPatchesPerCycle, + "next_cycle_in", startupProbeReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/startup_probe_reconcile_test.go b/src/pkg/hub/startup_probe_reconcile_test.go new file mode 100644 index 0000000000..a250e9af05 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile_test.go @@ -0,0 +1,136 @@ +package hub + +import ( + "encoding/json" + "strings" + "testing" +) + +type testPatchOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value json.RawMessage `json:"value"` +} + +func decodeStartupProbePatch(t *testing.T, patch string) []testPatchOp { + t.Helper() + var ops []testPatchOp + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + return ops +} + +func TestStartupProbePatchAddsMissingProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[{"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}], + "containers":[ + {"name":"sidecar","image":"busybox"}, + {"name":"hive","image":"ghcr.io/hivecommons/hive:v4-latest", + "livenessProbe":{"httpGet":{"path":"/api/livez","port":3002},"periodSeconds":30,"failureThreshold":3}} + ]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != "/spec/template/spec/containers/1/startupProbe" { + t.Fatalf("unexpected patch: %s", patch) + } + var got templateStartupProbe + if err := json.Unmarshal(ops[0].Value, &got); err != nil { + t.Fatal(err) + } + if got != wantHostedStartupProbe() { + t.Fatalf("patched probe %+v != template %+v", got, wantHostedStartupProbe()) + } + if got.HTTPGet.Path != "/api/health" || got.HTTPGet.Port != dashboardPort || got.FailureThreshold != hostedStartupProbeFailureThreshold { + t.Fatalf("probe does not match the template: %+v", got) + } + if len(changes) != 1 || changes[0] != "startupProbe" { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestStartupProbePatchReplacesDriftedProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive", + "startupProbe":{"httpGet":{"path":"/api/health","port":3002},"initialDelaySeconds":10,"periodSeconds":5,"failureThreshold":30}}]}}}}`) + patch, _, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if ops := decodeStartupProbePatch(t, patch); len(ops) != 1 || ops[0].Path != "/spec/template/spec/containers/0/startupProbe" { + t.Fatalf("drifted probe must be replaced: %s", patch) + } +} + +func TestStartupProbePatchNoopWhenConverged(t *testing.T) { + want := wantHostedStartupProbe() + // API-server defaulted fields (scheme, timeoutSeconds, successThreshold) + // must not make a converged Deployment look drifted. + probe := map[string]any{ + "httpGet": map[string]any{"path": want.HTTPGet.Path, "port": want.HTTPGet.Port, "scheme": "HTTP"}, + "initialDelaySeconds": want.InitialDelaySeconds, + "periodSeconds": want.PeriodSeconds, + "failureThreshold": want.FailureThreshold, + "timeoutSeconds": 1, + "successThreshold": 1, + } + dep := map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "initContainers": []any{map[string]any{"name": "init-permissions", "image": "ghcr.io/hivecommons/hive:v4-latest"}}, + "containers": []any{map[string]any{"name": "hive", "startupProbe": probe}}, + }}}} + live, _ := json.Marshal(dep) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if patch != "" || len(changes) != 0 { + t.Fatalf("converged deployment must not be patched, got %s (%v)", patch, changes) + } +} + +func TestStartupProbePatchRewritesLegacyInitImageKeepingTag(t *testing.T) { + want, _ := json.Marshal(wantHostedStartupProbe()) + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[ + {"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}, + {"name":"init-permissions","image":"ghcr.io/kubestellar/hive:v2-latest"} + ], + "containers":[{"name":"hive","startupProbe":` + string(want) + `}]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "replace" || ops[0].Path != "/spec/template/spec/initContainers/1/image" { + t.Fatalf("unexpected patch: %s", patch) + } + var img string + if err := json.Unmarshal(ops[0].Value, &img); err != nil { + t.Fatal(err) + } + if img != "ghcr.io/hivecommons/hive:v2-latest" { + t.Fatalf("legacy init image must move to the template repo keeping its tag, got %q", img) + } + if len(changes) != 1 || !strings.Contains(changes[0], "init-permissions") { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestRewriteLegacyHiveImage(t *testing.T) { + for in, want := range map[string]string{ + "ghcr.io/kubestellar/hive:v2-latest": "ghcr.io/hivecommons/hive:v2-latest", + "ghcr.io/kubestellar/hive": "ghcr.io/hivecommons/hive", + "ghcr.io/kubestellar/hive@sha256:abc": "ghcr.io/hivecommons/hive@sha256:abc", + } { + if got, ok := rewriteLegacyHiveImage(in); !ok || got != want { + t.Errorf("%s: got %q,%v want %q", in, got, ok, want) + } + } + for _, in := range []string{"ghcr.io/hivecommons/hive:v4-latest", "ghcr.io/kubestellar/hive-other:v1", "busybox"} { + if _, ok := rewriteLegacyHiveImage(in); ok { + t.Errorf("%s must not be rewritten", in) + } + } +} From 51874aea4ac637dbb07218511c4be1d8e916db2b Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 03:18:56 +0000 Subject: [PATCH 052/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-knowledge-plaud-source.md | 1 - ...ixed-dashboard-bind-not-gated-on-advisory-ensure.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-knowledge-plaud-source.md delete mode 100644 changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 102ac33d96..05b761601f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.0) + +### Added + +- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. + +### Fixed + +- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). + ## 2026-11-12 (v5.227.1) ### Fixed diff --git a/changelog.d/added-knowledge-plaud-source.md b/changelog.d/added-knowledge-plaud-source.md deleted file mode 100644 index c7cf5b02d6..0000000000 --- a/changelog.d/added-knowledge-plaud-source.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. diff --git a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md b/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md deleted file mode 100644 index b7cfafe241..0000000000 --- a/changelog.d/fixed-dashboard-bind-not-gated-on-advisory-ensure.md +++ /dev/null @@ -1 +0,0 @@ -- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). From 3abbe43b26ad6cc145df9382a28bd115c8af0107 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 20:24:26 -0700 Subject: [PATCH 053/166] =?UTF-8?q?=F0=9F=90=9B=20hub:=20negatively=20cach?= =?UTF-8?q?e=20rejected=20bearer=20tokens;=20stop=20re-probing=20exhausted?= =?UTF-8?q?=20user=20quota=20(#17837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateGitHubToken re-hit GET /user with the caller's own token on every request once the token was rejected or rate-limited, charging the owner's exhausted core bucket and prolonging the 401 outage. Remember 401/403/429 for 60 s (or until X-RateLimit-Reset, capped at 15 min), record the probe in REST accounting under the pass-through caller agent:hub-bearer-validate, and route hub release-notes contents reads through the shared ETag/accounting transport. Document the fleet-wide operator steps for getting personal tokens out of cluster secrets (ARC runner scale set -> GitHub App auth). Signed-off-by: Andrew Anderson --- .../fixed-hub-bearer-negative-cache.md | 1 + .../operator-reference/runtime-and-quota.md | 51 ++++++ src/pkg/hub/release_notes.go | 5 +- src/pkg/hub/saas_auth.go | 70 +++++++- src/pkg/hub/saas_auth_bearer_cache_test.go | 159 ++++++++++++++++++ 5 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-hub-bearer-negative-cache.md create mode 100644 src/pkg/hub/saas_auth_bearer_cache_test.go diff --git a/changelog.d/fixed-hub-bearer-negative-cache.md b/changelog.d/fixed-hub-bearer-negative-cache.md new file mode 100644 index 0000000000..2fde88abb7 --- /dev/null +++ b/changelog.d/fixed-hub-bearer-negative-cache.md @@ -0,0 +1 @@ +- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index dd3d67c23f..f85565eff8 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -183,6 +183,57 @@ loops, so the proxy enforces two safeguards: once. Bodies are stored gzip-compressed, so the cap is several times its size in raw JSON. +### Hub bearer validation and exhausted user tokens + +The hub validates a `Bearer` token it has not seen before with one +`GET /user` **using that token**, then caches the login for 5 minutes. A token +GitHub rejects (401, or 403/429 with `X-RateLimit-Remaining: 0`) is now also +remembered: for 60 s by default, or until the token's `X-RateLimit-Reset` +(capped at 15 minutes) when it is rate-limited. Without that negative entry a +client retrying with an exhausted token made the hub re-probe GitHub on every +request, charging the token owner's own core bucket while it was already at +zero. The probe is recorded in `/api/gh-rate-limits` under caller +`agent:hub-bearer-validate`, so a flood of 401s from one dead token is visible +on the hub admin page; being a pass-through credential it never updates the +hub's own core-bucket snapshot. + +### Keep personal tokens out of cluster secrets + +GitHub's 5,000 req/h core limit is **per user**, shared by every PAT and +OAuth token (`gho_…`, `ghp_…`, `github_pat_…`) that user owns, from every +host. A `gh auth token` pasted into a cluster secret therefore shares its +owner's bucket with their laptop `gh`, the hub's bearer validation of that +user, and any other service holding a copy. The hub itself authenticates as +the GitHub App (`HIVE_HUB_GITHUB_APP_ID` / `HIVE_HUB_GITHUB_INSTALLATION_ID` / +`HIVE_HUB_GITHUB_APP_KEY_FILE`, see [env-vars.md](../env-vars.md)) and uses +`HIVE_HUB_GITHUB_TOKEN` only as a fallback, logging a startup `WARN` naming the +identity it ended up with. Audit the rest of the fleet the same way: + +1. **Find copies.** For every cluster context, compare the first 8 characters + of each GitHub-looking secret value and `/data/gh-user-token` file against + the owner's token (`kubectl get secret -A -o json`, decode, never print the + full value). Known holders in a hive deployment are the Actions Runner + Controller (`gha-runner-scale-set`) `githubConfigSecret`, its generated + `*-listener-config` secret, and any sidecar that scrapes GitHub (dibs). +2. **Move ARC to App auth.** The runner scale-set listener long-polls the + Actions broker and acknowledges every message; with a PAT those calls are + charged to the PAT owner (~3,000 req/h at 350 ephemeral runners/h). Grant + the GitHub App repository **Administration: read & write** (or organization + **Self-hosted runners: read & write** for an org-level runner group), + then replace the `githubConfigSecret` keys `github_token` with + `github_app_id`, `github_app_installation_id` and + `github_app_private_key` (the same PEM mounted on the hub as + `hive-hub-github-app-key`), and delete the listener pod so the controller + regenerates its `*-listener-config` from the new secret. App traffic lands + in the installation's own 15,000 req/h bucket. +3. **Remove the PAT from the hub.** With the three App variables set, unset + `HIVE_HUB_GITHUB_TOKEN` on `deploy/hive-hub`; the startup log then reads + `hub GitHub API identity selected mode=app` instead of the `WARN`, and + `GET /api/gh-rate-limits` reports the App installation's `core.limit`. +4. **Rotate.** Once no cluster holds it, revoke the leaked user token + (`gh auth refresh` mints a new `gho_`), so stale copies fail closed rather + than silently sharing the bucket again. + The proxy returns JSON shaped for the GitHub CLI: ```json diff --git a/src/pkg/hub/release_notes.go b/src/pkg/hub/release_notes.go index 82bc94d3e2..8e621cd090 100644 --- a/src/pkg/hub/release_notes.go +++ b/src/pkg/hub/release_notes.go @@ -13,6 +13,7 @@ import ( "sync" "time" + hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/releasenotes" ) @@ -30,7 +31,9 @@ const ( var ( hubReleaseNotesRawBaseURL = "https://raw.githubusercontent.com" - hubReleaseNotesHTTP = &http.Client{Timeout: 10 * time.Second} + // Contents reads go through pkg/github's transport so repeat fetches of an + // unchanged ref are ETag 304s and the calls show up in /api/gh-rate-limits. + hubReleaseNotesHTTP = &http.Client{Transport: hgithub.NewHTTPTransport(nil), Timeout: 10 * time.Second} hubReleaseNotesSHAPattern = regexp.MustCompile(`^[0-9a-fA-F]{7,40}$`) ) diff --git a/src/pkg/hub/saas_auth.go b/src/pkg/hub/saas_auth.go index d493b6fee6..5c0a7d5281 100644 --- a/src/pkg/hub/saas_auth.go +++ b/src/pkg/hub/saas_auth.go @@ -8,11 +8,14 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "sync" "time" "golang.org/x/net/publicsuffix" + + hgithub "github.com/hivecommons/hive/pkg/github" ) func (s *HubServer) requireAuth(next http.HandlerFunc) http.HandlerFunc { @@ -691,13 +694,60 @@ var ( ghTokenCache = map[string]ghTokenCacheEntry{} ) -const ghTokenCacheTTL = 5 * time.Minute +const ( + ghTokenCacheTTL = 5 * time.Minute + // ghTokenNegativeTTL bounds how long a rejected bearer token (401/403, + // including a rate-limited one) is remembered without re-probing + // GET /user. Without it every request carrying a dead or exhausted token + // re-hit GitHub, charging the token owner's own core quota while it was + // already at zero and stretching the outage. + ghTokenNegativeTTL = 60 * time.Second + // ghTokenNegativeMaxTTL caps how far a 403's X-RateLimit-Reset may push + // the negative entry so a skewed clock cannot lock a user out for long. + ghTokenNegativeMaxTTL = 15 * time.Minute + ghTokenValidateTimeout = 10 * time.Second + ghTokenValidateCaller = "agent:hub-bearer-validate" +) type ghTokenCacheEntry struct { username string expiresAt time.Time } +// ghTokenValidateHTTPClient probes GET /user for bearer validation. It is a +// plain client on purpose: the probe carries the CALLER's token, and routing +// it through pkg/github's shared transport chain would let one exhausted user +// bucket push the hub's own App traffic into rate-limit caution. The probe is +// still recorded in REST accounting (see validateGitHubToken). +func ghTokenValidateHTTPClient() *http.Client { + return &http.Client{Timeout: ghTokenValidateTimeout} +} + +// ghTokenNegativeExpiry picks the expiry of a negative cache entry for a +// rejected probe: the default short TTL, extended up to ghTokenNegativeMaxTTL +// when GitHub says the token is rate-limited until X-RateLimit-Reset. +func ghTokenNegativeExpiry(status int, hdr http.Header, now time.Time) time.Time { + exp := now.Add(ghTokenNegativeTTL) + if status != http.StatusForbidden && status != http.StatusTooManyRequests { + return exp + } + if hdr.Get("X-RateLimit-Remaining") != "0" { + return exp + } + reset, err := strconv.ParseInt(strings.TrimSpace(hdr.Get("X-RateLimit-Reset")), 10, 64) + if err != nil { + return exp + } + until := time.Unix(reset, 0) + if until.After(now.Add(ghTokenNegativeMaxTTL)) { + until = now.Add(ghTokenNegativeMaxTTL) + } + if until.After(exp) { + exp = until + } + return exp +} + func (s *HubServer) validateGitHubToken(token string) string { if token == "" { return "" @@ -710,19 +760,31 @@ func (s *HubServer) validateGitHubToken(token string) string { } ghTokenCacheMu.RUnlock() - client := &http.Client{Timeout: 10 * time.Second} + client := ghTokenValidateHTTPClient() // Hub always validates tokens against github.com (the hub is a SaaS service). - req, err := http.NewRequest("GET", defaultGHUserURL, nil) + req, err := http.NewRequest(http.MethodGet, defaultGHUserURL, nil) if err != nil { return "" } req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Accept", "application/vnd.github+json") resp, err := client.Do(req) - if err != nil || resp.StatusCode != 200 { + if err != nil { return "" } defer func() { _ = resp.Body.Close() }() + // Attributed under the pass-through ("agent:") caller prefix: visible in + // /api/gh-rate-limits, but the caller's rate-limit headers never overwrite + // the hub's own core-bucket snapshot. + hgithub.RecordRESTRequest(ghTokenValidateCaller, req.Method, req.URL.Path, resp.StatusCode, resp.Header) + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + ghTokenCacheMu.Lock() + ghTokenCache[token] = ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())} + ghTokenCacheMu.Unlock() + } + return "" + } var user struct { Login string `json:"login"` } diff --git a/src/pkg/hub/saas_auth_bearer_cache_test.go b/src/pkg/hub/saas_auth_bearer_cache_test.go new file mode 100644 index 0000000000..b9c62866e5 --- /dev/null +++ b/src/pkg/hub/saas_auth_bearer_cache_test.go @@ -0,0 +1,159 @@ +package hub + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" + + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func withBearerProbeServer(t *testing.T, handler http.HandlerFunc) *int32 { + t.Helper() + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + handler(w, r) + })) + oldURL := defaultGHUserURL + defaultGHUserURL = srv.URL + "/user" + t.Cleanup(func() { + defaultGHUserURL = oldURL + srv.Close() + }) + return &hits +} + +func clearGHTokenCache(t *testing.T, tokens ...string) { + t.Helper() + clear := func() { + ghTokenCacheMu.Lock() + for _, tok := range tokens { + delete(ghTokenCache, tok) + } + ghTokenCacheMu.Unlock() + } + clear() + t.Cleanup(clear) +} + +// A rejected bearer token must not re-probe GET /user on every request while +// the negative entry is fresh: that re-probe charged the token owner's own +// core quota while it was already exhausted (the 401 storm in the PAT outage). +func TestValidateGitHubToken_RejectedTokenIsNegativelyCached(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + const token = "dead-token-negative-cache" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + for i := 0; i < 3; i++ { + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("call %d: rejected token resolved to %q", i, got) + } + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times for a rejected token; want 1", n) + } + ghTokenCacheMu.RLock() + entry, ok := ghTokenCache[token] + ghTokenCacheMu.RUnlock() + if !ok || entry.username != "" || !entry.expiresAt.After(time.Now()) { + t.Fatalf("negative entry = %+v, ok=%v; want an unexpired entry with empty username", entry, ok) + } +} + +// An expired negative entry probes again, and a token that has since become +// valid resolves — the negative cache is a short hold, not a ban. +func TestValidateGitHubToken_ExpiredNegativeEntryReprobes(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "revived"}) + }) + const token = "revived-token" + clearGHTokenCache(t, token) + ghTokenCacheMu.Lock() + ghTokenCache[token] = ghTokenCacheEntry{expiresAt: time.Now().Add(-time.Second)} + ghTokenCacheMu.Unlock() + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "revived" { + t.Fatalf("validateGitHubToken = %q, want revived", got) + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times; want 1", n) + } +} + +func TestGhTokenNegativeExpiry(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + hdr := func(remaining string, reset time.Time) http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", remaining) + h.Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + return h + } + cases := []struct { + name string + status int + hdr http.Header + want time.Time + }{ + {"401 uses the short hold", http.StatusUnauthorized, http.Header{}, now.Add(ghTokenNegativeTTL)}, + {"403 with quota left is not a rate limit", http.StatusForbidden, hdr("12", now.Add(time.Hour)), now.Add(ghTokenNegativeTTL)}, + {"403 exhausted holds until reset", http.StatusForbidden, hdr("0", now.Add(5*time.Minute)), now.Add(5 * time.Minute)}, + {"429 exhausted holds until reset", http.StatusTooManyRequests, hdr("0", now.Add(3*time.Minute)), now.Add(3 * time.Minute)}, + {"reset far away is capped", http.StatusForbidden, hdr("0", now.Add(2*time.Hour)), now.Add(ghTokenNegativeMaxTTL)}, + {"reset already passed keeps the short hold", http.StatusForbidden, hdr("0", now.Add(-time.Minute)), now.Add(ghTokenNegativeTTL)}, + {"garbage reset keeps the short hold", http.StatusForbidden, func() http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", "0") + h.Set("X-RateLimit-Reset", "soon") + return h + }(), now.Add(ghTokenNegativeTTL)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ghTokenNegativeExpiry(tc.status, tc.hdr, now); !got.Equal(tc.want) { + t.Fatalf("ghTokenNegativeExpiry = %v, want %v", got, tc.want) + } + }) + } +} + +// The probe is attributed in /api/gh-rate-limits under the pass-through +// ("agent:") caller so an exhausted user token is visible on the hub admin +// page without being mistaken for the hub's own App credential. +func TestValidateGitHubToken_ProbeIsAccountedAsPassThroughCaller(t *testing.T) { + withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10)) + w.WriteHeader(http.StatusForbidden) + }) + const token = "accounted-probe-token" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("validateGitHubToken = %q, want empty", got) + } + var found *hgithub.RESTConsumer + for _, c := range hgithub.RESTTopConsumers(100) { + if c.Caller == ghTokenValidateCaller && c.Endpoint == "/user" { + c := c + found = &c + break + } + } + if found == nil { + t.Fatalf("caller %q not found in REST accounting: %+v", ghTokenValidateCaller, hgithub.RESTTopConsumers(100)) + } + if found.RateLimited == 0 { + t.Fatalf("probe not recorded as rate-limited: %+v", *found) + } +} From e51d635ffe6bd1fdc8642e7d3b612c0dd2f6b4dd Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:50:12 +0000 Subject: [PATCH 054/166] =?UTF-8?q?=F0=9F=90=9B=20crux=20serve:=20count=20?= =?UTF-8?q?auth-rejected=20requests=20in=20crux.serve.auth.rejections=20(#?= =?UTF-8?q?17839)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instrument sits inside authenticate, so requests refused with a 401 never reached crux.serve.requests or crux.serve.request.duration. Wrapping authenticate with instrument would label them with the unmatched route, so a separate counter, crux.serve.auth.rejections, is incremented in each 401 branch of authenticate. Its only attribute is a fixed-enum reason (missing_or_invalid_token, project_forbidden, no_token_configured); never a token, path or remote address. Closes hivecommons/crux#146 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-crux-serve-auth-rejections.md | 1 + src/crux/serve/metrics.go | 24 ++++++++++- src/crux/serve/metrics_test.go | 41 +++++++++++++++++++ src/crux/serve/serve.go | 3 ++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-crux-serve-auth-rejections.md diff --git a/changelog.d/fixed-crux-serve-auth-rejections.md b/changelog.d/fixed-crux-serve-auth-rejections.md new file mode 100644 index 0000000000..ba4e30efba --- /dev/null +++ b/changelog.d/fixed-crux-serve-auth-rejections.md @@ -0,0 +1 @@ +- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. diff --git a/src/crux/serve/metrics.go b/src/crux/serve/metrics.go index 4ea8b016c7..e21088a79c 100644 --- a/src/crux/serve/metrics.go +++ b/src/crux/serve/metrics.go @@ -26,6 +26,7 @@ const ( requestDurationMetric = "crux.serve.request.duration" requestCountMetric = "crux.serve.requests" + authRejectionMetric = "crux.serve.auth.rejections" routeUnmatched = "unmatched" @@ -38,16 +39,28 @@ const ( methodOther = "_OTHER" ) +// Why authenticate refused a request: a fixed enum, so the rejection counter's +// reason attribute stays bounded. +const ( + rejectProjectForbidden = "project_forbidden" + rejectNoTokenConfigured = "no_token_configured" + rejectInvalidToken = "missing_or_invalid_token" +) + var ( + attrReason = attribute.Key("reason") attrMethod = attribute.Key("method") attrRoute = attribute.Key("route") attrStatusClass = attribute.Key("status_class") ) -// requestMetrics holds the two serve request instruments. +// requestMetrics holds the serve request and auth rejection instruments. type requestMetrics struct { duration metric.Float64Histogram requests metric.Int64Counter + // rejections counts requests authenticate refused with a 401. They never + // reach instrument, which sits inside authenticate. + rejections metric.Int64Counter } func newRequestMetrics(meter metric.Meter) requestMetrics { @@ -58,7 +71,9 @@ func newRequestMetrics(meter metric.Meter) requestMetrics { metric.WithDescription("Time crux serve took to answer a request.")) requests, _ := meter.Int64Counter(requestCountMetric, metric.WithDescription("crux serve requests, by method, route pattern and status class.")) - return requestMetrics{duration: duration, requests: requests} + rejections, _ := meter.Int64Counter(authRejectionMetric, + metric.WithDescription("crux serve requests refused by authentication, by reason.")) + return requestMetrics{duration: duration, requests: requests, rejections: rejections} } var globalRequestMetrics = sync.OnceValue(func() requestMetrics { @@ -168,6 +183,11 @@ func instrument(next http.Handler) http.Handler { }) } +// recordAuthRejection counts one 401 with only the bounded reason attribute. +func recordAuthRejection(ctx context.Context, reason string) { + globalRequestMetrics().rejections.Add(ctx, 1, metric.WithAttributes(attrReason.String(reason))) +} + func (m requestMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { opt := metric.WithAttributes(attrs...) m.duration.Record(ctx, d.Seconds(), opt) diff --git a/src/crux/serve/metrics_test.go b/src/crux/serve/metrics_test.go index 823724d53a..256fbeb7ef 100644 --- a/src/crux/serve/metrics_test.go +++ b/src/crux/serve/metrics_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux/api" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" sdkmetric "go.opentelemetry.io/otel/sdk/metric" @@ -334,3 +336,42 @@ func collectStatusClasses(t *testing.T, reader *sdkmetric.ManualReader) []string } return classes } + +func TestAuthRejectionsCountedByReason(t *testing.T) { + reader := useTestMetrics(t) + _, h := principalFixture(t) + + callWith(h, api.PathHealth, "wrong-secret-token") + callWith(h, api.PathHealth, "") + callWith(h, api.PathHealth, testProjectToken) + callWith(h, api.PathHealth, "another-wrong-token") + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + got := map[string]int64{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != authRejectionMetric { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + if dp.Attributes.Len() != 1 { + t.Errorf("attributes = %v, want only reason", dp.Attributes) + } + reason, _ := dp.Attributes.Value(attrReason) + got[reason.AsString()] = dp.Value + } + } + } + want := map[string]int64{rejectInvalidToken: 3, rejectProjectForbidden: 1} + if len(got) != len(want) { + t.Fatalf("rejections = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("rejections[%s] = %d, want %d", k, got[k], v) + } + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 490eff2750..de7d07870a 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -209,6 +209,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if project, projectToken := s.LaneAuth.Project(token); projectToken { if !s.isWorkService(r.URL.Path) { + recordAuthRejection(r.Context(), rejectProjectForbidden) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") return } @@ -217,6 +218,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } if s.Token == "" { if network != networkUnix { + recordAuthRejection(r.Context(), rejectNoTokenConfigured) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") return } @@ -225,6 +227,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if !ok || subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + recordAuthRejection(r.Context(), rejectInvalidToken) w.Header().Set("WWW-Authenticate", `Bearer realm="crux"`) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") return From d4fcb55bb7532124fcce97de0385296c804cf7bc Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 03:52:41 +0000 Subject: [PATCH 055/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-hub-reconcile-startup-probe.md | 1 - changelog.d/fixed-substrate-snapshot-on-idle-gc.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-hub-reconcile-startup-probe.md delete mode 100644 changelog.d/fixed-substrate-snapshot-on-idle-gc.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b761601f..ca2c0d1e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.1) + +### Fixed + +- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) +- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). + ## 2026-11-12 (v5.228.0) ### Added diff --git a/changelog.d/fixed-hub-reconcile-startup-probe.md b/changelog.d/fixed-hub-reconcile-startup-probe.md deleted file mode 100644 index 71a7a69c72..0000000000 --- a/changelog.d/fixed-hub-reconcile-startup-probe.md +++ /dev/null @@ -1 +0,0 @@ -- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) diff --git a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md b/changelog.d/fixed-substrate-snapshot-on-idle-gc.md deleted file mode 100644 index 97a4d8f193..0000000000 --- a/changelog.d/fixed-substrate-snapshot-on-idle-gc.md +++ /dev/null @@ -1 +0,0 @@ -- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). From e1fd7b0bbd7d65716b6ccd4fe8162a62b3e29acd Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:16:53 +0100 Subject: [PATCH 056/166] fix(hub): bound bearer token validation cache (#17846) Sweep expired entries at capacity and skip new entries when all slots remain occupied. Apply the same limit to accepted and rejected tokens. Hive-Run: hivecommons/hive#17844 Hive-Plan: issue-17844-cap-and-expired-sweep Hive-Spec: issue-17844#recommendation Signed-off-by: Karibhou Co-authored-by: Karibhou --- changelog.d/security-17844-bearer-cache.md | 1 + src/pkg/hub/saas_auth.go | 33 ++++++++--- src/pkg/hub/saas_auth_bearer_cache_test.go | 68 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 changelog.d/security-17844-bearer-cache.md diff --git a/changelog.d/security-17844-bearer-cache.md b/changelog.d/security-17844-bearer-cache.md new file mode 100644 index 0000000000..8d537d4e6e --- /dev/null +++ b/changelog.d/security-17844-bearer-cache.md @@ -0,0 +1 @@ +- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. diff --git a/src/pkg/hub/saas_auth.go b/src/pkg/hub/saas_auth.go index 5c0a7d5281..2daf88d43a 100644 --- a/src/pkg/hub/saas_auth.go +++ b/src/pkg/hub/saas_auth.go @@ -695,7 +695,8 @@ var ( ) const ( - ghTokenCacheTTL = 5 * time.Minute + ghTokenCacheMaxEntries = 10_000 + ghTokenCacheTTL = 5 * time.Minute // ghTokenNegativeTTL bounds how long a rejected bearer token (401/403, // including a rate-limited one) is remembered without re-probing // GET /user. Without it every request carrying a dead or exhausted token @@ -714,6 +715,28 @@ type ghTokenCacheEntry struct { expiresAt time.Time } +// cacheGHToken bounds both positive and negative entries. At capacity, expired +// entries are reclaimed; if all entries are live, new tokens are not cached. +func cacheGHToken(token string, entry ghTokenCacheEntry) { + ghTokenCacheMu.Lock() + defer ghTokenCacheMu.Unlock() + + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + now := time.Now() + for key, cached := range ghTokenCache { + if !now.Before(cached.expiresAt) { + delete(ghTokenCache, key) + } + } + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + if _, exists := ghTokenCache[token]; !exists { + return + } + } + } + ghTokenCache[token] = entry +} + // ghTokenValidateHTTPClient probes GET /user for bearer validation. It is a // plain client on purpose: the probe carries the CALLER's token, and routing // it through pkg/github's shared transport chain would let one exhausted user @@ -779,9 +802,7 @@ func (s *HubServer) validateGitHubToken(token string) string { hgithub.RecordRESTRequest(ghTokenValidateCaller, req.Method, req.URL.Path, resp.StatusCode, resp.Header) if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { - ghTokenCacheMu.Lock() - ghTokenCache[token] = ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())} - ghTokenCacheMu.Unlock() + cacheGHToken(token, ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())}) } return "" } @@ -792,9 +813,7 @@ func (s *HubServer) validateGitHubToken(token string) string { return "" } - ghTokenCacheMu.Lock() - ghTokenCache[token] = ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)} - ghTokenCacheMu.Unlock() + cacheGHToken(token, ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)}) return user.Login } diff --git a/src/pkg/hub/saas_auth_bearer_cache_test.go b/src/pkg/hub/saas_auth_bearer_cache_test.go index b9c62866e5..942b8ba717 100644 --- a/src/pkg/hub/saas_auth_bearer_cache_test.go +++ b/src/pkg/hub/saas_auth_bearer_cache_test.go @@ -68,6 +68,74 @@ func TestValidateGitHubToken_RejectedTokenIsNegativelyCached(t *testing.T) { } } +func TestValidateGitHubToken_CacheCapacityAndExpiry(t *testing.T) { + ghTokenCacheMu.Lock() + original := ghTokenCache + ghTokenCache = make(map[string]ghTokenCacheEntry) + ghTokenCacheMu.Unlock() + t.Cleanup(func() { + ghTokenCacheMu.Lock() + ghTokenCache = original + ghTokenCacheMu.Unlock() + }) + + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer valid-overflow" { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "valid-user"}) + return + } + w.WriteHeader(http.StatusUnauthorized) + }) + srv := newHubServerForTest(t) + for i := range ghTokenCacheMaxEntries + 1 { + token := "rejected-capacity-" + strconv.Itoa(i) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("rejected token resolved to %q", got) + } + } + ghTokenCacheMu.RLock() + size := len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if size != ghTokenCacheMaxEntries { + t.Fatalf("cache has %d entries, want %d", size, ghTokenCacheMaxEntries) + } + + // Both rejected and accepted tokens still probe when the cache is full. + for range 2 { + if got := srv.validateGitHubToken("rejected-capacity-" + strconv.Itoa(ghTokenCacheMaxEntries)); got != "" { + t.Fatalf("overflow rejected token resolved to %q", got) + } + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("overflow accepted token resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+5); got != want { + t.Fatalf("GET /user probed %d times, want %d", got, want) + } + ghTokenCacheMu.Lock() + entry := ghTokenCache["rejected-capacity-0"] + entry.expiresAt = time.Now().Add(-time.Second) + ghTokenCache["rejected-capacity-0"] = entry + ghTokenCacheMu.Unlock() + + // A full cache sweeps expired entries and makes room for a new positive entry. + for range 2 { + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("accepted token after expiry resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+6); got != want { + t.Fatalf("GET /user probed %d times after expiry, want %d", got, want) + } + ghTokenCacheMu.RLock() + _, expiredPresent := ghTokenCache["rejected-capacity-0"] + size = len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if expiredPresent || size != ghTokenCacheMaxEntries { + t.Fatalf("after sweep: expired present = %t, cache size = %d", expiredPresent, size) + } +} + // An expired negative entry probes again, and a token that has since become // valid resolves — the negative cache is a short hold, not a ban. func TestValidateGitHubToken_ExpiredNegativeEntryReprobes(t *testing.T) { From 4542de28d2cf9b3b44f4efdbbe4d82869eb6523b Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 04:23:04 +0000 Subject: [PATCH 057/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-crux-serve-auth-rejections.md | 1 - changelog.d/fixed-hub-bearer-negative-cache.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-crux-serve-auth-rejections.md delete mode 100644 changelog.d/fixed-hub-bearer-negative-cache.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2c0d1e0b..6d84059b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.2) + +### Fixed + +- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. +- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. + ## 2026-11-12 (v5.228.1) ### Fixed diff --git a/changelog.d/fixed-crux-serve-auth-rejections.md b/changelog.d/fixed-crux-serve-auth-rejections.md deleted file mode 100644 index ba4e30efba..0000000000 --- a/changelog.d/fixed-crux-serve-auth-rejections.md +++ /dev/null @@ -1 +0,0 @@ -- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. diff --git a/changelog.d/fixed-hub-bearer-negative-cache.md b/changelog.d/fixed-hub-bearer-negative-cache.md deleted file mode 100644 index 2fde88abb7..0000000000 --- a/changelog.d/fixed-hub-bearer-negative-cache.md +++ /dev/null @@ -1 +0,0 @@ -- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. From 5befa566c6af7050f660e47901f519637be353aa Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:42:17 +0000 Subject: [PATCH 058/166] =?UTF-8?q?=F0=9F=94=92=20knowledge:=20start=20the?= =?UTF-8?q?=20Plaud=20MCP=20server=20with=20an=20allowlisted=20env,=20not?= =?UTF-8?q?=20the=20hive=20process=20env=20(#17845)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stdioMCPDialer passed os.Environ() to a third-party MCP server fetched at run time (npx -y @plaud-ai/mcp@latest), so every hub-injected credential the hive process carries — DASHBOARD_AUTH_TOKEN (server-verified owner on the dashboard API), HIVE_TERMINAL_KEY, HIVE_HEARTBEAT_KEY, HIVE_HUB_SECRET, HIVE_UPSTREAM_PROXY_URL (proxy Basic credential), GitHub/Linear/provider keys — reached code the hive does not control. crux/agent strips the same set from agent panes (#17657); the MCP dialer bypassed that. The child now gets an allowlist: PATH, HOME, locale, TMPDIR, TERM, the proxy and CA-bundle variables Node needs to reach the registry through the egress proxy, NODE_*/npm_config_*/NPM_CONFIG_* and PLAUD_*. Any name that contains TOKEN, SECRET, PASSWORD or CREDENTIAL is dropped even when it matches a prefix. Fixes #17843 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-mcp-stdio-env-allowlist.md | 1 + src/docs/knowledge-connectors.md | 8 ++ src/pkg/knowledge/connector/mcpstdio.go | 57 +++++++++++++- src/pkg/knowledge/connector/mcpstdio_test.go | 76 +++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 changelog.d/security-mcp-stdio-env-allowlist.md diff --git a/changelog.d/security-mcp-stdio-env-allowlist.md b/changelog.d/security-mcp-stdio-env-allowlist.md new file mode 100644 index 0000000000..6d09e3f87a --- /dev/null +++ b/changelog.d/security-mcp-stdio-env-allowlist.md @@ -0,0 +1 @@ +- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index d9328667f9..cd39352198 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -256,6 +256,14 @@ stores or logs the token. By default the connector points the MCP server's so the token survives restarts; set `home: inherit` to use the hive process's HOME instead. +The MCP server is third-party code fetched at run time, so it does not inherit +the hive process environment. It is started with an allowlist only — `PATH`, +`HOME`, locale, `TMPDIR`, `TERM`, the `HTTP(S)_PROXY`/`NO_PROXY` and CA-bundle +variables, `NODE_*`/`npm_config_*` and `PLAUD_*` — and never the hub-injected +credentials (`DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY`, `HIVE_UPSTREAM_PROXY_URL`, +provider API keys). Pin `scope.command` to a specific `@plaud-ai/mcp@` +if you do not want a sync to pull a new upstream release unreviewed. + ### Setup 1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run diff --git a/src/pkg/knowledge/connector/mcpstdio.go b/src/pkg/knowledge/connector/mcpstdio.go index 6cc36e2f15..5d916fcc72 100644 --- a/src/pkg/knowledge/connector/mcpstdio.go +++ b/src/pkg/knowledge/connector/mcpstdio.go @@ -266,9 +266,62 @@ func (b *lockedBuffer) Tail(n int) string { return s } +// mcpChildEnvNames are the exact environment variables a stdio MCP server +// inherits from the hive process; mcpChildEnvPrefixes extend the set by +// prefix. The server is third-party code fetched at run time (the Plaud +// default is `npx -y @plaud-ai/mcp@latest`), so it gets what a Node process +// needs to start and reach the registry through the egress proxy — never the +// hub-injected credentials the hive process carries (DASHBOARD_AUTH_TOKEN, +// HIVE_TERMINAL_KEY, HIVE_UPSTREAM_PROXY_URL, provider API keys, …), which +// crux/agent strips from agent panes for the same reason. +var mcpChildEnvNames = map[string]bool{ + "PATH": true, "HOME": true, "USER": true, "LOGNAME": true, "SHELL": true, + "TMPDIR": true, "TMP": true, "TEMP": true, "TZ": true, "TERM": true, "LANG": true, "LANGUAGE": true, + "HTTP_PROXY": true, "HTTPS_PROXY": true, "NO_PROXY": true, + "http_proxy": true, "https_proxy": true, "no_proxy": true, + "SSL_CERT_FILE": true, "SSL_CERT_DIR": true, "NODE_EXTRA_CA_CERTS": true, +} + +var mcpChildEnvPrefixes = []string{"LC_", "XDG_", "NODE_", "NPM_CONFIG_", "npm_config_", "PLAUD_"} + +// mcpChildEnv filters base (KEY=VALUE pairs) down to the allowlisted names. +// A name that looks like a credential is dropped even when it matches, so a +// future prefix cannot readmit one. +func mcpChildEnv(base []string) []string { + out := make([]string, 0, len(base)) + for _, entry := range base { + key, _, ok := strings.Cut(entry, "=") + if !ok || key == "" || mcpCredentialName(key) { + continue + } + if mcpChildEnvNames[key] { + out = append(out, entry) + continue + } + for _, p := range mcpChildEnvPrefixes { + if strings.HasPrefix(key, p) { + out = append(out, entry) + break + } + } + } + return out +} + +func mcpCredentialName(key string) bool { + u := strings.ToUpper(key) + for _, marker := range []string{"TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"} { + if strings.Contains(u, marker) { + return true + } + } + return false +} + // stdioMCPDialer starts command as a child process per session. home, when // non-empty, overrides HOME so the server keeps its own token store in a -// persistent directory. +// persistent directory. The child's environment is the allowlisted subset +// mcpChildEnv keeps, not the hive process environment. func stdioMCPDialer(command, home string) mcpDialer { return func(ctx context.Context) (mcpClient, error) { argv, err := splitCommandLine(command) @@ -279,7 +332,7 @@ func stdioMCPDialer(command, home string) mcpDialer { return nil, errors.New("mcp: command is empty") } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) - cmd.Env = os.Environ() + cmd.Env = mcpChildEnv(os.Environ()) if home != "" { if err := os.MkdirAll(home, 0o700); err != nil { return nil, fmt.Errorf("mcp: creating home %s: %w", home, err) diff --git a/src/pkg/knowledge/connector/mcpstdio_test.go b/src/pkg/knowledge/connector/mcpstdio_test.go index 97a63e0df1..2fce7e8932 100644 --- a/src/pkg/knowledge/connector/mcpstdio_test.go +++ b/src/pkg/knowledge/connector/mcpstdio_test.go @@ -180,3 +180,79 @@ func TestStdioMCPDialerSpawnsProcess(t *testing.T) { t.Fatalf("HOME dir must be created: %v", err) } } + +func TestMCPChildEnvDropsHiveSecrets(t *testing.T) { + base := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + "DASHBOARD_AUTH_TOKEN=owner-token", + "HIVE_DASHBOARD_TOKEN=owner-token", + "HIVE_TERMINAL_KEY=terminal-key", + "HIVE_HEARTBEAT_KEY=heartbeat-key", + "HIVE_HUB_SECRET=hub-secret", + "HIVE_UPSTREAM_PROXY_URL=http://user:pass@corp:3128", + "HIVE_GITHUB_TOKEN=ghs_x", + "GITHUB_TOKEN=ghp_x", + "ANTHROPIC_API_KEY=sk-ant", + "LINEAR_CLIENT_SECRET=lin", + // Allowlisted prefix but credential-shaped: must still be dropped. + "NODE_AUTH_TOKEN=npm-token", + "PLAUD_SECRET=plaud", + "malformed-no-equals", + } + got := mcpChildEnv(base) + want := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("mcpChildEnv =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// The spawned MCP server must not see the hub-injected credentials the hive +// process carries (hivecommons/hive#17843): the same secrets crux/agent +// strips from agent panes. +func TestStdioMCPDialerDoesNotLeakHiveEnv(t *testing.T) { + t.Setenv("DASHBOARD_AUTH_TOKEN", "dash_auth_leaked") + t.Setenv("HIVE_TERMINAL_KEY", "terminal_key_leaked") + t.Setenv("HIVE_UPSTREAM_PROXY_URL", "http://u:p@corp:3128") + t.Setenv("PLAUD_REGION", "eu") + + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + // Answers initialize, then reports the env it sees on the first tools/call. + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\n" + + "printf '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"dash=%s term=%s proxy=%s region=%s path_set=%s\"}]}}\\n' " + + "\"$DASHBOARD_AUTH_TOKEN\" \"$HIVE_TERMINAL_KEY\" \"$HIVE_UPSTREAM_PROXY_URL\" \"$PLAUD_REGION\" \"${PATH:+yes}\"\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + got, err := c.Call(ctx, "env", nil) + if err != nil { + t.Fatalf("call: %v", err) + } + if got != "dash= term= proxy= region=eu path_set=yes" { + t.Fatalf("child env = %q; hub secrets must be absent, PLAUD_* and PATH present", got) + } +} From 8f650a52ee81e6b9c7dc6c07630948e5643efd88 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 04:52:03 +0000 Subject: [PATCH 059/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.3 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/security-17844-bearer-cache.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-17844-bearer-cache.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d84059b50..6c86909221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.3) + +### Security + +- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. + ## 2026-11-12 (v5.228.2) ### Fixed diff --git a/changelog.d/security-17844-bearer-cache.md b/changelog.d/security-17844-bearer-cache.md deleted file mode 100644 index 8d537d4e6e..0000000000 --- a/changelog.d/security-17844-bearer-cache.md +++ /dev/null @@ -1 +0,0 @@ -- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. From a9ac7910699ddbf4a32197d38d094a3ba77449c6 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 05:21:17 +0000 Subject: [PATCH 060/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.4 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/security-mcp-stdio-env-allowlist.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-mcp-stdio-env-allowlist.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c86909221..bf097b5b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.4) + +### Security + +- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). + ## 2026-11-12 (v5.228.3) ### Security diff --git a/changelog.d/security-mcp-stdio-env-allowlist.md b/changelog.d/security-mcp-stdio-env-allowlist.md deleted file mode 100644 index 6d09e3f87a..0000000000 --- a/changelog.d/security-mcp-stdio-env-allowlist.md +++ /dev/null @@ -1 +0,0 @@ -- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). From 6a554fd9e305299cd30b22c94c98d622dfe8a4bd Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 22:38:25 -0700 Subject: [PATCH 061/166] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20omp=201?= =?UTF-8?q?8.18.0=20->=2018.19.0=20(#17850)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=18.19.0 SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 Signed-off-by: hive-release[bot] Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] --- src/Dockerfile | 6 +++--- src/Dockerfile.contributor | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 65c9b3a47d..70476b97bc 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -411,9 +411,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index be037588fc..0b4f73bf04 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -465,9 +465,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ From 160d683985b938798d2b4c00baa0439659d1575e Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 22:52:26 -0700 Subject: [PATCH 062/166] =?UTF-8?q?=F0=9F=90=9B=20army:=20stop=20attributi?= =?UTF-8?q?ng=20adopted=20PRs=20to=20a=20remote=20agent's=20rework=20queue?= =?UTF-8?q?=20and=20show=20why=20a=20fix=20is=20not=20dispatched=20(#17853?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contribute fleet card listed every stage-store record by author in 072 or 074 as the relay's pending rework, so merged/closed PRs still carrying stage/074-adopted-by-hive (the store prunes them only after 14 days) and PRs the hive had taken over piled up as '33 PRs awaiting rework'. Only open, not-adopted, not-dispatched PRs count now: the forge cache decides open vs merged/closed when it knows the PR, otherwise a record the stage lane has not re-observed for an hour is treated as gone. A queued fix whose PR is gone leaves the fix queue instead of being handed to the relay. The idle-with-queue stall: a fix dispatched to a socket that dropped stays booked in flight with nobody holding it, so capacity 1 is busy forever and the relay gets neither the fix nor new work (task_unavailable fix_pr_in_flight, which the card hid behind 'idle · N awaiting rework'). The relay's next ready now releases such orphans (no attempt counted) and a task_complete for the dispatch on a reconnected socket settles it. The card prints the dispatcher's actual reason (pending_rework_blocked) instead of 'not yet dispatched', and links at most three PRs inline with the rest behind a show-N-more disclosure. Signed-off-by: Andrew Anderson --- changelog.d/fixed-army-rework-queue.md | 1 + src/docs/contributor-relay.md | 5 +- .../stage-labels.md | 3 +- src/pkg/dashboard/contribute_fixqueue.go | 67 +++++- src/pkg/dashboard/contribute_fleet.go | 24 ++- src/pkg/dashboard/contribute_fleet_rework.go | 174 +++++++++++++--- .../dashboard/contribute_fleet_rework_test.go | 190 +++++++++++++++++- src/pkg/dashboard/contribute_landing.go | 28 ++- src/pkg/dashboard/contribute_ws.go | 9 + src/pkg/fixqueue/fixqueue.go | 21 +- src/pkg/fixqueue/fixqueue_test.go | 25 +++ 11 files changed, 500 insertions(+), 47 deletions(-) create mode 100644 changelog.d/fixed-army-rework-queue.md diff --git a/changelog.d/fixed-army-rework-queue.md b/changelog.d/fixed-army-rework-queue.md new file mode 100644 index 0000000000..93eaad0192 --- /dev/null +++ b/changelog.d/fixed-army-rework-queue.md @@ -0,0 +1 @@ +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 7f16ed2f28..c97d5a24cc 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1486,9 +1486,10 @@ Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/op | `fix_mode` | `spawn` or `preempt`, the interrupt mode that capacity implies. | | `in_flight[]` | One entry per work item on the connection — the primary task plus every `fix-pr` sub-session — each with `kind` (`new` or `rework`), `task_kind`, `repo`, `number`, `title`, `url`, `stage` (the item's `stage/NNN-*` label as the hive last observed it), `mode` and `started_at`. The row shows one line per item with a **NEW** / **REWORK** badge. A relay running only a boundary-dispatched fix therefore shows as *working*, not *idle*. | | `fix_queue_depth` | Red PRs waiting in the contributor's server-side fix queue (not dispatched). | -| `pending_rework`, `pending_rework_prs[]` | The contributor's open PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. The row prints a red **N PRs awaiting rework (not yet dispatched)** line, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework`, `pending_rework_prs[]` | The contributor's **open** PRs in `stage/072-returned-to-contributor` that have **not** been dispatched to them. Merged and closed PRs, PRs the hive adopted (074) and stage-store records the stage lane has stopped observing are never listed or counted. The row prints a red **N PRs awaiting rework (…)** line with at most three PRs linked inline and the rest behind a *show N more* disclosure, and an idle relay reads **idle · N PRs awaiting rework** instead of `idle: no matching work`. | +| `pending_rework_blocked` | Why the pending rework is not on its way to the relay right now, in the dispatcher's own order: relay offline, no `fix_pr_interrupt_v1` capability, repository scope not acknowledged, not in this hub's fix queue, an earlier fix dispatch still booked on a dropped connection, retry backoff, focus window, slots busy, or simply waiting for the relay's next `ready`. Printed in place of the old *not yet dispatched*. Empty when nothing pends. | -`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections. A non-zero `pending_rework` on an idle row is the signal to look at — typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, or a contributor who is offline. +`kind: rework` means the item repairs one of the contributor's own PRs: any `fix-pr` task, or a task whose target the stage store has in a repair stage (`070-fixing`, `071-needs-author`, `072-returned-to-contributor`, `074-adopted-by-hive`). Everything else is `new`. `pending_rework` is derived without a GitHub call: the stage-label store (which remembers each PR's author) ∪ the fix queue, minus the fix tasks in flight on any of the contributor's connections. A non-zero `pending_rework` on an idle row is the signal to look at — `pending_rework_blocked` names the cause (typically a relay that did not advertise `fix_pr_interrupt_v1`, a preempt-mode relay inside its `contribute.min_focus_window`, a fix dispatched to a socket that dropped before the relay reported on it, or a contributor who is offline). Whether a PR is still open comes from the forge cache the hive already holds; when the cache has no entry, a stage-store record the lane has not re-observed for an hour is treated as closed. A queued fix whose PR turns out merged or closed leaves the fix queue instead of being handed to the relay, and a fix booked in flight that no live connection holds is released at the relay's next `ready`. ## Reconnecting without losing in-flight work diff --git a/src/docs/labels-and-control-signals/stage-labels.md b/src/docs/labels-and-control-signals/stage-labels.md index a82ae1d89e..2dea077700 100644 --- a/src/docs/labels-and-control-signals/stage-labels.md +++ b/src/docs/labels-and-control-signals/stage-labels.md @@ -1256,7 +1256,8 @@ When a contributor-agent PR goes red the hive does not just relabel it (a human- in-place adoption their push makes the hive yield (above). The contribute page's "PRs awaiting rework" list for a relay shows `adopting (contributor offline)` on a nudged PR whose relay connection is - stale, and `adopted by hive` once the PR is in 074. + stale; once the PR is in 074 it is the hive's and leaves that relay's + list. 5. Backpressure — the per-contributor fix budget. Capacity = the relay's advertised `max_parallel_fixes` (fix sub-sessions beside the primary), clamped by `contribute.max_parallel_fixes` (default 2); an unadvertised diff --git a/src/pkg/dashboard/contribute_fixqueue.go b/src/pkg/dashboard/contribute_fixqueue.go index 787f819da1..bba745e1f2 100644 --- a/src/pkg/dashboard/contribute_fixqueue.go +++ b/src/pkg/dashboard/contribute_fixqueue.go @@ -476,6 +476,12 @@ func (h *ContributeWSHub) selectFixTask(c *ContributorConnection) *WSMessage { } user := c.profile.GitHubUsername sched := h.fixQueue() + // A `ready` says nothing is running on this relay. A fix dispatch the + // scheduler still books for this contributor that no live connection + // holds (its socket dropped before the relay reported on it) would + // otherwise occupy the only slot forever: nothing dispatched, new work + // refused with fix_pr_in_flight. Put it back at the head of the queue. + h.releaseOrphanedFixes(user, sched) if !sched.Pending(user) { return nil } @@ -496,13 +502,34 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin delete(c.fixTasksAt, taskID) } c.mu.Unlock() - if !ok || c.profile == nil { + if c.profile == nil { return ok } user := c.profile.GitHubUsername sched := h.fixQueue() + var prs []WSFixPR + if ok { + prs = assign.FixPRs + } else { + // The relay finished a fix it was handed on an earlier socket (the + // 1006 flap): this connection never saw the task_assign, but the + // scheduler still books the dispatch under this contributor. Settle + // it there rather than leaving the slot occupied for good. + for _, d := range sched.InFlight(user) { + if d.ID != taskID { + continue + } + ok = true + for _, it := range d.Items { + prs = append(prs, WSFixPR{Repo: it.Repo, Number: it.Number}) + } + } + if !ok { + return false + } + } if sched.CompleteDispatch(user, taskID) { - for _, pr := range assign.FixPRs { + for _, pr := range prs { h.forgetFixOwner(user, fixqueue.Key(pr.Repo, pr.Number)) } // Refill the freed slot with the next queued fix. @@ -513,6 +540,42 @@ func (h *ContributeWSHub) completeFixTask(c *ContributorConnection, taskID strin return true } +// heldFixTaskIDs returns the fix-pr task IDs every live connection of user +// currently holds. +func (h *ContributeWSHub) heldFixTaskIDs(user string) map[string]bool { + held := map[string]bool{} + h.mu.RLock() + defer h.mu.RUnlock() + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && strings.EqualFold(c.profile.GitHubUsername, user) { + for id := range c.fixTasks { + held[id] = true + } + } + c.mu.Unlock() + } + return held +} + +// releaseOrphanedFixes returns to user's queue every dispatch the scheduler +// books in flight that no live connection holds. Called at a task boundary, +// when the relay has just said it is running nothing. +func (h *ContributeWSHub) releaseOrphanedFixes(user string, sched *fixqueue.Scheduler) { + var held map[string]bool + for _, d := range sched.InFlight(user) { + if held == nil { + held = h.heldFixTaskIDs(user) + } + if held[d.ID] { + continue + } + if items := sched.Release(user, d.ID); len(items) > 0 { + h.logger.Warn("[contribute-ws] fix-pr dispatch orphaned by a dropped connection; requeued", "username", user, "task", d.ID, "prs", len(items)) + } + } +} + // failFixTask settles a fix-pr task the relay reported failed: its PRs go // back in the queue with backoff, and a PR that has failed // fixTaskMaxAttempts times leaves the queue for the stage engine's re-nudge diff --git a/src/pkg/dashboard/contribute_fleet.go b/src/pkg/dashboard/contribute_fleet.go index 838d34f2f0..3b3465d68f 100644 --- a/src/pkg/dashboard/contribute_fleet.go +++ b/src/pkg/dashboard/contribute_fleet.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "time" ) @@ -259,6 +260,23 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { Clankers: make([]FleetClanker, 0, len(h.connections)), Work: make([]FleetWorkItem, 0), } + // Fix-pr task IDs held per contributor across all their live sockets, + // gathered before any row lock is taken so no two connections are ever + // locked at once (fleetReworkFor names a dispatch nobody holds). + heldFix := map[string]map[string]bool{} + for _, c := range h.connections { + c.mu.Lock() + if c.profile != nil && len(c.fixTasks) > 0 { + user := strings.ToLower(c.profile.GitHubUsername) + if heldFix[user] == nil { + heldFix[user] = map[string]bool{} + } + for id := range c.fixTasks { + heldFix[user][id] = true + } + } + c.mu.Unlock() + } for _, c := range h.connections { c.mu.Lock() fc := FleetClanker{ @@ -352,7 +370,11 @@ func (h *ContributeWSHub) FleetSnapshot() FleetSnapshot { if task == nil { fc.IdleReason = c.lastIdleReason } - fc.FleetRework = h.fleetReworkFor(c, fc.Stale) + var held map[string]bool + if c.profile != nil { + held = heldFix[strings.ToLower(c.profile.GitHubUsername)] + } + fc.FleetRework = h.fleetReworkFor(c, fc.Stale, held) c.mu.Unlock() fc.CurrentTask = task fc.PromptPreview = promptPreview diff --git a/src/pkg/dashboard/contribute_fleet_rework.go b/src/pkg/dashboard/contribute_fleet_rework.go index cc44a499a2..7ed5d1d2cf 100644 --- a/src/pkg/dashboard/contribute_fleet_rework.go +++ b/src/pkg/dashboard/contribute_fleet_rework.go @@ -1,6 +1,7 @@ package dashboard import ( + "fmt" "sort" "strings" "time" @@ -51,8 +52,8 @@ type FleetInFlightItem struct { StartedAt string `json:"started_at,omitempty"` } -// FleetReworkPR is one of the contributor's PRs waiting in stage 072 that has -// not been dispatched to them. +// FleetReworkPR is one of the contributor's OPEN PRs waiting in stage 072 +// that has not been dispatched to them. type FleetReworkPR struct { Repo string `json:"repo"` Number int `json:"number"` @@ -63,16 +64,39 @@ type FleetReworkPR struct { Since string `json:"since,omitempty"` // Cause is the failing check / review ask, when the fix queue knows it. Cause string `json:"cause,omitempty"` - // State explains what the hive is doing about a PR the contributor is - // not working: "adopting (contributor offline)" while the connection is - // stale and the nudge went unanswered; "adopted by hive" once stage 074 - // took it over (hivecommons/hive#12436). + // State is "adopting (contributor offline)" while the connection is + // stale and the nudge went unanswered (hivecommons/hive#12436). A PR + // stage 074 already took over is not the contributor's any more and is + // not listed at all. State string `json:"state,omitempty"` } const ( FleetReworkStateAdopting = "adopting (contributor offline)" - FleetReworkStateAdopted = "adopted by hive" + // FleetReworkStateAdopted names the hand-off in logs and tests; such a + // PR no longer appears on the contributor's card. + FleetReworkStateAdopted = "adopted by hive" +) + +// fleetReworkSeenWithin bounds how long a stage-store record may go +// unobserved and still count as an open PR. The stage lane re-observes every +// open PR each sweep (about a minute); a record it has not touched for this +// long describes a PR that left the open listing (merged or closed) and is +// only waiting for the store's 14-day prune. +const fleetReworkSeenWithin = time.Hour + +// Why a queued fix is not on its way to the relay (FleetRework.PendingReworkBlocked). +const ( + FleetReworkBlockedOffline = "relay offline: no heartbeat within the presence window; the hive adopts the PR if the nudge stays unanswered" + FleetReworkBlockedNoFixPR = "relay does not accept fix-pr tasks (no fix_pr_interrupt_v1 capability); the PR only gets a GitHub comment" + FleetReworkBlockedScope = "relay's repository scope is not acknowledged (%s); the task boundary serves nothing until it is" + FleetReworkBlockedOrphan = "earlier fix dispatch %s is still booked in flight on a dropped connection; released at the relay's next ready" + FleetReworkBlockedBackoff = "retry backoff until %s after %d failed dispatch(es)" + FleetReworkBlockedFocus = "primary task protected by the focus window until %s; served at the next task boundary" + FleetReworkBlockedSlotsBusy = "all %d fix slot(s) busy" + FleetReworkBlockedAwaitReady = "waiting for the relay's next ready (fixes are served before new work)" + FleetReworkBlockedNotQueued = "not in this hub's fix queue; queued when the stage engine re-nudges" + fleetReworkBlockedIdleReasonFm = "%s; relay last reported idle: %s" ) // FleetRework is the per-contributor rework view spliced onto FleetClanker. @@ -88,14 +112,19 @@ type FleetRework struct { // InFlight lists every item this connection is working: the primary task // plus any fix-pr tasks running beside / instead of it. InFlight []FleetInFlightItem `json:"in_flight,omitempty"` - // FixQueueDepth is how many red PRs wait in the contributor's fix queue. + // FixQueueDepth is how many open red PRs wait in the contributor's fix queue. FixQueueDepth int `json:"fix_queue_depth"` - // PendingRework counts the contributor's PRs in + // PendingRework counts the contributor's OPEN PRs in // stage/072-returned-to-contributor that have NOT been dispatched to them - // (fix queue ∪ stage store, minus in-flight). Non-zero while the relay is - // idle is the operator's tell that rework is not reaching the agent. + // (fix queue ∪ stage store, minus in-flight, minus anything merged, + // closed, adopted by the hive or no longer observed by the stage lane). + // Non-zero while the relay is idle is the operator's tell that rework is + // not reaching the agent. PendingRework int `json:"pending_rework"` PendingReworkPRs []FleetReworkPR `json:"pending_rework_prs,omitempty"` + // PendingReworkBlocked says why the pending rework is not on its way to + // the relay right now (FleetReworkBlocked*); empty when nothing pends. + PendingReworkBlocked string `json:"pending_rework_blocked,omitempty"` } // stageStoreOrShared returns the injected stage store or the process-wide one. @@ -106,6 +135,44 @@ func (h *ContributeWSHub) stageStoreOrShared() *stagelabels.Store { return stagelabels.Shared() } +// prOpenState answers whether repo#number is still open from the forge +// cache the hive already holds (never a GitHub call): known=false when the +// cache has no entry. prOpenFn is the test seam. +func (h *ContributeWSHub) prOpenState(repo string, number int) (open, known bool) { + if h.prOpenFn != nil { + return h.prOpenFn(repo, number) + } + if h.server == nil || h.server.deps == nil || h.server.deps.GHClient == nil { + return false, false + } + fs := h.server.deps.GHClient.ForgeState() + if fs == nil { + return false, false + } + item, ok := fs.PR(repo, number) + if !ok { + return false, false + } + return item.Open(), true +} + +// reworkCandidateOpen reports whether a rework candidate is still an open +// PR of the contributor's: the forge cache decides when it knows the PR; +// otherwise a stage-store record the lane stopped observing means closed. +// A record stage 074 took over is the hive's now, not the contributor's. +func (h *ContributeWSHub) reworkCandidateOpen(repo string, number int, rec stagelabels.Record, hasRec bool, now time.Time) bool { + if hasRec && rec.Stage == stagelabels.AdoptedByHive { + return false + } + if open, known := h.prOpenState(repo, number); known { + return open + } + if hasRec && !rec.LastSeen.IsZero() && now.Sub(rec.LastSeen) > fleetReworkSeenWithin { + return false + } + return true +} + // isReworkStage reports whether a stage means "this PR is being repaired". func isReworkStage(st stagelabels.Stage) bool { switch st { @@ -140,9 +207,12 @@ func workKindFor(store *stagelabels.Store, taskKind, repo string, number int) (k // fleetReworkFor builds the rework view for one connection. Call with c.mu // held; it takes only the fix scheduler's and stage store's own locks. -func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) FleetRework { +// heldFixIDs are the fix-pr task IDs every live connection of this +// contributor holds (FleetSnapshot gathers them before locking c). +func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool, heldFixIDs map[string]bool) FleetRework { var out FleetRework store := h.stageStoreOrShared() + now := time.Now() cap := h.fixCapacity(c) switch { case cap.Parallel: @@ -202,8 +272,6 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F return out } sched := h.fixQueue() - queued := sched.Queued(user) - out.FixQueueDepth = len(queued) // Dispatched on ANY of this contributor's connections counts as in flight. for _, d := range sched.InFlight(user) { for _, it := range d.Items { @@ -211,7 +279,16 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F } } pending := map[string]FleetReworkPR{} - for _, it := range queued { + for _, it := range sched.Queued(user) { + rec, hasRec := store.Get(stagelabels.Key(it.Repo, it.Number)) + if !h.reworkCandidateOpen(it.Repo, it.Number, rec, hasRec, now) { + // Merged, closed or adopted while queued: nothing left for the + // relay to fix, and handing it out would waste the agent's turn. + sched.Remove(user, it.Repo, it.Number) + h.forgetFixOwner(user, it.Key()) + continue + } + out.FixQueueDepth++ if inFlightKeys[it.Key()] { continue } @@ -236,6 +313,9 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F if repo == "" || number == 0 { continue } + if !h.reworkCandidateOpen(repo, number, ai.Record, true, now) { + continue + } pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label()} if !ai.Record.EnteredAt.IsZero() { pr.Since = ai.Record.EnteredAt.UTC().Format(time.RFC3339) @@ -245,20 +325,6 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F } pending[ai.Key] = pr } - for _, ai := range store.ByAuthor(user, stagelabels.AdoptedByHive) { - if _, dup := pending[ai.Key]; dup { - continue - } - repo, number := splitStageKey(ai.Key) - if repo == "" || number == 0 { - continue - } - pr := FleetReworkPR{Repo: repo, Number: number, Stage: ai.Record.Stage.Label(), State: FleetReworkStateAdopted} - if !ai.Record.EnteredAt.IsZero() { - pr.Since = ai.Record.EnteredAt.UTC().Format(time.RFC3339) - } - pending[ai.Key] = pr - } keys := make([]string, 0, len(pending)) for k := range pending { keys = append(keys, k) @@ -268,9 +334,59 @@ func (h *ContributeWSHub) fleetReworkFor(c *ContributorConnection, stale bool) F out.PendingReworkPRs = append(out.PendingReworkPRs, pending[k]) } out.PendingRework = len(out.PendingReworkPRs) + if out.PendingRework > 0 { + out.PendingReworkBlocked = h.fleetReworkBlockReason(c, stale, user, cap, sched, heldFixIDs, now) + } return out } +// fleetReworkBlockReason explains why user's pending rework is not on its +// way to the relay, in the order the dispatcher itself refuses. Call with +// c.mu held. +func (h *ContributeWSHub) fleetReworkBlockReason(c *ContributorConnection, stale bool, user string, cap fixqueue.Capacity, sched *fixqueue.Scheduler, heldFixIDs map[string]bool, now time.Time) string { + reason := func(r string) string { + if c.currentTask == nil && c.lastIdleReason != "" { + return fmt.Sprintf(fleetReworkBlockedIdleReasonFm, r, c.lastIdleReason) + } + return r + } + if stale { + return FleetReworkBlockedOffline + } + if !relaySupportsFixPR(c) { + return FleetReworkBlockedNoFixPR + } + if gate := repoScopeGateReasonLocked(c); gate != "" { + return reason(fmt.Sprintf(FleetReworkBlockedScope, gate)) + } + queued := sched.Queued(user) + if len(queued) == 0 { + return reason(FleetReworkBlockedNotQueued) + } + inflight := sched.InFlight(user) + for _, d := range inflight { + if !heldFixIDs[d.ID] { + return reason(fmt.Sprintf(FleetReworkBlockedOrphan, d.ID)) + } + } + if head := queued[0]; now.Before(head.NotBefore) { + return reason(fmt.Sprintf(FleetReworkBlockedBackoff, head.NotBefore.UTC().Format(time.RFC3339), head.Attempts)) + } + st := sched.Status(user, cap) + if c.currentTask != nil { + if st.FocusUntil != nil && now.Before(*st.FocusUntil) { + return fmt.Sprintf(FleetReworkBlockedFocus, st.FocusUntil.UTC().Format(time.RFC3339)) + } + if len(inflight) >= cap.Slots() { + return fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.Slots()) + } + } + if len(inflight) >= cap.BoundarySlots() { + return reason(fmt.Sprintf(FleetReworkBlockedSlotsBusy, cap.BoundarySlots())) + } + return reason(FleetReworkBlockedAwaitReady) +} + // splitStageKey inverts stagelabels.Key ("owner/repo#N"). func splitStageKey(key string) (repo string, number int) { i := strings.LastIndex(key, "#") diff --git a/src/pkg/dashboard/contribute_fleet_rework_test.go b/src/pkg/dashboard/contribute_fleet_rework_test.go index 704a383c41..f6038c07b9 100644 --- a/src/pkg/dashboard/contribute_fleet_rework_test.go +++ b/src/pkg/dashboard/contribute_fleet_rework_test.go @@ -3,6 +3,7 @@ package dashboard import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os/exec" @@ -266,6 +267,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { ".work-kind.new{", ".work-kind.rework{", ".clanker-idle-rework,.clanker-rework-pending{color:var(--status-error)", + ".clanker-rework-more>summary{", } { if !strings.Contains(body, want) { t.Errorf("operations page missing %q", want) @@ -273,7 +275,7 @@ func TestOpsClankerRowRendersCapacityAndRework(t *testing.T) { } script := ` const assert = require('node:assert/strict'); -var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false; +var ccDeferredClankers = null, ccKnownClankers = {}, adminEnabled = false, ccOpenPanes = {}, CLANKER_REWORK_INLINE_MAX = 3; const fleet = { querySelector() { return null; }, set innerHTML(v) { this.html = v; } }; const document = { getElementById(id) { return id === 'clanker-list' ? fleet : {}; } }; function ccUpdateArmy() {} @@ -327,6 +329,29 @@ assert.doesNotMatch(html, /idle: no matching work/, 'idle reason replaced while assert.match(html, /class="clanker-sub clanker-rework-pending" title="org\/r#2 \(build\), org\/r#3">2 PRs awaiting rework \(not yet dispatched\) · fix queue 1: org\/r#2<\/a>, org\/r#3<\/a>earlier fix dispatch fix-erin-1 is still booked[^<]*<\/span>\) · fix queue 1: org\/r#9<\/a>/, 'block reason'); +assert.doesNotMatch(html, /not yet dispatched/, 'generic wording gone when the hub explains'); + +// Five pending: three inline, the other two behind a closed disclosure. +var five = [1,2,3,4,5].map(function(n){return {repo: 'org/r', number: n, url: 'https://x/' + n};}); +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +html = fleet.html; +assert.equal((html.match(/class="cc-issue-link work-ref"/g) || []).length, 5, 'every PR is still linked'); +var moreAt = html.indexOf('
show 2 more'); +assert.ok(moreAt > 0, 'disclosure present and closed'); +assert.ok(html.indexOf('org/r#3') < moreAt && html.indexOf('org/r#4') > moreAt, 'first three inline, rest behind the summary'); +ccOpenPanes['frank:rework'] = true; +renderClankers([{github_username: 'frank', fix_queue_depth: 5, pending_rework: 5, pending_rework_prs: five}]); +assert.match(fleet.html, /clanker-rework-more" data-pane="frank:rework" open>/, 'disclosure stays open across re-renders'); +delete ccOpenPanes['frank:rework']; +renderClankers([{github_username: 'gina', fix_queue_depth: 3, pending_rework: 3, pending_rework_prs: five.slice(0,3)}]); +assert.doesNotMatch(fleet.html, /clanker-rework-more/, 'three or fewer: no disclosure'); + // Idle, nothing pending: the pre-#12502 idle reason line is unchanged. renderClankers([{github_username: 'carol', idle_reason: 'no_matching_work', pending_rework: 0}]); html = fleet.html; @@ -359,14 +384,15 @@ func TestFleetReworkAdoptionState(t *testing.T) { store.NoteAuthor(stagelabels.Key("org/r", 3), "alice") fc := snapshotFor(t, hub, "alice") - if fc.Stale || len(fc.PendingReworkPRs) != 3 { + // The 074 PR is the hive's now: not listed, not counted. + if fc.Stale || fc.PendingRework != 2 || len(fc.PendingReworkPRs) != 2 { t.Fatalf("live: %+v", fc.FleetRework) } byNumber := map[int]FleetReworkPR{} for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - if byNumber[1].State != "" || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted || byNumber[3].Stage != stagelabels.AdoptedByHive.Label() { + if _, adopted := byNumber[3]; adopted || byNumber[1].State != "" || byNumber[2].State != "" { t.Fatalf("live states: %+v", fc.PendingReworkPRs) } @@ -381,10 +407,13 @@ func TestFleetReworkAdoptionState(t *testing.T) { for _, pr := range fc.PendingReworkPRs { byNumber[pr.Number] = pr } - // Nudged and unanswered → adopting; never nudged → plain 072; 074 → adopted. - if byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" || byNumber[3].State != FleetReworkStateAdopted { + // Nudged and unanswered → adopting; never nudged → plain 072; 074 → gone. + if _, adopted := byNumber[3]; adopted || byNumber[1].State != FleetReworkStateAdopting || byNumber[2].State != "" { t.Fatalf("stale states: %+v", fc.PendingReworkPRs) } + if fc.PendingReworkBlocked != FleetReworkBlockedOffline { + t.Fatalf("offline relay must say so: %q", fc.PendingReworkBlocked) + } raw, _ := json.Marshal(fc.PendingReworkPRs) if !strings.Contains(string(raw), `"state":"adopting (contributor offline)"`) { t.Fatalf("json: %s", raw) @@ -416,3 +445,154 @@ func TestContributeHubPresence(t *testing.T) { t.Fatalf("offline relay: %+v", p) } } + +// Only the contributor's OPEN PRs are their rework: a PR the forge cache +// reports merged or closed, or a stage-store record the lane stopped +// observing, is dropped from the count and (when queued) from the fix +// queue, so the relay is never handed a fix for a PR that is gone. +func TestFleetReworkDropsMergedClosedAndUnobserved(t *testing.T) { + hub, _, store := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + + closed := map[string]bool{"org/r#2": true, "org/r#3": true} + hub.prOpenFn = func(repo string, number int) (bool, bool) { + k := fixqueue.Key(repo, number) + if closed[k] { + return false, true + } + if k == "org/r#1" { + return true, true + } + return false, false + } + for _, n := range []int{1, 2, 3, 4} { + store.Observe(stagelabels.Key("org/r", n), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", n), "alice") + } + // Queue #2 (closed) and #4 (unknown to the cache, fresh in the store). + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 2, Title: "closed"}) + hub.fixQueue().Enqueue("alice", fixqueue.Item{Repo: "org/r", Number: 4, Title: "open"}) + hub.rememberFixOwner("alice", "org/r#2") + hub.rememberFixOwner("alice", "org/r#4") + + fc := snapshotFor(t, hub, "alice") + got := map[int]bool{} + for _, pr := range fc.PendingReworkPRs { + got[pr.Number] = true + } + if fc.PendingRework != 2 || !got[1] || !got[4] || got[2] || got[3] { + t.Fatalf("pending: %+v", fc.FleetRework) + } + if fc.FixQueueDepth != 1 { + t.Fatalf("closed PR must leave the fix queue: %+v", fc.FleetRework) + } + if q := hub.fixQueue().Queued("alice"); len(q) != 1 || q[0].Number != 4 { + t.Fatalf("scheduler still holds the closed PR: %+v", q) + } + if fc.PendingReworkBlocked != FleetReworkBlockedAwaitReady { + t.Fatalf("idle relay with a queued fix waits for ready: %q", fc.PendingReworkBlocked) + } +} + +// A stage-store record the lane has not re-observed within +// fleetReworkSeenWithin describes a PR that left the open listing. +func TestFleetReworkDropsRecordTheLaneStoppedObserving(t *testing.T) { + hub, _, _ := reworkHub(t) + now := time.Now() + fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + clock := now.Add(-2 * fleetReworkSeenWithin) + store := stagelabels.NewStore("", func() time.Time { return clock }) + hub.stageStore = store + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + clock = now + store.Observe(stagelabels.Key("org/r", 2), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 2), "alice") + + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 { + t.Fatalf("unobserved #1 must be dropped: %+v", fc.FleetRework) + } + // With no fix queue entry the hub can only wait for the engine's re-nudge. + if fc.PendingReworkBlocked != FleetReworkBlockedNotQueued { + t.Fatalf("block reason: %q", fc.PendingReworkBlocked) + } +} + +// A fix dispatched to a socket that then dropped stays booked in the +// scheduler with nobody holding it: capacity 1 is "busy" for good, so the +// relay is served nothing — not the fix, not new work. The snapshot names +// the orphan, and the relay's next `ready` releases and re-dispatches it. +func TestFleetReworkOrphanedDispatchIsNamedAndReleasedAtReady(t *testing.T) { + hub, _, store := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + store.Observe(stagelabels.Key("org/r", 1), stagelabels.ReturnedToContributor, "check-failing:build") + store.NoteAuthor(stagelabels.Key("org/r", 1), "alice") + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + orphanID := (*got)[0].TaskID + + // The socket that took the fix dies; the relay comes back on a new one. + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + c2.lastIdleReason = taskUnavailableFixInFlight + + // Another red PR queues behind the orphan. + hub.Nudge(context.Background(), fixNudge("r", 2, "lint")) + fc := snapshotFor(t, hub, "alice") + if fc.PendingRework != 1 || fc.PendingReworkPRs[0].Number != 2 || len(fc.InFlight) != 0 { + t.Fatalf("pending: %+v", fc.FleetRework) + } + want := fmt.Sprintf(FleetReworkBlockedOrphan, orphanID) + "; relay last reported idle: " + taskUnavailableFixInFlight + if fc.PendingReworkBlocked != want { + t.Fatalf("block reason:\n got %q\nwant %q", fc.PendingReworkBlocked, want) + } + + // `ready` on the new socket: the orphan is released (no attempt counted) + // and the queue head goes out as a boundary fix. + msg := hub.selectFixTask(c2) + if msg == nil || msg.Kind != taskKindFixPR || msg.Mode != "boundary" { + t.Fatalf("boundary fix expected after release: %+v", msg) + } + if n := len(hub.fixQueue().InFlight("alice")); n != 1 { + t.Fatalf("exactly the new dispatch in flight, got %d", n) + } + for _, it := range hub.fixQueue().Queued("alice") { + if it.Attempts != 0 { + t.Fatalf("release must not count an attempt: %+v", it) + } + } +} + +// A relay that finished a fix handed out on an earlier socket reports +// task_complete on the new one; the scheduler settles it all the same. +func TestFleetReworkCompleteFixOnReconnectedSocket(t *testing.T) { + hub, _, _ := reworkHub(t) + got := captureFixDispatch(hub) + now := time.Now() + c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now) + hub.Nudge(context.Background(), fixNudge("r", 1, "build")) + if len(*got) != 1 { + t.Fatalf("preempt dispatch expected, got %d", len(*got)) + } + hub.mu.Lock() + delete(hub.connections, c.connID) + hub.mu.Unlock() + c2 := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), now.Add(time.Second)) + if !hub.completeFixTask(c2, (*got)[0].TaskID) { + t.Fatal("completion on the new socket must settle the scheduler's dispatch") + } + if st := hub.FixQueueStatus("alice"); st.InFlight != 0 || st.Queued != 0 { + t.Fatalf("dispatch should be settled: %+v", st) + } + if hub.completeFixTask(c2, "fix-alice-999") { + t.Fatal("an unknown task id is not a fix task") + } +} diff --git a/src/pkg/dashboard/contribute_landing.go b/src/pkg/dashboard/contribute_landing.go index c9fc370cb5..035b9511b4 100644 --- a/src/pkg/dashboard/contribute_landing.go +++ b/src/pkg/dashboard/contribute_landing.go @@ -472,6 +472,11 @@ code{background:var(--surface-0);padding:var(--sp-1) var(--sp-4);border-radius:v .work-kind.rework{background:color-mix(in srgb,var(--status-attention) var(--component-tint),transparent);color:var(--status-attention);border-color:color-mix(in srgb,var(--status-attention) var(--component-border),transparent)} .clanker-idle-rework,.clanker-rework-pending{color:var(--status-error);font-weight:600} .clanker-rework-pending .cc-issue-link{color:var(--status-error)} +.clanker-rework-pending .clanker-rework-why{font-weight:400;color:var(--text-muted)} +.clanker-rework-more{display:inline;font-weight:400} +.clanker-rework-more>summary{display:inline;cursor:pointer;color:var(--text-faint);user-select:none;list-style:none} +.clanker-rework-more>summary::-webkit-details-marker{display:none} +.clanker-rework-more[open]>summary{display:none} .clanker-pane{margin-top:var(--sp-3);font-size:.72rem} .clanker-pane>summary{cursor:pointer;color:var(--text-faint);font-family:ui-monospace,SFMono-Regular,Menlo,monospace;user-select:none;list-style:none} .clanker-pane>summary::before{content:"\25B8";display:inline-block;width:1em;color:var(--text-muted)} @@ -5518,14 +5523,29 @@ function clankerIdleLine(c){ if(pend>0)return '
idle · '+pend+' PR'+(pend===1?'':'s')+' awaiting rework
'; return c.idle_reason?('
idle: '+esc(idleReasonLabel(c.idle_reason))+'
'):''; } +// At most this many pending PRs are linked inline; the rest sit behind a +// "show N more" disclosure (the card is a status line, not a backlog). +var CLANKER_REWORK_INLINE_MAX=3; function clankerPendingReworkLine(c){ var pend=c.pending_rework|0; if(pend<=0)return ''; - var prs=(c.pending_rework_prs||[]).map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); - var links=(c.pending_rework_prs||[]).map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}).join(', '); + var items=c.pending_rework_prs||[]; + var prs=items.map(function(p){return (p.repo||'')+'#'+(p.number||'')+(p.cause?(' ('+p.cause+')'):'')+(p.state?(' — '+p.state):'');}).join(', '); + var refs=items.map(function(p){return clankerWorkRef(p)+(p.state?(' '+esc(p.state)+''):'');}); + var links=refs.slice(0,CLANKER_REWORK_INLINE_MAX).join(', '); + var rest=refs.slice(CLANKER_REWORK_INLINE_MAX); + if(rest.length){ + var key=(c.github_username||c.contributor_id||'').toLowerCase()+':rework'; + var open=ccOpenPanes[key]?' open':''; + links+=',
show '+rest.length+' more'+rest.join(', ')+'
'; + } var depth=c.fix_queue_depth|0; var q=depth>0?(' · fix queue '+depth):''; - return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework (not yet dispatched)'+q+(links?(': '+links):'')+'
'; + // The server's own account of why the dispatcher is holding the fix + // (offline, no fix-pr capability, orphaned dispatch, backoff, focus + // window, waiting for the relay's next ready); older hubs send none. + var why=c.pending_rework_blocked?('not dispatched: '+esc(c.pending_rework_blocked)+''):'not yet dispatched'; + return '
'+pend+' PR'+(pend===1?'':'s')+' awaiting rework ('+why+')'+q+(links?(': '+links):'')+'
'; } // #7317 item 3 helpers. Both return '' when there is nothing to show so a // healthy row is byte-for-byte what it was before. @@ -5546,7 +5566,7 @@ function clankerPaneBlock(lines,key){ return '
agent pane · '+lines.length+' line'+(lines.length===1?'':'s')+'
'+esc(lines.join('\n'))+'
'; } onEl('clanker-list','toggle',function(e){ - var d=e.target;if(!d||!d.classList||!d.classList.contains('clanker-pane'))return; + var d=e.target;if(!d||!d.classList||!(d.classList.contains('clanker-pane')||d.classList.contains('clanker-rework-more')))return; var k=d.getAttribute('data-pane');if(!k)return; if(d.open)ccOpenPanes[k]=true;else delete ccOpenPanes[k]; },true); diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 87219cdb1f..f9c4e86eed 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -903,6 +903,9 @@ type ContributeWSHub struct { fixSched *fixqueue.Scheduler fixOwners map[string]map[string]bool fixDispatched func(*ContributorConnection, WSMessage) + // prOpenFn overrides the forge-cache lookup the fleet snapshot uses to + // drop merged/closed PRs from a contributor's rework; a test hook. + prOpenFn func(repo string, number int) (open, known bool) // fixFailedSig is closed (and replaced) whenever a relay reports a // fix-pr task failed, waking a Nudge that is waiting to learn whether its // dispatch was refused (hivecommons/hive#14668). fixAckGrace overrides @@ -3577,6 +3580,12 @@ func repoScopeGateReason(c *ContributorConnection) string { } c.mu.Lock() defer c.mu.Unlock() + return repoScopeGateReasonLocked(c) +} + +// repoScopeGateReasonLocked is repoScopeGateReason for a caller that already +// holds c.mu (the fleet snapshot explains an undispatched fix with it). +func repoScopeGateReasonLocked(c *ContributorConnection) string { if c.repoScopeInvalid { return taskUnavailableRepoScopeInvalid } diff --git a/src/pkg/fixqueue/fixqueue.go b/src/pkg/fixqueue/fixqueue.go index d0078def6d..863544914e 100644 --- a/src/pkg/fixqueue/fixqueue.go +++ b/src/pkg/fixqueue/fixqueue.go @@ -362,6 +362,19 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool { // straight back to a relay that just refused it. It returns the requeued // items; nil when id is not in flight. func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { + return s.unwind(user, id, backoff, true, "requeue") +} + +// Release puts an in-flight dispatch back at the head of user's queue +// without counting an attempt or booking a backoff: the relay never reported +// on it (its socket dropped, or it asked for new work while the dispatch was +// still booked), so nothing about the PR itself failed. nil when id is not +// in flight. +func (s *Scheduler) Release(user, id string) []Item { + return s.unwind(user, id, 0, false, "release") +} + +func (s *Scheduler) unwind(user, id string, backoff time.Duration, countAttempt bool, event string) []Item { s.mu.Lock() defer s.mu.Unlock() c, ok := s.by[norm(user)] @@ -380,12 +393,14 @@ func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item { out := d.public().Items notBefore := s.opts.Now().Add(backoff) for j := range out { - out[j].Attempts++ - out[j].NotBefore = notBefore + if countAttempt { + out[j].Attempts++ + out[j].NotBefore = notBefore + } c.queue = append(c.queue, out[j]) } sortQueue(c.queue) - s.log("requeue", user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) + s.log(event, user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue)) return out } return nil diff --git a/src/pkg/fixqueue/fixqueue_test.go b/src/pkg/fixqueue/fixqueue_test.go index 94c3c5f7b6..42906a962e 100644 --- a/src/pkg/fixqueue/fixqueue_test.go +++ b/src/pkg/fixqueue/fixqueue_test.go @@ -418,3 +418,28 @@ func TestDispatchAt(t *testing.T) { t.Fatalf("inflight=%+v", got) } } + +// Release puts an in-flight dispatch back without an attempt or a backoff: +// the relay never reported on it, so the PR is immediately dispatchable. +func TestReleaseReturnsDispatchWithoutAttempt(t *testing.T) { + c := newClock() + s := newSched(c) + s.Enqueue("ida", red(1, time.Hour, c)) + d := s.Dispatch("ida", Capacity{Max: 1}) + if d == nil { + t.Fatal("dispatch expected") + } + if s.Release("ida", "nope") != nil || s.Release("zed", d.ID) != nil { + t.Fatal("unknown dispatch released") + } + got := s.Release("ida", d.ID) + if len(got) != 1 || got[0].Attempts != 0 || !got[0].NotBefore.IsZero() { + t.Fatalf("released = %+v", got) + } + if st := s.Status("ida", Capacity{Max: 1}); st.Queued != 1 || st.InFlight != 0 || st.FocusUntil != nil { + t.Fatalf("status after release = %+v", st) + } + if s.Dispatch("ida", Capacity{Max: 1}) == nil { + t.Fatal("released item must be dispatchable at once") + } +} From c9575cab9d5285cc0a7f27a124bcaa6fb3f9ca59 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:13:04 +0100 Subject: [PATCH 063/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20detect=20missing?= =?UTF-8?q?=20GitHub=20App=20repository=20coverage=20(#17733)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: detect missing GitHub App repository coverage Hive-Run: hivecommons/hive#17696 Hive-Plan: repository access and advisory recovery Hive-Spec: issue-17696#expected Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: Karibhou * fix: add App repository coverage changelog Signed-off-by: Karibhou * fix: bound boot coverage and probe large installations Signed-off-by: Karibhou * docs: clarify GitHub App coverage polling Signed-off-by: Karibhou --------- Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-17696-app-repo-coverage.md | 1 + src/cmd/hive/boot_dashboard_api_deps.go | 15 +- src/cmd/hive/boot_dashboard_api_deps_test.go | 36 +++++ src/cmd/hive/boot_github_deps.go | 8 +- src/cmd/hive/boot_github_deps_test.go | 49 ++++++ src/cmd/hive/main.go | 33 ++++- src/cmd/hive/main_github_app.go | 16 ++ src/docs/github-app-setup.md | 27 +++- src/pkg/dashboard/api_governor.go | 75 +++++++--- .../dashboard/repo_check_access_probe_test.go | 139 ++++++++++++++++-- .../static-src/js/linear-repos-save.js | 19 +-- src/pkg/dashboard/static/index.html | 19 +-- src/pkg/github/advisory.go | 3 + src/pkg/github/advisory_retry_test.go | 81 ++++++++++ src/pkg/github/app_coverage.go | 48 ++++++ 15 files changed, 488 insertions(+), 81 deletions(-) create mode 100644 changelog.d/fixed-17696-app-repo-coverage.md create mode 100644 src/pkg/github/advisory_retry_test.go diff --git a/changelog.d/fixed-17696-app-repo-coverage.md b/changelog.d/fixed-17696-app-repo-coverage.md new file mode 100644 index 0000000000..8855f73672 --- /dev/null +++ b/changelog.d/fixed-17696-app-repo-coverage.md @@ -0,0 +1 @@ +- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). diff --git a/src/cmd/hive/boot_dashboard_api_deps.go b/src/cmd/hive/boot_dashboard_api_deps.go index 0d99bb23ee..b173ec14de 100644 --- a/src/cmd/hive/boot_dashboard_api_deps.go +++ b/src/cmd/hive/boot_dashboard_api_deps.go @@ -2,9 +2,11 @@ package main import ( "context" + "log/slog" "time" "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" ) // bootDashboardAPIDeps are the long-lived effects bootDashboardAPI performs @@ -22,13 +24,12 @@ import ( type bootDashboardAPIDeps struct { registerAPI func(srv *dashboard.Server, d *dashboard.Dependencies) setForgeAppInventory func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) + classifyRepoCoverage func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) // startInstallDiscovery runs tryDiscover once immediately and then every // githubAppDiscoveryInterval until ctx is done. startInstallDiscovery func(ctx context.Context, tryDiscover func()) // startSelfHeal runs heal every githubAppSelfHealInterval until ctx is - // done. It does NOT run immediately: boot has just applied the classified - // App state and the first tick is cheap because heal no-ops while the - // banner is not showing. + // done. It does NOT run immediately: boot has just checked coverage. startSelfHeal func(ctx context.Context, heal func()) startInceptionWatcher func(ctx context.Context, w *dashboard.InceptionWatcher) } @@ -39,15 +40,13 @@ type bootDashboardAPIDeps struct { // requiring anyone to paste it. const githubAppDiscoveryInterval = 5 * time.Minute -// githubAppSelfHealInterval mirrors the heartbeat cadence so a stale banner -// clears within one heartbeat window of the app becoming healthy, without -// adding meaningful GitHub API load (the check only runs while the banner is -// actually showing). +// githubAppSelfHealInterval paces both coverage checks and banner healing. const githubAppSelfHealInterval = 2 * time.Minute func defaultBootDashboardAPIDeps() bootDashboardAPIDeps { return bootDashboardAPIDeps{ - registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, + registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, setForgeAppInventory: func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) { srv.SetForgeAppInventoryFn(fn) }, diff --git a/src/cmd/hive/boot_dashboard_api_deps_test.go b/src/cmd/hive/boot_dashboard_api_deps_test.go index 076ef3cdc0..bea7683d49 100644 --- a/src/cmd/hive/boot_dashboard_api_deps_test.go +++ b/src/cmd/hive/boot_dashboard_api_deps_test.go @@ -37,6 +37,9 @@ type bootDashboardAPIFake struct { func newBootDashboardAPIFake() *bootDashboardAPIFake { f := &bootDashboardAPIFake{} f.deps = bootDashboardAPIDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { f.registeredOn, f.registered = srv, d }, @@ -221,6 +224,39 @@ func TestBootDashboardAPIWith_TickBodiesAreCheapWhenNothingToDo(t *testing.T) { } } +func TestSelfHealDetectsCoverageWithoutExistingBanner(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBootDashboardAPIFake() + cfg := bootDashboardAPIConfig() + cfg.Project.PrimaryRepo = "private" + b := newBootDashboardAPIBoot(t, f, cfg) + b.ghClient = fakeGitHubClient(t) + called := false + f.deps.classifyRepoCoverage = func(_ context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootDashboardAPIWith(f.deps) + f.healTick() + if !called || b.dashSrv.IsGitHubAppRequired() != tc.raise { + t.Fatalf("called=%v required=%v", called, b.dashSrv.IsGitHubAppRequired()) + } + }) + } +} + func TestBootDashboardAPIWith_ForgeInventoryResolvesTheActiveKey(t *testing.T) { f := newBootDashboardAPIFake() cfg := bootDashboardAPIConfig() diff --git a/src/cmd/hive/boot_github_deps.go b/src/cmd/hive/boot_github_deps.go index 1dbba48eb0..3f976f0c80 100644 --- a/src/cmd/hive/boot_github_deps.go +++ b/src/cmd/hive/boot_github_deps.go @@ -26,8 +26,9 @@ func defaultBootGitHubDeps() bootGitHubDeps { // notifier, ACMM inference, attribution hook, and on-disk default policy seed // run for real (cfg.Policies.LocalDir points a test at a temp dir). type bootAdvisoryDeps struct { - ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) - classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) + classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + classifyRepoCoverage func(ctx context.Context, appAuth *github.AppAuth, org string, repos []string, logger *slog.Logger) (bool, string, github.AppAuthState) } func defaultBootAdvisoryDeps() bootAdvisoryDeps { @@ -35,6 +36,7 @@ func defaultBootAdvisoryDeps() bootAdvisoryDeps { ensureAdvisoryIssue: func(ctx context.Context, c *github.Client, repo string) (int, error) { return c.EnsureAdvisoryIssue(ctx, repo) }, - classifyAppFailure: classifyGitHubAppFailure, + classifyAppFailure: classifyGitHubAppFailure, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, } } diff --git a/src/cmd/hive/boot_github_deps_test.go b/src/cmd/hive/boot_github_deps_test.go index 410e612bfa..1dc2faa30d 100644 --- a/src/cmd/hive/boot_github_deps_test.go +++ b/src/cmd/hive/boot_github_deps_test.go @@ -98,6 +98,9 @@ type bootAdvisoryFake struct { func newBootAdvisoryFake() *bootAdvisoryFake { f := &bootAdvisoryFake{ensureNum: 42} f.deps = bootAdvisoryDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, ensureAdvisoryIssue: func(_ context.Context, _ *github.Client, repo string) (int, error) { f.ensured = append(f.ensured, repo) return f.ensureNum, f.ensureErr @@ -122,6 +125,52 @@ func bootAdvisoryConfig(t *testing.T) *config.Config { return cfg } +func TestBootAdvisoryDetectsMissingRepoCoverage(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := bootAdvisoryConfig(t) + cfg.Project.PrimaryRepo = "private" + level := 0 + cfg.ACMMLevel = &level + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + called := false + var coverageCtx context.Context + f.deps.classifyRepoCoverage = func(ctx context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + coverageCtx = ctx + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > advisoryEnsureBudget { + t.Fatalf("coverage deadline = %v, present=%v", deadline, ok) + } + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootAdvisoryWith(f.deps) + if !called || b.githubAppRequired != tc.raise { + t.Fatalf("called=%v required=%v", called, b.githubAppRequired) + } + if coverageCtx.Err() != context.Canceled { + t.Fatalf("coverage context not canceled after check: %v", coverageCtx.Err()) + } + if tc.raise && (b.githubAppState != tc.state || b.githubAppDiag != "select private in the installation") { + t.Fatalf("coverage diagnosis: %v %q", b.githubAppState, b.githubAppDiag) + } + }) + } +} + func TestBootAdvisoryWithEnsuresIssueAndWritesPolicy(t *testing.T) { t.Setenv("HIVE_ADVISORY_ISSUE", "") cfg := bootAdvisoryConfig(t) diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index a7fca4a85b..51f69d747e 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1782,6 +1782,18 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { } } } + // Installation-level permissions do not establish that every configured + // repository is selected, including when advisory reads happen to succeed. + if b.ghClient != nil { + coverageCtx, cancelCoverage := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyRepoCoverage(coverageCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger) + cancelCoverage() + if raise { + b.githubAppRequired = true + b.githubAppDiag, b.githubAppState = diag, state + b.logger.Warn("GitHub App installation does not cover every configured repo", "state", state.String(), "detail", diag) + } + } b.advisoryStore = advisory.NewStore() @@ -3576,7 +3588,7 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // as "app not installed / no read" — sending the operator after // credentials that were never broken. Checking first means the // specific, correct message wins over the generic one. - if raise, diag, state := classifyGitHubAppRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.cfg.Project.Repos, b.logger); raise { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { b.dashSrv.SetGitHubAppPermIssue(diag) b.dashSrv.SetGitHubAppState(state.String()) b.logger.Warn("github app recheck: installation does not cover every configured repo", @@ -3665,12 +3677,9 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // so a later transient write failure that re-set the flag was never // re-evaluated — the banner stuck until the pod was restarted. // - // The loop therefore runs for the lifetime of the process (it does NOT - // return after the first success) and, whenever the banner is currently - // showing, re-runs the SAME read+write verification as the manual "Re-check" - // button (githubAppRecheckFn, which calls diagnoseGitHubApp) and clears - // the flag on success. When the banner is not showing there is nothing to do, - // so the tick is a cheap no-op that makes no GitHub API calls. + // The loop runs for the lifetime of the process. It detects selection + // changes even when no banner is showing, and re-runs the manual re-check + // while a banner is present. Inconclusive coverage checks never raise one. { primaryRepo := b.cfg.Project.PrimaryRepo if primaryRepo == "" && len(b.cfg.Project.Repos) > 0 { @@ -3678,7 +3687,15 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { } if primaryRepo != "" { deps.startSelfHeal(b.ctx, func() { - // Nothing to heal unless the banner is showing. + if !b.dashSrv.IsGitHubAppRequired() && b.ghClient != nil { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { + b.dashSrv.SetGitHubAppRequired(true) + b.dashSrv.SetGitHubAppPermIssue(diag) + b.dashSrv.SetGitHubAppState(state.String()) + b.logger.Warn("github app self-heal: installation does not cover every configured repo", "state", state.String(), "detail", diag) + return + } + } if !b.dashSrv.IsGitHubAppRequired() { return } diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go index 5bb5fa99dd..b747bdca20 100644 --- a/src/cmd/hive/main_github_app.go +++ b/src/cmd/hive/main_github_app.go @@ -487,6 +487,22 @@ func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) } +// appCoverageRepos includes the primary repo even when it is not in the +// watched list; it is also used for advisory writes. +func appCoverageRepos(cfg *config.Config) []string { + repos := cfg.Project.Repos + primary := cfg.Project.PrimaryRepo + if primary == "" { + return repos + } + for _, repo := range repos { + if strings.EqualFold(repo, primary) || strings.EqualFold(repo, cfg.Project.Org+"/"+primary) || strings.EqualFold(cfg.Project.Org+"/"+repo, primary) { + return repos + } + } + return append(append([]string(nil), repos...), primary) +} + // The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). // These wrappers keep the existing call sites unchanged; advisoryPostGate is // now an owned instance rather than a package-level struct tests reset. diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index b9b5794076..2976b95c7b 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -208,11 +208,12 @@ that has not approved a permission update is the one that still looks fine, and GitHub keeps an App on its old permissions until an org owner accepts, so a partly approved fleet is otherwise invisible. -Verdicts are computed where a GitHub call has failed and on the dashboard's -**Re-check** button, not on every eval cycle, so this adds no API calls to a -healthy hive. To read a specific installation's grants on demand — for example -to confirm the demo/canary install — press Re-check on that hive and read the -line above from its logs. +Permission verdicts are computed where a GitHub call has failed and on the +dashboard's **Re-check** button, not on every eval cycle. Repository coverage +is checked separately on the self-heal loop, including on a healthy hive, as +described below. To read a specific installation's grants on demand — for +example to confirm the demo/canary install — press Re-check on that hive and +read the line above from its logs. ## Install and configure Hive @@ -312,6 +313,22 @@ installation rather than creating a new one, so the `installation_id` in that redirect is one the hive already holds — the failed page does not mean App authentication is broken. +An installation can authenticate successfully while excluding a private +repository under **Only select repositories**. When adding a repo in the +dashboard, use the reported installation settings link to add it to the +selection, then re-check access. For repos already configured, the Forge App +banner identifies missing repository coverage; update the installation and +re-check it. A rejected advisory issue creation can then be retried without +waiting for the next UTC day. + +Boot coverage checks have a time limit; a timeout is inconclusive. The +background self-heal loop rechecks installation coverage every two minutes, +including on a healthy hive, using the cached installation token. Each check +walks up to 10 pages of `GET /installation/repositories`. If that bounded +listing is incomplete, the dashboard checks the repository's App installation +directly. Temporary GitHub errors still block adding the repository until a +check succeeds. + ## `/gh-setup` flow When the app's Setup URL points at `https:///gh-setup`, GitHub redirects back with `setup_action` and, for install/update, `installation_id`. diff --git a/src/pkg/dashboard/api_governor.go b/src/pkg/dashboard/api_governor.go index 096cb82a7e..6b5262c7d1 100644 --- a/src/pkg/dashboard/api_governor.go +++ b/src/pkg/dashboard/api_governor.go @@ -3366,20 +3366,6 @@ func firstRepoPathSegment(ref string) string { // a repo the operator is about to add, and — when it cannot — returns the // per-forge App install/authorize URL so the Repos tab can guide the user to // grant access before the repo is finally accepted (requirement #3). -// -// The check today probes at ORG granularity: DiscoverInstallationID(org) returns -// ErrNoInstallationForOrg when the App is not installed on the repo's org at all, -// which is the common "new org the App was never installed on" case and the one -// that hard-blocks reads and writes. On a positive result we report ok:true. -// -// TODO(repo-access-probe): tighten to REPO granularity. An App installed on the -// org "Only select repositories" may still lack THIS repo (see #2353's -// write-forbidden case). The deeper probe should mint an installation token and -// GET /repos/{org}/{repo} (s.deps.GHClient.GetRepo) — a 404/403 there means the -// installation exists but this repo is not in its selected set, which needs the -// same "manage installation" URL to add the repo to the selection. That probe is -// scoped out of this change to keep it focused; the org-level check plus the -// install workflow already covers the "App not installed on the org" footgun. func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Request) { var body struct { Repo string `json:"repo"` @@ -3420,6 +3406,18 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re jsonError(w, "cannot determine the org for this repo — use org/repo format", http.StatusBadRequest) return } + if len(parts) == 1 { + parts = []string{org, parts[0]} + } + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } + repo := strings.TrimSuffix(parts[1], ".git") + if repo == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } // An App-less hive (advisory-only, no private key) cannot and need not probe // installation access: it never writes. Treat it as "no access check needed" @@ -3433,13 +3431,13 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re if ctx == nil { ctx = r.Context() } - if _, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org); err != nil { - // Not installed on this org (or discovery failed). Surface the correct - // per-forge install URL and arm the existing pending-install mechanism so - // the banner/recheck plumbing the App-required flow already uses drives - // this to completion. AppInstallURL() returns "" for a GHE host whose App - // slug was never configured — the UI renders that as a config message - // rather than a dead link. + installationID, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org) + if err != nil { + if !errors.Is(err, github.ErrNoInstallationForOrg) { + s.logger.Warn("repo add: installation discovery failed", "org", org, "error", err) + jsonError(w, "could not check GitHub App installation access; please retry", http.StatusServiceUnavailable) + return + } installURL := "" if s.deps.Config != nil { installURL = s.deps.Config.GitHub.AppInstallURL() @@ -3457,6 +3455,41 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re }) return } + coverage, err := s.deps.GHAppAuth.InstallationCoverageForID(ctx, installationID) + if err != nil { + s.logger.Warn("repo add: installation repository coverage failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if _, found := coverage.Repos[strings.ToLower(org+"/"+repo)]; !found { + if coverage.Truncated { + covered, err := s.deps.GHAppAuth.RepositoryCoveredByInstallation(ctx, installationID, org, repo) + if err != nil { + s.logger.Warn("repo add: repository installation probe failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if covered { + okResponse(w, map[string]string{"status": "ok", "org": org}) + return + } + } + installURL := (github.AppAuthDiagnosis{ + ExpectedAccount: org, + InstallationID: installationID, + APIURL: s.deps.Config.GitHub.ResolvedAPIURL(), + }).InstallationSettingsURL() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "ok": false, + "needsInstall": true, + "org": org, + "installUrl": installURL, + "error": fmt.Sprintf("the Hive GitHub App installation cannot access %s/%s. Add the repository to the installation's repository access, then re-check.", org, repo), + }) + return + } okResponse(w, map[string]string{"status": "ok", "org": org}) } diff --git a/src/pkg/dashboard/repo_check_access_probe_test.go b/src/pkg/dashboard/repo_check_access_probe_test.go index 755d9047fc..3215153692 100644 --- a/src/pkg/dashboard/repo_check_access_probe_test.go +++ b/src/pkg/dashboard/repo_check_access_probe_test.go @@ -11,20 +11,14 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" ghpkg "github.com/hivecommons/hive/pkg/github" ) -// These tests cover the App-present half of handleGovernorRepoCheckAccess — -// the DiscoverInstallationID probe the endpoint exists for. The App-less, -// cross-forge, and no-org guards are covered in repo_add_form_test.go; here a -// real *ghpkg.AppAuth (generated RSA key) talks to a stub GitHub API so both -// probe outcomes run deterministically and offline: -// - installed → 200 {ok:true, status:"ok", org} -// - not installed → 409 {ok:false, needsInstall:true, org, installUrl} and -// the pending-install banner mechanism is armed. +// App-present access checks use a real AppAuth and an offline GitHub stub. // newDiscoveryAppAuth builds an AppAuth whose JWT generation succeeds and whose // API calls hit the given handler. Discovery results are memoised process-wide @@ -59,8 +53,7 @@ func newDiscoveryAppAuth(t *testing.T, handler http.HandlerFunc) *ghpkg.AppAuth return auth } -// installedOrgHandler answers GET /orgs/{org}/installation affirmatively for -// exactly one org and 404s everything else (including the listing fallback). +// installedOrgHandler supplies an installation covering widget. func installedOrgHandler(org string, id int64) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/orgs/"+org+"/installation" { @@ -71,6 +64,16 @@ func installedOrgHandler(org string, id int64) http.HandlerFunc { }) return } + if r.URL.Path == "/app/installations/"+strconv.FormatInt(id, 10)+"/access_tokens" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"installation-token","expires_at":"2099-01-01T00:00:00Z"}`)) + return + } + if r.URL.Path == "/installation/repositories" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"repositories": []any{map[string]any{"full_name": org + "/widget"}}}) + return + } if r.URL.Path == "/app/installations" { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[]`)) @@ -130,6 +133,16 @@ func TestGovernorRepoCheckAccess_FullURLResolvesOrg(t *testing.T) { } } +func TestGovernorRepoCheckAccess_BareRepoUsesConfiguredOrg(t *testing.T) { + srv := newFullServer(t) + srv.deps.Config.Project.Org = "bare-org" + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, installedOrgHandler("bare-org", 123)) + w := checkAccessRequest(t, srv, `{"repo":"widget"}`) + if w.Code != http.StatusOK { + t.Fatalf("code = %d, want 200; body: %s", w.Code, w.Body.String()) + } +} + // App NOT installed on the org: 409 with needsInstall, the resolved org, the // per-forge install URL, and the pending-install banner mechanism armed so the // existing recheck plumbing drives the flow to completion. @@ -179,6 +192,112 @@ func TestGovernorRepoCheckAccess_NotInstalled409(t *testing.T) { } } +func TestGovernorRepoCheckAccess_RepoSelection(t *testing.T) { + for _, tc := range []struct { + name string + listStatus int + listBody string + wantStatus int + }{ + {"unselected", http.StatusOK, `{"repositories":[{"full_name":"selection-org/other"}]}`, http.StatusConflict}, + {"list failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + {"case insensitive", http.StatusOK, `{"repositories":[{"full_name":"SELECTION-ORG/WIDGET"}]}`, http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("selection-org", 4242) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/installation/repositories" { + if r.Header.Get("Authorization") != "Bearer installation-token" { + t.Errorf("repository listing did not use installation token") + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.listStatus) + _, _ = w.Write([]byte(tc.listBody)) + return + } + base(w, r) + }) + w := checkAccessRequest(t, srv, `{"repo":"selection-org/widget"}`) + if w.Code != tc.wantStatus { + t.Fatalf("code = %d, want %d; body: %s", w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict { + var resp struct { + NeedsInstall bool `json:"needsInstall"` + InstallURL string `json:"installUrl"` + Error string `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.NeedsInstall || !strings.Contains(resp.InstallURL, "/organizations/selection-org/settings/installations/4242") || !strings.Contains(resp.Error, "widget") { + t.Errorf("unexpected response: %+v", resp) + } + } + }) + } +} + +func TestGovernorRepoCheckAccess_DiscoveryFailure(t *testing.T) { + srv := newFullServer(t) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unavailable", http.StatusInternalServerError) + }) + w := checkAccessRequest(t, srv, `{"repo":"discovery-failure/widget"}`) + if w.Code != http.StatusServiceUnavailable || srv.IsPendingGitHubAppInstall() { + t.Fatalf("unexpected response %d %s; pending=%v", w.Code, w.Body.String(), srv.IsPendingGitHubAppInstall()) + } +} + +func TestGovernorRepoCheckAccess_TruncatedCoverageUsesRepositoryInstallation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + wantStatus int + }{ + {"selected", http.StatusOK, `{"id":4242}`, http.StatusOK}, + {"not selected", http.StatusNotFound, `{"message":"Not Found"}`, http.StatusConflict}, + {"different installation", http.StatusOK, `{"id":2}`, http.StatusServiceUnavailable}, + {"missing installation id", http.StatusOK, `{}`, http.StatusServiceUnavailable}, + {"forbidden", http.StatusForbidden, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"rate limited", http.StatusTooManyRequests, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"server failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("truncated-org", 4242) + probed := false + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/installation/repositories": + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Link", `; rel="next"`) + _, _ = w.Write([]byte(`{"repositories":[]}`)) + case "/repos/truncated-org/widget/installation": + probed = true + if auth := r.Header.Get("Authorization"); !strings.HasPrefix(auth, "Bearer ") || auth == "Bearer installation-token" { + t.Errorf("repository installation probe did not use App JWT: %q", auth) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + default: + base(w, r) + } + }) + w := checkAccessRequest(t, srv, `{"repo":"truncated-org/widget"}`) + if !probed || w.Code != tc.wantStatus { + t.Fatalf("probed=%v code=%d, want %d; body: %s", probed, w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict && !strings.Contains(w.Body.String(), "/organizations/truncated-org/settings/installations/4242") { + t.Fatalf("missing installation settings link: %s", w.Body.String()) + } + }) + } +} + // Malformed body and blank repo are rejected before any probe runs. func TestGovernorRepoCheckAccess_BadInput(t *testing.T) { srv := newFullServer(t) diff --git a/src/pkg/dashboard/static-src/js/linear-repos-save.js b/src/pkg/dashboard/static-src/js/linear-repos-save.js index cd3a3852c8..5dc703eaa0 100644 --- a/src/pkg/dashboard/static-src/js/linear-repos-save.js +++ b/src/pkg/dashboard/static-src/js/linear-repos-save.js @@ -342,7 +342,7 @@ return; } - // Requirement #3: verify the App can access the repo's org. + // Verify the App installation covers this repository. setStatus('Checking GitHub App access…', 'var(--muted)'); try { const res = await fetch('/api/config/governor/repos/check-access', { @@ -351,14 +351,11 @@ }); const d = await res.json().catch(() => ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? 'Install/authorize the App for ' + esc(d.org || '') + ' ↗' - : 'This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.'; + ? 'Manage App access for ' + esc(d.org || '') + ' ↗' + : 'An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '
' + installBtn + '  ·  Re-check & add', 'var(--amber)'); return; } @@ -367,11 +364,7 @@ return; } } catch (e) { - // Network error probing access is non-fatal: the server re-validates the - // single-host rule and the default-repo rule on save, so let the add - // proceed rather than hard-block on a transient dashboard hiccup. - _appendRepoAccepted(val); - setStatus('Added (App-access check could not be reached; the server will re-validate on save).', 'var(--muted)'); + setStatus('✕ Could not check GitHub App access. Please retry.', 'var(--red)'); return; } _appendRepoAccepted(val); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 0b3097ddd5..2917b2797f 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -71539,7 +71539,7 @@

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'} ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? 'Install/authorize the App for ' + esc(d.org || '') + ' ↗' - : 'This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.'; + ? 'Manage App access for ' + esc(d.org || '') + ' ↗' + : 'An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '
' + installBtn + '  ·  Re-check & add', 'var(--amber)'); return; } @@ -71564,11 +71561,7 @@

${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}

Date: Thu, 12 Nov 2026 06:24:44 +0000 Subject: [PATCH 064/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.5 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-army-rework-queue.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-army-rework-queue.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bf097b5b21..da7895396c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.5) + +### Fixed + +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. + ## 2026-11-12 (v5.228.4) ### Security diff --git a/changelog.d/fixed-army-rework-queue.md b/changelog.d/fixed-army-rework-queue.md deleted file mode 100644 index 93eaad0192..0000000000 --- a/changelog.d/fixed-army-rework-queue.md +++ /dev/null @@ -1 +0,0 @@ -- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. From e089cd7e6888979c3801ecc6830ab6e4f42f675c Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 11 Nov 2026 23:45:01 -0700 Subject: [PATCH 065/166] =?UTF-8?q?=E2=9C=A8=20knowledge:=20Wispr=20Flow?= =?UTF-8?q?=20dictation=20history=20as=20a=20knowledge=20source=20(#17855)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: Wispr Flow dictation history as a knowledge source Adds knowledge.connectors[].type: wisprflow. Reads the desktop app's local flow.sqlite read-only through the sqlite3 CLI (text columns only, never the audio/screenshot BLOBs) or a folder of exported .json/.ndjson/.csv/.md files, and emits one fact per finalised dictation (or per day) with the dictation time, source app and URL. Incremental by timestamp with a 2h overlap; min_words and apps filters; tags. Dashboard: Knowledge -> Integrations card (path, status, last sync, count), Add-source entry pointing out that new dictations need no connector at all (dictate into + New Fact). Docs section next to Plaud's; changelog fragment. Fixes #17854 Signed-off-by: Andrew Anderson * 🐛 knowledge: wisprflow tests run without the sqlite3 CLI Sync tests now use a fake sqlite3 runner fed from testdata/wisprflow/history.json so the SQLite path is exercised (and counted for coverage) on runners without the CLI; the real-CLI test skips when sqlite3 is absent. TestWisprValidate passes an explicit path so it no longer depends on a macOS default location. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- .../added-knowledge-wisprflow-source.md | 1 + src/docs/knowledge-connectors.md | 97 ++- .../knowledge_integrations_modal_test.go | 42 ++ .../dashboard/static-src/js/config-state.js | 1 + .../dashboard/static-src/js/config-tabs.js | 9 +- src/pkg/dashboard/static-src/js/knowledge.js | 16 + src/pkg/dashboard/static/index.html | 26 +- src/pkg/knowledge/connector/connector.go | 3 +- src/pkg/knowledge/connector/connector_test.go | 2 +- .../export/2026-11-11-1430-cursor.md | 3 + .../wisprflow/export/2026-11-11.ndjson | 3 + .../testdata/wisprflow/export/dictations.csv | 2 + .../testdata/wisprflow/export/dictations.json | 4 + .../testdata/wisprflow/export/ignored.bin | 1 + .../wisprflow/export/nested/frontmatter.md | 6 + .../connector/testdata/wisprflow/history.json | 6 + .../connector/testdata/wisprflow/history.sql | 14 + src/pkg/knowledge/connector/wisprflow.go | 577 ++++++++++++++++++ .../knowledge/connector/wisprflow_export.go | 259 ++++++++ src/pkg/knowledge/connector/wisprflow_test.go | 350 +++++++++++ 20 files changed, 1412 insertions(+), 10 deletions(-) create mode 100644 changelog.d/added-knowledge-wisprflow-source.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/history.json create mode 100644 src/pkg/knowledge/connector/testdata/wisprflow/history.sql create mode 100644 src/pkg/knowledge/connector/wisprflow.go create mode 100644 src/pkg/knowledge/connector/wisprflow_export.go create mode 100644 src/pkg/knowledge/connector/wisprflow_test.go diff --git a/changelog.d/added-knowledge-wisprflow-source.md b/changelog.d/added-knowledge-wisprflow-source.md new file mode 100644 index 0000000000..3e63a6018a --- /dev/null +++ b/changelog.d/added-knowledge-wisprflow-source.md @@ -0,0 +1 @@ +- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index cd39352198..6f84bf0979 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -1,6 +1,6 @@ # Knowledge connectors -External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis and Plaud voice notes) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. +External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis, Plaud voice notes and Wispr Flow dictation) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. For the shared configuration fields, the fact format and how connectors relate to `git_sources` and `documents`, see [knowledge-curator.md → Connectors](knowledge-curator.md#connectors-knowledgeconnectors). @@ -10,7 +10,7 @@ For the shared configuration fields, the fact format and how connectors relate t knowledge: connectors: - name: team-wiki # lowercase letters, digits, dashes - type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud + type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud | wisprflow enabled: true interval: 15m # minimum 1m; blank = 15m layer: project # personal | project | org | community @@ -311,6 +311,99 @@ if you do not want a sync to pull a new upstream release unreviewed. `document` connector (`file_path`). No code, still $0, and no MCP sign-in. - Everything in this section costs **$0** beyond your Plaud subscription. +## Wispr Flow (dictation, `type: wisprflow`) + +[Wispr Flow](https://wisprflow.ai) is dictation into whatever field has focus: +hold the Flow key, speak, release, and polished text is pasted where the +cursor is. That gives two $0 paths into the hive. + +**Direct dictation — no connector.** Put the cursor in **Knowledge → + New +Fact** (title or body), the Import box, an agent prompt, or any other text +field in the dashboard and dictate. Nothing to configure. This is the +recommended way to get *new* thoughts in. + +**History connector.** The desktop app keeps every dictation in a local +SQLite file. The `wisprflow` connector reads it read-only and turns each +dictation (what you said, when, and into which app) into a fact, so ideas you +dictated into Slack or your editor last week are searchable by agents too. +There is no official export API for dictation history — Wispr's remote MCP +server covers Notetaker meetings only and states it cannot read dictation +history, and the Voice Interface API is invite-only and does speech-to-text, +not history — so the local file is the only free source. The schema is +community-documented (`History` table: `transcriptEntityId`, `asrText`, +`formattedText`, `editedText`, `timestamp`, `app`, `url`, `duration`, +`numWords`, `status`, `isArchived`) and can change with an app update. + +```yaml +knowledge: + connectors: + - name: wispr + type: wisprflow + layer: personal + interval: 15m + scope: + # path: /Users/me/Library/Application Support/Wispr Flow/flow.sqlite # default on macOS; %APPDATA%\Wispr Flow\flow.sqlite on Windows + # path: /data/knowledge/wispr-export # or a folder of exported .json/.ndjson/.csv/.md files + since_days: "30" # how far back the first sync reaches + max_items: "500" # dictations read per sync + min_words: "3" # skip "ok", "yes", … + group: dictation # dictation = one fact each; day = one fact per calendar day + apps: "" # optional allow-list matched against the app bundle id, e.g. "slack, obsidian, vscode" + include_raw: "false" # also keep the raw ASR text under "Raw transcript" + tags: "voice, ideas" # extra tags written as attr_tags + # sqlite3: sqlite3 # CLI used to read the file; ships with macOS and the hive image +``` + +No `auth` block and no sign-in: the data is already on disk. The connector +shells out to the `sqlite3` CLI with `-readonly -json` and selects text +columns only (the table also holds audio and screenshot BLOBs, which are never +read). Rows with `status = formatted` and `isArchived = 0` are used; +`editedText` wins over `formattedText` over `asrText`. + +### Setup + +1. **Same machine as the hive (laptop hive):** add the connector from + **Knowledge → 🔌 Integrations → Wispr Flow dictation** and leave `path` + empty. On macOS the hive process may need **Full Disk Access** (System + Settings → Privacy & Security) to read `~/Library/Application Support`. +2. **Hive on a cluster or another host:** sync the file or an export to a + path the hive can read and set `path` to it. Any of these work: + - copy `flow.sqlite` (or one of the app's timestamped `backups/*.sqlite`) + into the knowledge PVC, e.g. with a cron `kubectl cp`; + - run a community exporter such as `wispr export -o ` + ([iamnikolie/wispr-cli](https://github.com/iamnikolie/wispr-cli), writes + `dictations/*.md`) or + [junxit/wispr-flow-exporter](https://github.com/junxit/wispr-flow-exporter) + (Markdown/JSON/NDJSON) and point `path` at the folder, or commit that + folder to a Git-synced vault and let the existing `git` connector pick + it up. +3. Click **Sync now**. New dictations are picked up every `interval`. + +Export folders are walked recursively; `.json` (array or +`{"dictations": [...]}`), `.ndjson`/`.jsonl`, `.csv` (header row) and `.md` +(one dictation per file; optional front matter `id:`, `timestamp:`, `app:`, +otherwise the date in the filename or the file mtime) are read, other files +are ignored. A file that fails to parse is reported as a partial sync and the +rest are still imported. + +### Behaviour + +- Incremental by `timestamp`: each sync re-reads from the newest dictation + seen minus a 2 h overlap (Flow updates a row in place as it finalises), or + `since_days` back on the first run. With `group: day` the current day's + fact is rebuilt whole. +- Facts carry `source: wisprflow`, `attr_wispr_transcript_id`, + `attr_dictated_at`, `attr_app` (bundle id), `attr_num_words`, + `attr_language` and the dictation date as the vault path + (`Wispr Flow/2026-11-11`). The title is the first ten words. +- The listing is incremental, so dictations you archive or delete in Flow + are not tombstoned automatically; remove the fact file or the connector. +- Everything here costs **$0** beyond your Wispr Flow plan; the free tier's + weekly word cap applies to dictating, not to reading history. Tip: add + your project vocabulary (`hive`, `dibs`, `spek`, `ACMM`, `crux`, repo + names) under Flow → Settings → Dictionary so it is spelled right the first + time. + ## Notion (`type: notion`) The Notion connector syncs pages and database rows through the public Notion diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 0e79b35a80..2c3cb41497 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -184,3 +184,45 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { } } } + +// TestKnowledgeIntegrationsModalWisprFlowCard guards the Wispr Flow entry: +// it is a plain local-file connector (no sign-in buttons), the card shows +// the configured path, the Knowledge pane links to it and documents the +// direct-dictation path, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalWisprFlowCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'wisprflow': { label: 'Wispr Flow dictation'`, + `'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)'`, + `['group', 'Fact per dictation or day', 'dictation']`, + `if (type === 'wisprflow') html += kbWisprFlowSourceLine(c);`, + `data-kb-wisprflow-note`, + `data-action="kbOpenWisprFlowIntegration"`, + `function kbWisprFlowSourceLine(c)`, + `function kbOpenWisprFlowIntegration()`, + `Wispr Flow dictation`, + `+ New Fact and hold the Flow key`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Wispr Flow card snippet %q", want) + } + } + if !strings.Contains(html, `'sharepoint', 'wisprflow']`) { + t.Error("wisprflow missing from KB_CONNECTOR_TYPE_ORDER") + } + start := strings.Index(html, "function kbWisprFlowSourceLine(c)") + end := strings.Index(html, "function kbOpenWisprFlowIntegration()") + if start < 0 || end < start { + t.Fatal("Wispr Flow block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Wispr Flow block uses a browser-native dialog %q", native) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 459c8a898a..162cf7e258 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -128,6 +128,7 @@ { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index fbd7bfa2ef..03d69b03f3 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -289,7 +289,7 @@ } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -300,7 +300,8 @@ 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -315,6 +316,7 @@ 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -332,7 +334,8 @@ 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index eb322b6c55..81daf98c72 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -1981,6 +1981,12 @@ html += '
'; html += '
'; + html += '
🗣️ Wispr Flow dictation'; + html += '
'; + html += '
Dictate straight into any field here — put the cursor in + New Fact and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -2428,6 +2434,7 @@ html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '
'; if (owner) { html += '
'; @@ -2468,6 +2475,7 @@ html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '

Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.

'; if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; @@ -2632,3 +2640,11 @@ } } function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '
Wispr Flow: ' + (path ? '' + escapeHtml(path) + '' : 'default app location on the hive host') + ' · ' + group + '
'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 2917b2797f..07b1ff15dc 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -45497,6 +45497,12 @@

Configure in Integrations'; html += '

'; + html += '
🗣️ Wispr Flow dictation'; + html += '
'; + html += '
Dictate straight into any field here — put the cursor in + New Fact and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.
'; + html += '
'; + html += '
'; + html += '
📚 Import from Context7'; html += '
'; html += '
Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").
'; @@ -45944,6 +45950,7 @@

How Agents Use Knowledge (Priming)

html += '
' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '
'; if (r.last_error) html += '
' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '
'; if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '
'; if (owner) { html += '
'; @@ -45984,6 +45991,7 @@

How Agents Use Knowledge (Priming)

html += '
'; html += '
' + kcScopeFieldsHTML(type, c.scope || {}) + '
'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '

Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.

'; if (type === 'plaud') html += '

Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.

'; html += ''; html += ''; @@ -46148,6 +46156,14 @@

How Agents Use Knowledge (Priming)

} } function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '
Wispr Flow: ' + (path ? '' + escapeHtml(path) + '' : 'default app location on the hive host') + ' · ' + group + '
'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } // ── Vault Management ── function kbOpenVaults() { @@ -54995,6 +55011,7 @@

Configure LiteLLM

{ tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; @@ -56275,7 +56292,7 @@

Configure LiteLLM

} // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -56286,7 +56303,8 @@

Configure LiteLLM

'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -56301,6 +56319,7 @@

Configure LiteLLM

'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -56318,7 +56337,8 @@

Configure LiteLLM

'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; diff --git a/src/pkg/knowledge/connector/connector.go b/src/pkg/knowledge/connector/connector.go index 777984bb2b..953b89907a 100644 --- a/src/pkg/knowledge/connector/connector.go +++ b/src/pkg/knowledge/connector/connector.go @@ -216,7 +216,7 @@ func NewRegistry() *Registry { // DefaultRegistry returns a registry with the built-in connector types // (`git`, `document`, `github-wiki`, `repo-wiki`, `confluence`, `notion`, -// `sharepoint`, `google-drive`, `context7` and `plaud`) registered. +// `sharepoint`, `google-drive`, `context7`, `plaud` and `wisprflow`) registered. func DefaultRegistry() *Registry { r := NewRegistry() _ = r.Register(TypeGit, newGitConnector) @@ -229,6 +229,7 @@ func DefaultRegistry() *Registry { _ = r.Register(TypeRepoWiki, newRepoWikiConnector) _ = r.Register(TypeContext7, newContext7Connector) _ = r.Register(TypePlaud, newPlaudConnector) + _ = r.Register(TypeWisprFlow, newWisprConnector) return r } diff --git a/src/pkg/knowledge/connector/connector_test.go b/src/pkg/knowledge/connector/connector_test.go index 33e132d104..4e0c8714fa 100644 --- a/src/pkg/knowledge/connector/connector_test.go +++ b/src/pkg/knowledge/connector/connector_test.go @@ -95,7 +95,7 @@ func TestRegistryRegister(t *testing.T) { if got := r.Types(); !reflect.DeepEqual(got, []string{"another", "fake"}) { t.Fatalf("Types() = %v", got) } - if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint}) { + if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint, TypeWisprFlow}) { t.Fatalf("DefaultRegistry().Types() = %v", got) } } diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md new file mode 100644 index 0000000000..edf494c416 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md @@ -0,0 +1,3 @@ +# Dictated into Cursor + +Refactor the syncer so the wisprflow connector shares the plaud cursor parser. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson new file mode 100644 index 0000000000..862be53254 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson @@ -0,0 +1,3 @@ +{"id":"n-1","text":"Remember to add spek, hive and dibs to the Wispr dictionary.","at":"2026-11-11 08:30:00.000 +00:00","app":"md.obsidian","words":11} + +{"id":"n-2","text":"Hi","at":"2026-11-11 08:31:00.000 +00:00","app":"com.apple.MobileSMS","words":1} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv new file mode 100644 index 0000000000..956dbdc67a --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv @@ -0,0 +1,2 @@ +id,text,timestamp,app,numWords +c-1,"Ask Greg about the KubeCon ops checklist, then file it in dibs.",2026-11-11T09:00:00Z,com.tinyspeck.slackmacgap,13 diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json new file mode 100644 index 0000000000..d9732572c6 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json @@ -0,0 +1,4 @@ +{"dictations": [ + {"transcriptEntityId": "j-1", "formattedText": "Spec idea: a wisprflow knowledge connector for hive.", "asrText": "spec idea a wispr flow knowledge connector for hive", "timestamp": "2026-11-10T12:00:00Z", "app": "com.electron.wispr-flow", "numWords": 8}, + {"transcriptEntityId": "j-old", "formattedText": "Too old to matter.", "timestamp": "2019-05-05T12:00:00Z", "app": "com.apple.Notes"} +]} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin new file mode 100644 index 0000000000..7c0582d8dd --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin @@ -0,0 +1 @@ +ignored binary diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md new file mode 100644 index 0000000000..0e1f51d3c3 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md @@ -0,0 +1,6 @@ +--- +id: m-fm +timestamp: 2026-11-10 16:20:00 +app: com.apple.Notes +--- +Front-matter note dictated into Apple Notes about the ACMM scorecard. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.json b/src/pkg/knowledge/connector/testdata/wisprflow/history.json new file mode 100644 index 0000000000..633157e859 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.json @@ -0,0 +1,6 @@ +[{"id":"d-1","text":"Add a retry with backoff around the upload call.","raw":"um so add a retry with backoff around the upload call","at":"2026-11-10 09:15:00.000 +00:00","app":"com.microsoft.VSCode","url":"","duration":6.5,"numWords":9,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-2","text":"Running late — start without me.","raw":"running late start without me","at":"2026-11-11 17:55:30.000 +00:00","app":"com.tinyspeck.slackmacgap","url":"","duration":2.1,"numWords":5,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-short","text":"Ok.","raw":"ok","at":"2026-11-11 18:00:00.000 +00:00","app":"com.apple.MobileSMS","url":"","duration":0.8,"numWords":1,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-raw","text":"still processing","raw":"still processing","at":"2026-11-11 18:05:00.000 +00:00","app":"com.google.Chrome","url":"","duration":3,"numWords":2,"language":"en","status":"raw_transcript","isArchived":0}, +{"id":"d-archived","text":"Archived thought about the hive dashboard.","raw":"archived thought about the hive dashboard","at":"2026-11-11 18:10:00.000 +00:00","app":"md.obsidian","url":"","duration":4,"numWords":6,"language":"en","status":"formatted","isArchived":1}, +{"id":"d-old","text":"Ancient dictation from long ago.","raw":"ancient dictation from long ago","at":"2020-01-01 10:00:00.000 +00:00","app":"com.google.Chrome","url":"https://example.com","duration":4,"numWords":5,"language":"en","status":"formatted","isArchived":0}] diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.sql b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql new file mode 100644 index 0000000000..1973358d97 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql @@ -0,0 +1,14 @@ +-- Minimal Wispr Flow History schema (community-documented, Sequelize DATETIME strings). +CREATE TABLE History ( + transcriptEntityId VARCHAR(36) PRIMARY KEY, + asrText TEXT, formattedText TEXT, editedText TEXT, + timestamp DATETIME, audio BLOB, screenshot BLOB, + status VARCHAR(255), app VARCHAR(255), url VARCHAR(255), + duration FLOAT, numWords INTEGER, language TEXT, isArchived TINYINT(1) DEFAULT 0 +); +INSERT INTO History VALUES ('d-1','um so add a retry with backoff around the upload call','Add a retry with backoff around the upload call.',NULL,'2026-11-10 09:15:00.000 +00:00',x'00',NULL,'formatted','com.microsoft.VSCode','',6.5,9,'en',0); +INSERT INTO History VALUES ('d-2','running late start without me','Running late, start without me.','Running late — start without me.','2026-11-11 17:55:30.000 +00:00',NULL,NULL,'formatted','com.tinyspeck.slackmacgap','',2.1,5,'en',0); +INSERT INTO History VALUES ('d-short','ok','Ok.',NULL,'2026-11-11 18:00:00.000 +00:00',NULL,NULL,'formatted','com.apple.MobileSMS','',0.8,1,'en',0); +INSERT INTO History VALUES ('d-raw','still processing','',NULL,'2026-11-11 18:05:00.000 +00:00',NULL,NULL,'raw_transcript','com.google.Chrome','',3,2,'en',0); +INSERT INTO History VALUES ('d-archived','archived thought about the hive dashboard','Archived thought about the hive dashboard.',NULL,'2026-11-11 18:10:00.000 +00:00',NULL,NULL,'formatted','md.obsidian','',4,6,'en',1); +INSERT INTO History VALUES ('d-old','ancient dictation from long ago','Ancient dictation from long ago.',NULL,'2020-01-01 10:00:00.000 +00:00',NULL,NULL,'formatted','com.google.Chrome','https://example.com',4,5,'en',0); diff --git a/src/pkg/knowledge/connector/wisprflow.go b/src/pkg/knowledge/connector/wisprflow.go new file mode 100644 index 0000000000..39f902748e --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow.go @@ -0,0 +1,577 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" +) + +// TypeWisprFlow is the connector type for Wispr Flow dictation history, +// read from the desktop app's local SQLite file or from an export folder. +const TypeWisprFlow = "wisprflow" + +const ( + // wisprDefaultSQLite3 is the sqlite3 CLI used to read flow.sqlite; it + // ships with macOS and the hive image, so no Go SQLite driver is needed. + wisprDefaultSQLite3 = "sqlite3" + // wisprDefaultSinceDays bounds the first sync. + wisprDefaultSinceDays = 30 + // wisprDefaultMaxItems caps dictations read per sync. + wisprDefaultMaxItems = 500 + // wisprDefaultMinWords drops one- and two-word dictations ("yes", "ok"). + wisprDefaultMinWords = 3 + // wisprRelistOverlap re-reads this far behind the cursor: Flow updates a + // row in place (raw_transcript → formatted), so a late finalisation is + // still picked up. + wisprRelistOverlap = 2 * time.Hour + // wisprGroupDictation emits one fact per dictation; wisprGroupDay one + // fact per calendar day. + wisprGroupDictation = "dictation" + wisprGroupDay = "day" + // wisprTitleWords is how many words of the text make the fact title. + wisprTitleWords = 10 + // wisprSQLTimeout bounds one sqlite3 invocation. + wisprSQLTimeout = 60 * time.Second + // wisprStatusFormatted is the terminal row status in Flow's History table. + wisprStatusFormatted = "formatted" +) + +// wisprDictation is one Wispr Flow history row, normalised from SQLite or +// an export file. +type wisprDictation struct { + ID string + Text string + RawText string + At time.Time + App string + URL string + Duration float64 + Words int + Language string +} + +// wisprConnector reads Wispr Flow's local dictation history. Scope keys: +// path (flow.sqlite, a backup copy, or an export folder), sqlite3, since_days, +// max_items, min_words, group (dictation|day), apps, include_raw, tags. +// No auth: the data is already on local disk. +type wisprConnector struct { + cfg ConnectorConfig + logger *slog.Logger + now func() time.Time + // query runs the sqlite3 CLI; tests replace it. + query func(ctx context.Context, dbPath, sql string) ([]byte, error) +} + +func newWisprConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &wisprConnector{cfg: cfg, logger: deps.logger(), now: time.Now} + bin := cfg.ScopeValue("sqlite3") + if bin == "" { + bin = wisprDefaultSQLite3 + } + c.query = wisprSQLite3Runner(bin) + return c, nil +} + +func (c *wisprConnector) Type() string { return TypeWisprFlow } + +// wisprDefaultPath is where the desktop app keeps its history on this OS. +func wisprDefaultPath() string { + switch runtime.GOOS { + case "darwin": + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, "Library", "Application Support", "Wispr Flow", "flow.sqlite") + case "windows": + if appdata := os.Getenv("APPDATA"); appdata != "" { + return filepath.Join(appdata, "Wispr Flow", "flow.sqlite") + } + } + return "" +} + +func (c *wisprConnector) path() string { + if p := c.cfg.ScopeValue("path"); p != "" { + return p + } + return wisprDefaultPath() +} + +// Validate checks the scope. The path may not exist yet (the app may be +// installed later); only its shape is checked. +func (c *wisprConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("wisprflow: auth.env/auth.file are not used; the connector reads Wispr Flow's local history file") + } + p := cfg.ScopeValue("path") + if p == "" && wisprDefaultPath() == "" { + return fmt.Errorf("wisprflow: scope.path is required on %s (no default Wispr Flow location)", runtime.GOOS) + } + if p != "" && !filepath.IsAbs(p) { + return fmt.Errorf("wisprflow: scope.path must be an absolute path to flow.sqlite or an export folder") + } + for _, key := range []string{"since_days", "max_items", "min_words"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 || (key != "min_words" && n == 0) { + return fmt.Errorf("wisprflow: scope.%s must be a positive integer", key) + } + } + } + if g := cfg.ScopeValue("group"); g != "" && g != wisprGroupDictation && g != wisprGroupDay { + return fmt.Errorf("wisprflow: scope.group must be %q or %q", wisprGroupDictation, wisprGroupDay) + } + if v := cfg.ScopeValue("include_raw"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("wisprflow: scope.include_raw must be true or false") + } + } + if bin := cfg.ScopeValue("sqlite3"); bin != "" { + if _, err := splitCommandLine(bin); err != nil { + return err + } + } + return nil +} + +func (c *wisprConnector) scopeInt(key string, def int) int { + v := c.cfg.ScopeValue(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return def + } + return n +} + +func (c *wisprConnector) group() string { + if c.cfg.ScopeValue("group") == wisprGroupDay { + return wisprGroupDay + } + return wisprGroupDictation +} + +// Sync reads dictations newer than the cursor (minus an overlap) and emits +// one page per dictation or per day. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *wisprConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + p := c.path() + if p == "" { + return cur, fmt.Errorf("wisprflow: no path configured") + } + info, err := os.Stat(p) + if err != nil { + return cur, fmt.Errorf("wisprflow: %w (install Wispr Flow, point scope.path at flow.sqlite or an export folder, or grant the hive process Full Disk Access on macOS)", err) + } + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", wisprDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-wisprRelistOverlap) + } + if c.group() == wisprGroupDay { + // A day page is rebuilt whole, so re-read from the start of that day. + y, m, d := since.UTC().Date() + since = time.Date(y, m, d, 0, 0, 0, 0, time.UTC) + } + maxItems := c.scopeInt("max_items", wisprDefaultMaxItems) + + var rows []wisprDictation + var partial error + if info.IsDir() { + rows, partial = readWisprExportDir(p, since, maxItems) + } else { + rows, err = c.readSQLite(ctx, p, since, maxItems) + if err != nil { + return cur, err + } + } + rows = c.filter(rows) + sort.Slice(rows, func(i, j int) bool { return rows[i].At.Before(rows[j].At) }) + + newest := parsePlaudCursor(cur) + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + } + pages := c.pages(rows) + for _, pg := range pages { + if err := ctx.Err(); err != nil { + return cur, err + } + if err := emit(pg); err != nil { + return cur, err + } + } + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if partial != nil { + return next, &PartialError{Err: partial} + } + return next, nil +} + +// filter applies min_words and the apps allow-list. +func (c *wisprConnector) filter(rows []wisprDictation) []wisprDictation { + minWords := c.scopeInt("min_words", wisprDefaultMinWords) + var apps []string + for _, a := range strings.Split(c.cfg.ScopeValue("apps"), ",") { + if a = strings.ToLower(strings.TrimSpace(a)); a != "" { + apps = append(apps, a) + } + } + out := rows[:0] + for _, r := range rows { + r.Text = strings.TrimSpace(r.Text) + if r.Text == "" || r.ID == "" { + continue + } + words := r.Words + if words <= 0 { + words = len(strings.Fields(r.Text)) + } + if words < minWords { + continue + } + if len(apps) > 0 && !wisprAppMatches(r.App, apps) { + continue + } + out = append(out, r) + } + return out +} + +func wisprAppMatches(app string, allow []string) bool { + app = strings.ToLower(app) + for _, a := range allow { + if strings.Contains(app, a) { + return true + } + } + return false +} + +// pages renders one page per dictation, or one per UTC day. +func (c *wisprConnector) pages(rows []wisprDictation) []Page { + includeRaw, _ := strconv.ParseBool(c.cfg.ScopeValue("include_raw")) + tags := c.cfg.ScopeValue("tags") + if c.group() == wisprGroupDictation { + out := make([]Page, 0, len(rows)) + for _, r := range rows { + out = append(out, c.dictationPage(r, includeRaw, tags)) + } + return out + } + byDay := map[string][]wisprDictation{} + var days []string + for _, r := range rows { + day := r.At.UTC().Format("2006-01-02") + if _, ok := byDay[day]; !ok { + days = append(days, day) + } + byDay[day] = append(byDay[day], r) + } + sort.Strings(days) + out := make([]Page, 0, len(days)) + for _, day := range days { + out = append(out, c.dayPage(day, byDay[day], includeRaw, tags)) + } + return out +} + +func wisprTitle(text string) string { + words := strings.Fields(text) + if len(words) > wisprTitleWords { + return strings.Join(words[:wisprTitleWords], " ") + "…" + } + return strings.Join(words, " ") +} + +// wisprAppName turns a bundle id like com.tinyspeck.slackmacgap into a +// readable label; unknown ids keep their last segment. +func wisprAppName(app string) string { + app = strings.TrimSpace(app) + if app == "" { + return "" + } + known := map[string]string{ + "com.tinyspeck.slackmacgap": "Slack", + "md.obsidian": "Obsidian", + "com.apple.MobileSMS": "Messages", + "com.microsoft.VSCode": "VS Code", + "com.google.Chrome": "Chrome", + "com.apple.Safari": "Safari", + "com.electron.wispr-flow": "Wispr Flow", + "com.openai.chat": "ChatGPT", + "com.anthropic.claudefordesktop": "Claude", + "com.apple.Notes": "Notes", + "com.apple.mail": "Mail", + "com.apple.Terminal": "Terminal", + "com.googlecode.iterm2": "iTerm2", + "com.todesktop.230313mzl4w4u92": "Cursor", + } + if n, ok := known[app]; ok { + return n + } + if i := strings.LastIndex(app, "."); i >= 0 && i < len(app)-1 { + return app[i+1:] + } + return app +} + +func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags string) Page { + appName := wisprAppName(r.App) + title := wisprTitle(r.Text) + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "- **Dictated:** %s\n", r.At.UTC().Format(time.RFC3339)) + if appName != "" { + fmt.Fprintf(&md, "- **App:** %s\n", appName) + } + if r.URL != "" { + fmt.Fprintf(&md, "- **URL:** %s\n", r.URL) + } + if r.Duration > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(r.Duration * float64(time.Second))).Round(time.Second)) + } + md.WriteString("\n") + md.WriteString(r.Text) + md.WriteString("\n") + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + md.WriteString("\n## Raw transcript\n\n") + md.WriteString(strings.TrimSpace(r.RawText)) + md.WriteString("\n") + } + attrs := map[string]string{ + "wispr_transcript_id": r.ID, + "dictated_at": r.At.UTC().Format(time.RFC3339), + } + if r.App != "" { + attrs["app"] = r.App + } + if r.Words > 0 { + attrs["num_words"] = strconv.Itoa(r.Words) + } + if r.Language != "" { + attrs["language"] = r.Language + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: r.ID, + Title: title, + URL: r.URL, + Markdown: md.String(), + UpdatedAt: r.At, + Path: []string{"Wispr Flow", r.At.UTC().Format("2006-01-02")}, + Attrs: attrs, + } +} + +func (c *wisprConnector) dayPage(day string, rows []wisprDictation, includeRaw bool, tags string) Page { + title := "Wispr Flow dictations " + day + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "%d dictation(s).\n\n", len(rows)) + var newest time.Time + words := 0 + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + words += r.Words + label := r.At.UTC().Format("15:04") + if n := wisprAppName(r.App); n != "" { + label += " · " + n + } + fmt.Fprintf(&md, "## %s\n\n%s\n\n", label, r.Text) + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + fmt.Fprintf(&md, "
Raw transcript\n\n%s\n\n
\n\n", strings.TrimSpace(r.RawText)) + } + } + attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows))} + if words > 0 { + attrs["num_words"] = strconv.Itoa(words) + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: "day-" + day, + Title: title, + Markdown: md.String(), + UpdatedAt: newest, + Path: []string{"Wispr Flow"}, + Attrs: attrs, + } +} + +// wisprSQLite3Runner executes the sqlite3 CLI read-only with JSON output. +// The History table holds audio and screenshot BLOBs, so only text columns +// are ever selected. +func wisprSQLite3Runner(bin string) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + return func(ctx context.Context, dbPath, sql string) ([]byte, error) { + argv, err := splitCommandLine(bin) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, wisprSQLTimeout) + defer cancel() + args := append(argv[1:], "-readonly", "-json", "-bail", dbPath, sql) + cmd := exec.CommandContext(ctx, argv[0], args...) + out, err := cmd.Output() + if err != nil { + var stderr string + if ee, ok := err.(*exec.ExitError); ok { + stderr = firstLine(string(ee.Stderr)) + } + if stderr == "" { + stderr = err.Error() + } + return nil, fmt.Errorf("wisprflow: sqlite3 failed: %s", stderr) + } + return out, nil + } +} + +// wisprHistorySQL selects finalised, unarchived dictations newer than since. +// Flow stores Sequelize DATETIME strings; datetime() normalises them so the +// comparison does not depend on the stored format. +func wisprHistorySQL(since time.Time, limit int) string { + s := strings.ReplaceAll(since.UTC().Format("2006-01-02 15:04:05"), "'", "''") + return fmt.Sprintf(`SELECT transcriptEntityId AS id, `+ + `COALESCE(NULLIF(editedText,''), NULLIF(formattedText,''), asrText) AS text, `+ + `asrText AS raw, timestamp AS at, COALESCE(app,'') AS app, COALESCE(url,'') AS url, `+ + `COALESCE(duration,0) AS duration, COALESCE(numWords,0) AS numWords, COALESCE(language,'') AS language `+ + `FROM History WHERE COALESCE(isArchived,0)=0 AND status='%s' AND datetime(timestamp) > datetime('%s') `+ + `ORDER BY datetime(timestamp) ASC LIMIT %d;`, wisprStatusFormatted, s, limit) +} + +func (c *wisprConnector) readSQLite(ctx context.Context, dbPath string, since time.Time, limit int) ([]wisprDictation, error) { + out, err := c.query(ctx, dbPath, wisprHistorySQL(since, limit)) + if err != nil { + return nil, err + } + return decodeWisprJSON(out) +} + +// decodeWisprJSON accepts the sqlite3 -json array (or any JSON array/object +// list of dictations, as written by community exporters). +func decodeWisprJSON(data []byte) ([]wisprDictation, error) { + text := strings.TrimSpace(string(data)) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return nil, fmt.Errorf("wisprflow: invalid JSON: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"dictations", "history", "items", "data", "results", "transcripts"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + items = []any{v} + } + } + out := make([]wisprDictation, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// wisprFromMap normalises one dictation object from SQLite JSON or an +// exporter's JSON, which name the same fields differently. +func wisprFromMap(m map[string]any) (wisprDictation, bool) { + d := wisprDictation{ + ID: plaudString(m, "id", "transcriptEntityId", "transcript_id", "uuid"), + Text: plaudString(m, "text", "editedText", "edited_text", "formattedText", "formatted_text", "content", "asrText"), + RawText: plaudString(m, "raw", "asrText", "asr_text", "raw_text"), + At: plaudTime(m, "at", "timestamp", "created_at", "createdAt", "date"), + App: plaudString(m, "app", "application", "bundle_id"), + URL: plaudString(m, "url"), + Language: plaudString(m, "language", "lang"), + } + d.Duration = wisprFloat(m, "duration", "duration_s", "seconds") + d.Words = int(wisprFloat(m, "numWords", "num_words", "words")) + if d.At.IsZero() { + d.At = wisprParseTime(plaudString(m, "at", "timestamp", "created_at", "createdAt", "date")) + } + if d.ID == "" || strings.TrimSpace(d.Text) == "" || d.At.IsZero() { + return d, false + } + return d, true +} + +// wisprParseTime handles Sequelize's "2026-11-11 14:05:09.123 +00:00" and +// the common ISO shapes; plaudTime covers RFC3339 and epoch numbers. +func wisprParseTime(s string) time.Time { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{} + } + for _, layout := range []string{ + "2006-01-02 15:04:05.000 -07:00", + "2006-01-02 15:04:05 -07:00", + "2006-01-02 15:04:05.000Z07:00", + "2006-01-02 15:04:05.000", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05.000Z07:00", + time.RFC3339Nano, + time.RFC3339, + "2006-01-02T15:04", + "2006-01-02 15:04", + "2006-01-02", + } { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// wisprFloat reads the first numeric value among keys, accepting JSON +// numbers and numeric strings (CSV exports). +func wisprFloat(m map[string]any, keys ...string) float64 { + for _, k := range keys { + switch v := m[k].(type) { + case float64: + return v + case string: + if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { + return f + } + } + } + return 0 +} diff --git a/src/pkg/knowledge/connector/wisprflow_export.go b/src/pkg/knowledge/connector/wisprflow_export.go new file mode 100644 index 0000000000..3a6d592b15 --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_export.go @@ -0,0 +1,259 @@ +package connector + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// wisprExportMaxFileBytes caps one export file read. +const wisprExportMaxFileBytes = 64 << 20 + +// wisprFilenameTimeRe pulls a date (and optional time) from an exported +// markdown filename such as 2026-11-11-1405-slack.md or 2026-11-11T14-05-09.md. +var wisprFilenameTimeRe = regexp.MustCompile(`(\d{4}-\d{2}-\d{2})(?:[T _-](\d{2})[-:h]?(\d{2})(?:[-:m]?(\d{2}))?)?`) + +// readWisprExportDir walks an export folder (as written by `wispr export`, +// wispr-flow-exporter or a hand-made dump) and returns the dictations newer +// than since. Supported files: .json (array or wrapped object), .ndjson / +// .jsonl (one object per line), .csv (header row) and .md (one dictation per +// file, optional YAML front matter). Unreadable files are reported together +// as one error while the readable ones are still returned. +func readWisprExportDir(dir string, since time.Time, limit int) ([]wisprDictation, error) { + var out []wisprDictation + var problems []string + seen := map[string]bool{} + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + problems = append(problems, fmt.Sprintf("%s: %v", path, walkErr)) + return nil + } + name := d.Name() + if d.IsDir() { + if path != dir && strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + return nil + } + if strings.HasPrefix(name, ".") { + return nil + } + rows, err := readWisprExportFile(dir, path) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v", filepath.Base(path), err)) + return nil + } + for _, r := range rows { + if r.At.IsZero() || !r.At.After(since) || seen[r.ID] { + continue + } + seen[r.ID] = true + out = append(out, r) + if len(out) >= limit { + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return out, err + } + if len(problems) > 0 { + return out, fmt.Errorf("wisprflow: %d export file(s) skipped: %s", len(problems), strings.Join(problems, "; ")) + } + return out, nil +} + +// readWisprExportFile dispatches on extension; unknown extensions are ignored. +func readWisprExportFile(root, path string) ([]wisprDictation, error) { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".json", ".ndjson", ".jsonl", ".csv", ".md", ".markdown", ".txt": + default: + return nil, nil + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.Size() > wisprExportMaxFileBytes { + return nil, fmt.Errorf("file larger than %d bytes", wisprExportMaxFileBytes) + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + switch ext { + case ".json": + return decodeWisprJSON(data) + case ".ndjson", ".jsonl": + return decodeWisprNDJSON(data) + case ".csv": + return decodeWisprCSV(data) + default: + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = filepath.Base(path) + } + d, ok := decodeWisprMarkdown(rel, data, info.ModTime()) + if !ok { + return nil, nil + } + return []wisprDictation{d}, nil + } +} + +func decodeWisprNDJSON(data []byte) ([]wisprDictation, error) { + var out []wisprDictation + sc := bufio.NewScanner(strings.NewReader(string(data))) + sc.Buffer(make([]byte, 0, 64<<10), wisprExportMaxFileBytes) + line := 0 + for sc.Scan() { + line++ + text := strings.TrimSpace(sc.Text()) + if text == "" { + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err != nil { + return out, fmt.Errorf("line %d: %v", line, err) + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, sc.Err() +} + +// decodeWisprCSV maps header names onto the same keys wisprFromMap accepts. +func decodeWisprCSV(data []byte) ([]wisprDictation, error) { + r := csv.NewReader(strings.NewReader(string(data))) + r.FieldsPerRecord = -1 + r.LazyQuotes = true + header, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil + } + return nil, err + } + for i := range header { + header[i] = strings.TrimSpace(strings.TrimPrefix(header[i], "\ufeff")) + } + var out []wisprDictation + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return out, err + } + m := make(map[string]any, len(header)) + for i, h := range header { + if i < len(rec) && h != "" { + m[h] = rec[i] + } + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// decodeWisprMarkdown reads one dictation per markdown file. Front matter +// keys id, timestamp/date/created, app and url are honoured; otherwise the +// id is the relative path and the time comes from the filename or mtime. +func decodeWisprMarkdown(rel string, data []byte, modTime time.Time) (wisprDictation, bool) { + body := strings.ReplaceAll(string(data), "\r\n", "\n") + meta := map[string]string{} + if strings.HasPrefix(body, "---\n") { + if end := strings.Index(body[4:], "\n---"); end >= 0 { + for _, line := range strings.Split(body[4:4+end], "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + meta[strings.ToLower(strings.TrimSpace(k))] = strings.Trim(strings.TrimSpace(v), `"'`) + } + body = body[4+end+4:] + } + } + body = strings.TrimSpace(body) + // Drop a leading heading: the title is derived from the text. + if strings.HasPrefix(body, "# ") { + if i := strings.IndexByte(body, '\n'); i >= 0 { + body = strings.TrimSpace(body[i+1:]) + } else { + body = "" + } + } + d := wisprDictation{ + Text: body, + App: meta["app"], + URL: meta["url"], + Language: meta["language"], + } + d.ID = meta["id"] + if d.ID == "" { + d.ID = meta["transcriptentityid"] + } + if d.ID == "" { + d.ID = strings.TrimSuffix(filepath.ToSlash(rel), filepath.Ext(rel)) + } + for _, k := range []string{"timestamp", "dictated", "date", "created", "created_at", "createdat"} { + if v := meta[k]; v != "" { + d.At = wisprParseTime(v) + if d.At.IsZero() { + if t, ok := wisprTimeFromName(v); ok { + d.At = t + } + } + break + } + } + if d.At.IsZero() { + if t, ok := wisprTimeFromName(filepath.Base(rel)); ok { + d.At = t + } else { + d.At = modTime.UTC() + } + } + if d.Text == "" { + return d, false + } + return d, true +} + +// wisprTimeFromName extracts a UTC time from a filename-style date stamp. +func wisprTimeFromName(name string) (time.Time, bool) { + m := wisprFilenameTimeRe.FindStringSubmatch(name) + if m == nil { + return time.Time{}, false + } + stamp := m[1] + layout := "2006-01-02" + if m[2] != "" { + stamp += " " + m[2] + ":" + m[3] + layout = "2006-01-02 15:04" + if m[4] != "" { + stamp += ":" + m[4] + layout = "2006-01-02 15:04:05" + } + } + t, err := time.Parse(layout, stamp) + if err != nil { + return time.Time{}, false + } + return t.UTC(), true +} diff --git a/src/pkg/knowledge/connector/wisprflow_test.go b/src/pkg/knowledge/connector/wisprflow_test.go new file mode 100644 index 0000000000..c20df12fda --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_test.go @@ -0,0 +1,350 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func wisprTestNow() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + +func newTestWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + cfg := ConnectorConfig{Name: "wispr", Type: TypeWisprFlow, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + wc := c.(*wisprConnector) + wc.now = wisprTestNow + return wc +} + +// wisprFixtureDB builds flow.sqlite from testdata/wisprflow/history.sql with +// the sqlite3 CLI the connector itself shells out to; callers skip when the +// CLI is absent (CI runners), so the sync tests use wisprFakeQuery instead. +func wisprFixtureDB(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 CLI not installed") + } + sql, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.sql")) + if err != nil { + t.Fatal(err) + } + db := filepath.Join(t.TempDir(), "flow.sqlite") + cmd := exec.Command("sqlite3", db) + cmd.Stdin = strings.NewReader(string(sql)) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("building fixture db: %v: %s", err, out) + } + return db +} + +var wisprSinceRE = regexp.MustCompile(`datetime\('([^']+)'\)`) + +// wisprFakeQuery stands in for the sqlite3 CLI: it serves the rows of +// testdata/wisprflow/history.json (the same data as history.sql) and applies +// the WHERE clause of wisprHistorySQL — status, isArchived and the since +// bound — so the connector sees exactly what sqlite3 -json would return. +func wisprFakeQuery(t *testing.T) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.json")) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err := json.Unmarshal(data, &rows); err != nil { + t.Fatal(err) + } + return func(_ context.Context, dbPath, sql string) ([]byte, error) { + if _, err := os.Stat(dbPath); err != nil { + return nil, fmt.Errorf("wisprflow: sqlite3 failed: unable to open database") + } + m := wisprSinceRE.FindStringSubmatch(sql) + if m == nil || !strings.Contains(sql, "status='formatted'") { + return nil, fmt.Errorf("unexpected sql: %s", sql) + } + since, err := time.Parse("2006-01-02 15:04:05", m[1]) + if err != nil { + return nil, err + } + var out []map[string]any + for _, r := range rows { + at := wisprParseTime(r["at"].(string)) + if r["status"] != "formatted" || r["isArchived"] != float64(0) || !at.After(since) { + continue + } + out = append(out, r) + } + return json.Marshal(out) + } +} + +// newFakeWispr returns a connector whose sqlite3 runner is wisprFakeQuery +// and whose path points at an existing (empty) flow.sqlite file. +func newFakeWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + db := filepath.Join(t.TempDir(), "flow.sqlite") + if err := os.WriteFile(db, nil, 0o600); err != nil { + t.Fatal(err) + } + if scope == nil { + scope = map[string]string{} + } + scope["path"] = db + c := newTestWispr(t, scope) + c.query = wisprFakeQuery(t) + return c +} + +func collectWispr(t *testing.T, c *wisprConnector, cur Cursor) ([]Page, Cursor, error) { + t.Helper() + var pages []Page + next, err := c.Sync(context.Background(), cur, func(p Page) error { + pages = append(pages, p) + return nil + }) + return pages, next, err +} + +func wisprIDs(pages []Page) []string { + out := make([]string, 0, len(pages)) + for _, p := range pages { + out = append(out, p.ID) + } + return out +} + +func TestWisprValidate(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": "/x/flow.sqlite"}) + cases := []struct { + name string + cfg ConnectorConfig + want string + }{ + {"auth rejected", ConnectorConfig{Auth: Auth{Env: "X"}, Scope: map[string]string{"path": "/x/flow.sqlite"}}, "not used"}, + {"relative path", ConnectorConfig{Scope: map[string]string{"path": "flow.sqlite"}}, "absolute"}, + {"bad group", ConnectorConfig{Scope: map[string]string{"path": "/x", "group": "week"}}, "scope.group"}, + {"bad since", ConnectorConfig{Scope: map[string]string{"path": "/x", "since_days": "0"}}, "since_days"}, + {"bad include_raw", ConnectorConfig{Scope: map[string]string{"path": "/x", "include_raw": "maybe"}}, "include_raw"}, + } + for _, tc := range cases { + err := c.Validate(tc.cfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want containing %q", tc.name, err, tc.want) + } + } + ok := ConnectorConfig{Scope: map[string]string{"path": "/x/flow.sqlite", "group": "day", "since_days": "7", "min_words": "0", "include_raw": "true"}} + if err := c.Validate(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +func TestWisprSyncSQLite(t *testing.T) { + c := newFakeWispr(t, map[string]string{"tags": "voice", "include_raw": "true"}) + + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // d-short (1 word), d-raw (not formatted), d-archived and d-old (before + // since_days) are all excluded. + if got := wisprIDs(pages); strings.Join(got, ",") != "d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } + p := pages[1] + if p.Title != "Running late — start without me." { + t.Errorf("title = %q (editedText should win)", p.Title) + } + for _, want := range []string{"**App:** Slack", "**Dictated:** 2026-11-11T17:55:30Z", "## Raw transcript", "running late start without me"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["wispr_transcript_id"] != "d-2" || p.Attrs["app"] != "com.tinyspeck.slackmacgap" || p.Attrs["tags"] != "voice" || p.Attrs["num_words"] != "5" { + t.Errorf("attrs = %v", p.Attrs) + } + if strings.Join(p.Path, "/") != "Wispr Flow/2026-11-11" { + t.Errorf("path = %v", p.Path) + } + if _, ok := pages[0].Attrs["tags"]; !ok || strings.Contains(pages[0].Markdown, "## Raw transcript") == false { + t.Errorf("first page attrs/raw = %v", pages[0].Attrs) + } + + // Incremental: the overlap window re-reads d-2 only. + pages, cur2, err := collectWispr(t, c, cur) + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-2" { + t.Fatalf("incremental pages = %v", got) + } + if cur2 != cur { + t.Fatalf("cursor moved to %q", cur2) + } +} + +func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { + c := newFakeWispr(t, map[string]string{"apps": "slack, vscode", "group": "day", "since_days": "3650"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "day-2026-11-10,day-2026-11-11" { + t.Fatalf("pages = %v", got) + } + day := pages[1] + if day.Attrs["dictations"] != "1" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { + t.Errorf("day page = %v\n%s", day.Attrs, day.Markdown) + } + if strings.Contains(day.Markdown, "Ok.") { + t.Error("one-word dictation leaked into the day page") + } + // d-old is Chrome, so even with a 10-year window the apps filter drops it. + for _, p := range pages { + if strings.Contains(p.Markdown, "Ancient") { + t.Error("apps filter did not drop the Chrome dictation") + } + } +} + +// TestWisprSQLite3CLI runs the real sqlite3 CLI against the fixture database +// and checks it agrees with the fake used by the other tests. +func TestWisprSQLite3CLI(t *testing.T) { + db := wisprFixtureDB(t) + c := newTestWispr(t, map[string]string{"path": db, "since_days": "3650"}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-old,d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } +} + +func TestWisprSyncMissingPath(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": filepath.Join(t.TempDir(), "nope", "flow.sqlite")}) + _, cur, err := collectWispr(t, c, "2026-01-01T00:00:00Z") + if err == nil || !strings.Contains(err.Error(), "Full Disk Access") { + t.Fatalf("err = %v", err) + } + if cur != "2026-01-01T00:00:00Z" { + t.Fatalf("cursor changed on error: %q", cur) + } +} + +func TestWisprSyncSQLiteError(t *testing.T) { + abs, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "history.sql")) + c := newTestWispr(t, map[string]string{"path": abs}) + c.query = func(context.Context, string, string) ([]byte, error) { + return nil, &PartialError{Err: os.ErrPermission} + } + if _, _, err := collectWispr(t, c, ""); err == nil { + t.Fatal("expected sqlite error to surface") + } +} + +func TestWisprHistorySQL(t *testing.T) { + sql := wisprHistorySQL(time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC), 7) + for _, want := range []string{"status='formatted'", "datetime('2026-11-01 12:00:00')", "LIMIT 7", "COALESCE(isArchived,0)=0", "NULLIF(editedText,'')"} { + if !strings.Contains(sql, want) { + t.Errorf("sql missing %q: %s", want, sql) + } + } + if strings.Contains(sql, "audio") || strings.Contains(sql, "screenshot") { + t.Error("sql must not select BLOB columns") + } +} + +func TestWisprSyncExportDir(t *testing.T) { + dir, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "export")) + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // j-old is before since_days; n-2 is one word; ignored.bin is skipped. + if got := wisprIDs(pages); strings.Join(got, ",") != "j-1,m-fm,n-1,c-1,2026-11-11-1430-cursor" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T14:30:00Z" { + t.Fatalf("cursor = %q", cur) + } + byID := map[string]Page{} + for _, p := range pages { + byID[p.ID] = p + } + if p := byID["2026-11-11-1430-cursor"]; !strings.HasPrefix(p.Title, "Refactor the syncer") || strings.Contains(p.Markdown, "# Dictated into Cursor") { + t.Errorf("markdown export page = %q\n%s", p.Title, p.Markdown) + } + if p := byID["m-fm"]; p.Attrs["app"] != "com.apple.Notes" || !p.UpdatedAt.Equal(time.Date(2026, 11, 10, 16, 20, 0, 0, time.UTC)) { + t.Errorf("front-matter page = %v %v", p.Attrs, p.UpdatedAt) + } + if p := byID["c-1"]; p.Attrs["num_words"] != "13" || !strings.Contains(p.Markdown, "**App:** Slack") { + t.Errorf("csv page = %v\n%s", p.Attrs, p.Markdown) + } + if p := byID["j-1"]; !strings.Contains(p.Markdown, "**App:** Wispr Flow") { + t.Errorf("json page:\n%s", p.Markdown) + } +} + +func TestWisprExportDirPartial(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "bad.json"), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "good.ndjson"), []byte(`{"id":"g","text":"three words here","at":"2026-11-11T10:00:00Z"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if !IsPartial(err) || !strings.Contains(err.Error(), "bad.json") { + t.Fatalf("err = %v, want partial naming bad.json", err) + } + if len(pages) != 1 || pages[0].ID != "g" || cur != "2026-11-11T10:00:00Z" { + t.Fatalf("pages = %v cur = %q", wisprIDs(pages), cur) + } +} + +func TestWisprParseTime(t *testing.T) { + cases := map[string]time.Time{ + "2026-11-11 17:55:30.000 +00:00": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11 19:55:30.250 +02:00": time.Date(2026, 11, 11, 17, 55, 30, 250e6, time.UTC), + "2026-11-11T17:55:30Z": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + "garbage": {}, + } + for in, want := range cases { + if got := wisprParseTime(in); !got.Equal(want) { + t.Errorf("wisprParseTime(%q) = %v, want %v", in, got, want) + } + } + if got, ok := wisprTimeFromName("2026-11-11T14-05-09.md"); !ok || !got.Equal(time.Date(2026, 11, 11, 14, 5, 9, 0, time.UTC)) { + t.Errorf("wisprTimeFromName = %v %v", got, ok) + } + if _, ok := wisprTimeFromName("notes.md"); ok { + t.Error("wisprTimeFromName matched a name without a date") + } +} + +func TestWisprAppName(t *testing.T) { + for in, want := range map[string]string{"com.tinyspeck.slackmacgap": "Slack", "com.example.foo": "foo", "": "", "Cursor": "Cursor"} { + if got := wisprAppName(in); got != want { + t.Errorf("wisprAppName(%q) = %q, want %q", in, got, want) + } + } +} From 7352b74f9d784487e739f712476ff0b4f5a55197 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:03:09 +0000 Subject: [PATCH 066/166] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20replace=20?= =?UTF-8?q?unopenable=20pluk=20session=20log=20so=20the=20supervisor=20can?= =?UTF-8?q?=20create=20it=20(#17889)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-pluk-log-dir-permission.md | 1 + src/crux/agent/pluk_dirs.go | 12 +++++++ src/crux/agent/pluk_dirs_branches_test.go | 36 ++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 changelog.d/fixed-pluk-log-dir-permission.md diff --git a/changelog.d/fixed-pluk-log-dir-permission.md b/changelog.d/fixed-pluk-log-dir-permission.md new file mode 100644 index 0000000000..4056c1f6f9 --- /dev/null +++ b/changelog.d/fixed-pluk-log-dir-permission.md @@ -0,0 +1 @@ +- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). diff --git a/src/crux/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go index c005c1dc0e..a8bd56c0c2 100644 --- a/src/crux/agent/pluk_dirs.go +++ b/src/crux/agent/pluk_dirs.go @@ -1,7 +1,9 @@ package agent import ( + "errors" "fmt" + "io/fs" "os" "path/filepath" ) @@ -50,6 +52,16 @@ func ensurePlukLogFile(runDir, session string) (string, error) { path := plukSessionLogPath(runDir, session) f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + if errors.Is(err, fs.ErrPermission) { + // A log left behind by another uid (a pane shell's `>>` under a tight + // umask, or an earlier run as a different user) cannot be opened or + // chmodded by us, but the shared setgid logs dir is group-writable, so + // the stale file can be replaced. Replacing beats leaving the session's + // log unreadable to peers. + if rmErr := os.Remove(path); rmErr == nil { + f, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + } + } if err != nil { return "", fmt.Errorf("creating %s: %w", path, err) } diff --git a/src/crux/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go index 45e6c39179..5cac61004b 100644 --- a/src/crux/agent/pluk_dirs_branches_test.go +++ b/src/crux/agent/pluk_dirs_branches_test.go @@ -83,6 +83,42 @@ func TestEnsurePlukLogFileWidensExistingTightLog(t *testing.T) { } } +// TestEnsurePlukLogFileReplacesUnopenableLog pins the supervisor failure: a log +// left by another uid that we cannot open (modelled as mode 0000) is replaced +// in the group-writable logs dir instead of failing with permission denied. +func TestEnsurePlukLogFileReplacesUnopenableLog(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses file permission checks") + } + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + + path := plukSessionLogPath(runDir, "hive-supervisor") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("seed log: %v", err) + } + if err := os.Chmod(path, 0); err != nil { + t.Fatalf("lock log: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-supervisor") + if err != nil { + t.Fatalf("ensure pluk log file with unopenable log: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if perm := info.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + // TestEnsurePlukLogFileErrorWhenLogsDirMissing covers the OpenFile error // return: with no logs/ directory the create fails, the error is wrapped // (fs.ErrNotExist stays reachable through errors.Is), and it names the path. From 081284783216a0592c4260e07e271f7029c082df Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 07:05:44 +0000 Subject: [PATCH 067/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.228.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.228.6 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-17696-app-repo-coverage.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-17696-app-repo-coverage.md diff --git a/CHANGELOG.md b/CHANGELOG.md index da7895396c..a5948e1c87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.228.6) + +### Fixed + +- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). + ## 2026-11-12 (v5.228.5) ### Fixed diff --git a/changelog.d/fixed-17696-app-repo-coverage.md b/changelog.d/fixed-17696-app-repo-coverage.md deleted file mode 100644 index 8855f73672..0000000000 --- a/changelog.d/fixed-17696-app-repo-coverage.md +++ /dev/null @@ -1 +0,0 @@ -- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). From 48ff9724d990b084ad8ed7dcdb82e89313a8bde0 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Thu, 12 Nov 2026 00:10:12 -0700 Subject: [PATCH 068/166] =?UTF-8?q?=E2=9C=A8=20workflows:=20owner=20toggle?= =?UTF-8?q?=20for=20model-based=20stage=20derivation=20from=20the=20dashbo?= =?UTF-8?q?ard=20(#17886)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ workflows: owner toggle for model-based stage derivation from the dashboard workflow.enabled gates rules-based stage derivation and bound-model label provisioning, but could only be set by editing hive.yaml, which hosted hives cannot do. Add PUT /api/workflows/enabled {enabled} (owner-gated, persists hive.yaml, audited workflow_enabled / workflow_disabled) and a "Derive stages from bound models" switch in Settings → Workflows → Library that confirms before turning on. The stage engine and the tracker already read cfg.Workflow.Enabled live and the reloader mutates the same *Config in place, so no reload hook is needed. Turning on also provisions every bound repository's model labels at once through the sweep's memoised ensureStageLabels path (new automerge.Engine.EnsureStageLabels, wired as Dependencies.WorkflowProvisionLabels); failures come back as warnings and the next sweep retries. The Repos binding row says "columns only — turn on derivation in Workflows" when a binding exists while the flag is off. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson * chore: move changelog fragment to root changelog.d/ Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-workflow-enabled-toggle.md | 1 + dashboard/openapi.json | 72 +++++++++++++ src/cmd/hive/main.go | 11 +- src/cmd/hive/workflow_labels.go | 42 ++++++++ src/cmd/hive/workflow_labels_test.go | 28 +++++ src/docs/api-reference.md | 2 +- src/docs/api-reference/configuration.md | 1 + src/docs/design/workflow-models.md | 16 +++ src/hive.yaml.example | 11 ++ src/pkg/dashboard/api_workflows.go | 87 +++++++++++++++ src/pkg/dashboard/api_workflows_test.go | 102 ++++++++++++++++++ src/pkg/dashboard/deps.go | 4 + .../dashboard/static-src/css/workflows.css | 4 + .../static-src/js/config-workflows-library.js | 67 +++++++++++- .../static-src/js/config-workflows.js | 4 +- src/pkg/dashboard/static/index.html | 75 +++++++++++-- .../dashboard/workflows_library_ui_test.go | 56 ++++++++++ .../automerge/automerge_stage_labels_gc.go | 15 +++ .../automerge_stage_labels_workflow_test.go | 30 ++++++ 19 files changed, 610 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-workflow-enabled-toggle.md create mode 100644 src/cmd/hive/workflow_labels.go create mode 100644 src/cmd/hive/workflow_labels_test.go diff --git a/changelog.d/added-workflow-enabled-toggle.md b/changelog.d/added-workflow-enabled-toggle.md new file mode 100644 index 0000000000..44fc411700 --- /dev/null +++ b/changelog.d/added-workflow-enabled-toggle.md @@ -0,0 +1 @@ +- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index cc87eca843..8ecb48243e 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -36029,6 +36029,78 @@ } } }, + "/api/workflows/enabled": { + "put": { + "tags": [ + "Workflows" + ], + "summary": "Turn model-based stage derivation on or off", + "description": "Owner only. Sets workflow.enabled from the dashboard so a hosted hive with no hive.yaml access can enable it (hivecommons/hive#17620). Body {enabled}. Persists hive.yaml; the stage engine and the tracker read the live flag, so it takes effect on the next derivation pass without a restart. Turning it on also provisions every bound repository's model labels immediately; a repository that could not be provisioned is listed in warnings and the next sweep retries it. Audited as workflow_enabled / workflow_disabled; a no-op is not audited.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Flag after the change", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "changed": { + "type": "boolean" + }, + "bindings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowBinding" + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Bound repositories whose labels could not be provisioned now (only when turning on)" + } + } + } + } + } + }, + "400": { + "description": "Body is not {enabled: boolean}" + }, + "403": { + "description": "Not an owner" + }, + "500": { + "description": "hive.yaml could not be saved; the flag is left unchanged" + }, + "503": { + "description": "No config" + } + } + } + }, "/api/workflows/{name}/{version}": { "get": { "tags": [ diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 51f69d747e..9062bf39e5 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1094,11 +1094,12 @@ func (b *boot) wireBootClosures() { WorkflowItemSnapshot: func(ctx context.Context, repo string, number int) (github.WorkflowItemSnapshot, error) { return b.ghClient.WorkflowItemSnapshot(ctx, repo, number) }, - WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), - WorkflowClusterRunner: workflowClusterRunner(), - WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), - AgentMgr: b.agentMgr, - CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), + WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), + WorkflowProvisionLabels: workflowProvisionLabels(b.cfg, func() *github.Client { return b.ghClient }, b.logger), + WorkflowClusterRunner: workflowClusterRunner(), + WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), + AgentMgr: b.agentMgr, + CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that diff --git a/src/cmd/hive/workflow_labels.go b/src/cmd/hive/workflow_labels.go new file mode 100644 index 0000000000..df5f076da3 --- /dev/null +++ b/src/cmd/hive/workflow_labels.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/github/automerge" +) + +// workflowProvisionLabels is the dashboard's WorkflowProvisionLabels +// dependency (hivecommons/hive#17620): when an owner turns workflow.enabled +// on from Settings → Workflows, every bound repository's model labels are +// provisioned at once through the sweep's own memoised path +// (automerge.Engine.EnsureStageLabels over the process-shared stage engine +// options and tracker), rather than on the next PR that needs a label. +// Without a GitHub client the call reports an error and the handler +// surfaces it as a warning; the flag itself still flips. +func workflowProvisionLabels(cfg *config.Config, client func() *github.Client, logger *slog.Logger) func(ctx context.Context, repo string) error { + return func(ctx context.Context, repo string) error { + var gh *github.Client + if client != nil { + gh = client() + } + if gh == nil { + return errNoForgeClient + } + owner, name, ok := strings.Cut(config.QualifyRepo(cfg.Project.Org, repo), "/") + if !ok || owner == "" || name == "" { + return fmt.Errorf("repository %q is not owner/name", repo) + } + eng := automerge.New(gh, automerge.Options{ + Logger: logger, + StageLabelPolicy: stageLabelPolicyFunc(cfg), + StageEngine: stageEngineOptions(cfg, logger), + }) + return eng.EnsureStageLabels(ctx, owner, name) + } +} diff --git a/src/cmd/hive/workflow_labels_test.go b/src/cmd/hive/workflow_labels_test.go new file mode 100644 index 0000000000..205caf29a8 --- /dev/null +++ b/src/cmd/hive/workflow_labels_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +func TestWorkflowProvisionLabelsWithoutClient(t *testing.T) { + cfg := &config.Config{Project: config.ProjectConfig{Org: "acme"}} + provision := workflowProvisionLabels(cfg, func() *github.Client { return nil }, nil) + if err := provision(context.Background(), "widgets"); !errors.Is(err, errNoForgeClient) { + t.Fatalf("err = %v", err) + } +} + +func TestWorkflowProvisionLabelsRejectsBareRepoWithoutOrg(t *testing.T) { + cfg := &config.Config{} + provision := workflowProvisionLabels(cfg, func() *github.Client { return &github.Client{} }, nil) + err := provision(context.Background(), "widgets") + if err == nil || !strings.Contains(err.Error(), "owner/name") { + t.Fatalf("err = %v", err) + } +} diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 44a4feadf9..33ea39c580 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,7 +13,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Health, status, events | [`api-reference/health-status-events.md`](api-reference/health-status-events.md) | 50 | | Snapshots and style | [`api-reference/snapshots-and-style.md`](api-reference/snapshots-and-style.md) | 5 | | Auth and external accounts | [`api-reference/auth.md`](api-reference/auth.md) | 22 | -| Configuration | [`api-reference/configuration.md`](api-reference/configuration.md) | 105 | +| Configuration | [`api-reference/configuration.md`](api-reference/configuration.md) | 110 | | Agents and controls | [`api-reference/agents-and-controls.md`](api-reference/agents-and-controls.md) | 36 | | Packs and ACMM | [`api-reference/packs-and-acmm.md`](api-reference/packs-and-acmm.md) | 18 | | Cost, tokens, telemetry | [`api-reference/cost-tokens-telemetry.md`](api-reference/cost-tokens-telemetry.md) | 13 | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 00f31ea1e5..4941f8f958 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -117,3 +117,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:282` | | `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:297` | | `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:298` | +| `PUT` | `/api/workflows/enabled` | Owner only | Turn model-based stage derivation (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) on or off from the dashboard, for hosted hives with no `hive.yaml` access. Body `{enabled: bool}`; persists `hive.yaml`; the stage engine and tracker read the live flag so it applies on the next derivation pass without a restart. Turning on also provisions every bound repository's model labels now (`warnings[]` lists repositories that could not be provisioned; the next sweep retries). Returns `{enabled, changed, bindings[], warnings?[]}`. Audited `workflow_enabled` / `workflow_disabled`; `400` on a non-boolean body | `pkg/dashboard/api_workflows.go:28` | diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 08d1e7f5cd..cba8a0dd33 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -169,6 +169,22 @@ and `GET /api/workflows/{name}/{version}`: Duplicate (saves a copy under a new name at version 1, provenance `local`) and Delete; Delete is withheld while anything is bound (the API returns 409 anyway) and the row says `bound` instead. Export shows the model JSON in-app (copy to clipboard) for everyone. +- **Derivation switch** — the Library header carries an owner-only toggle, *Derive stages from + bound models* (`renderWorkflowEnabledToggle`, `PUT /api/workflows/enabled {enabled}`), so a + hosted hive with no `hive.yaml` access can turn `workflow.enabled` on or off. Off: a binding + only changes the Activity Monitor's columns. On: a bound repository's stages are derived by its + model's rules and the model's labels are provisioned. Turning on asks for confirmation (it + changes derivation for every bound repository and creates labels in them); turning off is + immediate. The handler sets `Config.Workflow.Enabled`, saves `hive.yaml`, audits + `workflow_enabled` / `workflow_disabled`, and — on the way on — calls + `Dependencies.WorkflowProvisionLabels` for each binding (cmd/hive wires it to + `automerge.Engine.EnsureStageLabels`, the sweep's memoised `ensureStageLabels` path over the + shared stage-engine options and tracker), so the bound models' labels exist at once rather than + on the next PR that needs a label write; a repository that fails is returned in `warnings[]` + and the next sweep retries. No reload hook is needed: `stageEngineWorkflowFor` and the tracker + read `cfg.Workflow.Enabled` on every call, and the hive.yaml reloader mutates the same + `*Config` in place. The Repos binding row shows *columns only — turn on derivation in + Workflows* when a binding exists while the flag is off. - **Detail** — facts, the diagram, diagnostics, citations and the raw model/profile JSON. `workflowDiagramSvg(model, profile)` is a pure function: the compiled profile's columns are lanes in display order, each state is a chip carrying its padded stage number, transitions are bezier diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 3277db7eba..cbd09db9a6 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1290,3 +1290,14 @@ data: claude_sessions_dir: /data/home/.claude/projects copilot_sessions_dir: /data/home/.copilot/session-state agents_dir: /data/agent-configs + +# Workflow models (hivecommons/hive#17620, docs/design/workflow-models.md). +# Repositories are bound to a model on Settings → Repos (project.workflow_bindings). +# A binding always changes the Activity Monitor's columns for that repository; +# `enabled` decides whether the bound model's rules also derive its stages and +# whether the model's labels are provisioned in the repository. Owners can flip +# it from Settings → Workflows → Library (PUT /api/workflows/enabled) without +# editing this file; the change applies on the next derivation pass. +# workflow: +# enabled: false # false (default): bindings are columns only +# label_gc: true # delete labels a previous binding provisioned once nothing wears them diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index d937eca0b9..71769ec9b8 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -1,6 +1,7 @@ package dashboard import ( + "context" "errors" "fmt" "net/http" @@ -24,6 +25,7 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("POST /api/workflows/ingest", s.handleWorkflowIngest) s.mux.HandleFunc("GET /api/workflows/bindings", s.handleWorkflowBindingsGet) s.mux.HandleFunc("PUT /api/workflows/bindings", s.handleWorkflowBindingPut) + s.mux.HandleFunc("PUT /api/workflows/enabled", s.handleWorkflowEnabledPut) s.mux.HandleFunc("GET /api/workflows/{name}/{version}", s.handleWorkflowGet) s.mux.HandleFunc("PUT /api/workflows/{name}/{version}", s.handleWorkflowSave) s.mux.HandleFunc("DELETE /api/workflows/{name}/{version}", s.handleWorkflowDelete) @@ -236,3 +238,88 @@ func (s *Server) handleWorkflowBindingPut(w http.ResponseWriter, r *http.Request } jsonResponse(w, map[string]interface{}{"changed": changed, "bindings": s.workflowBindings()}) } + +// workflowProvisionTimeout bounds the label provisioning PUT +// /api/workflows/enabled runs across every bound repository when it turns +// derivation on: one label list plus a create per missing label per +// repository, all within the request. +const workflowProvisionTimeout = 60 * time.Second + +// handleWorkflowEnabledPut flips workflow.enabled (hivecommons/hive#17620) +// from the dashboard so a hosted hive with no hive.yaml access can turn +// model-based stage derivation on or off. The stage engine +// (cmd/hive stageEngineWorkflowFor) and the tracker read the live config +// on every call, so the change takes effect on the next derivation pass +// with no restart or reload. Turning it on also provisions every bound +// repository's model labels now through Dependencies.WorkflowProvisionLabels; +// a provisioning failure is reported in the response as a warning, never as +// a failure of the flip itself, because the sweep retries it on its next +// pass. Audited as workflow_enabled / workflow_disabled; a no-op flip is +// not audited. +func (s *Server) handleWorkflowEnabledPut(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config unavailable", http.StatusServiceUnavailable) + return + } + var body struct { + Enabled *bool `json:"enabled"` + } + if err := decodeBody(r, &body); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if body.Enabled == nil { + jsonError(w, "body must be {\"enabled\": true|false}", http.StatusBadRequest) + return + } + changed := s.deps.Config.Workflow.Enabled != *body.Enabled + if changed { + s.deps.Config.Workflow.Enabled = *body.Enabled + if err := s.saveConfig(); err != nil { + s.deps.Config.Workflow.Enabled = !*body.Enabled + jsonError(w, "save config: "+err.Error(), http.StatusInternalServerError) + return + } + action := "workflow_disabled" + if *body.Enabled { + action = "workflow_enabled" + } + s.auditFromRequest(r, action, fmt.Sprintf("%d bound repositories", len(s.workflowBindings())), "") + } + resp := map[string]interface{}{ + "enabled": s.deps.Config.Workflow.Enabled, + "changed": changed, + "bindings": s.workflowBindings(), + } + if *body.Enabled { + if warnings := s.provisionBoundWorkflowLabels(r.Context()); len(warnings) > 0 { + resp["warnings"] = warnings + } + } + jsonResponse(w, resp) +} + +// provisionBoundWorkflowLabels runs WorkflowProvisionLabels for every bound +// repository and returns one message per repository it could not provision. +func (s *Server) provisionBoundWorkflowLabels(ctx context.Context) []string { + if s.deps.WorkflowProvisionLabels == nil { + return nil + } + bindings := s.workflowBindings() + if len(bindings) == 0 { + return nil + } + ctx, cancel := context.WithTimeout(ctx, workflowProvisionTimeout) + defer cancel() + var warnings []string + for _, b := range bindings { + if err := s.deps.WorkflowProvisionLabels(ctx, b.Repo); err != nil { + s.logger.Warn("workflow labels: provisioning on enable failed; the next sweep retries", "repo", b.Repo, "workflow", b.Key(), "err", err) + warnings = append(warnings, fmt.Sprintf("%s: %s", b.Repo, err.Error())) + } + } + return warnings +} diff --git a/src/pkg/dashboard/api_workflows_test.go b/src/pkg/dashboard/api_workflows_test.go index bb9e077957..7a7a7d9c59 100644 --- a/src/pkg/dashboard/api_workflows_test.go +++ b/src/pkg/dashboard/api_workflows_test.go @@ -2,12 +2,16 @@ package dashboard import ( "bytes" + "context" "encoding/json" + "errors" "io" "log/slog" "net/http" "net/http/httptest" "os" + "path/filepath" + "strings" "testing" "time" @@ -276,3 +280,101 @@ func TestStageTrackerWorkflowsBlock(t *testing.T) { t.Fatalf("%+v", resp.Workflows) } } + +// PUT /api/workflows/enabled flips workflow.enabled from the dashboard +// (hivecommons/hive#17620): owner-gated, persisted to hive.yaml, audited, +// and on the way on it provisions every bound repository's labels. +func TestWorkflowEnabledPut(t *testing.T) { + s := workflowTestServer(t, true) + s.deps.Config.SourcePath = filepath.Join(t.TempDir(), "hive.yaml") + var provisioned []string + s.deps.WorkflowProvisionLabels = func(_ context.Context, repo string) error { + provisioned = append(provisioned, repo) + if repo == "repo2" { + return errors.New("no such repository") + } + return nil + } + if rec := wfReq(s, http.MethodPost, "/api/workflows", techdocsModel(t), true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + for _, repo := range []string{"repo1", "repo2"} { + if _, err := s.deps.Config.SetWorkflowBinding(repo, "cncf-techdocs", "1"); err != nil { + t.Fatal(err) + } + } + + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d %s", rec.Code, rec.Body) + } + for _, bad := range []string{"{", "{}", `{"enabled":"yes"}`} { + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", bad, true); rec.Code != http.StatusBadRequest { + t.Fatalf("body %s: %d %s", bad, rec.Code, rec.Body) + } + } + if s.deps.Config.Workflow.Enabled || len(provisioned) != 0 { + t.Fatalf("rejected requests changed state: enabled=%v provisioned=%v", s.deps.Config.Workflow.Enabled, provisioned) + } + + rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true) + var out struct { + Enabled bool + Changed bool + Bindings []struct{ Repo string } + Warnings []string + } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || !out.Enabled || !out.Changed || len(out.Bindings) != 2 { + t.Fatalf("enable: %d %s", rec.Code, rec.Body) + } + if !s.deps.Config.Workflow.Enabled { + t.Fatal("flag not applied to the live config") + } + if len(provisioned) != 2 || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "repo2") { + t.Fatalf("provisioned=%v warnings=%v", provisioned, out.Warnings) + } + saved, err := os.ReadFile(s.deps.Config.SourcePath) + if err != nil || !bytes.Contains(saved, []byte("enabled: true")) { + t.Fatalf("hive.yaml not persisted: %v\n%s", err, saved) + } + rec = wfReq(s, http.MethodGet, "/api/workflows", nil, false) + if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte(`"enabled":true`)) { + t.Fatalf("list after enable: %d %s", rec.Code, rec.Body) + } + + // Idempotent: same value again is not a change, not audited, but still + // re-provisions (the memo makes that cheap). + rec = wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || !out.Enabled || out.Changed { + t.Fatalf("enable again: %d %s", rec.Code, rec.Body) + } + provisioned = nil + out.Warnings = nil + rec = wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": false}, true) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || out.Enabled || !out.Changed || len(out.Warnings) != 0 { + t.Fatalf("disable: %d %s", rec.Code, rec.Body) + } + if s.deps.Config.Workflow.Enabled || len(provisioned) != 0 { + t.Fatalf("disable: enabled=%v provisioned=%v", s.deps.Config.Workflow.Enabled, provisioned) + } + + var actions []string + for _, e := range s.audit.Recent(0) { + if e.Action == "workflow_enabled" || e.Action == "workflow_disabled" { + actions = append(actions, e.Action) + } + } + if strings.Join(actions, ",") != "workflow_disabled,workflow_enabled" && strings.Join(actions, ",") != "workflow_enabled,workflow_disabled" { + t.Fatalf("audit actions %v", actions) + } + + // A failed save leaves the live flag where it was. + s.deps.Config.SourcePath = filepath.Join(t.TempDir(), "missing", "hive.yaml") + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true); rec.Code != http.StatusInternalServerError || s.deps.Config.Workflow.Enabled { + t.Fatalf("unsaveable: %d %s enabled=%v", rec.Code, rec.Body, s.deps.Config.Workflow.Enabled) + } + + s.deps.Config = nil + if rec := wfReq(s, http.MethodPut, "/api/workflows/enabled", map[string]bool{"enabled": true}, true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no config: %d", rec.Code) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 251c12bddb..e0bff6c556 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -54,6 +54,10 @@ type Dependencies struct { // spec) for POST /api/workflows/ingest/spec {repo, path}. Nil disables // repo-path spec ingest. WorkflowRepoFile func(ctx context.Context, repo, path string) (string, error) + // WorkflowProvisionLabels creates (or recolours) one bound repository's + // stage and model labels now, for PUT /api/workflows/enabled turning + // derivation on. Nil leaves provisioning to the next sweep pass. + WorkflowProvisionLabels func(ctx context.Context, repo string) error // WorkflowClusterRunner runs kubectl for POST /api/workflows/ingest/cluster // (RFC #17620 phase 6). Nil means no kube access is configured on this // spoke and the route answers 422. diff --git a/src/pkg/dashboard/static-src/css/workflows.css b/src/pkg/dashboard/static-src/css/workflows.css index 7ae7bc94b8..0fcbaf339f 100644 --- a/src/pkg/dashboard/static-src/css/workflows.css +++ b/src/pkg/dashboard/static-src/css/workflows.css @@ -6,6 +6,10 @@ .wf-key { color: var(--text); text-decoration: none; font-weight: 700; } .wf-key:hover { text-decoration: underline; } .wf-desc { font-size: var(--fs-xs); margin-top: var(--sp-1); max-width: 36em; } + .wf-enabled-toggle { margin: var(--sp-3) 0 var(--sp-1); } + .wf-enabled-toggle .config-toggle-label { font-weight: 700; } + .wf-enabled-explain { margin: 0 0 var(--sp-4); max-width: 60em; } + .wf-derivation-off { color: var(--status-warn); } .wf-prov, .wf-compiled { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; white-space: nowrap; border: var(--line-width) solid var(--border); color: var(--muted); } .wf-prov-built-in { color: var(--status-info); border-color: color-mix(in srgb, var(--status-info) 40%, transparent); background: color-mix(in srgb, var(--status-info) 10%, transparent); } .wf-prov-curated { color: var(--status-ok); border-color: color-mix(in srgb, var(--status-ok) 40%, transparent); background: color-mix(in srgb, var(--status-ok) 10%, transparent); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-library.js b/src/pkg/dashboard/static-src/js/config-workflows-library.js index b4b9a4260a..c1f05464d1 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-library.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-library.js @@ -235,7 +235,7 @@ pr: { checks: 'unknown', review: 'pending', dco: 'unknown', conflicting: false } }; } function wfSimDefaultState() { return { kind: 'pr', repo: '', facts: wfSimDefaultFacts(), result: null, error: '', busy: false, open: false }; } - const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, + const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, enabledBusy: false, ingest: { repo: '', name: '', busy: false, result: null, error: '' }, sim: wfSimDefaultState() }; @@ -314,10 +314,30 @@ }).join(''); } - function renderWorkflowLibraryTable(workflows, bindings, owner) { + // renderWorkflowEnabledToggle is the Library header's owner switch for + // workflow.enabled (PUT /api/workflows/enabled). Pure: enabled is the + // live flag from GET /api/workflows, owner gates the control, busy dims + // it while a PUT is in flight. + function renderWorkflowEnabledToggle(enabled, owner, busy) { + const on = !!enabled; + const locked = !owner || !!busy; + const explain = on + ? 'On: a bound repository\'s stages are derived by its model\'s rules and the model\'s labels are provisioned in the repository.' + : 'Off: a binding only changes the Activity Monitor\'s columns; stages are still derived by the built-in engine.'; + return '
' + + '
' + + 'Derive stages from bound models (' + yamlHint('workflow.enabled') + ')' + + '
' + + '

' + explain + '

'; + } + + function renderWorkflowLibraryTable(workflows, bindings, owner, enabled, busy) { return '
' + '

Workflow library

' + - '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when ' + yamlHint('workflow.enabled') + ' is on, how its stages are derived.

' + + renderWorkflowEnabledToggle(enabled, owner, busy) + + '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when the switch above is on, how its stages are derived.

' + '' + '' + renderWorkflowLibraryRows(workflows, bindings, owner) + '
ModelProvenanceSourceCompileS / CBound
' + '
'; @@ -783,7 +803,7 @@ } else if (_wfLib.view === 'marketplace') { body = renderWorkflowMarketplaceView(owner); } else { - body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); + body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy); } return wfLibNav(_wfLib.view) + body; } @@ -829,6 +849,45 @@ openConfigDialog('governor', null, 'Workflows'); } + // wfToggleEnabled flips workflow.enabled through PUT /api/workflows/enabled. + // Turning it on changes how every bound repository's stages are derived + // and creates labels in those repositories, so it asks first; turning it + // off is immediate (columns stay, derivation reverts to the built-in + // engine). + async function wfToggleEnabled(el) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to change workflow.enabled.', 'error'); return; } + if (_wfLib.enabledBusy) return; + const next = !_workflowState.enabled; + if (next) { + const bound = (_workflowState.bindings || []).length; + const ok = await hiveConfirm({ + title: 'Derive stages from bound models?', + message: 'Stages for ' + (bound ? bound + ' bound repositor' + (bound === 1 ? 'y' : 'ies') : 'any repository you bind') + ' will be derived by the bound model\'s rules instead of the built-in engine, and each model\'s labels are created in the repository now. Unbound repositories are unaffected. You can turn this off again at any time.', + confirmLabel: 'Turn on', + }); + if (!ok) return; + } + _wfLib.enabledBusy = true; + wfLibRerender(); + try { + const res = await fetch('/api/workflows/enabled', { method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify({ enabled: next }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + _workflowState.enabled = !!data.enabled; + if (Array.isArray(data.bindings)) _workflowState.bindings = data.bindings; + if (Array.isArray(data.warnings) && data.warnings.length) { + showToast('Derivation is on, but some labels could not be provisioned yet (the next sweep retries): ' + data.warnings.join('; '), 'error'); + } else { + showToast(_workflowState.enabled ? 'Stages are now derived from bound models.' : 'Stage derivation is back on the built-in engine; bound columns stay.'); + } + } catch (err) { + showToast('Could not change workflow.enabled: ' + err.message, 'error'); + } finally { + _wfLib.enabledBusy = false; + wfLibRerender(); + } + } + async function wfLibReload() { _workflowState.loaded = false; await loadWorkflowLibrary(); diff --git a/src/pkg/dashboard/static-src/js/config-workflows.js b/src/pkg/dashboard/static-src/js/config-workflows.js index 407bed42df..74536c106b 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows.js +++ b/src/pkg/dashboard/static-src/js/config-workflows.js @@ -60,9 +60,9 @@ const label = key + (w.description ? ' — ' + w.description : '') + (compiled ? '' : ' (draft, does not compile)'); return ``; }).join(''); - const flag = _workflowState.enabled + const flag = _workflowState.enabled || !bound ? '' - : 'derivation off'; + : 'columns only — turn on derivation in Workflows'; return `
${flag}${bound ? `bound` : 'default'}
`; } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 07b1ff15dc..e75ead995d 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5053,6 +5053,10 @@ .wf-key { color: var(--text); text-decoration: none; font-weight: 700; } .wf-key:hover { text-decoration: underline; } .wf-desc { font-size: var(--fs-xs); margin-top: var(--sp-1); max-width: 36em; } + .wf-enabled-toggle { margin: var(--sp-3) 0 var(--sp-1); } + .wf-enabled-toggle .config-toggle-label { font-weight: 700; } + .wf-enabled-explain { margin: 0 0 var(--sp-4); max-width: 60em; } + .wf-derivation-off { color: var(--status-warn); } .wf-prov, .wf-compiled { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; white-space: nowrap; border: var(--line-width) solid var(--border); color: var(--muted); } .wf-prov-built-in { color: var(--status-info); border-color: color-mix(in srgb, var(--status-info) 40%, transparent); background: color-mix(in srgb, var(--status-info) 10%, transparent); } .wf-prov-curated { color: var(--status-ok); border-color: color-mix(in srgb, var(--status-ok) 40%, transparent); background: color-mix(in srgb, var(--status-ok) 10%, transparent); } @@ -62957,9 +62961,9 @@

Agent Watchdog

const label = key + (w.description ? ' — ' + w.description : '') + (compiled ? '' : ' (draft, does not compile)'); return ``; }).join(''); - const flag = _workflowState.enabled + const flag = _workflowState.enabled || !bound ? '' - : 'derivation off'; + : 'columns only — turn on derivation in Workflows'; return `
${flag}${bound ? `bound` : 'default'}
`; } @@ -63228,7 +63232,7 @@

Agent Watchdog

pr: { checks: 'unknown', review: 'pending', dco: 'unknown', conflicting: false } }; } function wfSimDefaultState() { return { kind: 'pr', repo: '', facts: wfSimDefaultFacts(), result: null, error: '', busy: false, open: false }; } - const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, + const _wfLib = { view: 'library', selected: null, detail: null, detailError: '', loadingKey: '', pending: null, enabledBusy: false, ingest: { repo: '', name: '', busy: false, result: null, error: '' }, sim: wfSimDefaultState() }; @@ -63307,10 +63311,30 @@

Agent Watchdog

}).join(''); } - function renderWorkflowLibraryTable(workflows, bindings, owner) { + // renderWorkflowEnabledToggle is the Library header's owner switch for + // workflow.enabled (PUT /api/workflows/enabled). Pure: enabled is the + // live flag from GET /api/workflows, owner gates the control, busy dims + // it while a PUT is in flight. + function renderWorkflowEnabledToggle(enabled, owner, busy) { + const on = !!enabled; + const locked = !owner || !!busy; + const explain = on + ? 'On: a bound repository\'s stages are derived by its model\'s rules and the model\'s labels are provisioned in the repository.' + : 'Off: a binding only changes the Activity Monitor\'s columns; stages are still derived by the built-in engine.'; + return '
' + + '
' + + 'Derive stages from bound models (' + yamlHint('workflow.enabled') + ')' + + '
' + + '

' + explain + '

'; + } + + function renderWorkflowLibraryTable(workflows, bindings, owner, enabled, busy) { return '
' + '

Workflow library

' + - '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when ' + yamlHint('workflow.enabled') + ' is on, how its stages are derived.

' + + renderWorkflowEnabledToggle(enabled, owner, busy) + + '

Every workflow model this hive knows: the built-in hive-default lifecycle, models saved here, and drafts ingested from repository pipelines. A repository follows a model only once it is bound on Settings → Repos; binding changes the Activity Monitor\'s columns for that repository and, when the switch above is on, how its stages are derived.

' + '' + '' + renderWorkflowLibraryRows(workflows, bindings, owner) + '
ModelProvenanceSourceCompileS / CBound
' + '
'; @@ -63776,7 +63800,7 @@

Agent Watchdog

} else if (_wfLib.view === 'marketplace') { body = renderWorkflowMarketplaceView(owner); } else { - body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); + body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy); } return wfLibNav(_wfLib.view) + body; } @@ -63822,6 +63846,45 @@

Agent Watchdog

openConfigDialog('governor', null, 'Workflows'); } + // wfToggleEnabled flips workflow.enabled through PUT /api/workflows/enabled. + // Turning it on changes how every bound repository's stages are derived + // and creates labels in those repositories, so it asks first; turning it + // off is immediate (columns stay, derivation reverts to the built-in + // engine). + async function wfToggleEnabled(el) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to change workflow.enabled.', 'error'); return; } + if (_wfLib.enabledBusy) return; + const next = !_workflowState.enabled; + if (next) { + const bound = (_workflowState.bindings || []).length; + const ok = await hiveConfirm({ + title: 'Derive stages from bound models?', + message: 'Stages for ' + (bound ? bound + ' bound repositor' + (bound === 1 ? 'y' : 'ies') : 'any repository you bind') + ' will be derived by the bound model\'s rules instead of the built-in engine, and each model\'s labels are created in the repository now. Unbound repositories are unaffected. You can turn this off again at any time.', + confirmLabel: 'Turn on', + }); + if (!ok) return; + } + _wfLib.enabledBusy = true; + wfLibRerender(); + try { + const res = await fetch('/api/workflows/enabled', { method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify({ enabled: next }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + _workflowState.enabled = !!data.enabled; + if (Array.isArray(data.bindings)) _workflowState.bindings = data.bindings; + if (Array.isArray(data.warnings) && data.warnings.length) { + showToast('Derivation is on, but some labels could not be provisioned yet (the next sweep retries): ' + data.warnings.join('; '), 'error'); + } else { + showToast(_workflowState.enabled ? 'Stages are now derived from bound models.' : 'Stage derivation is back on the built-in engine; bound columns stay.'); + } + } catch (err) { + showToast('Could not change workflow.enabled: ' + err.message, 'error'); + } finally { + _wfLib.enabledBusy = false; + wfLibRerender(); + } + } + async function wfLibReload() { _workflowState.loaded = false; await loadWorkflowLibrary(); diff --git a/src/pkg/dashboard/workflows_library_ui_test.go b/src/pkg/dashboard/workflows_library_ui_test.go index 9d02ec9922..e7f0f5156b 100644 --- a/src/pkg/dashboard/workflows_library_ui_test.go +++ b/src/pkg/dashboard/workflows_library_ui_test.go @@ -124,3 +124,59 @@ console.log('ok'); t.Fatalf("unexpected node output: %s", out) } } + +// The Library header's workflow.enabled switch (hivecommons/hive#17620): +// wired to PUT /api/workflows/enabled with a confirm on the way on, and the +// Repos binding row's "columns only" hint when a binding exists while +// derivation is off. +func TestWorkflowsEnabledToggleWiring(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + `function renderWorkflowEnabledToggle(enabled, owner, busy)`, + `async function wfToggleEnabled(el)`, + `fetch('/api/workflows/enabled', { method: 'PUT'`, + `title: 'Derive stages from bound models?'`, + `data-action="wfToggleEnabled"`, + `renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner, _workflowState.enabled, _wfLib.enabledBusy)`, + `columns only — turn on derivation in Workflows`, + `.wf-enabled-toggle {`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing workflow.enabled toggle wiring %q", want) + } + } +} + +func TestWorkflowsEnabledToggleNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +function esc(s) { return String(s).replace(/&/g, '&').replace(/' + p + ''; } +` + jsFunc(t, html, "renderWorkflowEnabledToggle") + ` +const off = renderWorkflowEnabledToggle(false, true, false); +assert.ok(off.includes('data-wf-enabled="off"') && off.includes('aria-checked="false"'), 'off state'); +assert.ok(off.includes('data-action="wfToggleEnabled"'), 'owner can flip'); +assert.ok(!off.includes('config-toggle-switch on'), 'switch not on'); +assert.ok(off.includes('Off: a binding only changes'), 'off explanation'); +assert.ok(off.includes('workflow.enabled'), 'names the yaml key'); + +const on = renderWorkflowEnabledToggle(true, true, false); +assert.ok(on.includes('config-toggle-switch on') && on.includes('aria-checked="true"'), 'on state'); +assert.ok(on.includes('On: a bound repository'), 'on explanation'); + +const viewer = renderWorkflowEnabledToggle(false, false, false); +assert.ok(!viewer.includes('data-action="wfToggleEnabled"') && viewer.includes('aria-disabled="true"') && viewer.includes(' disabled'), 'non-owner sees a locked switch'); +assert.ok(viewer.includes('Owner role is required'), 'non-owner title explains'); + +const busy = renderWorkflowEnabledToggle(true, true, true); +assert.ok(!busy.includes('data-action="wfToggleEnabled"') && busy.includes(' disabled'), 'busy switch is locked'); +` + cmd := exec.Command(node, "-e", script) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("node: %v\n%s", err, out) + } +} diff --git a/src/pkg/github/automerge/automerge_stage_labels_gc.go b/src/pkg/github/automerge/automerge_stage_labels_gc.go index d0d9747ea4..6861ca575b 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_gc.go +++ b/src/pkg/github/automerge/automerge_stage_labels_gc.go @@ -180,3 +180,18 @@ func (c *Engine) workflowLabelInUse(ctx context.Context, owner, repo, name strin } return res.GetTotal(), nil } + +// EnsureStageLabels provisions owner/repo's label vocabulary now — the +// built-in family plus the bound workflow model's LabelDefs — through the +// same memoised path the sweep's stage pass uses (ensureStageLabels), so a +// dashboard action that turns model-based derivation on can create the +// bound models' labels immediately instead of waiting for the next PR that +// needs a label write. The shared tracker's memo is honoured and updated: +// a repository already provisioned for the current profile hash is a +// no-op, and the sweep will not re-list it afterwards. +func (c *Engine) EnsureStageLabels(ctx context.Context, owner, repo string) error { + if c == nil || c.gh == nil { + return fmt.Errorf("stage labels: no GitHub transport") + } + return c.ensureStageLabels(ctx, owner, repo, c.stageTrackerOrShared(), c.stagePolicy().AuthorKindLabels) +} diff --git a/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go b/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go index ef0c3505e6..3247be7128 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go +++ b/src/pkg/github/automerge/automerge_stage_labels_workflow_test.go @@ -369,3 +369,33 @@ func TestEnsureStageLabelsGCNoStoreOrRecordIsInert(t *testing.T) { t.Fatalf("unbound repo touched GC routes: %v %v", gcAPI.searches, gcAPI.deleted) } } + +// EnsureStageLabels is the dashboard's on-demand entry into the same +// memoised path (PUT /api/workflows/enabled turning derivation on). +func TestEnsureStageLabelsExportedUsesEngineTrackerAndPolicy(t *testing.T) { + ctx := context.Background() + api := &stageAPI{labels: map[string]string{}} + srv := newStageAPI(t, api) + defer srv.Close() + profile := workflowTestProfile(t, workflowmodel.LabelEntry{Name: "docs/010-requested", State: "requested"}) + e, tracker := newWorkflowStageEngine(t, srv.URL, func(string, string) *workflowmodel.Profile { return profile }) + + if err := e.EnsureStageLabels(ctx, "acme", "widget"); err != nil { + t.Fatal(err) + } + if api.lists != 1 || !slices.Contains(api.creates, "docs/010-requested") || tracker.ensuredWorkflow["acme/widget"] != profile.Hash { + t.Fatalf("lists=%d creates=%v memo=%q", api.lists, api.creates, tracker.ensuredWorkflow["acme/widget"]) + } + // The sweep's own path then finds the memo and does nothing. + if err := e.ensureStageLabels(ctx, "acme", "widget", tracker, authorkind.DefaultNames); err != nil || api.lists != 1 { + t.Fatalf("sweep after dashboard provisioning: err=%v lists=%d", err, api.lists) + } + + var nilEngine *Engine + if err := nilEngine.EnsureStageLabels(ctx, "acme", "widget"); err == nil { + t.Fatal("nil engine must report an error") + } + if err := New(nil, Options{}).EnsureStageLabels(ctx, "acme", "widget"); err == nil { + t.Fatal("engine without a transport must report an error") + } +} From db5700d2a535ca84ef552c2670ffecf1a3ca6085 Mon Sep 17 00:00:00 2001 From: hive-release-bot Date: Thu, 12 Nov 2026 07:40:26 +0000 Subject: [PATCH 069/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.229.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.229.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-knowledge-wisprflow-source.md | 1 - changelog.d/fixed-pluk-log-dir-permission.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-knowledge-wisprflow-source.md delete mode 100644 changelog.d/fixed-pluk-log-dir-permission.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a5948e1c87..da5445321a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.229.0) + +### Added + +- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. + +### Fixed + +- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). + ## 2026-11-12 (v5.228.6) ### Fixed diff --git a/changelog.d/added-knowledge-wisprflow-source.md b/changelog.d/added-knowledge-wisprflow-source.md deleted file mode 100644 index 3e63a6018a..0000000000 --- a/changelog.d/added-knowledge-wisprflow-source.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. diff --git a/changelog.d/fixed-pluk-log-dir-permission.md b/changelog.d/fixed-pluk-log-dir-permission.md deleted file mode 100644 index 4056c1f6f9..0000000000 --- a/changelog.d/fixed-pluk-log-dir-permission.md +++ /dev/null @@ -1 +0,0 @@ -- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). From e132e7e532c0e819d176dff59bbcc542a77680ae Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:40:42 +0100 Subject: [PATCH 070/166] fix: classify plain HTTP proxy forwarding failures (#17864) Hive-Run: hivecommons/hive#17859 Hive-Plan: classify repeated proxy forward failures Hive-Spec: fleet-report#runtime-classes Signed-off-by: Karibhou Co-authored-by: Karibhou Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-proxy-plain-forward-report.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 1 + src/pkg/delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 2 ++ 6 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-proxy-plain-forward-report.md diff --git a/changelog.d/fixed-proxy-plain-forward-report.md b/changelog.d/fixed-proxy-plain-forward-report.md new file mode 100644 index 0000000000..2e525cf0dd --- /dev/null +++ b/changelog.d/fixed-proxy-plain-forward-report.md @@ -0,0 +1 @@ +- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index ad7f77ae2a..fcd9a05adc 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -153,6 +153,7 @@ title is `[selfreport] : `. | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `copilot_cli_hung` | a Copilot agent showed none of the CLI's startup chrome (input prompt, help hint, login request) 180 s after launch, outside kick delivery, so the hang detector killed its session, ran bare `copilot` to capture the error and relaunched the agent (`copilot hung with no CLI prompt, running diagnostic`, `crux/agent/manager_poll.go`), or that diagnostic saw no output before its own deadline and relaunched anyway (`diagnostic: timed out waiting for copilot error output`, `crux/agent/manager_credentials.go`; the line carries the pane's `output_tail`). The key is the message, so ≥5 in 10 min counts every Copilot agent on the hive together: its Copilot agents keep failing to reach their prompt. Typical causes are a stale Copilot token the CLI hangs on (the diagnostic restores or clears it when it sees an auth error), a blocked network path to the Copilot API, or a CLI build whose startup screen the detector does not recognise (`cliReadyIndicators`, `crux/agent/pane_classify.go`). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_plain_forward_failed` | the proxy could not complete an agent's plain-HTTP upstream request (`proxy: plain HTTP forward failed`, `crux/proxy/github_proxy_connect.go`), returning HTTP 502. Check the requested host and logged error, including DNS and outbound connectivity; this differs from a destination refused by the proxy policy. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium | | `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index e21278a332..377adbc3da 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -101,6 +101,7 @@ var selfreportOpsRemedies = map[string]string{ "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 4c20d4367e..9f0bdc0ae3 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -160,7 +160,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, + fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, } { diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 9e9b3c3c48..3946d8adb4 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -79,6 +79,7 @@ const ( ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" ClassInferenceUpstreamError = "inference_upstream_error" ClassInferenceCallerUnknown = "inference_caller_unidentified" @@ -201,6 +202,7 @@ var Classes = map[string]ClassSpec{ ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, + ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index da3b933e4a..bf9c2dafbf 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -456,6 +456,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, + {class: ClassProxyPlainForwardFailed, message: "proxy: plain HTTP forward failed"}, {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, {class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"}, {class: ClassInferenceNoRoute, message: "inference no route"}, @@ -1241,6 +1242,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, "inference no route": ClassInferenceNoRoute, "inference upstream error": ClassInferenceUpstreamError, From 0bc47b5f922d85cc93ab30612221316c9627f6c4 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Thu, 12 Nov 2026 01:00:54 -0700 Subject: [PATCH 071/166] =?UTF-8?q?=E2=9C=A8=20Inbox:=20voice=20recorders?= =?UTF-8?q?=20as=20first-class=20input=20sources=20(Plaud=20=E2=86=92=20sp?= =?UTF-8?q?ek=20RFC=20/=20dibs=20/=20knowledge)=20(#17885)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Inbox: voice recorders as first-class input sources Add a source-agnostic Inbox (sidebar → Intelligence) that lists recordings synced by voice knowledge connectors (plaud now, wisprflow next), keyed by (source, item_id), and routes each one inside the hive to a spektacular RFC with child issues (spektacular voice --publish, or a spoke-agent fallback), a dibs draft idea, or Knowledge only — single, bulk, or by auto-routing rules applied after every sync. - pkg/inbox: Item/RouteState model, JSON store, fact-backed source, rules, spek_rfc + dibs destinations, service with sweep. - config: inbox.{sources,rules,spek,dibs} + validation. - dashboard: GET /api/inbox, POST /api/inbox/{route,dismiss}, GET|POST /api/inbox/settings, Inbox section + modals, sidebar badge. - Plaud: POST /api/knowledge/connectors/{name}/auth/complete paste fallback for hosted hives (Plaud OAuth only redirects to localhost:8199); the Integrations card now shows connection state + Open Inbox only. - docs: inbox-voice-sources.md, knowledge-connectors.md, api-reference, dashboard-sections, openapi.json. Signed-off-by: Andrew Anderson * ✨ Inbox: default spek_rfc agent fallback to first enabled agent When the hive image has no spektacular binary and inbox.spek.agent is unset, hand the RFC job to the alphabetically first enabled, unpaused agent so candidate hives can file RFCs without extra config. Signed-off-by: Andrew Anderson * 🌱 Inbox: allowlist pkg/inbox dashboard import; sidebar order test includes inbox-section Signed-off-by: Andrew Anderson * 🌱 Inbox: cover service/store/source edge cases and inbox config validation Raises pkg/inbox to 92% and pkg/config back over the 90% coverage floor. Signed-off-by: Andrew Anderson --------- Signed-off-by: Andrew Anderson --- changelog.d/added-inbox-voice-sources.md | 1 + dashboard/openapi.json | 893 ++++++++++++++++++ src/cmd/hive/boot.go | 2 + src/cmd/hive/boot_inbox.go | 82 ++ src/cmd/hive/boot_knowledge_connectors.go | 8 +- .../hive/boot_knowledge_connectors_test.go | 4 +- src/cmd/hive/main.go | 1 + src/docs/api-reference/knowledge.md | 6 + src/docs/api-reference/packs-and-acmm.md | 2 +- src/docs/dashboard-sections.md | 6 + .../dashboard-sections/workflow-and-agents.md | 14 + src/docs/documentation-map.md | 1 + src/docs/inbox-voice-sources.md | 94 ++ src/docs/knowledge-connectors.md | 16 +- .../testutil/dashboard_import_allowlist.txt | 3 + src/pkg/config/config.go | 5 +- src/pkg/config/inbox_config.go | 122 +++ .../config/inbox_config_default_agent_test.go | 51 + .../config/testdata/agent_runtime_golden.json | 4 + src/pkg/config/validate.go | 3 + src/pkg/dashboard/api.go | 1 + src/pkg/dashboard/api_inbox.go | 383 ++++++++ src/pkg/dashboard/api_inbox_test.go | 217 +++++ .../dashboard/dashboard_section_cards_test.go | 1 + .../dashboard_section_reorder_9062_test.go | 10 +- .../dashboard_section_summaries_test.go | 2 +- src/pkg/dashboard/deps.go | 4 + src/pkg/dashboard/import_count_test.go | 7 +- .../knowledge_integrations_modal_test.go | 4 +- .../dashboard/settings_parity_allowlist.txt | 6 + src/pkg/dashboard/static-src/css/inbox.css | 17 + src/pkg/dashboard/static-src/html/inbox.html | 6 + .../dashboard/static-src/html/sidebar.html | 1 + src/pkg/dashboard/static-src/js/inbox.js | 296 ++++++ src/pkg/dashboard/static-src/js/knowledge.js | 4 +- src/pkg/dashboard/static-src/js/layout.js | 2 +- src/pkg/dashboard/static-src/js/refresh.js | 3 +- src/pkg/dashboard/static-src/js/sections.js | 4 +- src/pkg/dashboard/static-src/manifest.txt | 3 + src/pkg/dashboard/static/index.html | 333 ++++++- .../dashboard/webstatic/static_index_test.go | 2 +- src/pkg/inbox/dibs.go | 113 +++ src/pkg/inbox/edges_test.go | 276 ++++++ src/pkg/inbox/factsource.go | 209 ++++ src/pkg/inbox/inbox.go | 123 +++ src/pkg/inbox/inbox_test.go | 398 ++++++++ src/pkg/inbox/rules.go | 62 ++ src/pkg/inbox/service.go | 279 ++++++ src/pkg/inbox/spek.go | 257 +++++ src/pkg/inbox/store.go | 218 +++++ src/pkg/knowledge/connector/plaud.go | 8 + .../connector/plaud_auth_complete.go | 159 ++++ .../connector/plaud_auth_complete_test.go | 79 ++ 53 files changed, 4773 insertions(+), 32 deletions(-) create mode 100644 changelog.d/added-inbox-voice-sources.md create mode 100644 src/cmd/hive/boot_inbox.go create mode 100644 src/docs/inbox-voice-sources.md create mode 100644 src/pkg/config/inbox_config.go create mode 100644 src/pkg/config/inbox_config_default_agent_test.go create mode 100644 src/pkg/dashboard/api_inbox.go create mode 100644 src/pkg/dashboard/api_inbox_test.go create mode 100644 src/pkg/dashboard/static-src/css/inbox.css create mode 100644 src/pkg/dashboard/static-src/html/inbox.html create mode 100644 src/pkg/dashboard/static-src/js/inbox.js create mode 100644 src/pkg/inbox/dibs.go create mode 100644 src/pkg/inbox/edges_test.go create mode 100644 src/pkg/inbox/factsource.go create mode 100644 src/pkg/inbox/inbox.go create mode 100644 src/pkg/inbox/inbox_test.go create mode 100644 src/pkg/inbox/rules.go create mode 100644 src/pkg/inbox/service.go create mode 100644 src/pkg/inbox/spek.go create mode 100644 src/pkg/inbox/store.go create mode 100644 src/pkg/knowledge/connector/plaud_auth_complete.go create mode 100644 src/pkg/knowledge/connector/plaud_auth_complete_test.go diff --git a/changelog.d/added-inbox-voice-sources.md b/changelog.d/added-inbox-voice-sources.md new file mode 100644 index 0000000000..a375f8cb62 --- /dev/null +++ b/changelog.d/added-inbox-voice-sources.md @@ -0,0 +1 @@ +- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 8ecb48243e..651888249f 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -37169,7 +37169,900 @@ } } } + }, + "/api/inbox": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "List inbox items (voice recordings and dictations) with their triage state", + "description": "Items are the facts written by voice knowledge connectors (`plaud`, `wisprflow`), newest first, keyed by `(source, id)`, with per-destination routing state, source connection status, candidate repositories and the auto-routing settings.", + "responses": { + "200": { + "description": "Inbox listing", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "description": "An incoming voice recording or dictation. Every item is already a Knowledge fact; the inbox only tracks triage.", + "properties": { + "source": { + "type": "string", + "description": "Connector type (plaud, wisprflow, …)" + }, + "connector": { + "type": "string" + }, + "id": { + "type": "string", + "description": "Source item id (recording id)" + }, + "title": { + "type": "string" + }, + "excerpt": { + "type": "string" + }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "duration_ms": { + "type": "integer" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "fact_slug": { + "type": "string" + }, + "fact_path": { + "type": "string" + }, + "layer": { + "type": "string" + }, + "routes": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dismissed": { + "type": "boolean", + "description": "Kept as a Knowledge fact only" + }, + "processed": { + "type": "boolean", + "description": "Dismissed or routed to at least one destination" + } + } + } + }, + "unprocessed": { + "type": "integer" + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "connector": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "connected": { + "type": "boolean" + }, + "account": { + "type": "string" + } + } + } + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Repositories this hive works on (RFC targets)" + }, + "settings": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/route": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Route an inbox item to a destination", + "description": "Owner-only. `spek_rfc` runs `spektacular voice --publish` inside the hive (or hands a prompt to the configured spoke agent) and files one RFC issue plus child issues on `repo` (default `inbox.spek.repo`); `dibs` posts the recording as a private draft idea to the configured dibs instance; `dismiss` keeps the item as a Knowledge fact only. Returns 202 with the queued state; poll `GET /api/inbox` for completion.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "source", + "item_id", + "destination" + ], + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string", + "description": "owner/repo for spek_rfc" + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Route accepted", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "state": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, + "400": { + "description": "Invalid request, unknown destination, missing repo, or destination not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown item", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Route already queued or running", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/dismiss": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Mark inbox items as Knowledge only (or undo)", + "description": "Owner-only bulk dismiss. Dismissed items stay searchable as Knowledge facts and leave the unprocessed count; `undo: true` returns them to the inbox.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "items" + ], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + } + } + } + }, + "undo": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Items updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "dismissed": { + "type": "integer" + } + } + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/settings": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Read inbox auto-routing settings", + "responses": { + "200": { + "description": "Settings", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Update inbox auto-routing settings", + "description": "Owner-only. Persists `inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` to the runtime config; omitted fields are left unchanged. Rules apply to items that arrive after the change.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Settings saved", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "400": { + "description": "Invalid rule or repository", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/complete": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Finish a connector sign-in from pasted material (hosted hives)", + "description": "Owner-only. Plaud's OAuth client only redirects to `localhost:8199`, which a hive running in a cluster never receives. The operator pastes either the `redirect_url` the browser landed on after approving (forwarded to the in-pod callback while the login started by `/auth/connect` is still pending, 10 minutes) or the `tokens` JSON from a laptop's `~/.plaud/tokens-mcp.json`; the hive stores it and re-checks the account.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "redirect_url": { + "type": "string", + "format": "uri" + }, + "tokens": { + "type": "string", + "description": "Raw JSON contents of tokens-mcp.json" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Sign-in state after completion", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector does not support pasted completion, bad body, or no pending login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "Upstream rejected the material", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } } + } }, "components": { "schemas": { diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 96abb844ce..91d87cde38 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -27,6 +27,7 @@ import ( "github.com/hivecommons/hive/pkg/github/automerge" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" "github.com/hivecommons/hive/pkg/notify" @@ -215,6 +216,7 @@ type boot struct { // bootKnowledge gitSyncer *knowledge.GitSyncer knowledgeConnectors *connector.Syncer + inboxSvc *inbox.Service knowledgePublish *connector.Mirror promotionScheduler *knowledge.PromotionScheduler quotaAccount string diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go new file mode 100644 index 0000000000..552c650911 --- /dev/null +++ b/src/cmd/hive/boot_inbox.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. +const inboxSweepTimeout = 2 * time.Minute + +// bootInbox builds the Inbox service when at least one configured knowledge +// connector is a voice source (inbox.sources, default plaud/wisprflow). +// Items are read from the connector vaults under /connectors; +// route state persists at /connector-state/_inbox.json (connector +// names cannot start with "_", so it never collides with a connector). +func (b *boot) bootInbox(baseDir string) { + sources := b.cfg.Inbox.EffectiveSources() + configured := map[string]bool{} + for _, c := range b.cfg.Knowledge.Connectors { + configured[c.Type] = true + } + var srcs []inbox.Source + for _, typ := range sources { + if configured[typ] { + srcs = append(srcs, &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ}) + } + } + if len(srcs) == 0 { + return + } + token := func(ctx context.Context, _ string) (string, error) { + if b.appAuth == nil { + return "", nil + } + return b.appAuth.Token(ctx) + } + kick := func(agent, prompt string) error { + if b.agentMgr == nil { + return fmt.Errorf("agent runtime is not available") + } + started, err := b.agentMgr.SendKickAsync(agent, prompt) + if err != nil { + return err + } + if !started { + return fmt.Errorf("agent %s did not accept the kick", agent) + } + return nil + } + store := inbox.NewStore(filepath.Join(baseDir, "connector-state", "_inbox.json")) + svc := inbox.NewService(b.cfg.Inbox, store, srcs, dashboard.NewInboxDestinations(b.cfg, token, kick), b.logger) + svc.SetReadiness(dashboard.InboxReadiness(b.cfg)) + b.inboxSvc = svc + b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) +} + +// inboxAfterSync runs the auto-routing rules once a voice connector synced. +func (b *boot) inboxAfterSync(st connector.Status) { + svc := b.inboxSvc + if svc == nil { + return + } + for _, typ := range b.cfg.Inbox.EffectiveSources() { + if typ != st.Type { + continue + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), inboxSweepTimeout) + defer cancel() + if n, err := svc.Sweep(ctx); err != nil { + b.logger.Warn("inbox sweep", "error", err, "routed", n) + } + }() + return + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 0110c8a04b..3d9cbdc64f 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -40,7 +40,7 @@ func connectorConfigs(entries []config.KnowledgeConnector) []connector.Connector // searchable like any other vault. Cursors and status persist under // /connector-state. A nil reg uses connector.DefaultRegistry(). // Returns nil when no connectors are configured. -func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger) (*connector.Syncer, error) { +func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger, afterSync func(connector.Status)) (*connector.Syncer, error) { if len(entries) == 0 { return nil, nil } @@ -79,6 +79,9 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne if store := api.GetVaultStore(filepath.Join(baseDir, "connectors", st.Layer)); store != nil { store.Reindex() } + if afterSync != nil { + afterSync(st) + } }, }) } @@ -86,7 +89,7 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne // bootKnowledgeConnectors builds and starts the connector syncer. A failure // is logged and leaves connectors off; it never blocks the rest of boot. func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { - s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger) + s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) return @@ -99,6 +102,7 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { deps.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(b.cfg.Knowledge.Connectors)) + b.bootInbox(knowledge.BaseDir()) } // knowledgeConnectorRuntime exposes the syncer to the dashboard. A nil diff --git a/src/cmd/hive/boot_knowledge_connectors_test.go b/src/cmd/hive/boot_knowledge_connectors_test.go index 048b2ea905..b0ca37dcaa 100644 --- a/src/cmd/hive/boot_knowledge_connectors_test.go +++ b/src/cmd/hive/boot_knowledge_connectors_test.go @@ -44,7 +44,7 @@ func (bootFakeConnector) Sync(_ context.Context, _ connector.Cursor, emit func(c func TestNewKnowledgeConnectorSyncer(t *testing.T) { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger); s != nil || err != nil { + if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger, nil); s != nil || err != nil { t.Fatalf("no connectors = %v, %v; want nil, nil", s, err) } @@ -58,7 +58,7 @@ func TestNewKnowledgeConnectorSyncer(t *testing.T) { api := knowledge.NewKnowledgeAPI(nil, knowledge.KnowledgeConfig{Enabled: true, Engine: "file"}, logger) s, err := newKnowledgeConnectorSyncer([]config.KnowledgeConnector{ {Name: "guide", Type: "fake", Layer: "project"}, - }, reg, api, base, logger) + }, reg, api, base, logger, nil) if err != nil || s == nil { t.Fatalf("newKnowledgeConnectorSyncer = %v, %v", s, err) } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 9062bf39e5..2cbba7bd59 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1146,6 +1146,7 @@ func (b *boot) wireBootClosures() { BeadSynthesizer: b.beadSynth, KnowledgeConnectors: b.knowledgeConnectorRuntime(), KnowledgePublish: b.knowledgePublishRuntime(), + Inbox: b.inboxSvc, BeadStores: b.beadStores, BeadStoreLoadFailures: b.beadStoreLoadFailures, AuditLedger: auditLedger, diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 1228e273fb..e406cdb1a3 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -65,3 +65,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | | `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | | `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | +| `POST` | `/api/knowledge/connectors/{name}/auth/complete` | Owner only | Finish a sign-in from pasted material on a hosted hive: `{redirect_url}` (the `localhost:8199/auth/callback?…` URL the browser landed on, forwarded in-pod while the login is pending) or `{tokens}` (contents of `~/.plaud/tokens-mcp.json`); see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:29` | +| `GET` | `/api/inbox` | Dashboard auth/session | Inbox items (voice recordings/dictations keyed by `(source, id)`) with per-destination route state, source connection status, candidate repos and auto-routing settings; see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:24` | +| `POST` | `/api/inbox/route` | Owner only | Route one item: `{source, item_id, destination: spek_rfc\|dibs\|dismiss, repo?}` (202 queued; 400 unconfigured destination, 404 unknown item, 409 already queued/running) | `pkg/dashboard/api_inbox.go:25` | +| `POST` | `/api/inbox/dismiss` | Owner only | Bulk "Knowledge only": `{items: [{source, item_id}], undo?}` | `pkg/dashboard/api_inbox.go:26` | +| `GET` | `/api/inbox/settings` | Dashboard auth/session | Auto-routing rules, default RFC repo, dibs URL, destination readiness | `pkg/dashboard/api_inbox.go:27` | +| `POST` | `/api/inbox/settings` | Owner only | Update `inbox.rules` / `inbox.spek.repo` / `inbox.dibs.url` in the runtime config (omitted fields unchanged) | `pkg/dashboard/api_inbox.go:28` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index f01162b768..7dbfa576e4 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:556`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:557`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index aebdf8bdaa..22b306f356 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -184,6 +184,12 @@ The facts Hive has learned about your projects, where they came from, and how mu Full entry: [Knowledge](dashboard-sections/workflow-and-agents.md#knowledge). +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +Full entry: [Inbox](dashboard-sections/workflow-and-agents.md#inbox). + ## Contributors The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them. diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index f55f9d7aa3..c49a26bbd1 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -202,6 +202,20 @@ The facts Hive has learned about your projects, where they came from, and how mu **What it is good for.** Check it when a worker keeps making the same mistake. Correct the wrong fact here. +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +**What it tells you.** Every recording a voice source (Plaud, Wispr Flow) syncs is already a Knowledge fact. The Inbox lists the ones you have not triaged yet, with the source, date, length and a short excerpt. + +**How the numbers are worked out.** + +- The sidebar count and the summary count items with no destination yet and not marked "Knowledge only". +- Each item shows one pill per destination it was sent to: queued, running, filed (with a link) or failed. +- Auto-routing rules run once per new item after a sync; the first matching rule wins. + +**What it is good for.** Record an idea on your recorder, then turn it into an RFC with child issues (→ RFC), a dibs idea (→ dibs), or just keep it searchable (Knowledge only) without leaving the dashboard. See [Inbox and voice sources](../inbox-voice-sources.md). + **Example.** A fact says tests run with `make test`, but your project uses `go test`. You edit the fact. **When it appears.** Always. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 64a28aa6f7..1a5ec111b1 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -172,6 +172,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Portable AgentDefinition format](../AGENT-DEFINITION.md) — standalone YAML schema for importing/exporting agent definitions. - [How Hive injects knowledge and memory into agents](agent-knowledge-and-memory.md) — the three context layers (Hive knowledge primer, CLI conversation, CLI persistent memory), the kick-primer path and its scoping/caps, why repo `AGENTS.md` is injected rather than read natively, which per-agent home directories are fleet-shared, and what survives a kick, `/clear`, pod restart or a new agent. - [Knowledge connectors](knowledge-connectors.md) — **Owner configuration:** connect, validate, and sync external document/wiki sources; includes scope and API reference. +- [Inbox and voice sources](inbox-voice-sources.md) — triage Plaud/Wispr Flow recordings into spektacular RFCs, dibs ideas, or knowledge only; hosted-hive Plaud sign-in. - [Knowledge curator](knowledge-curator.md) — configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) — interpret confidence badges and score explanations. - [Knowledge lifecycle states](knowledge-lifecycle.md) — approve, deprecate, or supersede facts and keep stale knowledge out of prompts. diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md new file mode 100644 index 0000000000..bb5c95792f --- /dev/null +++ b/src/docs/inbox-voice-sources.md @@ -0,0 +1,94 @@ +# Inbox and voice sources + +The **Inbox** (sidebar → Intelligence → 📥 Inbox) is where recordings and +dictations from voice sources land so you can turn them into work without +leaving the dashboard. It is source-agnostic: Plaud is the first source, +Wispr Flow (`type: wisprflow`) plugs in with the same model, and later sources +(file drop, webhook, Otter.ai, Fireflies, …) add nothing to the UI. + +Tracker RFC: *Voice recorders as first-class input sources (Inbox)* — +see the issue linked from the changelog entry. + +## Model + +- **Source** — a knowledge connector whose facts are recordings (`plaud`, + `wisprflow`). Connecting a source happens in **Knowledge → 🔌 Integrations**; + that card shows only the connection state and an **Open Inbox** button. +- **Item** — one recording, keyed `(source, item_id)` (for Plaud, the + `attr_plaud_file_id`). Every item **is already a Knowledge fact**; the Inbox + only tracks what you did with it. "📚 Fact" opens the fact in Knowledge. +- **Destination** — where an item goes: + - **→ RFC** (`spek_rfc`): the hive runs `spektacular voice --from stdin + --title --publish --repo <owner/repo>` on the transcript and + files one RFC issue plus native child issues with the hive handshake + marker. The hive's normal scanner then claims the children. If no + `spektacular` binary is on the host, the hive hands the same job to a + spoke agent as a kick prompt (state `delegated`): `inbox.spek.agent` if + set, otherwise the first enabled, unpaused agent by name. + - **→ dibs** (`dibs`): posts title, summary, transcript and the + `plaud://recordings/<id>` provenance to `POST {inbox.dibs.url}/api/ideas` + as a private draft idea (Bearer token from `inbox.dibs.token_env`). + - **Knowledge only** (`dismiss`): leaves the fact where it is and drops the + item from the unprocessed count. +- **State** per item per destination: `queued → running → filed | failed` + (or `delegated` for the agent fallback), with the RFC/idea URL and child + issue URLs. Stored in `connector-state/_inbox.json`. +- **Auto-routing rules** (⚙ Auto-routing in the Inbox, or `inbox.rules`): + `source?`, `title_prefix` or `tag`, `destination`, `repo?`. Rules run once + for each item that arrives after a connector sync; the first match wins; + items already in the Inbox are never re-routed. + +## Configuration + +```yaml +inbox: + sources: [plaud, wisprflow] # default: every configured voice connector + spek: + repo: hivecommons/hive # default RFC target + binary: spektacular # or an absolute path; empty = look on PATH + agent: "" # fallback agent; default: first enabled agent + dibs: + url: https://dibs.example.org + token_env: DIBS_TOKEN + rules: + - title_prefix: "RFC:" + destination: spek_rfc + - title_prefix: "Idea:" + destination: dibs +``` + +`inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` are editable from the +dashboard (`POST /api/inbox/settings`); the token is environment-only. + +## Hosted hives: signing in to Plaud + +Plaud's OAuth client only redirects to `http://localhost:8199/auth/callback`, +so a hive running in a cluster never receives the browser callback. The +Plaud card therefore offers **Finish sign-in by pasting**: + +1. Click **Connect Plaud**; open the authorization link and approve. +2. The browser lands on `localhost:8199/auth/callback?code=…` (a connection + error page is expected). Copy that URL and paste it into the card within + 10 minutes; the hive forwards it to the login session running in the pod. +3. Alternative: on a laptop run `npx -y @plaud-ai/mcp@latest install`, sign + in, and paste the contents of `~/.plaud/tokens-mcp.json`. + +Both go through `POST /api/knowledge/connectors/{name}/auth/complete`. +A browser-only redirect to the hive's public URL is not possible until Plaud +allows additional redirect URIs for its client. + +## Requirements on the hive image + +- Node.js ≥ 20 for the Plaud MCP server (the image ships Node 26). +- `spektacular` for → RFC: not in the image yet (tracked as a child of the + Inbox RFC); until then the agent fallback runs by default, or install it on + the host (`go install github.com/hivecommons/spektacular/cmd/spektacular@latest`). +- A GitHub token with issue-write on the target repo (the hive's app token + is injected as `GITHUB_TOKEN`). + +## API + +See [api-reference/knowledge.md](api-reference/knowledge.md) for +`GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, +`GET|POST /api/inbox/settings` and +`POST /api/knowledge/connectors/{name}/auth/complete`. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 6f84bf0979..e5a24b695b 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -276,12 +276,16 @@ if you do not want a sync to pull a new upstream release unreviewed. remote hive forward that port first (`ssh -L 8199:localhost:8199 user@host`) and open the URL from the forwarding machine. The login session stays open for 10 minutes; click **Check** to confirm "connected as …". - - Alternative when port-forwarding is impractical: authorize once on your - laptop with `npx -y @plaud-ai/mcp@latest install`, then copy - `~/.plaud/tokens-mcp.json` into the connector's `home` directory - (`…/connector-state/<name>/home/.plaud/`). The file is Plaud's own - token store, not a hive secret. -4. Click **Sync now**. New recordings are picked up every `interval`. + - Hosted hive (no port forward): open **Finish sign-in by pasting** on the + card and paste either the `localhost:8199/auth/callback?…` URL the + browser landed on after approving, or the contents of + `~/.plaud/tokens-mcp.json` from a laptop where `npx -y + @plaud-ai/mcp@latest install` signed in. See + [Inbox and voice sources](inbox-voice-sources.md#hosted-hives-signing-in-to-plaud). +4. Click **Sync now**. New recordings are picked up every `interval` and + appear in the **📥 Inbox**, where you file them as a spektacular RFC, send + them to dibs, or keep them as knowledge only + ([Inbox and voice sources](inbox-voice-sources.md)). ### Behaviour diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 63c7035f19..f0882b392c 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -49,6 +49,9 @@ github.com/hivecommons/hive/pkg/housestyle github.com/hivecommons/hive/pkg/hub github.com/hivecommons/hive/pkg/hubactivity github.com/hivecommons/hive/pkg/hubbackup +# pkg/inbox: Inbox triage API (hivecommons/hive#17876) — dashboard routes +# /api/inbox* wrap the inbox service; no existing seam covers it. +github.com/hivecommons/hive/pkg/inbox github.com/hivecommons/hive/pkg/inferencehealth github.com/hivecommons/hive/pkg/ioscan github.com/hivecommons/hive/pkg/knowledge diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 6c28e87550..bbeb01a2f8 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -103,7 +103,10 @@ type Config struct { Data DataConfig `yaml:"data"` Deployment DeploymentConfig `yaml:"deployment,omitempty" json:"deployment,omitempty"` Knowledge KnowledgeConfig `yaml:"knowledge"` - Hub HubConfig `yaml:"hub"` + // Inbox routes voice-recorder items (Plaud, Wispr Flow) to spek RFCs or + // dibs from the dashboard Inbox. + Inbox InboxConfig `yaml:"inbox,omitempty" json:"inbox,omitempty"` + Hub HubConfig `yaml:"hub"` // Issues tunes issue lifecycle automation. Zero value keeps the safe // defaults: close issues after their fix PR merges, and backfill hourly. Issues IssuesConfig `yaml:"issues,omitempty" json:"issues,omitempty"` diff --git a/src/pkg/config/inbox_config.go b/src/pkg/config/inbox_config.go new file mode 100644 index 0000000000..a40f0b312e --- /dev/null +++ b/src/pkg/config/inbox_config.go @@ -0,0 +1,122 @@ +package config + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Inbox destinations. `knowledge` is implicit: every item is already a +// Knowledge fact when it reaches the Inbox. +const ( + InboxDestinationSpekRFC = "spek_rfc" + InboxDestinationDibs = "dibs" + InboxDestinationDismiss = "dismiss" +) + +// DefaultInboxSources are the connector types whose facts surface as Inbox +// items when inbox.sources is empty. +var DefaultInboxSources = []string{"plaud", "wisprflow"} + +// InboxConfig is the `inbox:` block: which connector types feed the Inbox, +// how items are auto-routed, and how each destination is reached. +type InboxConfig struct { + // Sources lists connector types (plaud, wisprflow, ...) whose facts are + // Inbox items. Empty uses DefaultInboxSources. + Sources []string `yaml:"sources,omitempty" json:"sources,omitempty"` + // Rules are evaluated in order against every new item; the first match + // routes it automatically. + Rules []InboxRule `yaml:"rules,omitempty" json:"rules,omitempty"` + Spek InboxSpek `yaml:"spek,omitempty" json:"spek,omitempty"` + Dibs InboxDibs `yaml:"dibs,omitempty" json:"dibs,omitempty"` +} + +// InboxRule routes items whose source and title/tag match. +type InboxRule struct { + // Source restricts the rule to one connector type; empty matches all. + Source string `yaml:"source,omitempty" json:"source,omitempty"` + // TitlePrefix matches case-insensitively against the item title. + TitlePrefix string `yaml:"title_prefix,omitempty" json:"title_prefix,omitempty"` + // Tag matches any item tag exactly (case-insensitive). + Tag string `yaml:"tag,omitempty" json:"tag,omitempty"` + // Destination is spek_rfc, dibs or dismiss. + Destination string `yaml:"destination" json:"destination"` + // Repo (owner/repo) overrides inbox.spek.repo for spek_rfc. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` +} + +// InboxSpek configures the spek_rfc destination. +type InboxSpek struct { + // Repo is the default owner/repo RFCs are filed to. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` + // Binary is the spektacular executable (default runs.spektacular.binary). + Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` + // Agent names a hive agent to kick with a spek-new/publish prompt when + // the binary is unavailable in the hive image. Empty picks the first + // enabled, unpaused agent by name (see InboxSpekAgent). + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` +} + +// InboxSpekAgent returns the agent the spek_rfc destination falls back to +// when the spektacular binary is absent: inbox.spek.agent if set, else the +// alphabetically first enabled, unpaused agent, else "". +func (c *Config) InboxSpekAgent() string { + if a := strings.TrimSpace(c.Inbox.Spek.Agent); a != "" { + return a + } + names := make([]string, 0, len(c.Agents)) + for name, a := range c.Agents { + if a.Enabled && !a.Paused { + names = append(names, name) + } + } + sort.Strings(names) + if len(names) == 0 { + return "" + } + return names[0] +} + +// InboxDibs configures the dibs destination. +type InboxDibs struct { + // URL is the dibs instance base URL. + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // TokenEnv names the environment variable holding the hub bearer token. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` +} + +var inboxRepoRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`) + +// EffectiveSources returns Sources or DefaultInboxSources. +func (c InboxConfig) EffectiveSources() []string { + if len(c.Sources) == 0 { + return append([]string(nil), DefaultInboxSources...) + } + return c.Sources +} + +// ValidateInbox rejects unknown destinations, malformed repos and rules +// without a filter (a filter-less rule would route every item). +func ValidateInbox(c InboxConfig) error { + if c.Spek.Repo != "" && !inboxRepoRe.MatchString(c.Spek.Repo) { + return fmt.Errorf("inbox.spek.repo %q: must be owner/repo", c.Spek.Repo) + } + if c.Dibs.URL != "" && !strings.HasPrefix(c.Dibs.URL, "http://") && !strings.HasPrefix(c.Dibs.URL, "https://") { + return fmt.Errorf("inbox.dibs.url %q: must start with http:// or https://", c.Dibs.URL) + } + for i, r := range c.Rules { + switch r.Destination { + case InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss: + default: + return fmt.Errorf("inbox.rules[%d].destination %q: must be %s, %s or %s", i, r.Destination, InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss) + } + if strings.TrimSpace(r.TitlePrefix) == "" && strings.TrimSpace(r.Tag) == "" { + return fmt.Errorf("inbox.rules[%d]: title_prefix or tag is required", i) + } + if r.Repo != "" && !inboxRepoRe.MatchString(r.Repo) { + return fmt.Errorf("inbox.rules[%d].repo %q: must be owner/repo", i, r.Repo) + } + } + return nil +} diff --git a/src/pkg/config/inbox_config_default_agent_test.go b/src/pkg/config/inbox_config_default_agent_test.go new file mode 100644 index 0000000000..a3193c4e13 --- /dev/null +++ b/src/pkg/config/inbox_config_default_agent_test.go @@ -0,0 +1,51 @@ +package config + +import "testing" + +func TestInboxSpekAgentDefaultsToFirstEnabledAgent(t *testing.T) { + c := &Config{AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "zed": {Enabled: true}, + "alpha": {Enabled: true, Paused: true}, + "bravo": {Enabled: true}, + "charly": {Enabled: false}, + }}} + if got := c.InboxSpekAgent(); got != "bravo" { + t.Fatalf("default agent = %q, want bravo (first enabled, unpaused)", got) + } + c.Inbox.Spek.Agent = " custom " + if got := c.InboxSpekAgent(); got != "custom" { + t.Fatalf("explicit agent = %q, want custom", got) + } + if got := (&Config{}).InboxSpekAgent(); got != "" { + t.Fatalf("no agents should give empty, got %q", got) + } +} + +func TestInboxEffectiveSourcesAndValidate(t *testing.T) { + if got := (InboxConfig{}).EffectiveSources(); len(got) != len(DefaultInboxSources) || got[0] != DefaultInboxSources[0] { + t.Fatalf("default sources = %v", got) + } + if got := (InboxConfig{Sources: []string{"wisprflow"}}).EffectiveSources(); len(got) != 1 || got[0] != "wisprflow" { + t.Fatalf("explicit sources = %v", got) + } + ok := InboxConfig{ + Spek: InboxSpek{Repo: "o/r"}, + Dibs: InboxDibs{URL: "https://dibs.example"}, + Rules: []InboxRule{{TitlePrefix: "RFC:", Destination: InboxDestinationSpekRFC, Repo: "o/r"}, {Tag: "idea", Destination: InboxDestinationDibs}}, + } + if err := ValidateInbox(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } + bad := map[string]InboxConfig{ + "spek repo": {Spek: InboxSpek{Repo: "not-a-repo"}}, + "dibs url": {Dibs: InboxDibs{URL: "dibs.example"}}, + "rule destination": {Rules: []InboxRule{{TitlePrefix: "x", Destination: "email"}}}, + "rule filter": {Rules: []InboxRule{{Destination: InboxDestinationDismiss}}}, + "rule repo": {Rules: []InboxRule{{TitlePrefix: "x", Destination: InboxDestinationSpekRFC, Repo: "nope"}}}, + } + for name, c := range bad { + if err := ValidateInbox(c); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index a70784ec8f..f61ae46f36 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -570,6 +570,10 @@ "OnError": "" } }, + "inbox": { + "spek": {}, + "dibs": {} + }, "Hub": { "Enabled": false, "URL": "https://hive.hivecommons.dev", diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8aab8ad642..8812f3e6cf 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -208,6 +208,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { for _, w := range advisorWarnings { log.Printf("WARNING: %s", w) } + if err := ValidateInbox(c.Inbox); err != nil { + return err + } if err := ValidateKnowledgeConnectors(c.Knowledge.Connectors); err != nil { return err } diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index af8380cc4a..ceebd002c0 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -549,6 +549,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() s.registerKnowledgeConnectorAuthRoutes() + s.registerInboxRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) // Per-agent Variables tab (#11458, api_variables.go). s.registerAgentVariableRoutes() diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go new file mode 100644 index 0000000000..1ca6dc14bd --- /dev/null +++ b/src/pkg/dashboard/api_inbox.go @@ -0,0 +1,383 @@ +package dashboard + +import ( + "context" + "errors" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxListTimeout bounds one Inbox listing (reads connector facts). +const inboxListTimeout = 20 * time.Second + +// Inbox (#voice-sources): source-agnostic triage of voice-recorder items +// that knowledge connectors ingest, with routes to spek RFCs and dibs. +func (s *Server) registerInboxRoutes() { + s.mux.HandleFunc("GET /api/inbox", s.handleInboxList) + s.mux.HandleFunc("POST /api/inbox/route", s.handleInboxRoute) + s.mux.HandleFunc("POST /api/inbox/dismiss", s.handleInboxDismiss) + s.mux.HandleFunc("GET /api/inbox/settings", s.handleInboxSettingsGet) + s.mux.HandleFunc("POST /api/inbox/settings", s.handleInboxSettingsPost) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/complete", s.handleKnowledgeConnectorAuthComplete) +} + +// InboxSourceStatus is one configured source as the Inbox header shows it. +type InboxSourceStatus struct { + Type string `json:"type"` + Connector string `json:"connector,omitempty"` + // Configured is true when a knowledge.connectors entry of this type exists. + Configured bool `json:"configured"` + // Connected mirrors the connector's AuthState when it signs in interactively. + Connected bool `json:"connected"` + Account string `json:"account,omitempty"` +} + +// InboxListResponse is GET /api/inbox. +type InboxListResponse struct { + Items []inbox.Item `json:"items"` + Unprocessed int `json:"unprocessed"` + Sources []InboxSourceStatus `json:"sources"` + // Repos are the hive's configured repos for the RFC repo picker. + Repos []string `json:"repos"` + Settings inbox.Settings `json:"settings"` + Error string `json:"error,omitempty"` +} + +func (s *Server) inboxService(w http.ResponseWriter) (*inbox.Service, bool) { + if s == nil || s.deps == nil || s.deps.Inbox == nil { + jsonError(w, "inbox is not available: configure a voice knowledge connector (plaud, wisprflow)", http.StatusNotFound) + return nil, false + } + return s.deps.Inbox, true +} + +func (s *Server) handleInboxList(w http.ResponseWriter, r *http.Request) { + svc, ok := s.inboxService(w) + if !ok { + return + } + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + items, err := svc.Items(ctx) + resp := InboxListResponse{ + Items: items, + Unprocessed: inbox.Unprocessed(items), + Sources: s.inboxSources(ctx, svc.Config().EffectiveSources()), + Repos: s.inboxRepos(), + Settings: svc.Settings(), + } + if resp.Items == nil { + resp.Items = []inbox.Item{} + } + if err != nil { + resp.Error = err.Error() + } + jsonResponse(w, resp) +} + +// inboxSources pairs each source type with its connector's cached auth state. +func (s *Server) inboxSources(ctx context.Context, types []string) []InboxSourceStatus { + out := make([]InboxSourceStatus, 0, len(types)) + rt := s.knowledgeConnectors() + var statuses []connector.Status + if rt != nil { + statuses = rt.Statuses() + } + lookup, _ := rt.(knowledgeConnectorLookup) + for _, typ := range types { + st := InboxSourceStatus{Type: typ} + for _, cs := range statuses { + if cs.Type != typ { + continue + } + st.Configured, st.Connector = true, cs.Name + if lookup != nil { + if c, ok := lookup.Connector(cs.Name); ok { + if auth, ok := c.(connector.Authenticator); ok { + a := auth.AuthStatus(ctx, false) + st.Connected, st.Account = a.Connected, a.Account + } + } + } + break + } + out = append(out, st) + } + return out +} + +func (s *Server) inboxRepos() []string { + if s.deps == nil || s.deps.Config == nil { + return []string{} + } + repos := append([]string(nil), s.deps.Config.Project.Repos...) + if def := s.deps.Config.Inbox.Spek.Repo; def != "" { + found := false + for _, r := range repos { + if r == def { + found = true + } + } + if !found { + repos = append([]string{def}, repos...) + } + } + if repos == nil { + repos = []string{} + } + return repos +} + +type inboxRouteRequest struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + Repo string `json:"repo,omitempty"` +} + +// InboxRouteResponse is POST /api/inbox/route. +type InboxRouteResponse struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + State inbox.RouteState `json:"state"` +} + +// handleInboxRoute (owner) queues one item for one destination; progress is +// read back from GET /api/inbox. +func (s *Server) handleInboxRoute(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxRouteRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + req.Source, req.ItemID, req.Destination, req.Repo = strings.TrimSpace(req.Source), strings.TrimSpace(req.ItemID), strings.TrimSpace(req.Destination), strings.TrimSpace(req.Repo) + if req.Source == "" || req.ItemID == "" || req.Destination == "" { + jsonError(w, "source, item_id and destination are required", http.StatusBadRequest) + return + } + if req.Repo != "" && !inboxRepoOK(req.Repo) { + jsonError(w, "repo must be owner/repo", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + it, found, err := svc.Find(ctx, req.Source, req.ItemID) + if !found { + msg := "unknown inbox item" + if err != nil { + msg += ": " + err.Error() + } + jsonError(w, msg, http.StatusNotFound) + return + } + st, err := svc.Route(ctx, it, inbox.Request{Destination: req.Destination, Repo: req.Repo}) + switch { + case errors.Is(err, inbox.ErrRouteBusy): + jsonError(w, "that route is already queued or running", http.StatusConflict) + return + case errors.Is(err, inbox.ErrUnknownDestination): + jsonError(w, err.Error(), http.StatusBadRequest) + return + case err != nil: + jsonError(w, "route failed: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_route", auditDetail("source", req.Source, "item", req.ItemID, "destination", req.Destination, "repo", req.Repo), "") + jsonStatusResponse(w, http.StatusAccepted, InboxRouteResponse{Source: req.Source, ItemID: req.ItemID, Destination: req.Destination, State: st}) +} + +func inboxRepoOK(repo string) bool { + owner, name, ok := strings.Cut(repo, "/") + return ok && owner != "" && name != "" && !strings.ContainsAny(repo, " \t\n") +} + +type inboxDismissRequest struct { + Items []struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + } `json:"items"` + Undo bool `json:"undo,omitempty"` +} + +// InboxDismissResponse is POST /api/inbox/dismiss. +type InboxDismissResponse struct { + Dismissed int `json:"dismissed"` +} + +// handleInboxDismiss (owner) marks items "knowledge only" (bulk). +func (s *Server) handleInboxDismiss(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxDismissRequest + if err := decodeBody(r, &req); err != nil || len(req.Items) == 0 { + jsonError(w, "items[] is required", http.StatusBadRequest) + return + } + n := 0 + for _, it := range req.Items { + if it.Source == "" || it.ItemID == "" { + continue + } + if err := svc.Dismiss(it.Source, it.ItemID, !req.Undo); err != nil { + jsonError(w, "dismiss failed: "+err.Error(), http.StatusInternalServerError) + return + } + n++ + } + s.auditFromRequest(r, "inbox_dismiss", auditDetail("count", strconv.Itoa(n)), "") + jsonResponse(w, InboxDismissResponse{Dismissed: n}) +} + +func (s *Server) handleInboxSettingsGet(w http.ResponseWriter, r *http.Request) { + svc, ok := s.inboxService(w) + if !ok { + return + } + jsonResponse(w, svc.Settings()) +} + +type inboxSettingsRequest struct { + Rules *[]config.InboxRule `json:"rules,omitempty"` + SpekRepo *string `json:"spek_repo,omitempty"` + DibsURL *string `json:"dibs_url,omitempty"` +} + +// handleInboxSettingsPost (owner) updates inbox.rules / inbox.spek.repo / +// inbox.dibs.url and persists hive.yaml. +func (s *Server) handleInboxSettingsPost(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + if s.deps.Config == nil { + jsonError(w, "config is not available", http.StatusServiceUnavailable) + return + } + var req inboxSettingsRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + next := s.deps.Config.Inbox + if req.Rules != nil { + next.Rules = *req.Rules + } + if req.SpekRepo != nil { + next.Spek.Repo = strings.TrimSpace(*req.SpekRepo) + } + if req.DibsURL != nil { + next.Dibs.URL = strings.TrimRight(strings.TrimSpace(*req.DibsURL), "/") + } + if err := config.ValidateInbox(next); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + s.deps.Config.Inbox = next + svc.UpdateConfig(next) + if err := s.saveConfig(); err != nil { + jsonError(w, "saving config: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_settings", auditDetail("rules", strconv.Itoa(len(next.Rules)), "spek_repo", next.Spek.Repo, "dibs_url", next.Dibs.URL), "") + jsonResponse(w, svc.Settings()) +} + +// handleKnowledgeConnectorAuthComplete (owner) finishes a sign-in whose +// OAuth callback cannot reach a hosted hive: the operator pastes either the +// redirect URL the browser landed on or the vendor token file contents. +func (s *Server) handleKnowledgeConnectorAuthComplete(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + completer, ok := auth.(connector.AuthCompleter) + if !ok { + jsonError(w, "this connector does not support pasted sign-in completion", http.StatusBadRequest) + return + } + var comp connector.AuthCompletion + if err := decodeBody(r, &comp); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := completer.CompleteAuth(ctx, comp) + if err != nil { + jsonError(w, err.Error(), http.StatusBadGateway) + return + } + mode := "redirect_url" + if comp.Tokens != "" { + mode = "tokens" + } + s.auditFromRequest(r, "knowledge_connector_auth_complete", auditDetail("connector", name, "mode", mode), "") + jsonResponse(w, st) +} + +// NewInboxDestinations builds the spek_rfc and dibs destinations from +// config; token minting and agent kicks come from the dashboard deps. +func NewInboxDestinations(cfg *config.Config, token func(ctx context.Context, repo string) (string, error), kick inbox.Kicker) []inbox.Destination { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek := &inbox.SpekDestination{ + Binary: binary, + DefaultRepo: cfg.Inbox.Spek.Repo, + Token: token, + Agent: cfg.InboxSpekAgent(), + Kick: kick, + } + dibs := &inbox.DibsDestination{URL: cfg.Inbox.Dibs.URL} + if env := cfg.Inbox.Dibs.TokenEnv; env != "" { + dibs.Token = os.Getenv(env) + } + return []inbox.Destination{spek, dibs} +} + +// InboxReadiness returns the probes the settings view reports. +func InboxReadiness(cfg *config.Config) (spek func() bool, dibs func() bool) { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek = func() bool { + if _, err := exec.LookPath(binary); err == nil { + return true + } + return cfg.InboxSpekAgent() != "" + } + dibs = func() bool { + return cfg.Inbox.Dibs.TokenEnv != "" && os.Getenv(cfg.Inbox.Dibs.TokenEnv) != "" + } + return spek, dibs +} diff --git a/src/pkg/dashboard/api_inbox_test.go b/src/pkg/dashboard/api_inbox_test.go new file mode 100644 index 0000000000..150f36c188 --- /dev/null +++ b/src/pkg/dashboard/api_inbox_test.go @@ -0,0 +1,217 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +const inboxTestFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +source: plaud +connector: plaud +source_id: rec-1 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +synthesized: 2026-11-10T08:58:00Z +--- + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +Hello hive +` + +type inboxTestDest struct { + name string + got []inbox.Request + err error +} + +func (d *inboxTestDest) Name() string { return d.name } +func (d *inboxTestDest) Deliver(_ context.Context, _ inbox.Item, _ inbox.Content, req inbox.Request) (inbox.Result, error) { + d.got = append(d.got, req) + if d.err != nil { + return inbox.Result{}, d.err + } + return inbox.Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, nil +} + +func newInboxTestService(t *testing.T, cfg config.InboxConfig, dests ...inbox.Destination) *inbox.Service { + t.Helper() + base := t.TempDir() + dir := filepath.Join(base, "connectors", "personal") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "plaud-plaud-rec-1.md"), []byte(inboxTestFact), 0o644); err != nil { + t.Fatal(err) + } + src := &inbox.FactSource{ConnectorsDir: filepath.Join(base, "connectors"), SourceType: "plaud"} + svc := inbox.NewService(cfg, inbox.NewStore(filepath.Join(base, "inbox.json")), []inbox.Source{src}, dests, nil) + svc.Run = func(fn func()) { fn() } + return svc +} + +func decodeInboxList(t *testing.T, body []byte) InboxListResponse { + t.Helper() + var resp InboxListResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return resp +} + +func TestInboxEndpoints(t *testing.T) { + s, deps := apiServer(t) + if rec := doGet(s, "/api/inbox"); rec.Code != http.StatusNotFound { + t.Fatalf("inbox without service = %d, want 404", rec.Code) + } + spek := &inboxTestDest{name: inbox.DestSpekRFC} + dibs := &inboxTestDest{name: inbox.DestDibs, err: errors.New("dibs down")} + deps.Config.Inbox = config.InboxConfig{Spek: config.InboxSpek{Repo: "myorg/rfcs"}} + deps.Inbox = newInboxTestService(t, deps.Config.Inbox, spek, dibs) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}}) + + rec := doGet(s, "/api/inbox") + if rec.Code != http.StatusOK { + t.Fatalf("list = %d: %s", rec.Code, rec.Body.String()) + } + resp := decodeInboxList(t, rec.Body.Bytes()) + if len(resp.Items) != 1 || resp.Items[0].ID != "rec-1" || resp.Items[0].FactSlug != "plaud-plaud-rec-1" || resp.Unprocessed != 1 { + t.Fatalf("list = %+v", resp) + } + if len(resp.Repos) != 2 || resp.Repos[0] != "myorg/rfcs" || resp.Repos[1] != "repo1" { + t.Fatalf("repos = %v", resp.Repos) + } + if len(resp.Sources) != 2 || resp.Sources[0].Type != "plaud" || resp.Sources[0].Configured || resp.Settings.SpekRepo != "myorg/rfcs" { + t.Fatalf("sources/settings = %+v %+v", resp.Sources, resp.Settings) + } + + if rec := doPostNoOwner(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner route = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "nope", "destination": "spek_rfc"}); rec.Code != http.StatusNotFound { + t.Fatalf("unknown item = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc", "repo": "bad repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "nowhere"}); rec.Code != http.StatusBadRequest { + t.Fatalf("unknown destination = %d: %s", rec.Code, rec.Body.String()) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("route = %d: %s", rec.Code, rec.Body.String()) + } + if len(spek.got) != 1 || spek.got[0].Repo != "myorg/rfcs" { + t.Fatalf("spek requests = %+v (default repo must apply)", spek.got) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + r := resp.Items[0].Routes[inbox.DestSpekRFC] + if r == nil || r.Status != inbox.StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || resp.Unprocessed != 0 { + t.Fatalf("spek route = %+v unprocessed=%d", r, resp.Unprocessed) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "dibs"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("dibs route = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if d := resp.Items[0].Routes[inbox.DestDibs]; d == nil || d.Status != inbox.StatusFailed || d.Error != "dibs down" { + t.Fatalf("dibs route = %+v", d) + } + + rec = doPost(s, "/api/inbox/dismiss", map[string]any{"items": []map[string]string{{"source": "plaud", "item_id": "rec-1"}}}) + if rec.Code != http.StatusOK { + t.Fatalf("dismiss = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if !resp.Items[0].Dismissed { + t.Fatalf("item not dismissed: %+v", resp.Items[0]) + } +} + +func TestInboxSettingsPersistRules(t *testing.T) { + s, deps := apiServer(t) + deps.Inbox = newInboxTestService(t, config.InboxConfig{}) + if rec := doPost(s, "/api/inbox/settings", map[string]any{"rules": []map[string]string{{"destination": "spek_rfc"}}}); rec.Code != http.StatusBadRequest { + t.Fatalf("rule without filter = %d, want 400: %s", rec.Code, rec.Body.String()) + } + if rec := doPost(s, "/api/inbox/settings", map[string]any{"spek_repo": "not-a-repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d, want 400", rec.Code) + } + rec := doPost(s, "/api/inbox/settings", map[string]any{ + "rules": []map[string]string{{"source": "plaud", "title_prefix": "RFC:", "destination": "spek_rfc", "repo": "o/r"}}, + "spek_repo": "o/r", + "dibs_url": "https://dibs.example/", + }) + if rec.Code != http.StatusOK { + t.Fatalf("settings = %d: %s", rec.Code, rec.Body.String()) + } + var st inbox.Settings + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if len(st.Rules) != 1 || st.Rules[0].TitlePrefix != "RFC:" || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" { + t.Fatalf("settings = %+v", st) + } + if deps.Config.Inbox.Spek.Repo != "o/r" || len(deps.Config.Inbox.Rules) != 1 || deps.Inbox.Config().Spek.Repo != "o/r" { + t.Fatalf("config not updated: %+v", deps.Config.Inbox) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusOK { + t.Fatalf("settings get = %d", rec.Code) + } +} + +// completerConn is an Authenticator that also accepts pasted completions. +type completerConn struct { + authConn + got connector.AuthCompletion +} + +func (c *completerConn) CompleteAuth(_ context.Context, comp connector.AuthCompletion) (connector.AuthState, error) { + c.got = comp + if comp.Tokens == "" && comp.RedirectURL == "" { + return c.state, errors.New("nothing pasted") + } + c.state = connector.AuthState{Connected: true, Account: "andy@example.com", CheckedAt: time.Now()} + return c.state, nil +} + +func TestKnowledgeConnectorAuthComplete(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{}) + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusBadRequest { + t.Fatalf("non-completer = %d, want 400: %s", rec.Code, rec.Body.String()) + } + conn := &completerConn{} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{}); rec.Code != http.StatusBadGateway { + t.Fatalf("empty completion = %d, want 502: %s", rec.Code, rec.Body.String()) + } + rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"redirect_url": "http://localhost:8199/auth/callback?code=x&state=y"}) + if rec.Code != http.StatusOK { + t.Fatalf("complete = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || conn.got.RedirectURL == "" { + t.Fatalf("state = %+v got = %+v", st, conn.got) + } +} diff --git a/src/pkg/dashboard/dashboard_section_cards_test.go b/src/pkg/dashboard/dashboard_section_cards_test.go index 11d6d11f5d..7fa8a5d0ac 100644 --- a/src/pkg/dashboard/dashboard_section_cards_test.go +++ b/src/pkg/dashboard/dashboard_section_cards_test.go @@ -274,6 +274,7 @@ func TestDashboardSectionRenderersRefreshSharedShell(t *testing.T) { "function renderFAQ()": "faq-section", "function renderInception()": "inception-section", "function renderKnowledge()": "knowledge-section", + "function renderInbox()": "inbox-section", "function renderDebugSection()": "debug-section", "function renderLogs()": "logs-section", } diff --git a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go index 8ee1af1a9d..04bf41a306 100644 --- a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go +++ b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go @@ -9,14 +9,14 @@ import ( ) var dashboardLayoutIDs9062 = []string{ - "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", + "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section", "faq-section", } var dashboardDefaultNavOrder9062 = []string{ - "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", + "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "faq-section", } @@ -26,7 +26,7 @@ var dashboardDefaultLayoutOrder9062 = append(append(append([]string{}, dashboard "faq-section", ) -const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` +const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` func jsArray9062(ids []string) string { var b strings.Builder @@ -165,7 +165,7 @@ func TestDashboardLayoutNormalizeFutureUnknownAndMissing(t *testing.T) { html := indexHTML(t) script := dashboardLayoutPreamble9062() + jsFunc(t, html, "dashboardLayoutAllIds") + "\n" + jsFunc(t, html, "dashboardLayoutNormalize") + ` const got = dashboardLayoutNormalize({v:99, main:['faq-section','beads-section','unknown','governor','faq-section']}); -const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; +const want = ` + jsArray9062([]string{"faq-section", "overview-section", "governor", "pr-throughput-section", "repos-section", "activity-monitor-section", "knowledge-section", "inbox-section", "review-queue-section", "contributors-section", "advisory-section", "token-panel", "cost-panel", "inception-section", "acmm-eval-section", "debug-section", "audit-section", "approvals-section", "nous-section", "logs-section", "agents-section", "agent-activity-section"}) + `; if (got.v !== 1) throw new Error('version not normalized: '+got.v); if (got.main.includes('beads-section')) throw new Error('legacy beads section survived normalization: '+JSON.stringify(got.main)); if (JSON.stringify(got.main) !== JSON.stringify(want)) throw new Error('normalized order '+JSON.stringify(got.main)); @@ -287,7 +287,7 @@ if (middle !== 'nous-section,logs-section,agents-section,agent-activity-section, if (new Set(got).size !== got.length) throw new Error('duplicate ids: '+JSON.stringify(got)); const moved = dashboardLayoutNormalize({v:1, main:['faq-section','cost-panel','overview-section']}).main; if (moved[0] !== 'faq-section') throw new Error('operator-moved FAQ not respected: '+JSON.stringify(moved)); -if (moved.length !== 21) throw new Error('missing sections not restored: '+moved.length); +if (moved.length !== 22) throw new Error('missing sections not restored: '+moved.length); ` cmd := exec.Command(node, "-e", script) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/src/pkg/dashboard/dashboard_section_summaries_test.go b/src/pkg/dashboard/dashboard_section_summaries_test.go index 53f124738c..e1f1a7f317 100644 --- a/src/pkg/dashboard/dashboard_section_summaries_test.go +++ b/src/pkg/dashboard/dashboard_section_summaries_test.go @@ -39,7 +39,7 @@ func TestDashboardSectionCardsHaveCollapsedSummaries(t *testing.T) { "overview-section", "advisory-section", "hive-advice-section", "fleet-report-section", "acmm-reco-section", "lifecycle-section", "pr-throughput-section", "repos-section", "acmm-eval-section", "approvals-section", "audit-section", - "review-queue-section", "nous-section", "inception-section", "knowledge-section", + "review-queue-section", "nous-section", "inception-section", "knowledge-section", "inbox-section", "contributors-section", "debug-section", "logs-section", "agents-section", "faq-section", } for _, id := range refreshRequired { diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index e0bff6c556..8ed04dbb1a 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -25,6 +25,7 @@ import ( "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hivestate" "github.com/hivecommons/hive/pkg/hooks" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/nous" "github.com/hivecommons/hive/pkg/review/autoscale" @@ -109,6 +110,9 @@ type Dependencies struct { // KnowledgePublish is the running knowledge.publish mirror; nil when // publishing is not configured or failed to start. KnowledgePublish KnowledgePublishRuntime + // Inbox is the voice-source triage service; nil when no voice + // connector is configured. + Inbox *inbox.Service // RotationMgr is the provider-rotation headroom reporter (RFC #3958). Nil // when rotation is disabled; the headroom endpoint then reports // enabled=false. diff --git a/src/pkg/dashboard/import_count_test.go b/src/pkg/dashboard/import_count_test.go index d2e43b1707..5274543e12 100644 --- a/src/pkg/dashboard/import_count_test.go +++ b/src/pkg/dashboard/import_count_test.go @@ -66,8 +66,11 @@ func TestDashboardInternalImportCountRatchet(t *testing.T) { // shared with the scheduler's kick injection (hivecommons/hive#17244), // plus pkg/workflowmodel/store, the spoke-local library of workflow // models behind the /api/workflows routes and the tracker's per-repo - // column block (RFC hivecommons/hive#17620, phase 2). - const maxDashboardInternalImports = 60 + // column block (RFC hivecommons/hive#17620, phase 2), plus pkg/inbox, + // the source-agnostic voice-recording triage service (items keyed by + // source + item id, spek_rfc/dibs destinations) behind the /api/inbox + // routes and the Inbox dashboard section. + const maxDashboardInternalImports = 61 entries, err := os.ReadDir(".") if err != nil { diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 2c3cb41497..8dc4184b81 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -155,7 +155,9 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { for _, want := range []string{ `'plaud': { label: 'Plaud voice notes'`, `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, - `if (type === 'plaud') html += kbPlaudAuthLine(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, + `data-action="kbPlaudPasteComplete"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete'`, `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, `data-action="kbPlaudConnect"`, `data-action="kbPlaudDisconnect"`, diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 718fbbf15f..0c0c2c8f71 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -618,3 +618,9 @@ project.workflow_bindings[].name # set through the per-repo workflow select on S project.workflow_bindings[].repo # see project.workflow_bindings[].name project.workflow_bindings[].version # see project.workflow_bindings[].name workflow.label_gc # operator safety knob for model-label garbage collection on rebind; yaml-only, no Settings control planned (RFC #17620) +inbox.sources[] # which voice connector types feed the Inbox; defaults to every configured plaud/wisprflow connector, yaml-only (Inbox RFC) +inbox.rules[].source # auto-routing rules are edited as whole rows in Inbox → Auto-routing via POST /api/inbox/settings, not leaf-path controls (Inbox RFC) +inbox.rules[].title_prefix # see inbox.rules[].source +inbox.rules[].tag # see inbox.rules[].source +inbox.rules[].destination # see inbox.rules[].source +inbox.rules[].repo # see inbox.rules[].source diff --git a/src/pkg/dashboard/static-src/css/inbox.css b/src/pkg/dashboard/static-src/css/inbox.css new file mode 100644 index 0000000000..c81646cfae --- /dev/null +++ b/src/pkg/dashboard/static-src/css/inbox.css @@ -0,0 +1,17 @@ + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } diff --git a/src/pkg/dashboard/static-src/html/inbox.html b/src/pkg/dashboard/static-src/html/inbox.html new file mode 100644 index 0000000000..8b462ea673 --- /dev/null +++ b/src/pkg/dashboard/static-src/html/inbox.html @@ -0,0 +1,6 @@ + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> diff --git a/src/pkg/dashboard/static-src/html/sidebar.html b/src/pkg/dashboard/static-src/html/sidebar.html index 434c15aea4..9ea8dbb310 100644 --- a/src/pkg/dashboard/static-src/html/sidebar.html +++ b/src/pkg/dashboard/static-src/html/sidebar.html @@ -42,6 +42,7 @@ <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js new file mode 100644 index 0000000000..4f38662937 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -0,0 +1,296 @@ + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) return; + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + function inboxReadRules() { + return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules() { + const host = document.getElementById('inbox-settings-rules'); + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div id="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + inboxRenderRules(); + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const errEl = document.getElementById('inbox-settings-error'); + const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 81daf98c72..10fdd87fd5 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2433,7 +2433,7 @@ html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { @@ -2572,7 +2572,7 @@ } let text; if (a.error) text = 'Plaud: ' + a.error; - else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a> (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; diff --git a/src/pkg/dashboard/static-src/js/layout.js b/src/pkg/dashboard/static-src/js/layout.js index 3819b9027b..4e6e3f3a67 100644 --- a/src/pkg/dashboard/static-src/js/layout.js +++ b/src/pkg/dashboard/static-src/js/layout.js @@ -4,7 +4,7 @@ var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; diff --git a/src/pkg/dashboard/static-src/js/refresh.js b/src/pkg/dashboard/static-src/js/refresh.js index 21511384cf..8e4e569066 100644 --- a/src/pkg/dashboard/static-src/js/refresh.js +++ b/src/pkg/dashboard/static-src/js/refresh.js @@ -562,13 +562,14 @@ ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index a289e2d9aa..35e32dd311 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -9,7 +9,7 @@ // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); @@ -121,6 +121,7 @@ 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -159,6 +160,7 @@ 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 45731eba3a..d6c8e81100 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -29,6 +29,7 @@ css/workflows-marketplace.css css/chat.css css/nous.css css/knowledge.css +css/inbox.css css/sections-collapse.css css/design-system.css css/responsive.css @@ -56,6 +57,7 @@ html/repos.html html/acmm-approvals-audit.html html/review-queue.html html/nous-inception-knowledge-contributors.html +html/inbox.html html/debug-logs.html html/agents.html html/tracker.html @@ -119,6 +121,7 @@ js/evidence.js js/approvals.js js/nous.js js/knowledge.js +js/inbox.js js/vault-audit.js js/review-queue.js js/contributors.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index e75ead995d..90af378a72 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5920,6 +5920,23 @@ .field-error { color: var(--red); font-size: var(--fs-xs); margin-top: var(--sp-1); } @media (max-width: 768px) { .kb-layout { grid-template-columns: 1fr; } } + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } /* Collapsible advisory sections (#1546) */ .collapse-chevron { display: inline-block; @@ -8560,6 +8577,7 @@ <h2 class="config-title">🐝 Getting Started<span id="welcome-progress" class=" <a class="oc-nav-item" data-section="acmm-eval-section" data-action="ocNavigate" data-arg0="acmm-eval-section"><span class="oc-nav-emoji">🎯</span><span class="oc-nav-text">ACMM Eval</span></a> <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> @@ -8998,6 +9016,12 @@ <h2 class="section-label m-0">👥 Contributors <span id="contributors-status-ba </div> + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> <div class="mt-6 mb-8" id="debug-section" data-dashboard-section="debug-section"> <h2 class="section-label section-header-toggle mb-5" data-action="toggleSection" data-arg0="debug-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Diagnostics section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 🩺 Diagnostics</h2> <div class="section-body"> @@ -10351,7 +10375,7 @@ <h3 class="faq-group-title">Where to get help</h3> // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section' ]); @@ -10463,6 +10487,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'nous-section': { title: '🧪 Experiment Lab (nous)', summary: '—', settingsHtml: sectionSettingsGear('Experiment Lab', 'openNousConfig', 'Experiment Lab Configuration') }, 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -10501,6 +10526,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'nous-section': { heading: 'Experiment Lab', help: 'An experimental planner that tries out code changes in your repositories and records what it learned.' }, 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, @@ -11925,7 +11951,7 @@ <h3 class="faq-group-title">Where to get help</h3> var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; @@ -45953,7 +45979,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name); + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { @@ -46092,7 +46118,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> } let text; if (a.error) text = 'Plaud: ' + a.error; - else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a> (remote hive: forward port 8199 first)' : ' (URL pending…)'); + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; @@ -46168,6 +46194,302 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> return '<div class="muted-xs" data-kb-wisprflow-source>Wispr Flow: ' + (path ? '<code>' + escapeHtml(path) + '</code>' : 'default app location on the hive host') + ' · ' + group + '</div>'; } function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) return; + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + function inboxReadRules() { + return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules() { + const host = document.getElementById('inbox-settings-rules'); + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div id="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + inboxRenderRules(); + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const errEl = document.getElementById('inbox-settings-error'); + const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } // ── Vault Management ── function kbOpenVaults() { @@ -48180,13 +48502,14 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); diff --git a/src/pkg/dashboard/webstatic/static_index_test.go b/src/pkg/dashboard/webstatic/static_index_test.go index 79db965065..9a1a9ad129 100644 --- a/src/pkg/dashboard/webstatic/static_index_test.go +++ b/src/pkg/dashboard/webstatic/static_index_test.go @@ -420,7 +420,7 @@ func TestStaticDashboardDefaultLayoutOrder(t *testing.T) { } html := rec.Body.String() - defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" + defaultOrder := "'overview-section','governor','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" if !strings.Contains(html, defaultOrder) { t.Fatalf("dashboard default layout/reset order no longer matches sidebar nav; missing %q", defaultOrder) } diff --git a/src/pkg/inbox/dibs.go b/src/pkg/inbox/dibs.go new file mode 100644 index 0000000000..334c2c203b --- /dev/null +++ b/src/pkg/inbox/dibs.go @@ -0,0 +1,113 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// dibsTimeout bounds one idea submission. +const dibsTimeout = 30 * time.Second + +// dibsMaxBody caps the idea body so a long transcript does not trip dibs' +// request limit; the full text stays in Knowledge. +const dibsMaxBody = 60_000 + +// DibsDestination posts an item as a private draft idea to a dibs instance +// (`POST {url}/api/ideas`, bearer hub token). +type DibsDestination struct { + URL string + Token string + HTTP *http.Client +} + +// Name implements Destination. +func (d *DibsDestination) Name() string { return DestDibs } + +// ErrDibsUnconfigured is returned when inbox.dibs.url is empty. +var ErrDibsUnconfigured = errors.New("dibs: inbox.dibs.url is not configured") + +type dibsIdea struct { + Title string `json:"title"` + Body string `json:"body"` + Visibility string `json:"visibility"` + Status string `json:"status"` +} + +// Deliver implements Destination. +func (d *DibsDestination) Deliver(ctx context.Context, it Item, c Content, _ Request) (Result, error) { + base := strings.TrimRight(strings.TrimSpace(d.URL), "/") + if base == "" { + return Result{}, ErrDibsUnconfigured + } + if strings.TrimSpace(d.Token) == "" { + return Result{}, errors.New("dibs: no token (set inbox.dibs.token_env to a variable holding a hub bearer token)") + } + body := DibsBody(it, c) + payload, _ := json.Marshal(dibsIdea{ + Title: firstNonEmpty(c.Title, it.Title), + Body: body, + Visibility: "private", + Status: "draft", + }) + rctx, cancel := context.WithTimeout(ctx, dibsTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodPost, base+"/api/ideas", bytes.NewReader(payload)) + if err != nil { + return Result{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+d.Token) + client := d.HTTP + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return Result{}, fmt.Errorf("dibs: %w", err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Result{}, fmt.Errorf("dibs: %s: %s", resp.Status, lastLines(string(data), 2)) + } + var idea struct { + ID string `json:"id"` + } + if err := json.Unmarshal(data, &idea); err != nil || idea.ID == "" { + return Result{}, fmt.Errorf("dibs: response has no idea id: %s", lastLines(string(data), 2)) + } + return Result{URL: base + "/api/ideas/" + idea.ID}, nil +} + +// DibsBody renders the idea body: summary, provenance, then transcript. +func DibsBody(it Item, c Content) string { + var b strings.Builder + if c.Summary != "" { + b.WriteString(c.Summary) + b.WriteString("\n\n") + } + fmt.Fprintf(&b, "_Source: %s_\n", c.Provenance) + if !it.RecordedAt.IsZero() { + fmt.Fprintf(&b, "_Recorded: %s_\n", it.RecordedAt.UTC().Format(time.RFC3339)) + } + text := c.Transcript + if text == "" && c.Summary == "" { + text = c.Markdown + } + if text != "" { + b.WriteString("\n## Transcript\n\n") + b.WriteString(text) + } + out := b.String() + if len(out) > dibsMaxBody { + out = out[:dibsMaxBody] + "\n\n_(truncated; full transcript in Knowledge)_" + } + return out +} diff --git a/src/pkg/inbox/edges_test.go b/src/pkg/inbox/edges_test.go new file mode 100644 index 0000000000..52ea44841a --- /dev/null +++ b/src/pkg/inbox/edges_test.go @@ -0,0 +1,276 @@ +package inbox + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestServiceSettingsFindAndConfigUpdate(t *testing.T) { + src := &fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}} + spek := &recordingDest{name: DestSpekRFC} + dibs := &recordingDest{name: DestDibs} + svc := newTestService(t, config.InboxConfig{}, src, spek, dibs) + if spek.Name() != DestSpekRFC || dibs.Name() != DestDibs { + t.Fatal("destination names") + } + + st := svc.Settings() + if st.Rules == nil || len(st.Rules) != 0 || st.SpekReady || st.DibsTokenSet { + t.Fatalf("default settings = %+v", st) + } + svc.SetReadiness(func() bool { return true }, func() bool { return true }) + svc.UpdateConfig(config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + Dibs: config.InboxDibs{URL: "https://dibs.example"}, + }) + st = svc.Settings() + if !st.SpekReady || !st.DibsTokenSet || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" || len(st.Rules) != 1 { + t.Fatalf("settings after update = %+v", st) + } + + it, ok, err := svc.Find(context.Background(), "plaud", "rec-1") + if err != nil || !ok || it.Title != "one" { + t.Fatalf("find = %+v %v %v", it, ok, err) + } + if _, ok, _ := svc.Find(context.Background(), "plaud", "missing"); ok { + t.Fatal("found a missing item") + } + svc.Wait() +} + +func TestServiceNowAndSortItems(t *testing.T) { + fixed := time.Date(2026, 3, 4, 5, 6, 7, 0, time.FixedZone("x", 3600)) + svc := newTestService(t, config.InboxConfig{}, &fakeSource{}, &recordingDest{name: DestSpekRFC}) + if !svc.now().Before(time.Now().Add(time.Second)) { + t.Fatal("now() without override should be wall clock") + } + svc.Now = func() time.Time { return fixed } + if got := svc.now(); got != fixed.UTC() { + t.Fatalf("now = %v", got) + } + items := []Item{ + {ID: "a", RecordedAt: fixed.Add(-time.Hour)}, + {ID: "b", RecordedAt: fixed}, + {ID: "c", RecordedAt: fixed.Add(-2 * time.Hour)}, + } + sortItems(items) + if items[0].ID != "b" || items[1].ID != "a" || items[2].ID != "c" { + t.Fatalf("sorted = %v %v %v", items[0].ID, items[1].ID, items[2].ID) + } +} + +type failingSource struct { + fakeSource + listErr error + fetchErr error +} + +func (f *failingSource) List(context.Context) ([]Item, error) { return f.items, f.listErr } +func (f *failingSource) Fetch(context.Context, Item) (Content, error) { + return f.body, f.fetchErr +} + +func TestServiceDeliverFailures(t *testing.T) { + src := &failingSource{fakeSource: fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}}, fetchErr: errors.New("boom")} + dest := &recordingDest{name: DestSpekRFC, res: Result{Note: "delegated to agent x"}} + svc := newTestService(t, config.InboxConfig{Spek: config.InboxSpek{Repo: "o/r"}}, src, dest) + + // Fetch error is recorded as failed. + items, _ := svc.Items(context.Background()) + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[0].Routes[DestSpekRFC]; r == nil || r.Status != StatusFailed || !strings.Contains(r.Error, "boom") { + t.Fatalf("fetch failure = %+v", r) + } + + // Unknown source is recorded as failed. + ghost := Item{Source: "ghost", ID: "g-1"} + if _, err := svc.Route(context.Background(), ghost, Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(ghost.Key(), DestSpekRFC); !ok || r.Status != StatusFailed || !strings.Contains(r.Error, "not configured") { + t.Fatalf("ghost route = %+v %v", r, ok) + } + + // A note-only result (no URL) becomes delegated. + src.fetchErr = nil + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(items[0].Key(), DestSpekRFC); !ok || r.Status != StatusDelegated || r.Error != "delegated to agent x" { + t.Fatalf("delegated route = %+v", r) + } + + // Items surfaces list errors but still returns items. + src.listErr = errors.New("list down") + items, err := svc.Items(context.Background()) + if err == nil || !strings.Contains(err.Error(), "list down") || len(items) != 1 { + t.Fatalf("items with list error = %d %v", len(items), err) + } + if _, err := svc.Sweep(context.Background()); err == nil { + t.Fatal("sweep should surface the list error") + } + if known := svc.Store.Known(); len(known) != 2 || known[0] != ghost.Key() { + t.Fatalf("known = %v", known) + } +} + +func TestStoreEdgeCases(t *testing.T) { + fixed := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + mem := NewStore("") + if mem.now().IsZero() { + t.Fatal("now") + } + mem.Now = func() time.Time { return fixed } + if st, err := mem.Begin("plaud/x", DestDibs, ""); err != nil || st.UpdatedAt != fixed { + t.Fatalf("in-memory begin = %+v %v", st, err) + } + if _, ok := mem.Route("plaud/x", DestSpekRFC); ok { + t.Fatal("route for another destination should be absent") + } + if _, ok := mem.Route("nope", DestDibs); ok { + t.Fatal("route for unknown key should be absent") + } + + // Corrupt file is tolerated; unwritable dir surfaces an error. + dir := t.TempDir() + bad := filepath.Join(dir, "bad.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if known := NewStore(bad).Known(); len(known) != 0 { + t.Fatalf("corrupt store known = %v", known) + } + blocker := filepath.Join(dir, "file") + if err := os.WriteFile(blocker, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewStore(filepath.Join(blocker, "inbox.json")).Begin("k", DestDibs, ""); err == nil { + t.Fatal("save under a file should fail") + } +} + +func TestFactSourceEdgeCases(t *testing.T) { + src := &FactSource{ConnectorsDir: filepath.Join(t.TempDir(), "missing"), SourceType: "wisprflow"} + if src.Type() != "wisprflow" { + t.Fatal("type") + } + if items, err := src.List(context.Background()); err != nil || items != nil { + t.Fatalf("missing dir = %v %v", items, err) + } + + dir := t.TempDir() + src.ConnectorsDir = dir + if err := os.WriteFile(filepath.Join(dir, "stray.txt"), nil, 0o600); err != nil { + t.Fatal(err) + } + writeFact(t, dir, "personal", "wisprflow-w-1.md", strings.Replace(plaudFact, "source: plaud", "source: wisprflow", 1)) + writeFact(t, dir, "personal", "wisprflow-w-2.md", "no front matter at all") + writeFact(t, dir, "personal", "wisprflow-w-3.md", "---\nsource: wisprflow\nunterminated") + writeFact(t, dir, "personal", "wisprflow-noid.md", "---\nsource: wisprflow\n---\nbody") + if err := os.Mkdir(filepath.Join(dir, "personal", "wisprflow-dir.md"), 0o755); err != nil { + t.Fatal(err) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 1 || items[0].Source != "wisprflow" { + t.Fatalf("items = %+v %v", items, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := src.List(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled list = %v", err) + } + + if _, err := src.Fetch(context.Background(), Item{FactPath: filepath.Join(dir, "nope.md")}); err == nil { + t.Fatal("fetch of a missing fact should fail") + } + // Fetch falls back to the item title when the fact has none. + path := filepath.Join(dir, "personal", "wisprflow-untitled.md") + if err := os.WriteFile(path, []byte("---\nsource: wisprflow\nsource_id: u\n---\nhello"), 0o600); err != nil { + t.Fatal(err) + } + c, err := src.Fetch(context.Background(), Item{ID: "u", Title: "from item", FactPath: path}) + if err != nil || c.Title != "from item" || c.Markdown != "hello" || c.Provenance != "wisprflow://recordings/u" { + t.Fatalf("fetch = %+v %v", c, err) + } + + fm, body := splitFrontMatter("---\na: 1\n---") + if fm["a"] != "1" || body != "" { + t.Fatalf("front matter without trailing newline = %v %q", fm, body) + } +} + +func TestSpekDestinationRealExecAndLookPath(t *testing.T) { + d := &SpekDestination{Binary: "definitely-not-a-binary-" + t.Name(), DefaultRepo: "o/r"} + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + _, err := d.Deliver(context.Background(), it, Content{Transcript: "hi"}, Request{}) + if err == nil || !strings.Contains(err.Error(), "not installed") { + t.Fatalf("missing binary = %v", err) + } + if d.lookPath("sh") != nil { + t.Skip("sh not on PATH") + } + out, err := d.exec(context.Background(), "sh", []string{"-c", "cat; echo done"}, os.Environ(), "input\n") + if err != nil || !strings.Contains(out, "input") || !strings.Contains(out, "done") { + t.Fatalf("exec = %q %v", out, err) + } + if got := lastLines("a\nb\nc\n", 2); got != "b\nc" { + t.Fatalf("lastLines = %q", got) + } + if got := (&SpekDestination{Prefixes: []string{"X:"}}).prefixes(); len(got) != 1 || got[0] != "X:" { + t.Fatalf("prefixes = %v", got) + } +} + +func TestSpekDestinationErrorPaths(t *testing.T) { + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + ok := func(string) error { return nil } + cases := []struct { + name string + d *SpekDestination + c Content + want string + }{ + {"no repo", &SpekDestination{LookPath: ok}, Content{Transcript: "t"}, ErrNoRepo.Error()}, + {"empty", &SpekDestination{LookPath: ok, DefaultRepo: "o/r"}, Content{}, "no transcript"}, + {"token", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Token: func(context.Context, string) (string, error) { return "", errors.New("mint") }}, Content{Transcript: "t"}, "github token: mint"}, + {"exec", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "l1\nl2\nboom", errors.New("exit 2") + }}, Content{Transcript: "t"}, "boom"}, + {"parse", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "nothing useful", nil + }}, Content{Transcript: "t"}, "spek_rfc:"}, + {"kick fails", &SpekDestination{LookPath: func(string) error { return errors.New("nope") }, DefaultRepo: "o/r", Agent: "a", Kick: func(string, string) error { return errors.New("kick") }}, Content{Transcript: "t"}, "kick to a failed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.d.Deliver(context.Background(), it, tc.c, Request{}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestNewServiceDefaultsLogger(t *testing.T) { + svc := NewService(config.InboxConfig{}, NewStore(""), nil, nil, nil) + if svc.Logger == nil { + t.Fatal("nil logger should default") + } + svc.Logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) + if items, err := svc.Items(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("items with no sources = %v %v", items, err) + } +} diff --git a/src/pkg/inbox/factsource.go b/src/pkg/inbox/factsource.go new file mode 100644 index 0000000000..5785290aac --- /dev/null +++ b/src/pkg/inbox/factsource.go @@ -0,0 +1,209 @@ +package inbox + +import ( + "context" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// excerptRunes bounds the list excerpt. +const excerptRunes = 240 + +// FactSource lists items of one connector type by reading the facts that +// type's connectors wrote under <ConnectorsDir>/<layer>/. It needs nothing +// from the vendor: any connector whose facts carry `source`/`source_id` +// front matter is an Inbox source. +type FactSource struct { + // ConnectorsDir is <knowledge base>/connectors. + ConnectorsDir string + // SourceType is the connector type (plaud, wisprflow, ...). + SourceType string +} + +// Type implements Source. +func (f *FactSource) Type() string { return f.SourceType } + +// List implements Source: newest first. +func (f *FactSource) List(ctx context.Context) ([]Item, error) { + layers, err := os.ReadDir(f.ConnectorsDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []Item + for _, layer := range layers { + if !layer.IsDir() { + continue + } + dir := filepath.Join(f.ConnectorsDir, layer.Name()) + files, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, fi := range files { + if ctx.Err() != nil { + return out, ctx.Err() + } + if fi.IsDir() || !strings.HasSuffix(fi.Name(), ".md") || !strings.HasPrefix(fi.Name(), f.SourceType+"-") { + continue + } + path := filepath.Join(dir, fi.Name()) + data, err := os.ReadFile(path) + if err != nil { + continue + } + fm, body := splitFrontMatter(string(data)) + if fm["source"] != f.SourceType || fm["source_id"] == "" || fm["status"] == "deprecated" { + continue + } + out = append(out, itemFromFact(fm, body, layer.Name(), path)) + } + } + sort.SliceStable(out, func(i, j int) bool { + if !out[i].RecordedAt.Equal(out[j].RecordedAt) { + return out[i].RecordedAt.After(out[j].RecordedAt) + } + return out[i].ID < out[j].ID + }) + return out, nil +} + +func itemFromFact(fm map[string]string, body, layer, path string) Item { + it := Item{ + Source: fm["source"], + Connector: fm["connector"], + ID: fm["source_id"], + Title: fm["title"], + FactSlug: strings.TrimSuffix(filepath.Base(path), ".md"), + FactPath: path, + Layer: layer, + Excerpt: excerpt(body), + } + if it.Title == "" { + it.Title = it.ID + } + for _, key := range []string{"attr_recorded_at", "synthesized"} { + if t, err := time.Parse(time.RFC3339, fm[key]); err == nil { + it.RecordedAt = t.UTC() + break + } + } + if ms, err := strconv.ParseInt(fm["attr_duration_ms"], 10, 64); err == nil { + it.DurationMS = ms + } + if tags := fm["attr_tags"]; tags != "" { + for _, t := range strings.Split(tags, ",") { + if t = strings.TrimSpace(t); t != "" { + it.Tags = append(it.Tags, t) + } + } + } + return it +} + +// Fetch implements Source by re-reading the fact and splitting the body +// into the connector's "## Summary" / "## Transcript" sections. +func (f *FactSource) Fetch(_ context.Context, it Item) (Content, error) { + data, err := os.ReadFile(it.FactPath) + if err != nil { + return Content{}, err + } + fm, body := splitFrontMatter(string(data)) + c := Content{ + Title: fm["title"], + Markdown: strings.TrimSpace(body), + Provenance: f.SourceType + "://recordings/" + it.ID, + } + if c.Title == "" { + c.Title = it.Title + } + c.Summary, c.Transcript = splitSections(body) + return c, nil +} + +// splitFrontMatter mirrors the connector fact layout: a `---` block of +// `key: value` lines, then the markdown body. +func splitFrontMatter(content string) (map[string]string, string) { + out := map[string]string{} + if !strings.HasPrefix(content, "---\n") { + return out, content + } + end := strings.Index(content[4:], "\n---") + if end < 0 { + return out, content + } + for _, l := range strings.Split(content[4:4+end], "\n") { + k, v, ok := strings.Cut(l, ":") + if !ok { + continue + } + out[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), "\"'") + } + rest := content[4+end+4:] + if i := strings.Index(rest, "\n"); i >= 0 { + rest = rest[i+1:] + } else { + rest = "" + } + return out, rest +} + +// splitSections returns the text under a "## Summary…" heading and under +// "## Transcript". Headings may carry a suffix such as "(Plaud)". +func splitSections(body string) (summary, transcript string) { + var cur *string + var sb, tb strings.Builder + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, "## ") { + h := strings.ToLower(line[3:]) + switch { + case strings.HasPrefix(h, "summary"): + cur = &summary + case strings.HasPrefix(h, "transcript"): + cur = &transcript + default: + cur = nil + } + continue + } + if cur == &summary { + sb.WriteString(line) + sb.WriteByte('\n') + } else if cur == &transcript { + tb.WriteString(line) + tb.WriteByte('\n') + } + } + return strings.TrimSpace(sb.String()), strings.TrimSpace(tb.String()) +} + +// excerpt returns the first prose lines of body, bounded to excerptRunes. +func excerpt(body string) string { + var b strings.Builder + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "**") { + continue + } + if b.Len() > 0 { + b.WriteByte(' ') + } + b.WriteString(line) + if utf8.RuneCountInString(b.String()) >= excerptRunes { + break + } + } + s := b.String() + if utf8.RuneCountInString(s) > excerptRunes { + r := []rune(s) + s = strings.TrimSpace(string(r[:excerptRunes])) + "…" + } + return s +} diff --git a/src/pkg/inbox/inbox.go b/src/pkg/inbox/inbox.go new file mode 100644 index 0000000000..bc6e194e16 --- /dev/null +++ b/src/pkg/inbox/inbox.go @@ -0,0 +1,123 @@ +// Package inbox turns voice-recorder items that knowledge connectors already +// ingest as facts (Plaud, Wispr Flow, ...) into a source-agnostic triage list +// with routes to spek RFCs, dibs ideas, or "knowledge only". +package inbox + +import ( + "context" + "time" +) + +// Route statuses for one (item, destination) pair. +const ( + StatusQueued = "queued" + StatusRunning = "running" + StatusFiled = "filed" + StatusFailed = "failed" +) + +// Destination names. Knowledge is implicit: items are Knowledge facts. +const ( + DestSpekRFC = "spek_rfc" + DestDibs = "dibs" + DestDismiss = "dismiss" +) + +// Item is one incoming recording/dictation as the Inbox lists it. +type Item struct { + // Source is the connector type (plaud, wisprflow, ...). + Source string `json:"source"` + // Connector is the knowledge.connectors[] entry name that synced it. + Connector string `json:"connector"` + // ID is the upstream id (Plaud file id, ...). (Source, ID) is the key. + ID string `json:"id"` + Title string `json:"title"` + Excerpt string `json:"excerpt,omitempty"` + RecordedAt time.Time `json:"recorded_at,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + Tags []string `json:"tags,omitempty"` + // FactSlug / FactPath / Layer locate the Knowledge fact backing the item. + FactSlug string `json:"fact_slug"` + FactPath string `json:"fact_path,omitempty"` + Layer string `json:"layer,omitempty"` + // Routes holds per-destination state; absent means never routed. + Routes map[string]*RouteState `json:"routes,omitempty"` + // Dismissed marks "knowledge only": the item stays a fact, no further route. + Dismissed bool `json:"dismissed,omitempty"` + // Processed is true once any route filed or the item was dismissed. + Processed bool `json:"processed"` +} + +// Key returns the stable "source/id" key used by the state store. +func (it Item) Key() string { return Key(it.Source, it.ID) } + +// Key joins a source and item id. +func Key(source, id string) string { return source + "/" + id } + +// RouteState is the lifecycle of one item on one destination. +type RouteState struct { + Status string `json:"status"` + // Repo is the owner/repo a spek_rfc route targeted. + Repo string `json:"repo,omitempty"` + // URL is the RFC issue or dibs idea once filed. + URL string `json:"url,omitempty"` + // Children are the RFC's child issue URLs. + Children []string `json:"children,omitempty"` + Error string `json:"error,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Content is the full item body a destination needs. +type Content struct { + Title string + Summary string + Transcript string + // Markdown is the whole fact body when Summary/Transcript are not split. + Markdown string + // Provenance is a stable reference such as plaud://recordings/<id>. + Provenance string +} + +// Text returns transcript+summary, falling back to the raw markdown. +func (c Content) Text() string { + if c.Transcript == "" && c.Summary == "" { + return c.Markdown + } + out := c.Transcript + if c.Summary != "" { + if out != "" { + out += "\n\n" + } + out += "Summary:\n" + c.Summary + } + return out +} + +// Source lists and fetches items of one type. The fact-backed source covers +// every connector type; vendor-specific sources can replace it later. +type Source interface { + Type() string + List(ctx context.Context) ([]Item, error) + Fetch(ctx context.Context, it Item) (Content, error) +} + +// Request is one routing request. +type Request struct { + Destination string + // Repo is the owner/repo for spek_rfc; empty uses the configured default. + Repo string +} + +// Result is what a destination filed. +type Result struct { + URL string + Children []string + // Note is set instead of URL when the work was delegated to an agent. + Note string +} + +// Destination files an item somewhere outside Knowledge. +type Destination interface { + Name() string + Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) +} diff --git a/src/pkg/inbox/inbox_test.go b/src/pkg/inbox/inbox_test.go new file mode 100644 index 0000000000..34868adcaa --- /dev/null +++ b/src/pkg/inbox/inbox_test.go @@ -0,0 +1,398 @@ +package inbox + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +const plaudFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +tags: [connector, plaud, plaud] +source: plaud +connector: plaud +source_id: rec-1 +source_path: Plaud +attr_duration_ms: 125000 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +attr_tags: voice, ideas +synthesized: 2026-11-10T08:58:00Z +synced_at: 2026-11-11T20:00:00Z +--- + +# RFC: Voice inbox + +**Recorded:** 2026-11-10T08:58:00Z + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +[00:02] Andy: Hello hive +[01:05] Andy: Second line +` + +func writeFact(t *testing.T, dir, layer, name, content string) string { + t.Helper() + p := filepath.Join(dir, layer, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestFactSourceListsAndFetchesConnectorFacts(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + writeFact(t, dir, "personal", "plaud-plaud-rec-0.md", strings.Replace(strings.Replace(plaudFact, "rec-1", "rec-0", -1), "status: active", "status: deprecated", 1)) + writeFact(t, dir, "personal", "notion-docs-x.md", strings.Replace(plaudFact, "source: plaud", "source: notion", 1)) + src := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + items, err := src.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 { + t.Fatalf("items = %+v", items) + } + it := items[0] + if it.ID != "rec-1" || it.Source != "plaud" || it.Connector != "plaud" || it.Title != "RFC: Voice inbox" || it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" { + t.Fatalf("item = %+v", it) + } + if it.DurationMS != 125000 || !it.RecordedAt.Equal(time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC)) || strings.Join(it.Tags, ",") != "voice,ideas" { + t.Fatalf("attrs = %+v", it) + } + if !strings.Contains(it.Excerpt, "ship the inbox") || strings.Contains(it.Excerpt, "Recorded:") { + t.Fatalf("excerpt = %q", it.Excerpt) + } + c, err := src.Fetch(context.Background(), it) + if err != nil { + t.Fatal(err) + } + if c.Summary != "- ship the inbox" || !strings.HasPrefix(c.Transcript, "[00:02] Andy: Hello hive") || c.Provenance != "plaud://recordings/rec-1" { + t.Fatalf("content = %+v", c) + } +} + +func TestRuleMatching(t *testing.T) { + it := Item{Source: "plaud", Title: "rfc: Build the thing", Tags: []string{"Ideas"}} + cases := []struct { + name string + rule Rule + want bool + }{ + {"prefix case-insensitive", Rule{TitlePrefix: "RFC:"}, true}, + {"prefix mismatch", Rule{TitlePrefix: "Hive:"}, false}, + {"tag", Rule{Tag: "ideas"}, true}, + {"source mismatch", Rule{Source: "wisprflow", TitlePrefix: "RFC:"}, false}, + {"source match", Rule{Source: "plaud", TitlePrefix: "RFC:"}, true}, + {"no filter never matches", Rule{Destination: DestDibs}, false}, + } + for _, tc := range cases { + if got := RuleMatches(tc.rule, it); got != tc.want { + t.Errorf("%s: got %v want %v", tc.name, got, tc.want) + } + } + if got := StripPrefix("RFC: Build the thing", DefaultTitlePrefixes); got != "Build the thing" { + t.Fatalf("StripPrefix = %q", got) + } + if got := StripPrefix("Plain", DefaultTitlePrefixes); got != "Plain" { + t.Fatalf("StripPrefix = %q", got) + } +} + +func TestStoreTransitions(t *testing.T) { + st := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + key := Key("plaud", "rec-1") + if _, err := st.Begin(key, DestSpekRFC, "o/r"); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("second Begin while queued: %v", err) + } + if _, err := st.Set(key, DestSpekRFC, StatusRunning, Result{}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("Begin while running: %v", err) + } + // dibs route on the same item is independent. + if _, err := st.Begin(key, DestDibs, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestSpekRFC, StatusFiled, Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestDibs, StatusFailed, Result{}, "boom"); err != nil { + t.Fatal(err) + } + // Reload from disk. + st2 := NewStore(st.Path) + items := []Item{{Source: "plaud", ID: "rec-1"}, {Source: "plaud", ID: "rec-9"}} + st2.Apply(items) + r := items[0].Routes[DestSpekRFC] + if r == nil || r.Status != StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || r.Repo != "o/r" { + t.Fatalf("spek route = %+v", r) + } + if d := items[0].Routes[DestDibs]; d == nil || d.Status != StatusFailed || d.Error != "boom" { + t.Fatalf("dibs route = %+v", d) + } + if !items[0].Processed || items[1].Processed { + t.Fatalf("processed flags = %v %v", items[0].Processed, items[1].Processed) + } + // A failed route may be retried; a filed one may be re-filed. + if _, err := st2.Begin(key, DestDibs, ""); err != nil { + t.Fatalf("retry after failure: %v", err) + } + if err := st2.Dismiss(Key("plaud", "rec-9"), true); err != nil { + t.Fatal(err) + } + st2.Apply(items) + if !items[1].Processed || !items[1].Dismissed { + t.Fatalf("dismissed item must be processed: %+v", items[1]) + } +} + +func TestParsePublishOutput(t *testing.T) { + jsonOut := `Publishing... +{"target":"github","repo":"o/r","rfc_url":"https://github.com/o/r/issues/10","rfc_number":10,"children":[{"id":"a","url":"https://github.com/o/r/issues/11","number":11},{"id":"b","url":"https://github.com/o/r/issues/12","number":12}],"created":3}` + res, err := ParsePublishOutput(jsonOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/10" || len(res.Children) != 2 || res.Children[1] != "https://github.com/o/r/issues/12" { + t.Fatalf("json result = %+v", res) + } + textOut := "Draft written to .spektacular/specs/voice.md\nRFC: https://github.com/o/r/issues/20\n child: https://github.com/o/r/issues/21\n child: https://github.com/o/r/issues/22 (dup) https://github.com/o/r/issues/22\n" + res, err = ParsePublishOutput(textOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/20" || len(res.Children) != 2 { + t.Fatalf("text result = %+v", res) + } + if _, err := ParsePublishOutput("error: nothing happened"); err == nil { + t.Fatal("no URL must be an error") + } +} + +func TestRenderAgentPrompt(t *testing.T) { + it := Item{Source: "plaud", ID: "rec-1"} + c := Content{Summary: "- ship it", Transcript: "Hello hive", Provenance: "plaud://recordings/rec-1"} + p := RenderAgentPrompt(it, c, "o/r", "Voice inbox v2") + for _, want := range []string{"spektacular publish voice-inbox-v2 --repo o/r --allow-draft --format json", "spektacular init", "Open questions", "plaud://recordings/rec-1", "## Summary\n- ship it", "## Transcript\nHello hive"} { + if !strings.Contains(p, want) { + t.Errorf("prompt missing %q:\n%s", want, p) + } + } +} + +type fakeSource struct { + items []Item + body Content +} + +func (f *fakeSource) Type() string { return "plaud" } +func (f *fakeSource) List(context.Context) ([]Item, error) { return f.items, nil } +func (f *fakeSource) Fetch(context.Context, Item) (Content, error) { return f.body, nil } + +func newTestService(t *testing.T, cfg config.InboxConfig, src Source, dests ...Destination) *Service { + t.Helper() + store := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + svc := NewService(cfg, store, []Source{src}, dests, slog.New(slog.NewTextHandler(os.Stderr, nil))) + svc.Run = func(fn func()) { fn() } + return svc +} + +func TestSpekDestinationRunsVoicePublish(t *testing.T) { + var gotArgs []string + var gotStdin string + var gotEnv []string + calls := 0 + d := &SpekDestination{ + Binary: "spektacular", + DefaultRepo: "o/r", + Token: func(context.Context, string) (string, error) { return "tok", nil }, + LookPath: func(string) error { return nil }, + Exec: func(_ context.Context, _ string, args, env []string, stdin string) (string, error) { + calls++ + if calls == 1 { + return "Error: unknown flag: --format", errors.New("exit status 1") + } + gotArgs, gotEnv, gotStdin = args, env, stdin + return "RFC: https://github.com/o/r/issues/5\nhttps://github.com/o/r/issues/6\n", nil + }, + } + it := Item{Source: "plaud", ID: "rec-1", Title: "RFC: Voice inbox"} + c := Content{Title: "RFC: Voice inbox", Transcript: "Hello hive", Summary: "- ship it", Provenance: "plaud://recordings/rec-1"} + res, err := d.Deliver(context.Background(), it, c, Request{Destination: DestSpekRFC}) + if err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("expected retry without --format json, calls=%d", calls) + } + want := []string{"voice", "--from", "stdin", "--title", "Voice inbox", "--publish", "--repo", "o/r"} + if strings.Join(gotArgs, " ") != strings.Join(want, " ") { + t.Fatalf("args = %v", gotArgs) + } + if !strings.Contains(gotStdin, "Hello hive") || !strings.Contains(gotStdin, "plaud://recordings/rec-1") { + t.Fatalf("stdin = %q", gotStdin) + } + hasTok := false + for _, e := range gotEnv { + if e == "GITHUB_TOKEN=tok" { + hasTok = true + } + } + if !hasTok { + t.Fatal("GITHUB_TOKEN not injected") + } + if res.URL != "https://github.com/o/r/issues/5" || len(res.Children) != 1 { + t.Fatalf("result = %+v", res) + } +} + +func TestSpekDestinationFallsBackToAgentKick(t *testing.T) { + var kicked, prompt string + d := &SpekDestination{ + LookPath: func(string) error { return errors.New("not found") }, + Agent: "builder", + Kick: func(agent, p string) error { kicked, prompt = agent, p; return nil }, + } + res, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x", Title: "Idea"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}) + if err != nil { + t.Fatal(err) + } + if kicked != "builder" || !strings.Contains(prompt, "--repo o/r") || res.Note == "" || res.URL != "" { + t.Fatalf("kick=%q note=%q prompt=%q", kicked, res.Note, prompt) + } + d.Agent = "" + if _, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}); err == nil { + t.Fatal("missing binary without agent must fail") + } + if _, err := (&SpekDestination{LookPath: func(string) error { return nil }}).Deliver(context.Background(), Item{}, Content{Markdown: "x"}, Request{}); !errors.Is(err, ErrNoRepo) { + t.Fatalf("no repo: %v", err) + } +} + +type recordingDest struct { + name string + mu sync.Mutex + got []Request + res Result + err error +} + +func (r *recordingDest) Name() string { return r.name } +func (r *recordingDest) Deliver(_ context.Context, _ Item, _ Content, req Request) (Result, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.got = append(r.got, req) + return r.res, r.err +} + +func TestServiceRouteAndSweep(t *testing.T) { + src := &fakeSource{items: []Item{ + {Source: "plaud", ID: "rec-1", Title: "RFC: one", RecordedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}, + {Source: "plaud", ID: "rec-2", Title: "note", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, body: Content{Markdown: "x"}} + spek := &recordingDest{name: DestSpekRFC, res: Result{URL: "https://github.com/o/r/issues/1"}} + dibs := &recordingDest{name: DestDibs, err: errors.New("dibs down")} + cfg := config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + } + svc := newTestService(t, cfg, src, spek, dibs) + + routed, err := svc.Sweep(context.Background()) + if err != nil || routed != 1 { + t.Fatalf("sweep routed=%d err=%v", routed, err) + } + if len(spek.got) != 1 || spek.got[0].Repo != "o/r" { + t.Fatalf("spek requests = %+v", spek.got) + } + items, _ := svc.Items(context.Background()) + if items[0].ID != "rec-1" || items[0].Routes[DestSpekRFC].Status != StatusFiled || !items[0].Processed { + t.Fatalf("rec-1 = %+v", items[0]) + } + if items[1].Processed || Unprocessed(items) != 1 { + t.Fatalf("rec-2 should be unprocessed: %+v", items[1]) + } + // A second sweep must not re-fire on seen items. + if routed, _ := svc.Sweep(context.Background()); routed != 0 { + t.Fatalf("second sweep routed %d", routed) + } + // Manual dibs route fails and records the error. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDibs}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[1].Routes[DestDibs]; r == nil || r.Status != StatusFailed || r.Error != "dibs down" { + t.Fatalf("dibs route = %+v", r) + } + if _, err := svc.Route(context.Background(), items[1], Request{Destination: "nope"}); !errors.Is(err, ErrUnknownDestination) { + t.Fatalf("unknown destination: %v", err) + } + // Dismiss = knowledge only. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDismiss}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if !items[1].Dismissed || Unprocessed(items) != 0 { + t.Fatalf("dismiss failed: %+v", items[1]) + } +} + +func TestDibsDestinationPostsIdea(t *testing.T) { + var gotAuth string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/ideas" || r.Method != http.MethodPost { + t.Errorf("unexpected %s %s", r.Method, r.URL.Path) + } + gotAuth = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"idea-7","title":"x"}`)) + })) + defer srv.Close() + d := &DibsDestination{URL: srv.URL + "/", Token: "hub-tok"} + it := Item{Source: "plaud", ID: "rec-1", Title: "Idea", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)} + res, err := d.Deliver(context.Background(), it, Content{Title: "Idea", Summary: "sum", Transcript: "words", Provenance: "plaud://recordings/rec-1"}, Request{}) + if err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer hub-tok" || gotBody["title"] != "Idea" || gotBody["visibility"] != "private" || gotBody["status"] != "draft" { + t.Fatalf("auth=%q body=%v", gotAuth, gotBody) + } + body, _ := gotBody["body"].(string) + if !strings.Contains(body, "plaud://recordings/rec-1") || !strings.Contains(body, "## Transcript") || !strings.HasPrefix(body, "sum") { + t.Fatalf("body = %q", body) + } + if res.URL != srv.URL+"/api/ideas/idea-7" { + t.Fatalf("url = %q", res.URL) + } + if _, err := (&DibsDestination{}).Deliver(context.Background(), it, Content{}, Request{}); !errors.Is(err, ErrDibsUnconfigured) { + t.Fatalf("unconfigured: %v", err) + } +} diff --git a/src/pkg/inbox/rules.go b/src/pkg/inbox/rules.go new file mode 100644 index 0000000000..61adf1a198 --- /dev/null +++ b/src/pkg/inbox/rules.go @@ -0,0 +1,62 @@ +package inbox + +import ( + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// DefaultTitlePrefixes are the prefixes the UI offers when a rule has none. +var DefaultTitlePrefixes = []string{"RFC:", "Hive:"} + +// Rule is one auto-routing rule (mirrors config.InboxRule). +type Rule = config.InboxRule + +// RuleMatches reports whether r applies to it: source (when set) must equal +// the item source, and at least one of title_prefix / tag must match. +// A rule with neither filter never matches, so a bad config cannot route +// every recording. +func RuleMatches(r Rule, it Item) bool { + if src := strings.TrimSpace(r.Source); src != "" && !strings.EqualFold(src, it.Source) { + return false + } + prefix := strings.TrimSpace(r.TitlePrefix) + tag := strings.TrimSpace(r.Tag) + if prefix == "" && tag == "" { + return false + } + if prefix != "" && strings.HasPrefix(strings.ToLower(strings.TrimSpace(it.Title)), strings.ToLower(prefix)) { + return true + } + if tag != "" { + for _, t := range it.Tags { + if strings.EqualFold(strings.TrimSpace(t), tag) { + return true + } + } + } + return false +} + +// FirstRule returns the first rule matching it. +func FirstRule(rules []Rule, it Item) (Rule, bool) { + for _, r := range rules { + if RuleMatches(r, it) { + return r, true + } + } + return Rule{}, false +} + +// StripPrefix removes a matched routing prefix ("RFC: Foo" → "Foo") so the +// filed title reads naturally; unmatched titles are returned trimmed. +func StripPrefix(title string, prefixes []string) string { + t := strings.TrimSpace(title) + for _, p := range prefixes { + p = strings.TrimSpace(p) + if p != "" && len(t) > len(p) && strings.EqualFold(t[:len(p)], p) { + return strings.TrimSpace(t[len(p):]) + } + } + return t +} diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go new file mode 100644 index 0000000000..ff1bcc3c92 --- /dev/null +++ b/src/pkg/inbox/service.go @@ -0,0 +1,279 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// routeTimeout bounds one background delivery. +const routeTimeout = 15 * time.Minute + +// Settings is the operator-editable part of the Inbox (mirrors inbox.* +// YAML minus binary/agent plumbing). +type Settings struct { + Sources []string `json:"sources"` + Rules []config.InboxRule `json:"rules"` + SpekRepo string `json:"spek_repo"` + DibsURL string `json:"dibs_url"` + // DibsTokenSet reports whether inbox.dibs.token_env resolves; the token + // itself is never returned. + DibsTokenSet bool `json:"dibs_token_set"` + SpekReady bool `json:"spek_ready"` +} + +// Service lists items across sources, applies persisted state, runs +// auto-routing rules and executes manual routes in the background. +type Service struct { + Store *Store + Logger *slog.Logger + // Now is injectable for tests. + Now func() time.Time + + mu sync.RWMutex + cfg config.InboxConfig + sources map[string]Source + dests map[string]Destination + wg sync.WaitGroup + // Run executes a delivery; nil runs it in a goroutine. Tests set it to + // run inline. + Run func(fn func()) + // Status snapshots for the settings view. + spekReady func() bool + dibsTokenSet func() bool +} + +// NewService builds a service for cfg with the given sources/destinations. +func NewService(cfg config.InboxConfig, store *Store, sources []Source, dests []Destination, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + s := &Service{Store: store, Logger: logger, cfg: cfg, sources: map[string]Source{}, dests: map[string]Destination{}} + for _, src := range sources { + s.sources[src.Type()] = src + } + for _, d := range dests { + s.dests[d.Name()] = d + } + return s +} + +// SetReadiness installs the probes behind Settings.SpekReady / DibsTokenSet. +func (s *Service) SetReadiness(spek, dibs func() bool) { + s.spekReady, s.dibsTokenSet = spek, dibs +} + +// Config returns the current inbox config. +func (s *Service) Config() config.InboxConfig { + s.mu.RLock() + defer s.mu.RUnlock() + return s.cfg +} + +// UpdateConfig replaces the routing config (after the dashboard saved it). +func (s *Service) UpdateConfig(cfg config.InboxConfig) { + s.mu.Lock() + s.cfg = cfg + s.mu.Unlock() +} + +// Settings returns the operator view of the config. +func (s *Service) Settings() Settings { + cfg := s.Config() + st := Settings{ + Sources: cfg.EffectiveSources(), + Rules: cfg.Rules, + SpekRepo: cfg.Spek.Repo, + DibsURL: cfg.Dibs.URL, + } + if st.Rules == nil { + st.Rules = []config.InboxRule{} + } + if s.spekReady != nil { + st.SpekReady = s.spekReady() + } + if s.dibsTokenSet != nil { + st.DibsTokenSet = s.dibsTokenSet() + } + return st +} + +// Items lists every item across the configured sources with state applied, +// newest first. +func (s *Service) Items(ctx context.Context) ([]Item, error) { + cfg := s.Config() + var out []Item + var errs []error + for _, typ := range cfg.EffectiveSources() { + src := s.source(typ) + if src == nil { + continue + } + items, err := src.List(ctx) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", typ, err)) + } + out = append(out, items...) + } + s.Store.Apply(out) + sortItems(out) + return out, errors.Join(errs...) +} + +func (s *Service) source(typ string) Source { + s.mu.RLock() + defer s.mu.RUnlock() + return s.sources[typ] +} + +// Find returns one item by key. +func (s *Service) Find(ctx context.Context, source, id string) (Item, bool, error) { + items, err := s.Items(ctx) + for _, it := range items { + if it.Source == source && it.ID == id { + return it, true, err + } + } + return Item{}, false, err +} + +// Unprocessed counts items with no filed route and not dismissed. +func Unprocessed(items []Item) int { + n := 0 + for _, it := range items { + if !it.Processed { + n++ + } + } + return n +} + +// Dismiss marks an item knowledge-only. +func (s *Service) Dismiss(source, id string, dismissed bool) error { + return s.Store.Dismiss(Key(source, id), dismissed) +} + +// ErrUnknownDestination is returned for a destination the service lacks. +var ErrUnknownDestination = errors.New("unknown destination") + +// Route queues it for dest and runs the delivery in the background. The +// returned state is the queued state; callers poll Items for progress. +func (s *Service) Route(ctx context.Context, it Item, req Request) (RouteState, error) { + if req.Destination == DestDismiss { + if err := s.Dismiss(it.Source, it.ID, true); err != nil { + return RouteState{}, err + } + return RouteState{Status: StatusFiled, UpdatedAt: s.now()}, nil + } + s.mu.RLock() + dest := s.dests[req.Destination] + s.mu.RUnlock() + if dest == nil { + return RouteState{}, fmt.Errorf("%w: %q", ErrUnknownDestination, req.Destination) + } + if req.Destination == DestSpekRFC && strings.TrimSpace(req.Repo) == "" { + req.Repo = s.Config().Spek.Repo + } + st, err := s.Store.Begin(it.Key(), req.Destination, req.Repo) + if err != nil { + return st, err + } + s.wg.Add(1) + run := s.Run + if run == nil { + run = func(fn func()) { go fn() } + } + run(func() { + defer s.wg.Done() + s.deliver(it, dest, req) + }) + return st, nil +} + +func (s *Service) deliver(it Item, dest Destination, req Request) { + key := it.Key() + ctx, cancel := context.WithTimeout(context.Background(), routeTimeout) + defer cancel() + if _, err := s.Store.Set(key, req.Destination, StatusRunning, Result{}, ""); err != nil { + s.Logger.Warn("inbox: persist running state", "item", key, "error", err) + } + src := s.source(it.Source) + if src == nil { + s.fail(key, req.Destination, fmt.Errorf("source %q is not configured", it.Source)) + return + } + content, err := src.Fetch(ctx, it) + if err != nil { + s.fail(key, req.Destination, fmt.Errorf("fetch: %w", err)) + return + } + res, err := dest.Deliver(ctx, it, content, req) + if err != nil { + s.fail(key, req.Destination, err) + return + } + status := StatusFiled + if res.URL == "" && res.Note != "" { + status = StatusDelegated + } + if _, err := s.Store.Set(key, req.Destination, status, res, res.Note); err != nil { + s.Logger.Warn("inbox: persist filed state", "item", key, "error", err) + } + s.Logger.Info("inbox: routed", "item", key, "destination", req.Destination, "status", status, "url", res.URL) +} + +func (s *Service) fail(key, dest string, err error) { + s.Logger.Warn("inbox: route failed", "item", key, "destination", dest, "error", err) + if _, perr := s.Store.Set(key, dest, StatusFailed, Result{}, err.Error()); perr != nil { + s.Logger.Warn("inbox: persist failed state", "item", key, "error", perr) + } +} + +// Sweep applies auto-routing rules to items the store has never seen and +// marks every listed item seen, so a rule added later never back-fires on +// old recordings. It is called after each connector sync. +func (s *Service) Sweep(ctx context.Context) (routed int, err error) { + items, lerr := s.Items(ctx) + rules := s.Config().Rules + for _, it := range items { + if s.Store.Seen(it.Key()) { + continue + } + rule, ok := FirstRule(rules, it) + if !ok { + _ = s.Store.MarkSeen(it.Key()) + continue + } + if _, rerr := s.Route(ctx, it, Request{Destination: rule.Destination, Repo: rule.Repo}); rerr != nil { + s.Logger.Warn("inbox: auto-route", "item", it.Key(), "error", rerr) + _ = s.Store.MarkSeen(it.Key()) + continue + } + routed++ + } + return routed, lerr +} + +// Wait blocks until background deliveries finish (tests, shutdown). +func (s *Service) Wait() { s.wg.Wait() } + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func sortItems(items []Item) { + for i := 1; i < len(items); i++ { + for j := i; j > 0 && items[j].RecordedAt.After(items[j-1].RecordedAt); j-- { + items[j], items[j-1] = items[j-1], items[j] + } + } +} diff --git a/src/pkg/inbox/spek.go b/src/pkg/inbox/spek.go new file mode 100644 index 0000000000..4a0e39df73 --- /dev/null +++ b/src/pkg/inbox/spek.go @@ -0,0 +1,257 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "text/template" + "time" +) + +// StatusDelegated means the route was handed to a hive agent (no binary in +// the image); the agent files the RFC and the operator links it manually. +const StatusDelegated = "delegated" + +// spekTimeout bounds one `spektacular voice --publish` run. +const spekTimeout = 10 * time.Minute + +// Kicker sends a prompt to a named hive agent (the dashboard's kick path). +type Kicker func(agent, prompt string) error + +// SpekDestination files an item as a spek RFC + child issues by running +// `spektacular voice --from stdin --publish` inside the hive. When the +// binary is missing and Agent is set, it kicks that agent with a prompt +// instead (fallback until the hive image ships spektacular). +type SpekDestination struct { + Binary string + DefaultRepo string + // Token mints a GitHub token for repo; empty/nil inherits the process env. + Token func(ctx context.Context, repo string) (string, error) + // Exec runs the command; nil uses os/exec. Tests inject it. + Exec func(ctx context.Context, binary string, args []string, env []string, stdin string) (string, error) + // LookPath reports whether binary is runnable; nil uses exec.LookPath. + LookPath func(binary string) error + Agent string + Kick Kicker + Prefixes []string +} + +// Name implements Destination. +func (d *SpekDestination) Name() string { return DestSpekRFC } + +// ErrNoRepo is returned when neither the request nor config names a repo. +var ErrNoRepo = errors.New("spek_rfc: no repo given and inbox.spek.repo is unset") + +// Deliver implements Destination. +func (d *SpekDestination) Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) { + repo := strings.TrimSpace(req.Repo) + if repo == "" { + repo = strings.TrimSpace(d.DefaultRepo) + } + if repo == "" { + return Result{}, ErrNoRepo + } + title := StripPrefix(firstNonEmpty(c.Title, it.Title), d.prefixes()) + text := c.Text() + if strings.TrimSpace(text) == "" { + return Result{}, fmt.Errorf("spek_rfc: item %s has no transcript or summary", it.Key()) + } + binary := firstNonEmpty(d.Binary, "spektacular") + if err := d.lookPath(binary); err != nil { + if d.Agent != "" && d.Kick != nil { + prompt := RenderAgentPrompt(it, c, repo, title) + if kerr := d.Kick(d.Agent, prompt); kerr != nil { + return Result{}, fmt.Errorf("spek_rfc: %s unavailable and kick to %s failed: %w", binary, d.Agent, kerr) + } + return Result{Note: "delegated to agent " + d.Agent}, nil + } + return Result{}, fmt.Errorf("spek_rfc: %s is not installed in this hive (set inbox.spek.binary or inbox.spek.agent): %w", binary, err) + } + stdin := text + "\n\nProvenance: " + c.Provenance + "\n" + env := os.Environ() + if d.Token != nil { + tok, err := d.Token(ctx, repo) + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: github token: %w", err) + } + if tok != "" { + env = append(env, "GITHUB_TOKEN="+tok, "GH_TOKEN="+tok) + } + } + rctx, cancel := context.WithTimeout(ctx, spekTimeout) + defer cancel() + args := []string{"voice", "--from", "stdin", "--title", title, "--publish", "--repo", repo, "--format", "json"} + out, err := d.exec(rctx, binary, args, env, stdin) + if err != nil && isUnknownFlag(out, err) { + out, err = d.exec(rctx, binary, args[:len(args)-2], env, stdin) + } + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: %w: %s", err, lastLines(out, 5)) + } + res, perr := ParsePublishOutput(out) + if perr != nil { + return Result{}, fmt.Errorf("spek_rfc: %w", perr) + } + return res, nil +} + +func (d *SpekDestination) prefixes() []string { + if len(d.Prefixes) > 0 { + return d.Prefixes + } + return DefaultTitlePrefixes +} + +func (d *SpekDestination) lookPath(binary string) error { + if d.LookPath != nil { + return d.LookPath(binary) + } + _, err := exec.LookPath(binary) + return err +} + +func (d *SpekDestination) exec(ctx context.Context, binary string, args, env []string, stdin string) (string, error) { + if d.Exec != nil { + return d.Exec(ctx, binary, args, env, stdin) + } + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Env = env + cmd.Stdin = strings.NewReader(stdin) + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + return buf.String(), err +} + +func isUnknownFlag(out string, err error) bool { + s := strings.ToLower(out + " " + err.Error()) + return strings.Contains(s, "unknown flag") || strings.Contains(s, "flag provided but not defined") || strings.Contains(s, "unknown shorthand flag") +} + +// publishJSON is the `spektacular publish --format json` envelope (see +// spektacular internal/publish.Result). +type publishJSON struct { + RFCURL string `json:"rfc_url"` + Children []struct { + URL string `json:"url"` + } `json:"children"` +} + +var issueURLRe = regexp.MustCompile(`https://[^\s"'<>)]+/[^\s"'<>)/]+/[^\s"'<>)/]+/(?:issues|pull)/\d+`) + +// ParsePublishOutput accepts either the JSON envelope (`rfc_url`, +// `children[].url`) or plain text where the first issue URL is the RFC and +// the rest are children (what `spektacular voice` prints). +func ParsePublishOutput(out string) (Result, error) { + trimmed := strings.TrimSpace(out) + if start := strings.Index(trimmed, "{"); start >= 0 { + var pj publishJSON + if dec := json.NewDecoder(strings.NewReader(trimmed[start:])); dec.Decode(&pj) == nil && pj.RFCURL != "" { + res := Result{URL: pj.RFCURL} + for _, ch := range pj.Children { + if ch.URL != "" { + res.Children = append(res.Children, ch.URL) + } + } + return res, nil + } + } + urls := issueURLRe.FindAllString(out, -1) + seen := map[string]bool{} + var uniq []string + for _, u := range urls { + if !seen[u] { + seen[u] = true + uniq = append(uniq, u) + } + } + if len(uniq) == 0 { + return Result{}, fmt.Errorf("no RFC URL in spektacular output: %s", lastLines(out, 3)) + } + return Result{URL: uniq[0], Children: uniq[1:]}, nil +} + +var agentPromptTmpl = template.Must(template.New("spek").Parse(`Turn the voice recording below into a spek RFC with child issues on {{.Repo}}. + +Steps: +1. Ensure the spektacular CLI is available ("go install github.com/hivecommons/spektacular@latest" or a release binary). +2. In the {{.Repo}} clone, run "spektacular init" if .spektacular/ is missing. +3. Create a spec named "{{.Name}}" seeded from the transcript and summary below. Fill it non-interactively using your judgment; list anything unclear under "Open questions". +4. Run: spektacular publish {{.Name}} --repo {{.Repo}} --allow-draft --format json +5. Reply with the rfc_url and the child issue URLs from the JSON output. + +Provenance: {{.Provenance}} +Title: {{.Title}} + +{{if .Summary}}## Summary +{{.Summary}} + +{{end}}## Transcript +{{.Transcript}} +`)) + +// RenderAgentPrompt is the fallback kick prompt used when the hive image +// lacks the spektacular binary. +func RenderAgentPrompt(it Item, c Content, repo, title string) string { + transcript := c.Transcript + if transcript == "" { + transcript = c.Markdown + } + var buf bytes.Buffer + _ = agentPromptTmpl.Execute(&buf, map[string]string{ + "Repo": repo, + "Name": specName(title), + "Title": title, + "Summary": c.Summary, + "Transcript": transcript, + "Provenance": c.Provenance, + }) + return buf.String() +} + +// specName derives a kebab-case spec name from a title. +func specName(title string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(title) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + dash = false + } else if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + name := strings.TrimRight(b.String(), "-") + if name == "" { + name = "voice-rfc" + } + if len(name) > 60 { + name = strings.TrimRight(name[:60], "-") + } + return name +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func lastLines(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} diff --git a/src/pkg/inbox/store.go b/src/pkg/inbox/store.go new file mode 100644 index 0000000000..17f3e0caec --- /dev/null +++ b/src/pkg/inbox/store.go @@ -0,0 +1,218 @@ +package inbox + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" +) + +// entryState is the persisted per-item state, keyed by Key(source, id). +type entryState struct { + Routes map[string]*RouteState `json:"routes,omitempty"` + Dismissed bool `json:"dismissed,omitempty"` +} + +// Store persists route states as one JSON file next to connector state. +type Store struct { + Path string + Now func() time.Time + + mu sync.Mutex + entries map[string]*entryState + loaded bool +} + +// NewStore returns a store backed by path (created on first save). +func NewStore(path string) *Store { + return &Store{Path: path} +} + +func (s *Store) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func (s *Store) load() { + if s.loaded { + return + } + s.loaded = true + s.entries = map[string]*entryState{} + if s.Path == "" { + return + } + data, err := os.ReadFile(s.Path) + if err != nil { + return + } + _ = json.Unmarshal(data, &s.entries) + if s.entries == nil { + s.entries = map[string]*entryState{} + } +} + +func (s *Store) save() error { + if s.Path == "" { + return nil + } + data, err := json.MarshalIndent(s.entries, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(s.Path), 0o755); err != nil { + return err + } + tmp := s.Path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, s.Path) +} + +func (s *Store) entry(key string) *entryState { + e := s.entries[key] + if e == nil { + e = &entryState{Routes: map[string]*RouteState{}} + s.entries[key] = e + } + if e.Routes == nil { + e.Routes = map[string]*RouteState{} + } + return e +} + +// Apply copies persisted state onto items and derives Processed. +func (s *Store) Apply(items []Item) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + for i := range items { + e := s.entries[items[i].Key()] + if e == nil { + items[i].Processed = false + continue + } + items[i].Dismissed = e.Dismissed + items[i].Routes = cloneRoutes(e.Routes) + items[i].Processed = e.Dismissed + for _, r := range e.Routes { + if r.Status == StatusFiled { + items[i].Processed = true + } + } + } +} + +func cloneRoutes(in map[string]*RouteState) map[string]*RouteState { + if len(in) == 0 { + return nil + } + out := make(map[string]*RouteState, len(in)) + for k, v := range in { + c := *v + c.Children = append([]string(nil), v.Children...) + out[k] = &c + } + return out +} + +// Route returns the current state for one destination, if any. +func (s *Store) Route(key, dest string) (RouteState, bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entries[key] + if e == nil || e.Routes[dest] == nil { + return RouteState{}, false + } + return *e.Routes[dest], true +} + +// ErrRouteBusy is returned when a route is queued or running already. +var ErrRouteBusy = fmt.Errorf("route already in progress") + +// Begin transitions key/dest to queued. A queued or running route is +// rejected; filed and failed routes may be retried (filed = re-file). +func (s *Store) Begin(key, dest, repo string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + if cur := e.Routes[dest]; cur != nil && (cur.Status == StatusQueued || cur.Status == StatusRunning) { + return *cur, ErrRouteBusy + } + st := &RouteState{Status: StatusQueued, Repo: repo, UpdatedAt: s.now()} + e.Routes[dest] = st + return *st, s.save() +} + +// Set replaces the status (and optional outcome) of key/dest. +func (s *Store) Set(key, dest, status string, res Result, errText string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + st := e.Routes[dest] + if st == nil { + st = &RouteState{} + e.Routes[dest] = st + } + st.Status = status + st.UpdatedAt = s.now() + st.Error = errText + if status == StatusFiled { + st.URL = res.URL + st.Children = append([]string(nil), res.Children...) + } + return *st, s.save() +} + +// Dismiss marks the item knowledge-only (or clears that mark). +func (s *Store) Dismiss(key string, dismissed bool) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + s.entry(key).Dismissed = dismissed + return s.save() +} + +// Known returns every key the store has seen, sorted (for tests/debugging). +func (s *Store) Known() []string { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + out := make([]string, 0, len(s.entries)) + for k := range s.entries { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// Seen reports whether the store has any record of key; Sweep uses it so +// auto-rules only fire on items that arrived after the rule existed. +func (s *Store) Seen(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + _, ok := s.entries[key] + return ok +} + +// MarkSeen records key without any route so a later Sweep skips it. +func (s *Store) MarkSeen(key string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + if _, ok := s.entries[key]; ok { + return nil + } + s.entry(key) + return s.save() +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go index de0c27e432..f0ea586090 100644 --- a/src/pkg/knowledge/connector/plaud.go +++ b/src/pkg/knowledge/connector/plaud.go @@ -48,6 +48,12 @@ type plaudConnector struct { dial mcpDialer now func() time.Time stateDir string + // home is the MCP server's HOME (token store); empty inherits the + // process HOME. + home string + // callbackBase is where a pasted OAuth redirect is forwarded while a + // login session is alive (the MCP server's fixed localhost listener). + callbackBase string mu sync.Mutex login *plaudLogin @@ -79,6 +85,8 @@ func newPlaudConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { command = plaudDefaultCommand } c.dial = stdioMCPDialer(command, home) + c.home = home + c.callbackBase = plaudCallbackBase return c, nil } diff --git a/src/pkg/knowledge/connector/plaud_auth_complete.go b/src/pkg/knowledge/connector/plaud_auth_complete.go new file mode 100644 index 0000000000..c129fb93bd --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete.go @@ -0,0 +1,159 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// AuthCompletion carries what the operator pasted to finish a sign-in whose +// OAuth callback cannot reach the hive: either the full redirect URL the +// browser landed on, or the vendor token file contents from a laptop login. +type AuthCompletion struct { + RedirectURL string `json:"redirect_url,omitempty"` + Tokens string `json:"tokens,omitempty"` +} + +// AuthCompleter is optionally implemented by Authenticators whose OAuth +// client only allows a fixed localhost redirect (Plaud: localhost:8199). A +// hosted hive never receives that callback, so the dashboard offers two +// completions: forward the pasted redirect URL to the in-pod listener while +// the login session is alive, or install pasted tokens directly. +type AuthCompleter interface { + CompleteAuth(ctx context.Context, c AuthCompletion) (AuthState, error) +} + +var _ AuthCompleter = (*plaudConnector)(nil) + +const ( + // plaudCallbackBase is the MCP server's fixed OAuth listener. + plaudCallbackBase = "http://127.0.0.1:8199" + // plaudCallbackPath is the path the vendor registered as redirect URI. + plaudCallbackPath = "/auth/callback" + // plaudTokenFile is the MCP server's token store relative to HOME. + plaudTokenFile = ".plaud/tokens-mcp.json" + // plaudCallbackTimeout bounds the forwarded callback request. + plaudCallbackTimeout = 20 * time.Second + // plaudCompleteSettle is how long CompleteAuth waits for the login + // goroutine to notice the new session before reporting state. + plaudCompleteSettle = 6 * time.Second +) + +// ErrNoPendingLogin is returned when a redirect URL is pasted but no login +// session is alive to receive it (Connect must be called first). +var ErrNoPendingLogin = errors.New("plaud: no sign-in is pending; press Connect first, then paste the redirect URL within the login window") + +// CompleteAuth implements AuthCompleter. +func (c *plaudConnector) CompleteAuth(ctx context.Context, comp AuthCompletion) (AuthState, error) { + switch { + case strings.TrimSpace(comp.Tokens) != "": + if err := c.installTokens(comp.Tokens); err != nil { + return c.AuthStatus(ctx, false), err + } + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.auth = AuthState{Pending: false, CheckedAt: c.now().UTC(), Message: "tokens installed"} + c.mu.Unlock() + return c.AuthStatus(ctx, true), nil + case strings.TrimSpace(comp.RedirectURL) != "": + return c.forwardRedirect(ctx, comp.RedirectURL) + default: + return c.AuthStatus(ctx, false), errors.New("plaud: redirect_url or tokens is required") + } +} + +// forwardRedirect replays the pasted callback against the MCP server's +// localhost listener inside the pod. +func (c *plaudConnector) forwardRedirect(ctx context.Context, raw string) (AuthState, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.RawQuery == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: paste the full URL the browser landed on (it contains ?code=...&state=...)") + } + if u.Query().Get("code") == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: the pasted URL has no code parameter") + } + c.mu.Lock() + l := c.login + c.mu.Unlock() + if l == nil { + return c.AuthStatus(ctx, false), ErrNoPendingLogin + } + select { + case <-l.done: + return c.AuthStatus(ctx, false), ErrNoPendingLogin + default: + } + target := strings.TrimRight(c.callbackBase, "/") + plaudCallbackPath + "?" + u.RawQuery + rctx, cancel := context.WithTimeout(ctx, plaudCallbackTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodGet, target, nil) + if err != nil { + return c.AuthStatus(ctx, false), err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: forwarding callback to the login session: %w", err) + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + if resp.StatusCode >= 400 { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: callback rejected (%s): %s", resp.Status, firstLine(string(body))) + } + // The login goroutine polls get_current_user and flips state itself; + // give it a moment so the caller sees the outcome. + settle, cancelSettle := context.WithTimeout(ctx, plaudCompleteSettle) + defer cancelSettle() + select { + case <-l.done: + case <-settle.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +// installTokens validates and writes a pasted tokens-mcp.json into the +// connector's HOME so the MCP server picks it up on next start. +func (c *plaudConnector) installTokens(raw string) error { + var tok struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + } + if err := json.Unmarshal([]byte(raw), &tok); err != nil { + return fmt.Errorf("plaud: tokens must be the JSON contents of ~/.plaud/tokens-mcp.json: %w", err) + } + if tok.AccessToken == "" && tok.RefreshToken == "" { + return errors.New("plaud: tokens JSON has neither access_token nor refresh_token") + } + home := c.home + if home == "" { + h, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("plaud: resolving HOME: %w", err) + } + home = h + } + path := filepath.Join(home, plaudTokenFile) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("plaud: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(raw)+"\n"), 0o600) +} + +// TokenPath returns where installTokens writes (tests). +func (c *plaudConnector) tokenPath() string { + home := c.home + if home == "" { + home, _ = os.UserHomeDir() + } + return filepath.Join(home, plaudTokenFile) +} diff --git a/src/pkg/knowledge/connector/plaud_auth_complete_test.go b/src/pkg/knowledge/connector/plaud_auth_complete_test.go new file mode 100644 index 0000000000..a4a86c5b1a --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete_test.go @@ -0,0 +1,79 @@ +package connector + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestPlaudCompleteAuthInstallsPastedTokens(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, nil) + raw := `{"access_token":"at","refresh_token":"rt","token_type":"Bearer","expires_at":1}` + st, err := pc.CompleteAuth(context.Background(), AuthCompletion{Tokens: raw}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(pc.tokenPath()) + if err != nil { + t.Fatalf("token file not written: %v", err) + } + if strings.TrimSpace(string(data)) != raw { + t.Fatalf("token file = %q", data) + } + if !strings.HasSuffix(pc.tokenPath(), ".plaud/tokens-mcp.json") || !strings.HasPrefix(pc.tokenPath(), pc.stateDir) { + t.Fatalf("token path %q should live under the connector home", pc.tokenPath()) + } + if !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("expected refreshed connected state, got %+v", st) + } +} + +func TestPlaudCompleteAuthRejectsBadInput(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), nil) + ctx := context.Background() + if _, err := pc.CompleteAuth(ctx, AuthCompletion{}); err == nil { + t.Fatal("empty completion must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: "not json"}); err == nil { + t.Fatal("non-JSON tokens must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: `{"token_type":"Bearer"}`}); err == nil { + t.Fatal("tokens without access/refresh token must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback"}); err == nil { + t.Fatal("redirect without query must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=x&state=y"}); err != ErrNoPendingLogin { + t.Fatalf("redirect without pending login: got %v, want ErrNoPendingLogin", err) + } +} + +func TestPlaudCompleteAuthForwardsRedirectToPendingLogin(t *testing.T) { + // A live login session: the fake callback listener plays the login + // goroutine, which finishes (closes done) once the callback lands. + l := &plaudLogin{done: make(chan struct{}), cancel: func() {}} + got := make(chan string, 1) + cb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.URL.Path + "?" + r.URL.RawQuery + close(l.done) + w.WriteHeader(http.StatusOK) + })) + defer cb.Close() + pc := newTestPlaud(t, newPlaudTestServer(), nil) + pc.callbackBase = cb.URL + pc.mu.Lock() + pc.login = l + pc.auth = AuthState{Pending: true} + pc.mu.Unlock() + _, err := pc.CompleteAuth(context.Background(), AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=abc&state=xyz"}) + if err != nil { + t.Fatal(err) + } + if fwd := <-got; fwd != "/auth/callback?code=abc&state=xyz" { + t.Fatalf("forwarded %q", fwd) + } +} From 71c59363763bad5e91868a7fb5b60f9780f78af5 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 08:19:04 +0000 Subject: [PATCH 072/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.230.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.230.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-workflow-enabled-toggle.md | 1 - changelog.d/fixed-proxy-plain-forward-report.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-workflow-enabled-toggle.md delete mode 100644 changelog.d/fixed-proxy-plain-forward-report.md diff --git a/CHANGELOG.md b/CHANGELOG.md index da5445321a..fd9b669559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.230.0) + +### Added + +- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. + +### Fixed + +- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. + ## 2026-11-12 (v5.229.0) ### Added diff --git a/changelog.d/added-workflow-enabled-toggle.md b/changelog.d/added-workflow-enabled-toggle.md deleted file mode 100644 index 44fc411700..0000000000 --- a/changelog.d/added-workflow-enabled-toggle.md +++ /dev/null @@ -1 +0,0 @@ -- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. diff --git a/changelog.d/fixed-proxy-plain-forward-report.md b/changelog.d/fixed-proxy-plain-forward-report.md deleted file mode 100644 index 2e525cf0dd..0000000000 --- a/changelog.d/fixed-proxy-plain-forward-report.md +++ /dev/null @@ -1 +0,0 @@ -- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. From a99b54035e9ca9dbbb300bc7e9b55fb750074332 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 01:43:15 -0700 Subject: [PATCH 073/166] =?UTF-8?q?=F0=9F=90=9B=20PR=20comments:=20overlap?= =?UTF-8?q?/mutation=20only=20on=20a=20finding,=20thank=20first-time=20con?= =?UTF-8?q?tributors=20only,=20no=20self-/lgtm=20relay=20on=20Prow=20PRs?= =?UTF-8?q?=20(#17870)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 PR comments: post overlap/mutation reports only on a finding, thank first-time contributors only, no self-/lgtm relay on Prow PRs Fixes #17863. - .github/scripts/pr-overlap.sh: create the <!-- hive-pr-overlap --> comment only when a sibling PR conflicts or shares a file; a later clean run edits the existing comment to a one-line Resolved note; never post for a PR that never had a finding. The step summary keeps the full report. - src/scripts/mutation-report.sh post: same for <!-- hive-mutation --> on a pass verdict; the report is appended to the step summary first. - src/scripts/feedback-gate.sh (+ test): decides whether the merge thank-you is posted — skips bot/App authors, scanner/* and hive/* branches, hive-origin bodies and returning contributors; FEEDBACK_* env knobs. Wired by the feedback.yml change in the next commit. - pkg/github: a dashboard approval on a Prow-managed, hive-authored PR relays /approve only (LGTMSkipped); Prow rejects a self-/lgtm every time and no hive lane reads the relayed /lgtm. - docs: notifications.md (new section + Prow config location), glossary, running-at-level-6; changelog fragment. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 workflows: gate the Feedback Wanted thank-you on src/scripts/feedback-gate.sh Refs #17863. Workflow-only commit: adds a gate job (self-test + decision) before the hivecommons/infra reusable-feedback call, which runs only when the gate outputs post=true. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 governance: re-pin TRUSTED_SCRIPTS_TREE for the feedback-gate and mutation-report scripts Refs #17863. src/scripts gained feedback-gate.sh / test-feedback-gate.sh and mutation-report.sh changed, so the pinned tree moves to 1f1eb9048. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .github/scripts/integration-governance.cjs | 2 +- .github/scripts/pr-overlap.sh | 27 ++++- .github/scripts/test-pr-overlap.sh | 27 ++++- .github/workflows/feedback.yml | 30 +++++ .../changed-17863-pr-comment-quartet.md | 1 + .../glossary-and-operations.md | 2 +- src/docs/notifications.md | 27 ++++- src/docs/running-at-level-6.md | 2 +- src/pkg/github/pr_decision_operator.go | 32 ++++- src/pkg/github/pr_decision_operator_test.go | 41 ++++++- src/scripts/feedback-gate.sh | 109 ++++++++++++++++++ src/scripts/mutation-report.sh | 22 ++++ src/scripts/test-feedback-gate.sh | 80 +++++++++++++ src/scripts/test-mutation-report.sh | 37 ++++++ 14 files changed, 423 insertions(+), 16 deletions(-) create mode 100644 changelog.d/changed-17863-pr-comment-quartet.md create mode 100755 src/scripts/feedback-gate.sh create mode 100755 src/scripts/test-feedback-gate.sh diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index a1cb7d505b..87ad4e2ce7 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '718b8f2888041981d0779cfd19d90484ac01b8a9'; +const TRUSTED_SCRIPTS_TREE = '1f1eb904839561af15c1772e6d9106a3b8d2b028'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/.github/scripts/pr-overlap.sh b/.github/scripts/pr-overlap.sh index f5fdaa2692..e354d26979 100755 --- a/.github/scripts/pr-overlap.sh +++ b/.github/scripts/pr-overlap.sh @@ -240,10 +240,19 @@ report_body() { } } +# One-line body the existing marker comment is edited to once a PR that had +# overlaps no longer has any. A PR that never had a finding gets no comment +# at all: the full clean report goes to the step summary only (#17863). +resolved_body() { + local base=$1 + # shellcheck disable=SC2016 # backticks are literal Markdown + printf '%s\n\n### Open PR overlap check\n\n✅ Resolved: no conflicting open PRs or same-file overlaps remain against `%s`.\n' "$MARKER" "$base" +} + upsert_comment_and_label() { - local pr=$1 body=$2 has_conflicts=$3 + local pr=$1 body=$2 has_conflicts=$3 has_any=${4:-1} base=${5:-} if ((DRY_RUN)); then - printf 'DRY RUN: would update PR #%s comment and label=%s\n' "$pr" "$has_conflicts" + printf 'DRY RUN: would update PR #%s comment and label=%s findings=%s\n' "$pr" "$has_conflicts" "$has_any" printf '%s\n' "$body" return 0 fi @@ -251,7 +260,13 @@ upsert_comment_and_label() { local comment_id comment_id=$(api "repos/${REPO}/issues/${pr}/comments?per_page=100" --paginate \ --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -n1) - if [[ -n "$comment_id" ]]; then + if [[ "$has_any" != '1' ]]; then + if [[ -n "$comment_id" ]]; then + api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$(resolved_body "$base")" >/dev/null + else + printf 'PR #%s: no overlap findings and no earlier comment; not posting\n' "$pr" + fi + elif [[ -n "$comment_id" ]]; then api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$body" >/dev/null else api -X POST "repos/${REPO}/issues/${pr}/comments" -f body="$body" >/dev/null @@ -276,12 +291,12 @@ write_summary() { run_for_one_pr() { local pr=$1 base=$2 writable=$3 truncated=${4:-0} - local rows has_conflicts _has_any base_conflict body - make_report_for_pr "$pr" "$base" rows has_conflicts _has_any base_conflict + local rows has_conflicts has_any base_conflict body + make_report_for_pr "$pr" "$base" rows has_conflicts has_any base_conflict body=$(report_body "$pr" "$base" "$rows" "$truncated" "$base_conflict") write_summary "$body" if [[ "$writable" == '1' ]]; then - upsert_comment_and_label "$pr" "$body" "$has_conflicts" + upsert_comment_and_label "$pr" "$body" "$has_conflicts" "$has_any" "$base" fi } diff --git a/.github/scripts/test-pr-overlap.sh b/.github/scripts/test-pr-overlap.sh index f3581977cc..3b553a279c 100755 --- a/.github/scripts/test-pr-overlap.sh +++ b/.github/scripts/test-pr-overlap.sh @@ -109,7 +109,7 @@ case "$*" in *'.head.repo.full_name'*) printf '%s\n' "$REPO_SLUG" ;; *'.head.sha'*) printf '%s\n' "${FALLBACK_SHA:-}" ;; *'.base.ref'*) printf 'v6\n' ;; - *'/comments?'*) : ;; + *'/comments?'*) if [[ -n "${EXISTING_COMMENT_ID:-}" ]]; then printf '%s\n' "$EXISTING_COMMENT_ID"; fi ;; *) echo "unexpected gh api call: $*" >&2; exit 2 ;; esac GH @@ -134,6 +134,8 @@ run_pr() { # pr -> sets OUT and WRITES LABEL_ADD="-X POST repos/${REPO_SLUG}/issues/%s/labels -f labels[]=conflicts-with-open-pr" LABEL_DEL="-X DELETE repos/${REPO_SLUG}/issues/%s/labels/conflicts-with-open-pr" +COMMENT_POST="-X POST repos/${REPO_SLUG}/issues/%s/comments -f body=" +COMMENT_PATCH="-X PATCH repos/${REPO_SLUG}/issues/comments/%s -f body=" echo 'PR 2: real sibling conflict with PR 3, stale PR 1 is not blamed' run_pr 2 @@ -144,6 +146,16 @@ assert_not_contains "$OUT" '| #1 |' 'does not blame PR 1 for its conflict with t assert_not_contains "$OUT" '| #4 |' 'does not link the unrelated stale PR 4' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_ADD" 2)" 'labels PR 2 conflicts-with-open-pr' +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'posts the overlap comment when there is a finding' + +echo 'PR 2: an existing marker comment is edited, not duplicated' +EXISTING_COMMENT_ID=4242 run_pr 2 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 4242)" 'edits the existing marker comment' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'does not post a second comment' echo 'PR 5: add/add of the same new path with different content' run_pr 5 @@ -167,9 +179,20 @@ assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 1)" 'removes the sibling-con echo 'PR 4: stale but clean, no overlap' run_pr 4 assert_exit0 -assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap' +assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap in the summary' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 4)" 'removes the sibling-conflict label' +assert_not_contains "$WRITES" '/comments' 'posts no comment when there is nothing to report (#17863)' +assert_contains "$OUT" 'no overlap findings and no earlier comment; not posting' 'logs the skipped post' + +echo 'PR 4: a clean run edits an earlier finding comment to a one-line resolved note' +EXISTING_COMMENT_ID=777 run_pr 4 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 777)<!-- hive-pr-overlap -->" 'edits the existing comment, keeping the marker' +assert_contains "$WRITES" 'Resolved: no conflicting open PRs or same-file overlaps remain against `v6`' 'collapses to the resolved line' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 4)" 'does not post a new comment' echo 'PR 3: missing pull ref falls back to the API head and replaces a stale local ref' export FALLBACK_SHA diff --git a/.github/workflows/feedback.yml b/.github/workflows/feedback.yml index 0d4f5de803..14872a8e7d 100644 --- a/.github/workflows/feedback.yml +++ b/.github/workflows/feedback.yml @@ -14,5 +14,35 @@ permissions: pull-requests: write jobs: + # The reusable workflow thanks every merged PR unconditionally; the gate + # keeps the comment for a person's first merged PR only (#17863). Rules and + # knobs: src/scripts/feedback-gate.sh. + gate: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + post: ${{ steps.gate.outputs.post }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Self-test feedback gate + run: bash src/scripts/test-feedback-gate.sh + - id: gate + name: Decide whether to thank the author + env: + GH_TOKEN: ${{ github.token }} + FEEDBACK_SKIP_BOTS: '1' + FEEDBACK_SKIP_BRANCHES: 'scanner/* hive/*' + FEEDBACK_SKIP_HIVE_ORIGIN: '1' + FEEDBACK_FIRST_TIME_ONLY: '1' + run: bash src/scripts/feedback-gate.sh + feedback: + needs: gate + if: needs.gate.outputs.post == 'true' uses: hivecommons/infra/.github/workflows/reusable-feedback.yml@f1c0b61675386f54e63c04cf07b3aa933cc963eb # hivecommons/infra main as of 2026-09-02 diff --git a/changelog.d/changed-17863-pr-comment-quartet.md b/changelog.d/changed-17863-pr-comment-quartet.md new file mode 100644 index 0000000000..eb5412ae90 --- /dev/null +++ b/changelog.d/changed-17863-pr-comment-quartet.md @@ -0,0 +1 @@ +- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 5c1d52e9ae..64a5606d00 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -225,7 +225,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str ## Non-label equivalents - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. -- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks an overlap notice, `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). +- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks the one overlap notice a PR gets (posted only when a sibling PR conflicts or shares a file, edited down to a "Resolved" line when that clears; [#17863](https://github.com/hivecommons/hive/issues/17863)), `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. - **Contributor queue holds** such as active leases, cooldowns, dependencies, and quota guard holds suppress contributor offers without touching GitHub labels. - **Jira/Linear labels** map only where the work-source adapter supports them: Linear skips held work using substring hold labels; Jira skips exact configured hold labels; GitHub Projects currently imports labels but does not use a hold gate. diff --git a/src/docs/notifications.md b/src/docs/notifications.md index d036a5c274..38427784fc 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -703,7 +703,32 @@ and exits non-zero instead of posting a possible duplicate. `.github/scripts/test-hive-status-card.sh` and `src/scripts/test-comment-merged-refs.sh` pin this behaviour. +### Per-PR workflow comments post only on a finding + +Three workflow comments used to land on every PR regardless of outcome, and +after the runtime's own reducers they were ~70% of the comments on a merged +PR ([#17863](https://github.com/hivecommons/hive/issues/17863)). Each now +posts only when it has something to say; the full report, including the +clean case, still goes to the job's step summary: + +| Workflow / script | Comment | Behaviour | +| --- | --- | --- | +| Open PR Overlap (`.github/workflows/pr-overlap.yml`, `.github/scripts/pr-overlap.sh`) | `<!-- hive-pr-overlap -->` table of conflicting / same-file open PRs | **Posts** only when at least one sibling PR conflicts or shares a file; **edits** the one marker comment in place on later pushes; when a later run finds nothing and a comment exists, **edits it** to a one-line "✅ Resolved" note; a PR that never had an overlap gets no comment. The `conflicts-with-open-pr` label is unchanged. | +| Mutation (advisory) (`.github/workflows/mutation-advisory.yml`, `src/scripts/mutation-report.sh post`) | `<!-- hive-mutation -->` surviving / uncovered mutant report | **Posts** only for a non-`pass` verdict (`lived`, `uncovered`, `misrouted`, `error`); **edits** the one marker comment on later pushes; a later `pass` **edits** the existing comment to a one-line "✅ Resolved" note; a PR whose every run passes gets no comment. | +| Feedback Wanted (`.github/workflows/feedback.yml` → `hivecommons/infra` `reusable-feedback.yml`, gated by `src/scripts/feedback-gate.sh`) | "Thank you for your contribution! Your PR has been merged." | **Posts** only for a person's first merged PR in the repo. Skipped for `[bot]` / App authors, hive-managed head branches (`scanner/*`, `hive/*`), PR bodies carrying a `<!-- hive:` marker (hive-agent PRs on any branch, e.g. `fix-*`), and any author with another merged PR. Knobs are the workflow's `env:` (`FEEDBACK_SKIP_BOTS`, `FEEDBACK_SKIP_BRANCHES`, `FEEDBACK_SKIP_HIVE_ORIGIN`, `FEEDBACK_SKIP_LOGINS`, `FEEDBACK_FIRST_TIME_ONLY`); a failed merged-PR lookup skips the comment rather than guessing. | + +`.github/scripts/test-pr-overlap.sh`, `src/scripts/test-mutation-report.sh` +and `src/scripts/test-feedback-gate.sh` pin these rules. + **Outside Hive's control.** Comments from external bots, such as Prow, tide and kubestellar-prow, are posted by their own services and cannot be edited, folded or skipped from this repository. Hive does not try to suppress them; -configure those bots in the project that runs them. +configure those bots in the project that runs them. For hivecommons repos +that configuration is `prow/plugins.yaml` and `prow/config.yaml` in +`kubestellar/infra` (org-wide `approve` with `require_self_approval: true` +and `lgtm` with `review_acts_as_lgtm: true`; tide requires both `lgtm` and +`approved`). What Hive does control is its own relay: a dashboard approval +on a Prow-managed PR the hive account opened posts `/approve` only, because +a `/lgtm` from the PR's own author draws Prow's "you cannot LGTM your own +PR" every time (`PullRequestApprovalEvent.LGTMSkipped`, #17863); no hive +lane reads the relayed `/lgtm`. diff --git a/src/docs/running-at-level-6.md b/src/docs/running-at-level-6.md index bbab0bb8f2..b7af5ce5fd 100644 --- a/src/docs/running-at-level-6.md +++ b/src/docs/running-at-level-6.md @@ -143,7 +143,7 @@ Config keys to cross-check: `project.issue_filter.require_labels`, `project.issu | `ci-flake` | CI/Hive scripts | Flake tracking, not an L6 hold itself. | Fix/quarantine the flake while keeping required checks meaningful. | | `user-feedback` | Dashboard/hub | Feedback provenance. | Triage reporter input; apply trust/hold policy as needed. | | `dco` / `no-dco` (Prow's `dco-signoff: yes|no` coexist on Prow-gated repos; the hive never adds or removes them) | Hive stage pass (commit check) | Merge-eligibility display on every managed repo. Where DCO is required (detected from required/head checks or configured via `governor.labels.stage.dco.required`), `no-dco` routes by author kind: `070` (hive/self-authored), `071` (human), or `072` (clanker); the 072 nudge includes the rebase command. On ungated repos it is informational only: the lane's stage is unchanged and no DCO rebase is requested. See [DCO routing](labels-and-control-signals/stage-labels.md#dco--no-dco--the-hive-owned-sign-off-signal). | Fix commit sign-offs (`git rebase --signoff`) rather than hand-editing labels. | -| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal. | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | +| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal (the companion `<!-- hive-pr-overlap -->` comment appears only when there is a conflict or same-file overlap). | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | | `has-open-pr` | Pre-dispatch guard | An open PR already links the issue with a closing keyword; the hive skipped the claim. | Nothing — the label is a tracker signal, not a park; see `scanner.skip_issues_with_open_pr` in [operator-reference.md](operator-reference.md#open-pr-dispatch-guard-and-duplicate-fix-sweep-scanner). | | `no-changelog` | Dependabot/maintainer | Repo changelog exception, not a Hive safety gate. | Apply only under repo policy. | | `human-only` remainder text (not a default label) | Agent/reviewer | Task-list sweep can park remaining issue work as `needs-human`. | Treat as a handoff; add PR `hold` if merging must stop. | diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index 6ca3c432df..8f0e30e6e2 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -94,6 +94,13 @@ type PullRequestApprovalEvent struct { // reply stays on the PR, so the next Approve takes the CommandsSkipped // path (hivecommons/hive#17475). ProwRejected bool + // LGTMSkipped is true when only `/approve` was relayed: the PR is + // Prow-managed and was opened by the hive's own account, so a `/lgtm` + // posted from that account is the PR author's and Prow's lgtm plugin + // answers every one with "you cannot LGTM your own PR" + // (hivecommons/hive#17863). No hive lane reads the relayed `/lgtm`; + // the `/approve` comment carries the attribution they use. + LGTMSkipped bool // ReportingError preserves partial progress: the native approval stands, // but comments, labels or stage reporting must be retried/reconciled. ReportingError string @@ -503,19 +510,40 @@ func (c *Client) reportOperatorPullRequestApproval(ctx context.Context, owner, n c.logger.Info("pull request approval commands not posted: Prow rejected the hive's earlier commands on this PR (bot is not an approver); the hive's own approval record stands", "repo", event.Repo, "pr", number, "actor", actor, "head", head) } } else { - for _, command := range PRApprovalCommands { + commands := PRApprovalCommands + if event.ProwManaged && selfAuthored { + commands = prApprovalCommandsWithoutLGTM(commands) + event.LGTMSkipped = true + } + // Each relayed command carries the prApprovalCommentMarkerPrefix + // marker bound to head, which is what prowRejectedHiveCommands and + // the sentinel/workflow-approval readers key on; once posted for a + // head it is not posted again (CommandsSkipped above). + for _, command := range commands { if err := c.CreateIssueComment(ctx, full, number, renderPRApprovalCommandComment(command, actor, head)); err != nil { return fmt.Errorf("posting %s on %s/%s#%d: %w", command, owner, name, number, err) } } if c.logger != nil { - c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head, "lgtm_skipped_self_authored", event.LGTMSkipped) } event.ProwRejected = c.awaitProwRejection(ctx, owner, name, number, event.ProwManaged) } return ctx.Err() } +// prApprovalCommandsWithoutLGTM returns commands minus `/lgtm`, for a +// Prow-managed PR the posting account itself authored (#17863). +func prApprovalCommandsWithoutLGTM(commands []string) []string { + out := make([]string, 0, len(commands)) + for _, command := range commands { + if command != "/lgtm" { + out = append(out, command) + } + } + return out +} + // ProwManagedLabels reports whether labels carry Prow's plugin markers: // the approve plugin's `approved`, size's `size/*`, the `do-not-merge/*` // family and trigger's `ok-to-test` / `needs-ok-to-test`. diff --git a/src/pkg/github/pr_decision_operator_test.go b/src/pkg/github/pr_decision_operator_test.go index c6a5746ad2..96afbdc68a 100644 --- a/src/pkg/github/pr_decision_operator_test.go +++ b/src/pkg/github/pr_decision_operator_test.go @@ -390,6 +390,42 @@ func TestRecordOperatorPullRequestApprovalSkipsCommandsProwRejected(t *testing.T } } +// On a Prow-managed PR the hive account opened, a `/lgtm` from that account +// is a self-LGTM Prow always rebukes; only `/approve` is relayed +// (hivecommons/hive#17863). A person's PR and a non-Prow repo keep both. +func TestRecordOperatorPullRequestApprovalSkipsSelfLGTMOnProwPR(t *testing.T) { + for _, tc := range []struct { + name string + pr unparkWireIssue + wantLGTM bool + wantSkipped bool + }{ + {"hive-authored prow-managed", parkedPR(liveDecisionPR, "hive[bot]", []string{PRNeedsDecisionStageLabel, "author/hive", "size/M"}), false, true}, + {"hive-authored not prow-managed", hiveParkedPR(), true, false}, + {"person-authored prow-managed", parkedPR(liveDecisionPR, "contributor", []string{PRNeedsDecisionStageLabel, "author/human", "size/M"}), true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + c, rec := newUnparkPRClient(t, []unparkWireIssue{tc.pr}, map[string]string{"maintainer": "write"}) + event, err := c.RecordOperatorPullRequestApproval(context.Background(), "org/repo", liveDecisionPR, "maintainer") + if err != nil { + t.Fatalf("approval: %v", err) + } + if event.LGTMSkipped != tc.wantSkipped { + t.Fatalf("event = %+v, want LGTMSkipped=%v", event, tc.wantSkipped) + } + posted := rec.posted[liveDecisionPR] + var gotLGTM, gotApprove bool + for _, body := range posted { + gotLGTM = gotLGTM || strings.HasPrefix(body, "/lgtm\n") + gotApprove = gotApprove || strings.HasPrefix(body, "/approve\n") + } + if gotLGTM != tc.wantLGTM || !gotApprove { + t.Fatalf("posted %v, want lgtm=%v and /approve", posted, tc.wantLGTM) + } + }) + } +} + // On a Prow repo where rejection has not been observed yet, Prow answers the // relayed commands after they are posted. The approval event reports that // reply instead of leaving the board at "approved" (hivecommons/hive#17475). @@ -418,8 +454,9 @@ func TestRecordOperatorPullRequestApprovalReportsProwRejectionAfterPost(t *testi if err != nil { t.Fatalf("approval: %v", err) } - if event.ProwRejected != tc.want || event.CommandsSkipped != "" || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands) { - t.Fatalf("event = %+v, want ProwRejected %v with commands posted", event, tc.want) + // Hive-authored and Prow-managed: only `/approve` is relayed (#17863). + if event.ProwRejected != tc.want || event.CommandsSkipped != "" || !event.LGTMSkipped || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands)-1 { + t.Fatalf("event = %+v, want ProwRejected %v with /approve posted", event, tc.want) } }) } diff --git a/src/scripts/feedback-gate.sh b/src/scripts/feedback-gate.sh new file mode 100755 index 0000000000..50c1c0e314 --- /dev/null +++ b/src/scripts/feedback-gate.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# feedback-gate.sh — decides whether the post-merge "Thank you for your +# contribution!" comment (.github/workflows/feedback.yml → +# hivecommons/infra reusable-feedback.yml) should be posted on a merged PR. +# +# The reusable workflow is unconditional, so before #17863 every merged PR +# got the comment: 38 of 40 in the 2026-11-11 spam sample were on hive-App or +# maintainer PRs that nobody reads it on. The comment is for people merging +# their first change, so the gate keeps it for exactly that case. +# +# Reads the pull_request event from $GITHUB_EVENT_PATH (or the FEEDBACK_PR_* +# overrides below, used by the self-test) and writes `post=true|false` and +# `reason=<why>` to $GITHUB_OUTPUT (stdout when unset). Exit status is 0 in +# both cases; only a broken event or API read exits non-zero, which the caller +# treats as "do not post". +# +# Configuration (workflow `env:`), each with its default: +# FEEDBACK_SKIP_BOTS=1 skip `[bot]` / type=Bot authors +# FEEDBACK_SKIP_BRANCHES='scanner/* hive/*' +# space-separated globs of hive-managed +# head branches to skip +# FEEDBACK_SKIP_HIVE_ORIGIN=1 skip PRs whose body carries a `<!-- hive:` +# marker (hive-agent PRs on any branch, +# e.g. `fix-*`) +# FEEDBACK_SKIP_LOGINS='' space-separated logins never thanked +# FEEDBACK_FIRST_TIME_ONLY=1 skip authors with another merged PR in +# the repo (needs GH_TOKEN; a failed lookup +# skips the comment rather than guessing) +# +# Overrides for tests / manual runs: FEEDBACK_PR_AUTHOR, FEEDBACK_PR_AUTHOR_TYPE, +# FEEDBACK_PR_HEAD_REF, FEEDBACK_PR_BODY, FEEDBACK_PR_MERGED, FEEDBACK_PR_NUMBER, +# FEEDBACK_MERGED_COUNT (skips the API lookup). +set -euo pipefail + +skip_bots="${FEEDBACK_SKIP_BOTS:-1}" +skip_branches="${FEEDBACK_SKIP_BRANCHES-scanner/* hive/*}" +skip_hive_origin="${FEEDBACK_SKIP_HIVE_ORIGIN:-1}" +skip_logins="${FEEDBACK_SKIP_LOGINS:-}" +first_time_only="${FEEDBACK_FIRST_TIME_ONLY:-1}" +repo="${GITHUB_REPOSITORY:-}" + +lower() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } + +event_field() { + local path=$1 + if [[ -n "${GITHUB_EVENT_PATH:-}" && -f "${GITHUB_EVENT_PATH}" ]]; then + jq -r "${path} // empty" "$GITHUB_EVENT_PATH" + fi +} + +author="${FEEDBACK_PR_AUTHOR:-$(event_field '.pull_request.user.login')}" +author_type="${FEEDBACK_PR_AUTHOR_TYPE:-$(event_field '.pull_request.user.type')}" +head_ref="${FEEDBACK_PR_HEAD_REF:-$(event_field '.pull_request.head.ref')}" +body="${FEEDBACK_PR_BODY-$(event_field '.pull_request.body')}" +merged="${FEEDBACK_PR_MERGED:-$(event_field '.pull_request.merged')}" +number="${FEEDBACK_PR_NUMBER:-$(event_field '.pull_request.number')}" + +decide() { + local post=$1 reason=$2 + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'post=%s\nreason=%s\n' "$post" "$reason" >> "$GITHUB_OUTPUT" + fi + printf 'feedback-gate: post=%s (%s)\n' "$post" "$reason" + exit 0 +} + +[[ -n "$author" ]] || { echo 'feedback-gate: no PR author in the event' >&2; exit 1; } +[[ "$merged" == 'true' ]] || decide false 'pr-not-merged' + +if [[ "$skip_bots" == '1' ]] && { [[ "$author" == *'[bot]' ]] || [[ "$(lower "$author_type")" == 'bot' ]]; }; then + decide false "bot-author:${author}" +fi + +for login in $skip_logins; do + if [[ "$(lower "$login")" == "$(lower "$author")" ]]; then + decide false "listed-login:${author}" + fi +done + +for pattern in $skip_branches; do + # shellcheck disable=SC2053 # the glob in $pattern is meant to expand + if [[ -n "$head_ref" && "$head_ref" == $pattern ]]; then + decide false "hive-branch:${head_ref}" + fi +done + +if [[ "$skip_hive_origin" == '1' && "$body" == *'<!-- hive:'* ]]; then + decide false 'hive-origin-body' +fi + +if [[ "$first_time_only" == '1' ]]; then + merged_count="${FEEDBACK_MERGED_COUNT:-}" + if [[ -z "$merged_count" ]]; then + [[ -n "$repo" ]] || { echo 'feedback-gate: GITHUB_REPOSITORY unset' >&2; exit 1; } + # The event fires after the merge, so the PR being judged is already one + # of the author's merged PRs: "first-time" means it is the only one. + merged_count=$(gh api -X GET search/issues -f "q=repo:${repo} is:pr is:merged author:${author}" -F per_page=1 --jq '.total_count') \ + || { echo "feedback-gate: merged-PR lookup for ${author} failed; not posting" >&2; exit 1; } + fi + if [[ ! "$merged_count" =~ ^[0-9]+$ ]]; then + echo "feedback-gate: unexpected merged count '${merged_count}'" >&2 + exit 1 + fi + if (( merged_count > 1 )); then + decide false "returning-contributor:${author}:${merged_count}-merged" + fi +fi + +decide true "first-merged-pr:${author}${number:+:#${number}}" diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index 6bac832ad0..8dcf9c2522 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -100,11 +100,33 @@ post_comment() { echo "PR_NUMBER/GITHUB_REPOSITORY unset; not posting a comment." return 0 fi + # The report always lands in the step summary; the PR comment is reserved + # for findings (any verdict but `pass`). A clean run never creates a + # comment, and one that follows an earlier finding edits that comment down + # to a one-line resolved note instead of posting again (#17863). + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$body" >> "$GITHUB_STEP_SUMMARY" + fi + local verdict="" resolved="" + [ -f "$verdict_file" ] && verdict=$(cat "$verdict_file") + if [ "$verdict" = pass ]; then + # shellcheck disable=SC2016 # backticks are literal Markdown + resolved="$(printf '%s\n### Mutation testing\n\n✅ Resolved: no surviving or uncovered mutants in changed files on the current head.\n\n_Verdict: `pass` (#17349)._\n' "$marker")" + fi # Only upsert the bot's own report so another participant cannot pre-seat or # get overwritten by a comment that merely starts with the marker. local id id=$(gh api --paginate "repos/${repo}/issues/${pr_number}/comments" \ --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"${marker}\"))) | .id" 2>/dev/null | head -n 1 || true) + if [ -n "$resolved" ]; then + if [ -z "$id" ]; then + echo "mutation: verdict pass and no earlier comment; not posting (report is in the step summary)." + return 0 + fi + gh api -X PATCH "repos/${repo}/issues/comments/${id}" -f "body=${resolved}" >/dev/null \ + || echo "::warning::could not edit mutation comment" + return 0 + fi if [ -n "$id" ]; then gh api -X PATCH "repos/${repo}/issues/comments/${id}" -F "body=@${body}" >/dev/null \ || echo "::warning::could not edit mutation comment" diff --git a/src/scripts/test-feedback-gate.sh b/src/scripts/test-feedback-gate.sh new file mode 100755 index 0000000000..3864f8ff04 --- /dev/null +++ b/src/scripts/test-feedback-gate.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# test-feedback-gate.sh — pins the gating rules of feedback-gate.sh (#17863) +# with a stub `gh` so no network access is needed. +set -u -o pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="${HERE}/feedback-gate.sh" +TMP_ROOT="${HERE}/../.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/feedback-gate.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +mkdir -p "$TMP/bin" +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *search/issues*) + [ -n "${GH_FAIL:-}" ] && exit 1 + printf '%s\n' "${GH_TOTAL:-1}" ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" + +# run <expected-post> <label> [VAR=value ...] +run() { + local want=$1 label=$2; shift 2 + local out="$TMP/output"; : > "$out" + local result rc + result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + FEEDBACK_PR_MERGED=true FEEDBACK_PR_AUTHOR=newcomer FEEDBACK_PR_AUTHOR_TYPE=User \ + FEEDBACK_PR_HEAD_REF=feat/thing FEEDBACK_PR_BODY='plain body' FEEDBACK_PR_NUMBER=7 \ + "$@" bash "$SCRIPT" 2>&1); rc=$? + if [ "$want" = error ]; then + [ "$rc" -ne 0 ] && ! grep -q '^post=' "$out" && pass "$label (exit $rc, no output)" || bad "$label: rc=$rc out=$(cat "$out") $result" + return + fi + if [ "$rc" -eq 0 ] && grep -qx "post=$want" "$out"; then pass "$label ($(grep '^reason=' "$out"))"; else bad "$label: rc=$rc out=$(cat "$out") $result"; fi +} + +echo "default rules" +run true 'human first merged PR is thanked' GH_TOTAL=1 +run false 'returning contributor is not thanked' GH_TOTAL=3 +run false 'App author is skipped' FEEDBACK_PR_AUTHOR='hivecommons-hive[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'type=Bot without [bot] suffix is skipped' FEEDBACK_PR_AUTHOR=somebot FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'scanner/* branch is skipped' FEEDBACK_PR_HEAD_REF=scanner/v6-claude-auth +run false 'hive/* branch is skipped' FEEDBACK_PR_HEAD_REF=hive/adopt-123 +run false 'hive-origin body is skipped on any branch' FEEDBACK_PR_HEAD_REF=fix-17863-feedback FEEDBACK_PR_BODY=$'Fixes #1\n<!-- hive:origin issue=1 -->' +run true 'fix-* branch by a person is still thanked' FEEDBACK_PR_HEAD_REF=fix-typo GH_TOTAL=1 +run false 'unmerged (closed) PR is never thanked' FEEDBACK_PR_MERGED=false +run error 'merged-PR lookup failure exits non-zero without an output' GH_FAIL=1 +run error 'garbage merged count exits non-zero' GH_TOTAL=lots + +echo "configuration" +run true 'FEEDBACK_SKIP_BOTS=0 thanks a bot' FEEDBACK_SKIP_BOTS=0 FEEDBACK_PR_AUTHOR='x[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot GH_TOTAL=1 +run true 'FEEDBACK_FIRST_TIME_ONLY=0 thanks a returning contributor' FEEDBACK_FIRST_TIME_ONLY=0 GH_TOTAL=9 +run false 'FEEDBACK_SKIP_BRANCHES adds a pattern' FEEDBACK_SKIP_BRANCHES='scanner/* release/*' FEEDBACK_PR_HEAD_REF=release/v5.1 +run true 'FEEDBACK_SKIP_BRANCHES= disables branch skipping' FEEDBACK_SKIP_BRANCHES= FEEDBACK_PR_HEAD_REF=scanner/x GH_TOTAL=1 +run false 'FEEDBACK_SKIP_LOGINS is case-insensitive' FEEDBACK_SKIP_LOGINS='Maintainer other' FEEDBACK_PR_AUTHOR=maintainer +run true 'FEEDBACK_MERGED_COUNT override skips the API' FEEDBACK_MERGED_COUNT=1 GH_FAIL=1 + +echo "event file" +cat > "$TMP/event.json" <<'EOF' +{"pull_request":{"number":17716,"merged":true,"user":{"login":"clubanderson","type":"User"},"head":{"ref":"fix/thing"},"body":"body"}} +EOF +out="$TMP/output"; : > "$out" +result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + GITHUB_EVENT_PATH="$TMP/event.json" GH_TOTAL=400 bash "$SCRIPT" 2>&1); rc=$? +[ "$rc" -eq 0 ] && grep -qx 'post=false' "$out" && grep -q 'returning-contributor:clubanderson:400-merged' "$out" \ + && pass "fields are read from GITHUB_EVENT_PATH" || bad "event file: rc=$rc out=$(cat "$out") $result" + +if [ "$fail" -ne 0 ]; then + echo "test-feedback-gate.sh: FAILED" + exit 1 +fi +echo "test-feedback-gate.sh: all checks passed" diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh index 3da43953bb..f41b6a9bdf 100755 --- a/src/scripts/test-mutation-report.sh +++ b/src/scripts/test-mutation-report.sh @@ -200,6 +200,43 @@ output=$(MUTATION_OUT_DIR="$TMP/missing" bash "$SCRIPT" verdict 2>&1); rc=$? [ "$rc" -eq 1 ] && out_grep -q '::error::mutation: no verdict at' \ && pass "missing verdict file fails the verdict phase" || bad "missing verdict rc=$rc: $output" +echo "post phase (#17863)" +# Stub gh: lists $GH_EXISTING_ID (if set) as the bot's marker comment and logs writes. +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *-X*) printf '%s\n' "$*" >> "$GH_WRITE_LOG"; exit 0 ;; + *'/comments'*) if [ -n "${GH_EXISTING_ID:-}" ]; then printf '%s\n' "$GH_EXISTING_ID"; fi; exit 0 ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" +run_post() { # run_post <out-dir> [existing-id] + GH_WRITE_LOG="$TMP/gh-writes.log"; : > "$GH_WRITE_LOG" + SUMMARY="$TMP/summary.md"; : > "$SUMMARY" + output=$(cd "$repo" && GH_WRITE_LOG="$GH_WRITE_LOG" GH_EXISTING_ID="${2:-}" GITHUB_STEP_SUMMARY="$SUMMARY" \ + MUTATION_OUT_DIR="$1" PR_NUMBER=42 GITHUB_REPOSITORY=example/hive bash "$SCRIPT" post 2>&1) + writes=$(cat "$GH_WRITE_LOG") +} +mkdir -p "$TMP/out-pass" "$TMP/out-lived" +printf '<!-- hive-mutation -->\n### Mutation testing\n\nAll 3 mutant(s) in changed files were killed.\n' > "$TMP/out-pass/comment.md" +echo pass > "$TMP/out-pass/verdict" +printf '<!-- hive-mutation -->\n### Mutation testing\n\ngremlins found **1 surviving mutant(s)**\n' > "$TMP/out-lived/comment.md" +echo lived > "$TMP/out-lived/verdict" +run_post "$TMP/out-pass" +[ -z "$writes" ] && pass "pass verdict with no earlier comment posts nothing" || bad "pass posted: $writes" +out_grep -q 'verdict pass and no earlier comment; not posting' && pass "skip is logged" || bad "skip not logged: $output" +grep -q 'All 3 mutant(s) in changed files were killed' "$SUMMARY" && pass "clean report still lands in the step summary" || bad "summary missing report" +run_post "$TMP/out-lived" +grep -q "^api -X POST repos/example/hive/issues/42/comments" <<<"$writes" && pass "lived verdict posts a new comment" || bad "lived did not post: $writes" +run_post "$TMP/out-lived" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99" <<<"$writes" && pass "lived verdict edits the existing comment" || bad "lived did not edit: $writes" +grep -q 'POST' <<<"$writes" && bad "lived posted a duplicate" || pass "no duplicate comment" +run_post "$TMP/out-pass" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99 -f body=<!-- hive-mutation -->" <<<"$writes" && pass "pass after a finding edits the existing comment, keeping the marker" || bad "pass did not edit: $writes" +grep -q 'Resolved: no surviving or uncovered mutants' <<<"$writes" && pass "edited body is the one-line resolved note" || bad "resolved note missing: $writes" +grep -q 'All 3 mutant' <<<"$writes" && bad "full report was written to the comment" || pass "full report stays out of the comment" + if [ "$fail" -ne 0 ]; then echo "test-mutation-report.sh: FAILED" exit 1 From f82abbab4adf04c4205565a2e6285a9de43ef745 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 01:46:33 -0700 Subject: [PATCH 074/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20atomic=20runtime?= =?UTF-8?q?=20config=20save=20+=20fresh=20legacy=20fallback=20(#17862)=20(?= =?UTF-8?q?#17867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: atomic runtime config save + fresh legacy fallback (#17862) Config.Save wrote /etc/hive/hive.yaml and /data/hive.yaml.runtime in place with O_TRUNC. A liveness SIGKILL between truncate and write left a 0-byte runtime file; the entrypoint's -s check then fell back to /data/hive.yaml.bak, frozen at the Aug 2026 rename on every hive, and vllmd-03 booted level 3 instead of 6. - writeFileAtomic: temp file in the target's dir, fsync, rename onto the EvalSymlinks-resolved target so the #17468 link survives; falls back to the old in-place write when rename is refused (single-file bind mounts). - Both Save writes use it; runtime stays 0600, source keeps its mode. - entrypoint: hive_refresh_legacy_fallback copies a YAML-valid runtime config over hive.yaml.bak on clean boots (tmp+mv), only when the new name is the boot source; hive_runtime_config_read WARNs loudly on fallback and flags a fallback older than the damaged runtime file. - Tests: writer unit tests incl. symlink + no-partial-read race; shell suite covers WARN, refresh, parse guard, legacy-boot no-op. Fixes #17862 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * test(config): cover atomic writer failure paths for coverage floor Adds tests for missing target dir, ENOTDIR resolve error, dangling symlink, rename+fallback double failure (no temp leak), in-place fallback, syncDir tolerance and sourceConfigMode, lifting pkg/config back over its 90% coverage floor. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .../fixed-17862-atomic-runtime-config-save.md | 1 + src/deploy/entrypoint.sh | 77 ++++- src/deploy/test_entrypoint_runtime_config.sh | 112 ++++++- src/pkg/config/atomic_write.go | 116 +++++++ src/pkg/config/atomic_write_test.go | 288 ++++++++++++++++++ src/pkg/config/config.go | 70 ++--- src/pkg/hub/saas_provision.go | 11 +- 7 files changed, 612 insertions(+), 63 deletions(-) create mode 100644 changelog.d/fixed-17862-atomic-runtime-config-save.md create mode 100644 src/pkg/config/atomic_write.go create mode 100644 src/pkg/config/atomic_write_test.go diff --git a/changelog.d/fixed-17862-atomic-runtime-config-save.md b/changelog.d/fixed-17862-atomic-runtime-config-save.md new file mode 100644 index 0000000000..0dad10650e --- /dev/null +++ b/changelog.d/fixed-17862-atomic-runtime-config-save.md @@ -0,0 +1 @@ +- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index 66e68db376..87e19e6667 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -64,7 +64,9 @@ EXIT_NET_ADMIN_REQUIRED=77 # file is the only copy of the live config, and mutating it at boot risks # losing owner customisations with no warning. The old name is picked up # read-only until the first save writes the new one, after which both -# exist and the new one wins. +# exist and the new one wins. Once the new name is the boot source, every +# clean boot refreshes the old name from it (#17862) so the fallback is +# never months stale; see hive_refresh_legacy_fallback. HIVE_CONFIG_PATH="${HIVE_CONFIG:-/etc/hive/hive.yaml}" HIVE_CONFIG_RUNTIME="/data/hive.yaml.runtime" HIVE_CONFIG_RUNTIME_LEGACY="/data/hive.yaml.bak" @@ -771,18 +773,89 @@ for _cfg in "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" /data/hive.yaml done unset _cfg +# hive_file_mtime echoes a file's mtime as epoch seconds (0 when unreadable). +# Both stat dialects are tried because the image is Debian but the helper +# must also behave on the BusyBox/macOS shells the tests run under. +hive_file_mtime() { + stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0 +} + # hive_runtime_config_read echoes the path to read the persisted runtime # config from: the new name when it is present and non-empty, else the # legacy name when that is, else empty. Read-only — it never creates, # renames or removes anything on the PVC. +# +# #17862: falling back to the legacy name is never routine any more. Every +# clean boot refreshes it from the runtime config (hive_refresh_legacy_fallback), +# so reaching it means the runtime config was lost or truncated since the +# last boot. Say so LOUDLY on stderr with both files' ages: a hive that +# silently booted a 3-month-old hive.yaml.bak (level 3 instead of 6) is the +# incident this guards against. When the legacy copy is OLDER than the +# runtime file it is replacing — i.e. the runtime file was written after +# the last refresh and then damaged — the WARN says the fallback is stale +# and names the overlay/audit paths an operator can reconstruct from. hive_runtime_config_read() { if [ -f "$HIVE_CONFIG_RUNTIME" ] && [ -s "$HIVE_CONFIG_RUNTIME" ]; then echo "$HIVE_CONFIG_RUNTIME" elif [ -f "$HIVE_CONFIG_RUNTIME_LEGACY" ] && [ -s "$HIVE_CONFIG_RUNTIME_LEGACY" ]; then + _rt_state="missing" + _rt_mtime=0 + if [ -f "$HIVE_CONFIG_RUNTIME" ]; then + _rt_state="present but EMPTY (0 bytes — likely truncated by a kill mid-save)" + _rt_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME")" + fi + _lg_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME_LEGACY")" + _now="$(date +%s)" + _lg_age_h=$(( (_now - _lg_mtime) / 3600 )) + echo "[entrypoint] WARN: runtime config $HIVE_CONFIG_RUNTIME is $_rt_state; FALLING BACK to legacy $HIVE_CONFIG_RUNTIME_LEGACY (last refreshed ${_lg_age_h}h ago)." >&2 + if [ "$_rt_mtime" -gt "$_lg_mtime" ] 2>/dev/null; then + echo "[entrypoint] WARN: the legacy fallback is OLDER than the damaged runtime config ($(( (_rt_mtime - _lg_mtime) / 3600 ))h behind). Settings saved since then (acmm_level, agent roster, pauses, models) are NOT in this file — check /data/hive.yaml.dashboard and /data/audit.jsonl before trusting the booted config. See hivecommons/hive#17862." >&2 + fi + unset _rt_state _rt_mtime _lg_mtime _now _lg_age_h echo "$HIVE_CONFIG_RUNTIME_LEGACY" fi } +# hive_refresh_legacy_fallback copies a GOOD runtime config over the legacy +# fallback name so the fallback is at most one boot stale (#17862). Before +# this, hive.yaml.bak was frozen at the rename (Aug 2026) on every hive, and +# a truncated runtime file booted a months-old config. +# +# Only runs when THIS boot is reading from the new name (so the legacy file +# is strictly older, never the only copy — the Docker/LXC worry behind the +# original never-write rule) and the runtime file parses as YAML with a +# project section (so a half-written file is never promoted to fallback). +# Copies to a temp name and renames so the fallback is never itself +# truncated by a kill. Best-effort: a failure here only means the fallback +# stays at its previous age. +hive_refresh_legacy_fallback() { + [ "$1" = "$HIVE_CONFIG_RUNTIME" ] || return 0 + [ -s "$HIVE_CONFIG_RUNTIME" ] || return 0 + if command -v python3 >/dev/null 2>&1; then + python3 -c ' +import sys, yaml +with open(sys.argv[1]) as f: + d = yaml.safe_load(f) +sys.exit(0 if isinstance(d, dict) and isinstance(d.get("project"), dict) else 1) +' "$HIVE_CONFIG_RUNTIME" 2>/dev/null || { + echo "[entrypoint] WARN: $HIVE_CONFIG_RUNTIME does not parse as a hive config; legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY left untouched" >&2 + return 0 + } + fi + if cmp -s "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + return 0 + fi + _lg_tmp="${HIVE_CONFIG_RUNTIME_LEGACY}.tmp.$$" + if cp "$HIVE_CONFIG_RUNTIME" "$_lg_tmp" 2>/dev/null && mv -f "$_lg_tmp" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + hive_harden_runtime_config "$HIVE_CONFIG_RUNTIME_LEGACY" + echo "[entrypoint] Legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY refreshed from $HIVE_CONFIG_RUNTIME" + else + rm -f "$_lg_tmp" 2>/dev/null + echo "[entrypoint] WARN: could not refresh legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY from $HIVE_CONFIG_RUNTIME" >&2 + fi + unset _lg_tmp +} + # hive_ghe_git_host echoes this hive's configured GitHub host when it is NOT # public github.com (i.e. a GitHub Enterprise instance such as github.ibm.com), # and nothing at all otherwise. Derived the same way the Go binary's @@ -1018,6 +1091,7 @@ if [ "$IS_KUBERNETES" = "true" ]; then # result equal to its own input on every boot. HIVE_CONFIG_BOOT="$(hive_runtime_config_read)" if [ -n "$HIVE_CONFIG_BOOT" ]; then + hive_refresh_legacy_fallback "$HIVE_CONFIG_BOOT" # Steady state: boot from what this hive last had. The ConfigMap seed is # deliberately NOT merged — it is frozen at provision and every field it # could contribute is either already here or re-asserted by the hub on @@ -1207,6 +1281,7 @@ else # first boot on new code seeds hive.yaml.runtime from the legacy content # without touching the legacy file itself. HIVE_CONFIG_SOURCE="$(hive_runtime_config_read)" + hive_refresh_legacy_fallback "$HIVE_CONFIG_SOURCE" if [ -f "$HIVE_CONFIG_PATH" ] && [ -s "$HIVE_CONFIG_PATH" ] && [ -z "$HIVE_CONFIG_SOURCE" ]; then # First boot: config exists but no PVC runtime config yet — seed it cp "$HIVE_CONFIG_PATH" "$HIVE_CONFIG_RUNTIME" diff --git a/src/deploy/test_entrypoint_runtime_config.sh b/src/deploy/test_entrypoint_runtime_config.sh index 4f97399d2c..6e21a20570 100755 --- a/src/deploy/test_entrypoint_runtime_config.sh +++ b/src/deploy/test_entrypoint_runtime_config.sh @@ -2,10 +2,13 @@ # Tests the hive.yaml.bak -> hive.yaml.runtime migration in entrypoint.sh. # # The rename is copy-forward: readers prefer /data/hive.yaml.runtime and fall -# back to the legacy /data/hive.yaml.bak, and NOTHING renames or removes the -# legacy file on the PVC. ~51 live hives carry only the legacy name, and on -# Docker/LXC that file is the single copy of the live config — mutating it at -# boot could lose owner customisations with no warning. +# back to the legacy /data/hive.yaml.bak, and NOTHING removes the legacy file +# on the PVC. ~51 live hives carry only the legacy name, and on Docker/LXC +# that file is the single copy of the live config. +# +# #17862 added one deliberate write: when the NEW name is the boot source and +# parses as a config, the legacy file is refreshed from it (temp + mv) so the +# fallback is never months stale, and falling back to it logs a loud WARN. # # Rather than grepping the script, this extracts the real resolver function and # executes it against real files, so it fails if the logic regresses. @@ -38,9 +41,9 @@ echo "=== entrypoint runtime-config migration tests ===" # Extract the resolver verbatim from the entrypoint so this tests the shipped # code, not a copy that can drift away from it. -RESOLVER="$(sed -n '/^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" -if [ -z "$RESOLVER" ]; then - echo " FAIL: could not extract hive_runtime_config_read from $ENTRYPOINT" +RESOLVER="$(sed -n '/^hive_file_mtime() {/,/^}/p; /^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" +if ! grep -q 'hive_runtime_config_read() {' <<<"$RESOLVER" || ! grep -q 'hive_file_mtime() {' <<<"$RESOLVER"; then + echo " FAIL: could not extract hive_runtime_config_read / hive_file_mtime from $ENTRYPOINT" exit 1 fi @@ -63,7 +66,7 @@ resolve() { export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY local out - out="$(eval "$RESOLVER"; hive_runtime_config_read)" + out="$(eval "$RESOLVER"; hive_runtime_config_read 2>/dev/null)" case "$out" in "$HIVE_CONFIG_RUNTIME") echo "new" ;; @@ -95,17 +98,96 @@ check "empty new + good legacy -> legacy" "legacy" "$(resolve "EMPTY" "old: cfg" # 6. Both empty is the same as neither. check "empty new + empty legacy -> empty" "none" "$(resolve "EMPTY" "EMPTY")" -# 7. The legacy file must never be renamed or deleted. On Docker/LXC it is the -# only copy of the live config, so a destructive migration could lose owner -# customisations silently. -if grep -Eq '(mv|rm)[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then - echo " FAIL: entrypoint moves or removes the legacy runtime config" - echo " the migration must be copy-forward only" +# 7. The legacy file must never be deleted, and the only thing allowed to +# overwrite it is hive_refresh_legacy_fallback (#17862), which copies a +# newer, validated runtime config over it. On Docker/LXC it can be the +# only copy of the live config, so nothing else may touch it. +if grep -Eq 'rm[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then + echo " FAIL: entrypoint removes the legacy runtime config" FAIL=$((FAIL + 1)) else - echo " PASS: legacy runtime config is never moved or removed" + echo " PASS: legacy runtime config is never removed" PASS=$((PASS + 1)) fi +REFRESH="$(sed -n '/^hive_refresh_legacy_fallback() {/,/^}/p' "$ENTRYPOINT")" +mv_total="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT" || true)" +mv_in_refresh="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' <<<"$REFRESH" || true)" +check "legacy runtime config is only overwritten inside hive_refresh_legacy_fallback" "$mv_total" "$mv_in_refresh" + +# ── #17862: the legacy fallback is refreshed on clean boots and WARNs loudly ── +# +# The incident: a kill mid-save left a 0-byte hive.yaml.runtime, the resolver +# fell back to a hive.yaml.bak frozen three months earlier, and the hive +# booted a stale ACMM level with no indication in the log. Two properties +# close it: (a) a clean boot from the new name refreshes the legacy copy so it +# is at most one boot old, and (b) a fallback is announced on stderr. + +echo +echo "=== #17862: legacy fallback freshness and WARN ===" + +# resolve_stderr <new-content> <legacy-content> — stderr the resolver emits. +resolve_stderr() { + local new_content="$1" legacy_content="$2" + local dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + [ -n "$new_content" ] && printf '%s' "$new_content" > "$HIVE_CONFIG_RUNTIME" + [ -n "$legacy_content" ] && printf '%s' "$legacy_content" > "$HIVE_CONFIG_RUNTIME_LEGACY" + [ "$new_content" = "EMPTY" ] && : > "$HIVE_CONFIG_RUNTIME" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY + (eval "$RESOLVER"; hive_runtime_config_read 2>&1 >/dev/null) || true +} + +# (b) Falling back is loud; the steady state is silent. +warn_out="$(resolve_stderr "EMPTY" "old: cfg")" +if grep -q 'WARN.*FALLING BACK to legacy' <<<"$warn_out" && grep -q '0 bytes' <<<"$warn_out"; then + echo " PASS: empty runtime + legacy fallback logs a loud WARN naming the truncation" + PASS=$((PASS + 1)) +else + echo " FAIL: fallback to legacy did not WARN (got: '$warn_out')" + FAIL=$((FAIL + 1)) +fi +check "steady-state boot from the new name is silent" "" "$(resolve_stderr "new: cfg" "old: cfg")" + +# (a) hive_refresh_legacy_fallback: the legacy copy is overwritten from a +# valid runtime file, never from an invalid one, and only when the new +# name is the boot source. +HARDEN_FN="$(sed -n '/^hive_harden_runtime_config() {/,/^}/p' "$ENTRYPOINT")" +refresh_case() { + # refresh_case <boot-source: new|legacy> <runtime-content> <legacy-content>; prints legacy content after. + local src="$1" rt="$2" lg="$3" dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + HIVE_RUNTIME_USER="$(id -un)"; HIVE_RUNTIME_GROUP="$(id -gn)" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY HIVE_RUNTIME_USER HIVE_RUNTIME_GROUP + printf '%s' "$rt" > "$HIVE_CONFIG_RUNTIME" + printf '%s' "$lg" > "$HIVE_CONFIG_RUNTIME_LEGACY" + local arg="$HIVE_CONFIG_RUNTIME" + [ "$src" = legacy ] && arg="$HIVE_CONFIG_RUNTIME_LEGACY" + (eval "$HARDEN_FN"; eval "$REFRESH"; hive_refresh_legacy_fallback "$arg" >/dev/null 2>&1) || true + cat "$HIVE_CONFIG_RUNTIME_LEGACY" +} +# Contents are single-line here so $(...) newline stripping cannot bite. +GOOD_RT='{project: {org: acme}, acmm_level: 6}' +STALE_LG='{acmm_level: 3}' +check "clean boot from new name refreshes the legacy fallback" \ + "$GOOD_RT" "$(refresh_case new "$GOOD_RT" "$STALE_LG")" +check "boot from the legacy name leaves it untouched" \ + "$STALE_LG" "$(refresh_case legacy "$GOOD_RT" "$STALE_LG")" +if command -v python3 >/dev/null 2>&1 && python3 -c 'import yaml' 2>/dev/null; then + check "unparsable runtime config never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new 'project: [unterminated' "$STALE_LG")" + check "runtime config without a project section never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new '{acmm_level: 0}' "$STALE_LG")" +else + hive_test_skip "python3+yaml unavailable — cannot exercise the parse guard in hive_refresh_legacy_fallback" +fi # 8. Every write of the config path to the PVC targets the NEW name. Counted, # not just grepped: there are two such writes (the K8s post-merge snapshot diff --git a/src/pkg/config/atomic_write.go b/src/pkg/config/atomic_write.go new file mode 100644 index 0000000000..bdd969964d --- /dev/null +++ b/src/pkg/config/atomic_write.go @@ -0,0 +1,116 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// writeFileAtomic replaces the contents of path without ever exposing a +// truncated or partial file to a concurrent reader or to the next boot. +// +// Motivation (#17862): Save used O_WRONLY|O_TRUNC in place on the PVC runtime +// config. A SIGKILL (liveness probe) landing between the truncate and the +// write left a 0-byte /data/hive.yaml.runtime; the entrypoint's `-s` check +// then fell back to a months-stale /data/hive.yaml.bak and a hive booted +// with a three-month-old ACMM level and roster. +// +// Strategy: write data to a temp file in the TARGET's directory, fsync it, +// then rename it onto the target. Two constraints shape the details: +// +// - path may be a symlink (hosted spokes link /etc/hive/hive.yaml to the +// PVC runtime config, #17468). The rename must land on the link's +// TARGET, not replace the link with a regular file, so the final path is +// resolved with EvalSymlinks first. Pinned by TestSaveThroughSymlinkKeepsLink. +// - path may be a single-file Docker bind mount, where rename over the +// mount point fails (EBUSY) and would also swap the inode the host sees. +// When the rename fails, fall back to the previous in-place write so +// those deployments keep working exactly as before; they never had +// atomicity and lose nothing here. +// +// mode is applied to the temp file before the rename, so a pre-existing +// world-readable inode does not carry its bits through (WriteFile's mode +// only applies on create). The fallback path chmods explicitly for the same +// reason. +func writeFileAtomic(path string, data []byte, mode os.FileMode) error { + target := path + if resolved, err := filepath.EvalSymlinks(path); err == nil { + target = resolved + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolving %s: %w", path, err) + } + + dir := filepath.Dir(target) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*") + if err != nil { + return fmt.Errorf("creating temp file in %s: %w", dir, err) + } + tmpPath := tmp.Name() + cleanup := func() { _ = os.Remove(tmpPath) } + + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("chmod temp file %s: %w", tmpPath, err) + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("writing temp file %s: %w", tmpPath, err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("syncing temp file %s: %w", tmpPath, err) + } + if err := tmp.Close(); err != nil { + cleanup() + return fmt.Errorf("closing temp file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, target); err != nil { + cleanup() + // Bind-mounted single file (or any filesystem that refuses the + // rename): keep the inode and write in place, as Save always did. + if inPlaceErr := writeFileInPlace(target, data, mode); inPlaceErr != nil { + return fmt.Errorf("renaming %s onto %s: %w (in-place fallback also failed: %v)", tmpPath, target, err, inPlaceErr) + } + return nil + } + syncDir(dir) + return nil +} + +// writeFileInPlace is the pre-#17862 O_TRUNC write, kept only as the fallback +// for targets that cannot be renamed over. It is NOT crash-safe. +func writeFileInPlace(target string, data []byte, mode os.FileMode) error { + f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Chmod(target, mode) +} + +// syncDir fsyncs a directory so the rename that just happened in it is +// durable. Best-effort: some filesystems (and all of Windows) refuse to open +// a directory for sync, and a failure here cannot un-do the rename. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} diff --git a/src/pkg/config/atomic_write_test.go b/src/pkg/config/atomic_write_test.go new file mode 100644 index 0000000000..dfdb561e23 --- /dev/null +++ b/src/pkg/config/atomic_write_test.go @@ -0,0 +1,288 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestWriteFileAtomicReplacesContentAndMode covers the plain case: an +// existing world-readable file is replaced wholesale and ends up with the +// requested mode, and no temp file is left behind. +func TestWriteFileAtomicReplacesContentAndMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml.runtime") + if err := os.WriteFile(path, []byte("old: content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(path, []byte("new: content\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new: content\n" { + t.Fatalf("content = %q, want new content", got) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 0600 (pre-existing 0644 bits must not survive)", fi.Mode().Perm()) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + t.Fatalf("temp file %s left behind", e.Name()) + } + } +} + +// TestWriteFileAtomicCreatesMissingFile pins that a missing target is +// created (Save's old create fallback) rather than failing on EvalSymlinks. +func TestWriteFileAtomicCreatesMissingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := writeFileAtomic(path, []byte("a: 1\n"), 0o644); err != nil { + t.Fatalf("writeFileAtomic on missing file: %v", err) + } + if got, _ := os.ReadFile(path); string(got) != "a: 1\n" { + t.Fatalf("content = %q", got) + } +} + +// TestWriteFileAtomicThroughSymlinkKeepsLink is the #17468 constraint at the +// writer level: the rename must land on the link TARGET so the symlink the +// copy-config init container created survives, and the target carries the +// new bytes. +func TestWriteFileAtomicThroughSymlinkKeepsLink(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "data", "hive.yaml.runtime") + link := filepath.Join(dir, "etc", "hive.yaml") + for _, d := range []string{filepath.Dir(target), filepath.Dir(link)} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(target, []byte("old\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(link, []byte("new\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic through link: %v", err) + } + fi, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("%s is no longer a symlink — rename replaced the link instead of its target", link) + } + if got, _ := os.ReadFile(target); string(got) != "new\n" { + t.Fatalf("target content = %q, want new", got) + } + // Nothing may be left in the link's directory: the temp file must be + // created next to the TARGET, or the rename would cross directories. + entries, _ := os.ReadDir(filepath.Dir(link)) + if len(entries) != 1 { + t.Fatalf("link dir has %d entries, want just the link", len(entries)) + } +} + +// TestSaveSurvivesKillMidWrite is the #17862 regression. It cannot SIGKILL +// the test process, so it pins the property that makes a kill harmless: +// at no point during Save does the runtime config path refer to a file +// with fewer bytes than the previous complete config. A temp-file+rename +// writer satisfies this trivially; the old O_TRUNC in-place write fails it +// because the file is observably 0 bytes between truncate and write. +// +// The check is made by racing a reader against repeated Saves and +// asserting it never observes an empty or partial file. +func TestSaveSurvivesKillMidWrite(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "hive.yaml") + runtimePath := filepath.Join(dir, "hive.yaml.runtime") + origOverlay, origRuntime := DashboardOverlayFile, RuntimeConfigFile + DashboardOverlayFile = filepath.Join(dir, "hive.yaml.dashboard") + RuntimeConfigFile = runtimePath + t.Cleanup(func() { DashboardOverlayFile, RuntimeConfigFile = origOverlay, origRuntime }) + + level := 6 + cfg := &Config{ + SourcePath: cfgPath, + Project: ProjectConfig{Org: "testorg", Name: "test"}, + AgentRuntimeConfig: AgentRuntimeConfig{ + Agents: map[string]AgentConfig{"scanner": {Role: "scanner"}}, + }, + ACMMLevel: &level, + } + if err := cfg.Save(); err != nil { + t.Fatalf("initial Save: %v", err) + } + full, err := os.ReadFile(runtimePath) + if err != nil || len(full) == 0 { + t.Fatalf("initial runtime config unreadable: %v", err) + } + + stop := make(chan struct{}) + bad := make(chan string, 1) + go func() { + for { + select { + case <-stop: + return + default: + } + for _, p := range []string{runtimePath, cfgPath} { + got, err := os.ReadFile(p) + if err != nil { + continue // a rename window may briefly ENOENT on some filesystems; never partial + } + if len(got) < len(full) { + select { + case bad <- p: + default: + } + return + } + } + } + }() + for i := 0; i < 200; i++ { + if err := cfg.Save(); err != nil { + t.Fatalf("Save %d: %v", i, err) + } + } + close(stop) + select { + case p := <-bad: + t.Fatalf("reader observed a truncated/partial config at %s during Save — write is not atomic", p) + default: + } +} + +func TestWriteFileAtomicMissingDirFails(t *testing.T) { + path := filepath.Join(t.TempDir(), "no-such-dir", "hive.yaml") + err := writeFileAtomic(path, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected error when target directory does not exist") + } + if !strings.Contains(err.Error(), "creating temp file") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicResolveErrorIsReported(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "regular") + if err := os.WriteFile(file, []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + // Using a regular file as a path component makes EvalSymlinks fail with + // ENOTDIR, which is not ErrNotExist and must surface as a resolve error. + err := writeFileAtomic(filepath.Join(file, "hive.yaml"), []byte("x"), 0o600) + if err == nil { + t.Fatal("expected resolve error") + } + if !strings.Contains(err.Error(), "resolving") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicDanglingSymlinkIsReplaced(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(dir, "hive.yaml") + if err := os.Symlink(filepath.Join(dir, "missing-target"), link); err != nil { + t.Skipf("symlink not supported: %v", err) + } + if err := writeFileAtomic(link, []byte("fresh"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(link) + if err != nil { + t.Fatal(err) + } + if string(got) != "fresh" { + t.Fatalf("got %q", got) + } +} + +func TestWriteFileAtomicRenameFailureFallsBackAndReportsBoth(t *testing.T) { + dir := t.TempDir() + // A non-empty directory as the target: rename of a file onto it fails, + // and the in-place O_TRUNC fallback cannot open a directory for writing. + target := filepath.Join(dir, "hive.yaml") + if err := os.MkdirAll(filepath.Join(target, "child"), 0o700); err != nil { + t.Fatal(err) + } + err := writeFileAtomic(target, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected rename + fallback failure") + } + if !strings.Contains(err.Error(), "in-place fallback also failed") { + t.Fatalf("unexpected error: %v", err) + } + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".hive.yaml.tmp-") { + t.Fatalf("temp file %s leaked after failure", e.Name()) + } + } +} + +func TestWriteFileInPlaceWritesAndSetsMode(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := os.WriteFile(path, []byte("old-longer-content"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileInPlace(path, []byte("new"), 0o600); err != nil { + t.Fatalf("writeFileInPlace: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new" { + t.Fatalf("got %q, want truncated rewrite", got) + } + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 600", st.Mode().Perm()) + } +} + +func TestWriteFileInPlaceOpenFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "hive.yaml") + if err := writeFileInPlace(path, []byte("x"), 0o600); err == nil { + t.Fatal("expected open failure for missing directory") + } +} + +func TestSyncDirToleratesMissingDir(t *testing.T) { + syncDir(filepath.Join(t.TempDir(), "nope")) + syncDir(t.TempDir()) +} + +func TestSourceConfigModePreservesExistingOrDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml") + if err := os.WriteFile(path, []byte("a: 1\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := sourceConfigMode(path); got != 0o600 { + t.Fatalf("existing mode = %o, want 600", got) + } + if got := sourceConfigMode(filepath.Join(dir, "absent.yaml")); got != 0o644 { + t.Fatalf("default mode = %o, want 644", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index bbeb01a2f8..6c80ab373b 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -7755,18 +7755,14 @@ func (c *Config) saveLocked() error { // layer this Save is about to persist. overlayErr := c.saveDashboardOverlay() - // Open the existing file (preserving its inode) rather than creating a - // temp file and renaming. Rename breaks Docker bind mounts because it - // replaces the inode — the host file is never updated, so acmm_level - // and other runtime changes are lost on container restart. + // Replace the primary config atomically (temp file + fsync + rename onto + // the resolved target) — see writeFileAtomic for why that is safe on the + // #17468 symlink and still works on single-file Docker bind mounts. // - // #17468: the same in-place write is what keeps hosted spokes durable. - // There the copy-config init container makes SourcePath - // (/etc/hive/hive.yaml, an emptyDir) a symlink to RuntimeConfigFile on - // the PVC; O_WRONLY|O_TRUNC follows the link and writes the PVC file, - // whereas a temp-file+rename would replace the link with a regular - // emptyDir file and silently revert the fix. Pinned by - // TestSaveThroughSymlinkKeepsLink — do not switch this to a rename. + // #17862: the previous O_WRONLY|O_TRUNC in-place write could be killed + // between truncate and write (liveness probe on a crash-looping hive), + // leaving a 0-byte runtime config that the entrypoint skipped in favour + // of a months-stale hive.yaml.bak. Pinned by TestSaveSurvivesKillMidWrite. // // #3961: a source-path failure must NOT abort the save. On deployments // that mount the config read-only (a ConfigMap mounted straight at @@ -7780,23 +7776,8 @@ func (c *Config) saveLocked() error { // writing the runtime layer, and report success iff the state will actually // survive a restart. var srcErr error - f, err := os.OpenFile(c.SourcePath, os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - // File may not exist yet — fall back to create. Continue below so - // the PVC backup and dashboard overlay are still written. - if writeErr := os.WriteFile(c.SourcePath, data, 0o644); writeErr != nil { - srcErr = fmt.Errorf("writing config (create fallback): %w", writeErr) - } - } else { - if _, err := f.Write(data); err != nil { - _ = f.Close() // best-effort cleanup; the write error is what's recorded - srcErr = fmt.Errorf("writing config: %w", err) - } else if err := f.Sync(); err != nil { - _ = f.Close() // best-effort cleanup; the sync error is what's recorded - srcErr = fmt.Errorf("syncing config: %w", err) - } else if err := f.Close(); err != nil { - srcErr = fmt.Errorf("closing config: %w", err) - } + if err := writeFileAtomic(c.SourcePath, data, sourceConfigMode(c.SourcePath)); err != nil { + srcErr = fmt.Errorf("writing config: %w", err) } // Persist the runtime config to the PVC. In K8s this is the boot input @@ -7820,11 +7801,11 @@ func (c *Config) saveLocked() error { if guardLivePVCPathUnderTest(runtimePath, defaultRuntimeConfigFile, "RuntimeConfigFile") { // Not written, so not persisted: keep the primary-path error visible. runtimeErr = errLivePVCPathGuarded - } else if err := os.WriteFile(runtimePath, data, 0o600); err != nil { + } else if err := writeFileAtomic(runtimePath, data, 0o600); err != nil { // Common cause: init container created the file as root, runtime user // can't overwrite. Remove and retry so runtime state is not silently lost. - _ = os.Remove(runtimePath) // best-effort; the retry's own WriteFile error is what's recorded below - if retryErr := os.WriteFile(runtimePath, data, 0o600); retryErr != nil { + _ = os.Remove(runtimePath) // best-effort; the retry's own error is what's recorded below + if retryErr := writeFileAtomic(runtimePath, data, 0o600); retryErr != nil { runtimeErr = retryErr log.Printf("[config] warning: failed to write PVC runtime config to %s (even after remove): %v", runtimePath, retryErr) } else { @@ -7832,12 +7813,6 @@ func (c *Config) saveLocked() error { } } else { log.Printf("[config] PVC runtime config written to %s", runtimePath) - // os.WriteFile's mode only applies when it CREATES the file; a - // pre-existing world-readable inode (every hive deployed before - // this fix) keeps its old 0644 bits, so tighten explicitly. - if chmodErr := os.Chmod(runtimePath, 0o600); chmodErr != nil { - log.Printf("[config] warning: failed to tighten permissions on %s: %v", runtimePath, chmodErr) - } } if srcErr == nil { @@ -7856,6 +7831,16 @@ func (c *Config) saveLocked() error { return srcErr } +// sourceConfigMode returns the mode Save should give the primary config +// path: the existing file's permission bits when it exists (an operator may +// have tightened them), else the historical 0644 default. +func sourceConfigMode(path string) os.FileMode { + if fi, err := os.Stat(path); err == nil { + return fi.Mode().Perm() + } + return 0o644 +} + // RuntimeConfigFile is where Save() persists the full runtime config on the // PVC. Its role differs by environment, which is exactly why the old // hive.yaml.bak name was misleading enough to cost debugging time: @@ -7880,11 +7865,12 @@ const defaultRuntimeConfigFile = "/data/hive.yaml.runtime" // RuntimeConfigFileLegacy is the pre-rename name of RuntimeConfigFile. // -// It is READ as a fallback and never written, renamed or removed: ~51 live -// hives carry only this file, and on Docker/LXC it is the single copy of -// their live configuration. Mutating it at boot could lose owner -// customisations with no warning, so the migration is copy-forward only — -// readers prefer RuntimeConfigFile and fall back to this one. +// It is READ as a fallback and never written by Save: ~51 live hives carry +// only this file, and on Docker/LXC it is the single copy of their live +// configuration. Readers prefer RuntimeConfigFile and fall back to this one. +// The entrypoint refreshes it from a good RuntimeConfigFile on every clean +// boot (#17862) — a fallback frozen at the Aug 2026 rename booted a +// months-old level/roster when a kill mid-save truncated the runtime file. // // Removable one release after every live hive has written the new name. const RuntimeConfigFileLegacy = "/data/hive.yaml.bak" diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index 8ef3b3db00..b5de4a99ca 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -3464,11 +3464,12 @@ spec: # Steady state (#17468): /etc/hive is an emptyDir, so a regular file # there is a throwaway cache and operator edits to it vanished on # restart. Symlink it to the PVC runtime config instead, so the path - # every runbook names IS the persisted boot source. Config.Save writes - # the config path in place (O_TRUNC, no rename), so the link survives - # saves. Only the NEW name is linked: the legacy .bak is never written - # (config.RuntimeConfigFileLegacy), and linking it would make every - # save write it. + # every runbook names IS the persisted boot source. Config.Save + # resolves the link and renames its temp file onto the TARGET + # (#17862), so the link survives saves. Only the NEW name is linked: + # the legacy .bak is refreshed by the entrypoint from a good runtime + # file on clean boots (never written by Save), and linking it would + # make every save write it. command: ["sh", "-c", "if [ -s /data/hive.yaml.runtime ]; then echo runtime-config-exists-for-recovery; ln -sf /data/hive.yaml.runtime /etc/hive/hive.yaml && echo config-path-linked-to-runtime || echo config-path-link-failed; elif [ -s /data/hive.yaml.bak ]; then echo legacy-runtime-config-exists-for-recovery; else cp /etc/hive-seed/hive.yaml /etc/hive/hive.yaml && echo configmap-copied-first-boot; fi"] volumeMounts: - name: config From a35c8902c0c345fe869cd4ddcd5d7a0912cb292a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 08:51:41 +0000 Subject: [PATCH 075/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.231.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.231.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-voice-sources.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-voice-sources.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9b669559..60d41bcc78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.231.0) + +### Added + +- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. + ## 2026-11-12 (v5.230.0) ### Added diff --git a/changelog.d/added-inbox-voice-sources.md b/changelog.d/added-inbox-voice-sources.md deleted file mode 100644 index a375f8cb62..0000000000 --- a/changelog.d/added-inbox-voice-sources.md +++ /dev/null @@ -1 +0,0 @@ -- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. From 2d0f9eac65456b9e2461e4b0534aa58dc78f75d3 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:13:00 +0100 Subject: [PATCH 076/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20wait=20for=20mark?= =?UTF-8?q?etplace=20refresh=20completion=20in=20auth=20test=20(#17868)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wait for a new refresh result rather than interpreting an idle refresher as proof that the newly accepted asynchronous run has finished. Hive-Run: hivecommons/hive#17865 Hive-Plan: marketplace-refresh-flake Hive-Spec: marketplace-refresh-flake#async-completion Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> --- src/pkg/hub/marketplace_test.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index e28ba73458..b364fea6dd 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -870,18 +870,29 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { // Spoke full run: 202 once per hour across callers; admin unaffected. rf := s.refresher() - waitIdle := func() { + waitRefresh := func(previous *marketplaceRefreshResult) { + t.Helper() testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() defer rf.mu.Unlock() - return !rf.running + return rf.last != previous && !rf.running }, "background refresh did not finish") } - rec = mktDo(s, spoke("/api/marketplace/refresh")) - if rec.Code != 202 { - t.Fatalf("spoke full: %d %s", rec.Code, rec.Body.String()) + refreshAndWait := func(req *http.Request, description string) { + t.Helper() + rf.mu.Lock() + previous := rf.last + rf.mu.Unlock() + rec := mktDo(s, req) + if rec.Code != http.StatusAccepted { + t.Fatalf("%s: %d %s", description, rec.Code, rec.Body.String()) + } + // Even a later fatal assertion must not restore fixture globals while + // this accepted refresh is still starting or running. + t.Cleanup(func() { waitRefresh(previous) }) + waitRefresh(previous) } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full") rf.mu.Lock() trigger := rf.last.Trigger rf.mu.Unlock() @@ -892,15 +903,9 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { if rec.Code != 429 || !strings.Contains(rec.Body.String(), "try again in") { t.Fatalf("spoke full again: %d %s", rec.Code, rec.Body.String()) } - if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("admin full while spoke limited: %d %s", rec.Code, rec.Body.String()) - } - waitIdle() + refreshAndWait(admin(http.MethodPost, "/api/marketplace/refresh"), "admin full while spoke limited") clock = clock.Add(marketplaceFullRefreshInterval) - if rec = mktDo(s, spoke("/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("spoke full after interval: %d", rec.Code) - } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full after interval") // Verbose status: anon 401, spoke and admin get the failing list. if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil)); rec.Code != 401 { From 533fe6258680363647779a095efcd3d7a7e3d343 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 09:22:46 +0000 Subject: [PATCH 077/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.231.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.231.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/changed-17863-pr-comment-quartet.md | 1 - changelog.d/fixed-17862-atomic-runtime-config-save.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/changed-17863-pr-comment-quartet.md delete mode 100644 changelog.d/fixed-17862-atomic-runtime-config-save.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d41bcc78..5c956ab5fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.231.1) + +### Changed + +- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. + +### Fixed + +- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). + ## 2026-11-12 (v5.231.0) ### Added diff --git a/changelog.d/changed-17863-pr-comment-quartet.md b/changelog.d/changed-17863-pr-comment-quartet.md deleted file mode 100644 index eb5412ae90..0000000000 --- a/changelog.d/changed-17863-pr-comment-quartet.md +++ /dev/null @@ -1 +0,0 @@ -- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `<!-- hive-pr-overlap -->` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `<!-- hive-mutation -->` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. diff --git a/changelog.d/fixed-17862-atomic-runtime-config-save.md b/changelog.d/fixed-17862-atomic-runtime-config-save.md deleted file mode 100644 index 0dad10650e..0000000000 --- a/changelog.d/fixed-17862-atomic-runtime-config-save.md +++ /dev/null @@ -1 +0,0 @@ -- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). From 61988eb41d9af4d3779f9f9db65cea500b065a0e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:55:44 +0000 Subject: [PATCH 078/166] =?UTF-8?q?=F0=9F=93=96=20docs:=20Inbox=20voice=20?= =?UTF-8?q?sources=20page=20+=20quickstart=20link=20(#17899)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: add Inbox voice sources page and quickstart links Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 📖 docs: use .row utility for Inbox quickstart link group TestStyleRatchet counts inline style= attributes in static/index.html; the new link group pushed it over the 1403 baseline. The existing .row utility (flex, gap var(--sp-4)) renders the same. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: link #17882 from the changelog fragment Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-inbox-voice-sources-docs.md | 1 + src/pkg/dashboard/static-src/js/knowledge.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added-inbox-voice-sources-docs.md diff --git a/changelog.d/added-inbox-voice-sources-docs.md b/changelog.d/added-inbox-voice-sources-docs.md new file mode 100644 index 0000000000..02a0ce3d1f --- /dev/null +++ b/changelog.d/added-inbox-voice-sources-docs.md @@ -0,0 +1 @@ +- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 10fdd87fd5..ac6380b949 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2454,7 +2454,7 @@ const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 90af378a72..dc5e519981 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46000,7 +46000,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; From e8185773a3b2a11847635cf2446cd792f8dc7155 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:08:24 +0000 Subject: [PATCH 079/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20Settings=20?= =?UTF-8?q?=E2=86=92=20Inbox=20section=20(default=20repo,=20dibs=20URL,=20?= =?UTF-8?q?rules)=20(#17901)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: Settings → Inbox section for default repo, dibs URL and rules Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> #17885 landed inbox.* config, validation, parity allowlist and /api/inbox/settings but no Settings tab section. Add Settings → Inbox reusing the Inbox drawer form and endpoint. Part of #17883. * 🐛 dashboard: scope the Inbox settings form to its active root The Inbox settings form renders twice — in the Inbox page modal and inline under Settings → Inbox — and both can be mounted at once: the settings tab stays in the DOM and the closed modal keeps its last body. inboxReadRules collected .inbox-rule rows from every #inbox-settings-rules, the repo/dibs inputs were read via getElementById (first in document) and the inline sniff kept the modal open on save, so saving from the modal after visiting Settings → Inbox posted the inline form's values and left the dialog up. Wrap the form in .inbox-settings-form[data-inbox-form=inline|modal], replace the duplicated ids with data-inbox-field attributes, and resolve one active root per operation (the modal while open, otherwise the inline section). Read, render, save and the close-modal decision all go through that root. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- changelog.d/added-inbox-settings-section.md | 1 + .../dashboard-sections/workflow-and-agents.md | 1 + src/docs/inbox-voice-sources.md | 3 +- .../dashboard/inbox_settings_section_test.go | 34 +++++++ .../dashboard/static-src/js/config-state.js | 3 +- .../dashboard/static-src/js/config-tabs.js | 2 + src/pkg/dashboard/static-src/js/inbox.js | 84 +++++++++++++---- src/pkg/dashboard/static/index.html | 89 ++++++++++++++----- 8 files changed, 176 insertions(+), 41 deletions(-) create mode 100644 changelog.d/added-inbox-settings-section.md create mode 100644 src/pkg/dashboard/inbox_settings_section_test.go diff --git a/changelog.d/added-inbox-settings-section.md b/changelog.d/added-inbox-settings-section.md new file mode 100644 index 0000000000..7199a8ab60 --- /dev/null +++ b/changelog.d/added-inbox-settings-section.md @@ -0,0 +1 @@ +- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index c49a26bbd1..bb9ec25b24 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -213,6 +213,7 @@ Voice recordings and dictations waiting for you to file them as an RFC, send to - The sidebar count and the summary count items with no destination yet and not marked "Knowledge only". - Each item shows one pill per destination it was sent to: queued, running, filed (with a link) or failed. - Auto-routing rules run once per new item after a sync; the first matching rule wins. +- The default RFC repository, dibs URL and rules can also be edited under Settings → Inbox. **What it is good for.** Record an idea on your recorder, then turn it into an RFC with child issues (→ RFC), a dibs idea (→ dibs), or just keep it searchable (Knowledge only) without leaving the dashboard. See [Inbox and voice sources](../inbox-voice-sources.md). diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index bb5c95792f..4f689aa9cd 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -36,7 +36,8 @@ see the issue linked from the changelog entry. - **Auto-routing rules** (⚙ Auto-routing in the Inbox, or `inbox.rules`): `source?`, `title_prefix` or `tag`, `destination`, `repo?`. Rules run once for each item that arrives after a connector sync; the first match wins; - items already in the Inbox are never re-routed. + items already in the Inbox are never re-routed. The same editor, with the + default RFC repository and dibs URL, is under Settings → Inbox. ## Configuration diff --git a/src/pkg/dashboard/inbox_settings_section_test.go b/src/pkg/dashboard/inbox_settings_section_test.go new file mode 100644 index 0000000000..147803693d --- /dev/null +++ b/src/pkg/dashboard/inbox_settings_section_test.go @@ -0,0 +1,34 @@ +package dashboard + +import ( + "strings" + "testing" +) + +func TestSettingsInboxSectionWiring(t *testing.T) { + raw, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + page := string(raw) + for _, want := range []string{ + "'Knowledge', 'Inbox', 'Project Observability'", + "case 'Inbox': return renderGovInbox();", + "function renderGovInbox", + "function loadInboxSettingsInline", + "loadInboxSettingsInline();", + "inbox-settings-inline", + "yamlHint('inbox.spek.repo')", + "yamlHint('inbox.dibs.url')", + "yamlHint('inbox.rules')", + "Default RFC repository", + "data-inbox-field=\"dibs-url\"", + "function inboxSettingsRoot", + "data-inbox-form=", + "key: 'inbox.spek.repo'", + } { + if !strings.Contains(page, want) { + t.Errorf("Settings → Inbox wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 162cf7e258..c3b76f8bfa 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -23,7 +23,7 @@ // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -82,6 +82,7 @@ { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index 03d69b03f3..b8ca800a08 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -53,6 +53,7 @@ // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -265,6 +266,7 @@ case 'Access': return renderGovSecurity(d.security || {}); case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Work Source': return renderGovWorkSource(d.work_source || {}); diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 4f38662937..9d2233d788 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -210,33 +210,77 @@ '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; } - function inboxReadRules() { - return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; }); } - function inboxRenderRules() { - const host = document.getElementById('inbox-settings-rules'); + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; if (!host) return; host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; } - function inboxOpenSettings() { - const settings = (_inboxCache || {}).settings || {}; - _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); - const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + '<div class="inbox-settings-grid">' + - '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + - '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + - '<div id="inbox-settings-rules"></div>' + - '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + - '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + - '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; - kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); inboxRenderRules(); } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } function inboxAddRule() { _inboxDraftRules = inboxReadRules(); _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); @@ -248,14 +292,16 @@ inboxRenderRules(); } async function inboxSaveSettings() { - const errEl = document.getElementById('inbox-settings-error'); - const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); - const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; try { const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); const data = await r.json().catch(() => ({})); if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); - kbCloseModal(); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); showToast('Inbox settings saved', 'success'); await fetchInbox(); } catch (e) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index dc5e519981..517974a26c 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46406,33 +46406,77 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; } - function inboxReadRules() { - return Array.from(document.querySelectorAll('#inbox-settings-rules .inbox-rule')).map(row => { + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; }); } - function inboxRenderRules() { - const host = document.getElementById('inbox-settings-rules'); + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; if (!host) return; host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; } - function inboxOpenSettings() { - const settings = (_inboxCache || {}).settings || {}; - _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); - const body = '<p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + '<div class="inbox-settings-grid">' + - '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" id="inbox-settings-spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + - '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" id="inbox-settings-dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + - '<div id="inbox-settings-rules"></div>' + - '<div class="hive-dialog-error" role="alert" id="inbox-settings-error"></div>' + - '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + - '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; - kbShowModal('⚙ Inbox auto-routing', body, { width: '720px' }); + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); inboxRenderRules(); } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } function inboxAddRule() { _inboxDraftRules = inboxReadRules(); _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); @@ -46444,14 +46488,16 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> inboxRenderRules(); } async function inboxSaveSettings() { - const errEl = document.getElementById('inbox-settings-error'); - const rules = inboxReadRules().map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); - const payload = { rules, spek_repo: (document.getElementById('inbox-settings-spek-repo') || {}).value || '', dibs_url: (document.getElementById('inbox-settings-dibs-url') || {}).value || '' }; + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; try { const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); const data = await r.json().catch(() => ({})); if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); - kbCloseModal(); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); showToast('Inbox settings saved', 'success'); await fetchInbox(); } catch (e) { @@ -55233,7 +55279,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Health', 'Sensing', 'Logging', 'Features', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -55292,6 +55338,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, @@ -56383,6 +56430,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -56595,6 +56643,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> case 'Access': return renderGovSecurity(d.security || {}); case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Work Source': return renderGovWorkSource(d.work_source || {}); From e7c935cf7304e4dcadb5e8016143fbfc52ddc7cf Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 10:24:22 +0000 Subject: [PATCH 080/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.232.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.232.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-voice-sources-docs.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-voice-sources-docs.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c956ab5fe..df3ae99e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.232.0) + +### Added + +- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). + ## 2026-11-12 (v5.231.1) ### Changed diff --git a/changelog.d/added-inbox-voice-sources-docs.md b/changelog.d/added-inbox-voice-sources-docs.md deleted file mode 100644 index 02a0ce3d1f..0000000000 --- a/changelog.d/added-inbox-voice-sources-docs.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). From be009e9567abb2b7e10a9d1625e2b76dfe43371f Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:30:07 +0000 Subject: [PATCH 081/166] =?UTF-8?q?=E2=9C=A8=20inbox:=20Plaud=20source=20o?= =?UTF-8?q?nto=20the=20Source=20interface=20(#17900)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ inbox: read Plaud recordings directly through the Source interface The Inbox only saw recordings through the facts the Plaud connector writes. Add a PlaudSource that lists/fetches straight from the connector (item id stays attr_plaud_file_id), preferred while Plaud is signed in with the fact-backed source as fallback. Refs #17877, part of #17875. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * 🐛 inbox: cache the direct Plaud listing off the poll path and keep fact fields PreferDirect.List spawned the Plaud MCP process on every Items/Find/Sweep call, so each dashboard poll paid for a process spawn plus list_files pagination and a slow spawn past the handler timeout silently flipped the UI back to facts. The direct listing is now cached for a TTL and refreshed by a single background goroutine with its own timeout; List serves the cache (or just facts while nothing is cached) and never blocks on Plaud. A failed refresh clears the cache and is not retried before the TTL. Direct items also dropped FactSlug/FactPath/Layer/Excerpt, hiding the Fact button and excerpt for recordings that do have facts. The fact listing is now merged by id onto the direct items, and facts Plaud no longer lists stay visible. Refs #17877. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 inbox: wait on the Plaud refresh with testutil.Eventually, not a sleep loop The sleep ratchet (TestSleepRatchet) rejects any new fixed time.Sleep in test files; poll the observable condition instead. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 inbox: note that direct-listed Plaud items may precede their fact Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../added-inbox-plaud-direct-source.md | 1 + src/cmd/hive/boot_inbox.go | 27 ++- src/docs/inbox-voice-sources.md | 6 +- src/pkg/inbox/plaudsource.go | 207 ++++++++++++++++++ src/pkg/inbox/plaudsource_test.go | 165 ++++++++++++++ src/pkg/knowledge/connector/plaud.go | 40 ++-- .../knowledge/connector/plaud_recordings.go | 83 +++++++ src/pkg/knowledge/connector/plaud_test.go | 27 +++ 8 files changed, 538 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-inbox-plaud-direct-source.md create mode 100644 src/pkg/inbox/plaudsource.go create mode 100644 src/pkg/inbox/plaudsource_test.go create mode 100644 src/pkg/knowledge/connector/plaud_recordings.go diff --git a/changelog.d/added-inbox-plaud-direct-source.md b/changelog.d/added-inbox-plaud-direct-source.md new file mode 100644 index 0000000000..161055f469 --- /dev/null +++ b/changelog.d/added-inbox-plaud-direct-source.md @@ -0,0 +1 @@ +- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index 552c650911..b30f16ec2b 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -28,7 +28,7 @@ func (b *boot) bootInbox(baseDir string) { var srcs []inbox.Source for _, typ := range sources { if configured[typ] { - srcs = append(srcs, &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ}) + srcs = append(srcs, b.inboxSource(baseDir, typ)) } } if len(srcs) == 0 { @@ -60,6 +60,31 @@ func (b *boot) bootInbox(baseDir string) { b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) } +// inboxSource returns the fact-backed source for typ. For Plaud it prefers +// reading recordings directly from the connector while it is signed in. +func (b *boot) inboxSource(baseDir, typ string) inbox.Source { + facts := &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ} + if typ != inbox.PlaudType || b.knowledgeConnectors == nil { + return facts + } + for _, c := range b.cfg.Knowledge.Connectors { + if c.Type != typ { + continue + } + conn, ok := b.knowledgeConnectors.Connector(c.Name) + reader, isReader := conn.(connector.PlaudReader) + if !ok || !isReader { + continue + } + direct := &inbox.PlaudSource{Name: c.Name, Reader: reader} + if auth, ok := conn.(connector.Authenticator); ok { + direct.Ready = func(ctx context.Context) bool { return auth.AuthStatus(ctx, false).Connected } + } + return &inbox.PreferDirect{Direct: direct, Facts: facts} + } + return facts +} + // inboxAfterSync runs the auto-routing rules once a voice connector synced. func (b *boot) inboxAfterSync(st connector.Status) { svc := b.inboxSvc diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index 4f689aa9cd..4f71b60ba2 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -15,8 +15,10 @@ see the issue linked from the changelog entry. `wisprflow`). Connecting a source happens in **Knowledge → 🔌 Integrations**; that card shows only the connection state and an **Open Inbox** button. - **Item** — one recording, keyed `(source, item_id)` (for Plaud, the - `attr_plaud_file_id`). Every item **is already a Knowledge fact**; the Inbox - only tracks what you did with it. "📚 Fact" opens the fact in Knowledge. + `attr_plaud_file_id`). When the source is connected the Inbox lists + recordings directly from it, so an item may appear before the sync has + written its Knowledge fact; the Inbox only tracks what you did with it. + "📚 Fact" opens the fact in Knowledge once that fact exists. - **Destination** — where an item goes: - **→ RFC** (`spek_rfc`): the hive runs `spektacular voice --from stdin --title <title> --publish --repo <owner/repo>` on the transcript and diff --git a/src/pkg/inbox/plaudsource.go b/src/pkg/inbox/plaudsource.go new file mode 100644 index 0000000000..2443d1b293 --- /dev/null +++ b/src/pkg/inbox/plaudsource.go @@ -0,0 +1,207 @@ +package inbox + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// PlaudType is the Source type for Plaud recordings. +const PlaudType = connector.TypePlaud + +// PlaudSource reads recordings straight from the Plaud connector. Item.ID is +// the Plaud file id (the `attr_plaud_file_id` of the fact the same sync +// writes), so route state carries over between this source and FactSource. +type PlaudSource struct { + // Name is the knowledge.connectors[] entry name. + Name string + Reader connector.PlaudReader + // Ready reports whether the connector is signed in; nil means always. + Ready func(ctx context.Context) bool +} + +// Type implements Source. +func (p *PlaudSource) Type() string { return PlaudType } + +// Available reports whether the connector can be read right now. +func (p *PlaudSource) Available(ctx context.Context) bool { + return p.Reader != nil && (p.Ready == nil || p.Ready(ctx)) +} + +// List implements Source: newest first. +func (p *PlaudSource) List(ctx context.Context) ([]Item, error) { + recs, err := p.Reader.ListRecordings(ctx) + if err != nil { + return nil, err + } + out := make([]Item, 0, len(recs)) + for _, r := range recs { + out = append(out, Item{ + Source: PlaudType, + Connector: p.Name, + ID: r.ID, + Title: r.Title, + RecordedAt: r.RecordedAt, + DurationMS: r.DurationMS, + Tags: r.Tags, + }) + } + sortItems(out) + return out, nil +} + +// Fetch implements Source. +func (p *PlaudSource) Fetch(ctx context.Context, it Item) (Content, error) { + c, err := p.Reader.ReadRecording(ctx, it.ID) + if err != nil { + return Content{}, err + } + var md strings.Builder + md.WriteString("# " + it.Title + "\n") + if c.Summary != "" { + md.WriteString("\n## Summary\n\n" + c.Summary + "\n") + } + if c.Transcript != "" { + md.WriteString("\n## Transcript\n\n" + c.Transcript + "\n") + } + return Content{ + Title: it.Title, + Summary: c.Summary, + Transcript: c.Transcript, + Markdown: strings.TrimSpace(md.String()), + Provenance: PlaudType + "://recordings/" + it.ID, + }, nil +} + +// directListTTL bounds how long a direct Plaud listing is reused before it +// is refreshed in the background. Listing spawns the Plaud MCP process, so +// the dashboard poll must never pay for it. +const directListTTL = 5 * time.Minute + +// directListTimeout bounds one background direct listing. +const directListTimeout = 2 * time.Minute + +// PreferDirect lists through Direct while it is available and healthy and +// falls back to Facts otherwise, so the Inbox keeps working before Plaud is +// signed in or when the MCP server is unreachable. +// +// The direct listing is cached for TTL and refreshed in the background: +// List returns whatever is cached (or just facts while nothing is) and +// never blocks on the Plaud process. Direct items are merged with the facts +// of the same id so they keep FactSlug/FactPath/Layer/Excerpt. +type PreferDirect struct { + Direct *PlaudSource + Facts Source + // TTL overrides directListTTL; zero means the default. + TTL time.Duration + + mu sync.Mutex + direct []Item + fetched time.Time + refreshing bool +} + +// Type implements Source. +func (s *PreferDirect) Type() string { return s.Facts.Type() } + +// List implements Source. +func (s *PreferDirect) List(ctx context.Context) ([]Item, error) { + facts, err := s.Facts.List(ctx) + if !s.Direct.Available(ctx) { + return facts, err + } + direct, stale := s.cached() + if stale { + s.refreshAsync() + } + if direct == nil { + return facts, err + } + return mergeDirect(direct, facts), err +} + +// cached returns the cached direct listing (nil when none or when the last +// refresh failed) and whether it is due for a refresh. +func (s *PreferDirect) cached() (items []Item, stale bool) { + s.mu.Lock() + defer s.mu.Unlock() + ttl := s.TTL + if ttl <= 0 { + ttl = directListTTL + } + return s.direct, !s.refreshing && time.Since(s.fetched) >= ttl +} + +// refreshAsync starts one background Refresh unless one is already running. +func (s *PreferDirect) refreshAsync() { + s.mu.Lock() + if s.refreshing { + s.mu.Unlock() + return + } + s.refreshing = true + s.mu.Unlock() + go func() { + ctx, cancel := context.WithTimeout(context.Background(), directListTimeout) + defer cancel() + _ = s.Refresh(ctx) + }() +} + +// Refresh lists Direct now and caches the result. A failure clears the +// cache so List falls back to Facts, and is not retried before TTL. +func (s *PreferDirect) Refresh(ctx context.Context) error { + items, err := s.Direct.List(ctx) + s.mu.Lock() + defer s.mu.Unlock() + s.refreshing = false + s.fetched = time.Now() + if err != nil { + s.direct = nil + return err + } + if items == nil { + items = []Item{} + } + s.direct = items + return nil +} + +// mergeDirect overlays the fact fields onto direct items with the same id +// and appends facts Plaud no longer lists (outside its since_days window). +func mergeDirect(direct, facts []Item) []Item { + byID := make(map[string]Item, len(facts)) + for _, f := range facts { + byID[f.ID] = f + } + out := make([]Item, 0, len(direct)+len(facts)) + for _, it := range direct { + if f, ok := byID[it.ID]; ok { + it.FactSlug, it.FactPath, it.Layer, it.Excerpt = f.FactSlug, f.FactPath, f.Layer, f.Excerpt + if it.Title == "" { + it.Title = f.Title + } + delete(byID, it.ID) + } + out = append(out, it) + } + for _, f := range facts { + if _, ok := byID[f.ID]; ok { + out = append(out, f) + } + } + sortItems(out) + return out +} + +// Fetch implements Source: fact-backed items (FactPath set) read their fact, +// direct items read Plaud. +func (s *PreferDirect) Fetch(ctx context.Context, it Item) (Content, error) { + if it.FactPath != "" { + return s.Facts.Fetch(ctx, it) + } + return s.Direct.Fetch(ctx, it) +} diff --git a/src/pkg/inbox/plaudsource_test.go b/src/pkg/inbox/plaudsource_test.go new file mode 100644 index 0000000000..7fb9953e2a --- /dev/null +++ b/src/pkg/inbox/plaudsource_test.go @@ -0,0 +1,165 @@ +package inbox + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +type fakePlaud struct { + recs []connector.PlaudRecording + content connector.PlaudContent + listErr error + reads []string +} + +func (f *fakePlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + return f.recs, f.listErr +} + +func (f *fakePlaud) ReadRecording(_ context.Context, id string) (connector.PlaudContent, error) { + f.reads = append(f.reads, id) + return f.content, nil +} + +func TestPlaudSourceListAndFetch(t *testing.T) { + older := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + newer := older.Add(24 * time.Hour) + fp := &fakePlaud{ + recs: []connector.PlaudRecording{ + {ID: "rec-1", Title: "Old", RecordedAt: older, DurationMS: 1000}, + {ID: "rec-2", Title: "New", RecordedAt: newer, Tags: []string{"voice"}}, + }, + content: connector.PlaudContent{Summary: "sum", Transcript: "words"}, + } + src := &PlaudSource{Name: "my-plaud", Reader: fp} + if src.Type() != "plaud" { + t.Fatalf("type = %q", src.Type()) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 2 { + t.Fatalf("items=%v err=%v", items, err) + } + if items[0].ID != "rec-2" || items[0].Connector != "my-plaud" || items[0].Source != "plaud" || items[0].Key() != "plaud/rec-2" { + t.Fatalf("first item = %+v", items[0]) + } + if items[1].DurationMS != 1000 || !items[0].RecordedAt.Equal(newer) { + t.Fatalf("mapping lost: %+v", items) + } + c, err := src.Fetch(context.Background(), items[0]) + if err != nil { + t.Fatal(err) + } + if c.Title != "New" || c.Summary != "sum" || c.Transcript != "words" || c.Provenance != "plaud://recordings/rec-2" { + t.Fatalf("content = %+v", c) + } + if len(fp.reads) != 1 || fp.reads[0] != "rec-2" { + t.Fatalf("reads = %v", fp.reads) + } +} + +func TestPreferDirectSelection(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &fakePlaud{recs: []connector.PlaudRecording{{ID: "direct-1", Title: "Direct", RecordedAt: time.Date(2026, 11, 11, 8, 0, 0, 0, time.UTC)}}} + ready := false + direct := &PlaudSource{Name: "plaud", Reader: fp, Ready: func(context.Context) bool { return ready }} + src := &PreferDirect{Direct: direct, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("not signed in should use facts: %v %v", items, err) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 0 { + t.Fatalf("fact item must not hit plaud: err=%v reads=%v", err, fp.reads) + } + + ready = true + if err := src.Refresh(ctx); err != nil { + t.Fatal(err) + } + items, err = src.List(ctx) + if err != nil || len(items) != 2 || items[0].ID != "direct-1" || items[1].ID != "rec-1" { + t.Fatalf("signed in should merge plaud with facts: %v %v", items, err) + } + if items[1].FactPath == "" || items[1].Excerpt == "" { + t.Fatalf("fact-only item lost fact fields: %+v", items[1]) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 1 { + t.Fatalf("direct item should read plaud: err=%v reads=%v", err, fp.reads) + } + + fp.listErr = errors.New("mcp down") + if err := src.Refresh(ctx); err == nil { + t.Fatal("refresh should report the plaud error") + } + items, err = src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("plaud failure should fall back to facts: %v %v", items, err) + } +} + +// List must never pay for the Plaud process: the first signed-in call +// returns facts and kicks off one background refresh. +func TestPreferDirectListDoesNotBlock(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &countingPlaud{recs: []connector.PlaudRecording{{ID: "rec-1", Title: "Direct title"}}, gate: make(chan struct{})} + src := &PreferDirect{Direct: &PlaudSource{Name: "plaud", Reader: fp}, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + for i := 0; i < 2; i++ { + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].Title != "RFC: Voice inbox" { + t.Fatalf("list %d while plaud is slow should serve facts: %v %v", i, items, err) + } + } + fp.release() + items := testutil.EventuallyValue(t, 5*time.Second, func() ([]Item, bool) { + got, _ := src.List(ctx) + return got, len(got) == 1 && got[0].Title == "Direct title" + }, "background refresh never landed") + if n := fp.calls.Load(); n != 1 { + t.Fatalf("ListRecordings calls = %d, want 1 (single-flight, cached for TTL)", n) + } + it := items[0] + if it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" || it.FactPath == "" || it.Excerpt == "" { + t.Fatalf("direct item lost fact fields: %+v", it) + } +} + +type countingPlaud struct { + recs []connector.PlaudRecording + calls atomic.Int32 + gate chan struct{} + once sync.Once +} + +func (c *countingPlaud) release() { + c.once.Do(func() { + if c.gate != nil { + close(c.gate) + } + }) +} + +func (c *countingPlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + c.calls.Add(1) + if c.gate != nil { + <-c.gate + } + return c.recs, nil +} + +func (c *countingPlaud) ReadRecording(context.Context, string) (connector.PlaudContent, error) { + return connector.PlaudContent{}, nil +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go index f0ea586090..02e959cbaf 100644 --- a/src/pkg/knowledge/connector/plaud.go +++ b/src/pkg/knowledge/connector/plaud.go @@ -238,21 +238,9 @@ func (c *plaudConnector) listFiles(ctx context.Context, client mcpClient, since // fetchPage builds the knowledge page for one recording. func (c *plaudConnector) fetchPage(ctx context.Context, client mcpClient, f plaudFile) (Page, error) { - noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": f.ID}) - transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": f.ID}) - if noteErr != nil && trErr != nil { - return Page{}, plaudAuthError(trErr) - } - if noteErr != nil { - noteText = "" - } - if trErr != nil { - transcriptText = "" - } - note := renderPlaudNote(noteText) - transcript := renderPlaudTranscript(transcriptText) - if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { - return Page{}, fmt.Errorf("no note or transcript available yet") + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, f.ID) + if err != nil { + return Page{}, err } audioURL := "" if c.includeAudio() { @@ -427,3 +415,25 @@ func plaudTime(m map[string]any, keys ...string) time.Time { } return time.Time{} } + +// fetchNoteAndTranscript renders the note and transcript of one recording. +// It fails only when neither is available. +func (c *plaudConnector) fetchNoteAndTranscript(ctx context.Context, client mcpClient, id string) (note, transcript string, err error) { + noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": id}) + transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": id}) + if noteErr != nil && trErr != nil { + return "", "", plaudAuthError(trErr) + } + if noteErr != nil { + noteText = "" + } + if trErr != nil { + transcriptText = "" + } + note = renderPlaudNote(noteText) + transcript = renderPlaudTranscript(transcriptText) + if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { + return "", "", fmt.Errorf("no note or transcript available yet") + } + return note, transcript, nil +} diff --git a/src/pkg/knowledge/connector/plaud_recordings.go b/src/pkg/knowledge/connector/plaud_recordings.go new file mode 100644 index 0000000000..291acb2f32 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_recordings.go @@ -0,0 +1,83 @@ +package connector + +import ( + "context" + "sort" + "strings" + "time" +) + +// PlaudRecording is one recording as the Plaud MCP server lists it. ID is +// the Plaud file id, the same value written to the `plaud_file_id` fact attr. +type PlaudRecording struct { + ID string + Title string + RecordedAt time.Time + DurationMS int64 + Tags []string +} + +// PlaudContent is the rendered note and transcript of one recording. +type PlaudContent struct { + Summary string + Transcript string +} + +// PlaudReader reads recordings straight from Plaud without going through +// the facts a sync writes. The plaud connector implements it. +type PlaudReader interface { + // ListRecordings lists recent recordings, newest first. + ListRecordings(ctx context.Context) ([]PlaudRecording, error) + // ReadRecording returns the note and transcript of one recording. + ReadRecording(ctx context.Context, id string) (PlaudContent, error) +} + +var _ PlaudReader = (*plaudConnector)(nil) + +// ListRecordings lists recordings within the connector's since_days window, +// capped at max_files, newest first. +func (c *plaudConnector) ListRecordings(ctx context.Context) ([]PlaudRecording, error) { + client, err := c.dial(ctx) + if err != nil { + return nil, err + } + defer func() { _ = client.Close() }() + since := c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return nil, err + } + var tags []string + for _, t := range strings.Split(c.cfg.ScopeValue("tags"), ",") { + if t = strings.TrimSpace(t); t != "" { + tags = append(tags, t) + } + } + out := make([]PlaudRecording, 0, len(files)) + for _, f := range files { + rec := PlaudRecording{ID: f.ID, Title: strings.TrimSpace(f.Name), RecordedAt: f.StartAt.UTC(), DurationMS: f.DurationMS, Tags: tags} + if f.StartAt.IsZero() { + rec.RecordedAt = f.CreatedAt.UTC() + } + if rec.Title == "" { + rec.Title = "Plaud recording " + f.ID + } + out = append(out, rec) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].RecordedAt.After(out[j].RecordedAt) }) + return out, nil +} + +// ReadRecording fetches the note and transcript of one recording. +func (c *plaudConnector) ReadRecording(ctx context.Context, id string) (PlaudContent, error) { + client, err := c.dial(ctx) + if err != nil { + return PlaudContent{}, err + } + defer func() { _ = client.Close() }() + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, id) + if err != nil { + return PlaudContent{}, err + } + return PlaudContent{Summary: strings.TrimSpace(note), Transcript: strings.TrimSpace(transcript)}, nil +} diff --git a/src/pkg/knowledge/connector/plaud_test.go b/src/pkg/knowledge/connector/plaud_test.go index eada7e56b8..c0f645393f 100644 --- a/src/pkg/knowledge/connector/plaud_test.go +++ b/src/pkg/knowledge/connector/plaud_test.go @@ -227,3 +227,30 @@ func TestPlaudRenderHelpers(t *testing.T) { t.Fatalf("auth error = %q", got) } } + +func TestPlaudListAndReadRecording(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), map[string]string{"tags": "voice, ideas"}) + recs, err := pc.ListRecordings(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 || recs[0].ID != "rec-2" || recs[1].ID != "rec-1" { + t.Fatalf("recs = %+v", recs) + } + if recs[1].Title != "Standup ideas" || recs[1].DurationMS != 125000 || len(recs[1].Tags) != 2 { + t.Fatalf("rec-1 = %+v", recs[1]) + } + if want := time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC); !recs[1].RecordedAt.Equal(want) { + t.Fatalf("recorded at = %v", recs[1].RecordedAt) + } + c, err := pc.ReadRecording(context.Background(), "rec-1") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(c.Summary, "ship plaud connector") || !strings.Contains(c.Transcript, "Hello hive") { + t.Fatalf("content = %+v", c) + } + if _, err := pc.ReadRecording(context.Background(), "rec-2"); err == nil { + t.Fatal("recording with no note or transcript should error") + } +} From e2230f97fcb0cb39c711effdf0fd5b49e54b2307 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 10:54:36 +0000 Subject: [PATCH 082/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.233.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.233.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-settings-section.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-settings-section.md diff --git a/CHANGELOG.md b/CHANGELOG.md index df3ae99e7e..f03c8e78de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.233.0) + +### Added + +- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. + ## 2026-11-12 (v5.232.0) ### Added diff --git a/changelog.d/added-inbox-settings-section.md b/changelog.d/added-inbox-settings-section.md deleted file mode 100644 index 7199a8ab60..0000000000 --- a/changelog.d/added-inbox-settings-section.md +++ /dev/null @@ -1 +0,0 @@ -- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. From 249e4fb898bb703f944fd6a1fc06b20cc64d2e79 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:14:06 +0000 Subject: [PATCH 083/166] =?UTF-8?q?=F0=9F=90=9B=20check-action-pins:=20onl?= =?UTF-8?q?y=20HTTP=20404/422=20is=20MISSING-SHA;=20retry=20transient=20AP?= =?UTF-8?q?I=20errors=20and=20report=20them=20as=20UNRESOLVED=20(#17905)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 check-action-pins: only 404/422 is MISSING-SHA; retry and report transient API errors as UNRESOLVED A rate limit, 5xx or dropped connection from gh api was printed as 'does not exist in that repository', which is a wrong verdict: the pin was fine and the visible next step (bump the pin) was the wrong one. Retry three times, then fail closed with a distinct UNRESOLVED line. The test double now answers a missing SHA with the real HTTP 422 text and gains a transient-error case that fails without this change. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 integration-governance: re-pin TRUSTED_SCRIPTS_TREE for the check-action-pins change src/scripts changed in this PR, so the enrolled .github closure must pin the new tree or the materializer contract test rejects it. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/integration-governance.cjs | 2 +- ...d-check-action-pins-transient-api-error.md | 1 + src/scripts/check-action-pins.sh | 39 +++++++++++++++++-- src/scripts/test-check-action-pins.sh | 35 +++++++++++++++-- 4 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-check-action-pins-transient-api-error.md diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 87ad4e2ce7..27e5a5c64b 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '1f1eb904839561af15c1772e6d9106a3b8d2b028'; +const TRUSTED_SCRIPTS_TREE = '08802cd3162cd8a179514afe3ddb5e088c90d5f0'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/changelog.d/fixed-check-action-pins-transient-api-error.md b/changelog.d/fixed-check-action-pins-transient-api-error.md new file mode 100644 index 0000000000..b158778fd9 --- /dev/null +++ b/changelog.d/fixed-check-action-pins-transient-api-error.md @@ -0,0 +1 @@ +- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). diff --git a/src/scripts/check-action-pins.sh b/src/scripts/check-action-pins.sh index e60dd905a0..a143d5f9e3 100755 --- a/src/scripts/check-action-pins.sh +++ b/src/scripts/check-action-pins.sh @@ -17,6 +17,7 @@ set -uo pipefail DIR="${1:-.github/workflows}" fail=0 +RESOLVE_ERR="" # Every `uses:` on a third-party action must be a 40-hex SHA, not a tag or # branch. A moving ref is a supply-chain hole; that part is shape-only and needs @@ -34,6 +35,29 @@ while IFS= read -r line; do # DIR, so `${line%%:*}` names the workflow and not its line number. done < <(grep -rHnE '^\s*(-\s*)?uses:\s*[a-zA-Z0-9._-]+/' "$DIR" 2>/dev/null | grep -v '^\s*#') +# resolve_pin OWNER/REPO SHA — 0 the commit exists, 1 the API says it does not +# (422 "No commit found for SHA" or 404 for the repository itself), 2 the API +# gave no verdict. Only a definite 404/422 may be reported as MISSING-SHA: +# a rate limit, a 5xx or a dropped connection says nothing about the pin, and +# calling it "does not exist" sent crux main red on a README-only commit whose +# setup-go pin was tagged v7.0.0 upstream all along. Transient errors are +# retried briefly, then reported as UNRESOLVED so a re-run — not a pin bump — +# is the visible next step. +resolve_pin() { + local repo="$1" sha="$2" err attempt + for attempt in 1 2 3; do + if err="$(gh api "repos/$repo/commits/$sha" --jq '.sha' 2>&1 >/dev/null)"; then + return 0 + fi + case "$err" in + *"(HTTP 404)"*|*"(HTTP 422)"*) return 1 ;; + esac + [ "$attempt" -lt 3 ] && sleep "$attempt" + done + RESOLVE_ERR="$(printf '%s' "$err" | head -1)" + return 2 +} + # Resolve each unique pin. Skipped without a token so the script stays usable # offline; CI always has one. if command -v gh >/dev/null 2>&1; then @@ -51,10 +75,17 @@ if command -v gh >/dev/null 2>&1; then [ -z "$ref" ] && continue sha="${ref#*@}"; path="${ref%@*}" repo="$(printf '%s' "$path" | cut -d/ -f1,2)" - if ! gh api "repos/$repo/commits/$sha" --jq '.sha' >/dev/null 2>&1; then - echo "MISSING-SHA $repo@$sha does not exist in that repository" - fail=1 - fi + resolve_pin "$repo" "$sha" + case "$?" in + 1) + echo "MISSING-SHA $repo@$sha does not exist in that repository" + fail=1 + ;; + 2) + echo "UNRESOLVED $repo@$sha: the GitHub API gave no verdict after 3 attempts ($RESOLVE_ERR); re-run, this is not a missing pin" + fail=1 + ;; + esac done <<<"$pins" fi diff --git a/src/scripts/test-check-action-pins.sh b/src/scripts/test-check-action-pins.sh index c796f1384b..0d70f4fcaa 100755 --- a/src/scripts/test-check-action-pins.sh +++ b/src/scripts/test-check-action-pins.sh @@ -48,14 +48,19 @@ assert_eq() { # Fake `gh`: only `gh api repos/<owner>/<repo>/commits/<sha> --jq .sha` is # ever called by the check. Every call is appended to $FAKE_GH_LOG; a call # whose SHA equals $FAKE_GH_MISSING fails like the real API does for a commit -# that does not exist in the repository. +# that does not exist in the repository (HTTP 422 on stderr), and one whose +# SHA equals $FAKE_GH_TRANSIENT fails like a rate limit (HTTP 403), which is +# not a verdict on the pin. BIN="$TMP/bin" mkdir -p "$BIN" cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 @@ -183,7 +188,8 @@ cat >"$BIN/gh" <<SH #!/usr/bin/env bash printf '%s\n' "\$*" >>"\${FAKE_GH_LOG:?}" case "\$2" in - */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) exit 1 ;; + */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) + echo "gh: No commit found for SHA: \${2##*/} (HTTP 422)" >&2; exit 1 ;; esac printf '%s\n' "\${2##*/}" exit 0 @@ -199,12 +205,33 @@ cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 SH +echo "case: an API error that is not 404/422 is not a missing SHA" +D="$TMP/transient" +workflow "$D" ci.yml <<EOF +jobs: + build: + steps: + - uses: actions/checkout@$SHA_A + - uses: actions/setup-go@$SHA_B # v7.0.0 +EOF +run_check transient FAKE_GH_TRANSIENT="$SHA_B" -- "$D" +assert_exit 1 "$RC" "an unresolved pin still fails closed" +assert_not_contains "$OUT" "MISSING-SHA" "a rate limit is not reported as a missing SHA" +assert_contains "$OUT" "UNRESOLVED actions/setup-go@$SHA_B" "reports the pin as UNRESOLVED instead" +assert_contains "$OUT" "(HTTP 403)" "quotes the API error" +assert_contains "$OUT" "action pin check FAILED" "prints the failure banner" +assert_eq 3 "$(grep -c "commits/$SHA_B" <<<"$GH_CALLS")" "the transient failure is retried three times" +assert_eq 1 "$(grep -c "commits/$SHA_A" <<<"$GH_CALLS")" "a pin that resolves first time is not retried" + echo "case: a directory with no third-party actions passes with gh present" D="$TMP/nopins" workflow "$D" ci.yml <<EOF From 0e058a47dfc771389fa32c0f92fbd5867c2cabf2 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 11:28:01 +0000 Subject: [PATCH 084/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.234.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.234.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-inbox-plaud-direct-source.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-inbox-plaud-direct-source.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f03c8e78de..e434f84c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.234.0) + +### Added + +- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. + ## 2026-11-12 (v5.233.0) ### Added diff --git a/changelog.d/added-inbox-plaud-direct-source.md b/changelog.d/added-inbox-plaud-direct-source.md deleted file mode 100644 index 161055f469..0000000000 --- a/changelog.d/added-inbox-plaud-direct-source.md +++ /dev/null @@ -1 +0,0 @@ -- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. From 6c760a36a3cf07a62b6de89466c65de8777ea1d9 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:48:12 +0100 Subject: [PATCH 085/166] =?UTF-8?q?=E2=9C=A8=20knowledge:=20prepare=20Wisp?= =?UTF-8?q?r=20Flow=20facts=20for=20Inbox=20(#17891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ knowledge: prepare Wispr Flow facts for Inbox Hive-Run: hivecommons/hive#17878 Hive-Plan: voice-recorders-inbox Hive-Spec: voice-recorders-inbox#source Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> * chore: retrigger review after correcting issue link The PR description now references #17878 rather than closing it because the Source adapter remains outstanding in #17885. Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-wisprflow-inbox-metadata.md | 1 + src/docs/knowledge-connectors.md | 11 ++++--- src/pkg/knowledge/connector/wisprflow.go | 5 ++-- src/pkg/knowledge/connector/wisprflow_test.go | 30 ++++++++++++++++++- 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added-wisprflow-inbox-metadata.md diff --git a/changelog.d/added-wisprflow-inbox-metadata.md b/changelog.d/added-wisprflow-inbox-metadata.md new file mode 100644 index 0000000000..70828adc48 --- /dev/null +++ b/changelog.d/added-wisprflow-inbox-metadata.md @@ -0,0 +1 @@ +- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index e5a24b695b..12dc561672 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -396,10 +396,13 @@ rest are still imported. seen minus a 2 h overlap (Flow updates a row in place as it finalises), or `since_days` back on the first run. With `group: day` the current day's fact is rebuilt whole. -- Facts carry `source: wisprflow`, `attr_wispr_transcript_id`, - `attr_dictated_at`, `attr_app` (bundle id), `attr_num_words`, - `attr_language` and the dictation date as the vault path - (`Wispr Flow/2026-11-11`). The title is the first ten words. +- Dictation facts carry `source: wisprflow`, `source_id`, + `attr_recorded_at`, `attr_wispr_transcript_id`, `attr_dictated_at`, + `attr_app` (bundle id), `attr_num_words`, `attr_language`, and the + dictation date as the vault path (`Wispr Flow/2026-11-11`). The title is + the first ten words, and the formatted text is under `## Transcript`. + With `group: day`, the fact represents the entire day and its + `attr_recorded_at` is the newest dictation's time. - The listing is incremental, so dictations you archive or delete in Flow are not tombstoned automatically; remove the fact file or the connector. - Everything here costs **$0** beyond your Wispr Flow plan; the free tier's diff --git a/src/pkg/knowledge/connector/wisprflow.go b/src/pkg/knowledge/connector/wisprflow.go index 39f902748e..9ac4fc239c 100644 --- a/src/pkg/knowledge/connector/wisprflow.go +++ b/src/pkg/knowledge/connector/wisprflow.go @@ -349,7 +349,7 @@ func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags s if r.Duration > 0 { fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(r.Duration * float64(time.Second))).Round(time.Second)) } - md.WriteString("\n") + md.WriteString("\n## Transcript\n\n") md.WriteString(r.Text) md.WriteString("\n") if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { @@ -360,6 +360,7 @@ func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags s attrs := map[string]string{ "wispr_transcript_id": r.ID, "dictated_at": r.At.UTC().Format(time.RFC3339), + "recorded_at": r.At.UTC().Format(time.RFC3339), } if r.App != "" { attrs["app"] = r.App @@ -405,7 +406,7 @@ func (c *wisprConnector) dayPage(day string, rows []wisprDictation, includeRaw b fmt.Fprintf(&md, "<details><summary>Raw transcript</summary>\n\n%s\n\n</details>\n\n", strings.TrimSpace(r.RawText)) } } - attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows))} + attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows)), "recorded_at": newest.UTC().Format(time.RFC3339)} if words > 0 { attrs["num_words"] = strconv.Itoa(words) } diff --git a/src/pkg/knowledge/connector/wisprflow_test.go b/src/pkg/knowledge/connector/wisprflow_test.go index c20df12fda..64545e2abd 100644 --- a/src/pkg/knowledge/connector/wisprflow_test.go +++ b/src/pkg/knowledge/connector/wisprflow_test.go @@ -195,6 +195,34 @@ func TestWisprSyncSQLite(t *testing.T) { } } +func TestWisprFactInboxMetadata(t *testing.T) { + c := newFakeWispr(t, map[string]string{"include_raw": "true"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + w := testWriter(t) + slug, _, err := w.Write(c.cfg, pages[1]) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(w.Dir, slug+".md")) + if err != nil { + t.Fatal(err) + } + fact := string(data) + for _, want := range []string{ + "source: wisprflow\n", + "source_id: d-2\n", + "attr_recorded_at: 2026-11-11T17:55:30Z\n", + "## Transcript\n\nRunning late — start without me.\n", + } { + if !strings.Contains(fact, want) { + t.Errorf("fact missing %q:\n%s", want, fact) + } + } +} + func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { c := newFakeWispr(t, map[string]string{"apps": "slack, vscode", "group": "day", "since_days": "3650"}) pages, _, err := collectWispr(t, c, "") @@ -205,7 +233,7 @@ func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { t.Fatalf("pages = %v", got) } day := pages[1] - if day.Attrs["dictations"] != "1" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { + if day.Attrs["dictations"] != "1" || day.Attrs["recorded_at"] != "2026-11-11T17:55:30Z" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { t.Errorf("day page = %v\n%s", day.Attrs, day.Markdown) } if strings.Contains(day.Markdown, "Ok.") { From 91a9ab10fee87b1e7e3f995159aa951fe2051686 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 11:59:18 +0000 Subject: [PATCH 086/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.234.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.234.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-check-action-pins-transient-api-error.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-check-action-pins-transient-api-error.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e434f84c45..6c6041a981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.234.1) + +### Fixed + +- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). + ## 2026-11-12 (v5.234.0) ### Added diff --git a/changelog.d/fixed-check-action-pins-transient-api-error.md b/changelog.d/fixed-check-action-pins-transient-api-error.md deleted file mode 100644 index b158778fd9..0000000000 --- a/changelog.d/fixed-check-action-pins-transient-api-error.md +++ /dev/null @@ -1 +0,0 @@ -- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED <repo>@<sha> … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). From e4494cb1e8b5921e766ed3790f0c2e70413c0a57 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 12:28:38 +0000 Subject: [PATCH 087/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.235.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.235.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-wisprflow-inbox-metadata.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-wisprflow-inbox-metadata.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6041a981..0337292185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.235.0) + +### Added + +- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. + ## 2026-11-12 (v5.234.1) ### Fixed diff --git a/changelog.d/added-wisprflow-inbox-metadata.md b/changelog.d/added-wisprflow-inbox-metadata.md deleted file mode 100644 index 70828adc48..0000000000 --- a/changelog.d/added-wisprflow-inbox-metadata.md +++ /dev/null @@ -1 +0,0 @@ -- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. From 9d8c9c5d0f952f3f5c37bd996f2f040978876a1f Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:48:25 +0100 Subject: [PATCH 088/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20avoid=20premature?= =?UTF-8?q?=20eval-cycle=20test=20timeout=20(#17872)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: avoid premature eval-cycle test timeout Hive-Run: hivecommons/hive#17866 Hive-Plan: eval-cycle-flake Hive-Spec: ci-flake#17866 Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * test: isolate eval cycle from rate-limit pacing Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * ci: retry transient action pin resolution The reported missing hivecommons/infra commit resolves through the GitHub API; retry the check without changing the valid pin. Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/cmd/hive/eval_cycle_run_7232_test.go | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/cmd/hive/eval_cycle_run_7232_test.go b/src/cmd/hive/eval_cycle_run_7232_test.go index 8631532df0..ea07cd3dde 100644 --- a/src/cmd/hive/eval_cycle_run_7232_test.go +++ b/src/cmd/hive/eval_cycle_run_7232_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -25,10 +26,12 @@ import ( // touches on a quiet org: every list endpoint is empty, every object endpoint // is a minimal document, and nothing ever errors. It records the paths hit so // a test can assert the cycle actually enumerated rather than bailed early. -func newEvalCycleAPI(t *testing.T, hits *[]string) *httptest.Server { +func newEvalCycleAPI(t *testing.T, mu *sync.Mutex, hits *[]string) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() *hits = append(*hits, r.Method+" "+r.URL.Path) + mu.Unlock() w.Header().Set("Content-Type", "application/json") switch { case r.URL.Path == "/rate_limit": @@ -63,6 +66,7 @@ type evalCycleFixture struct { dashSrv *dashboard.Server last *atomic.Pointer[github.ActionableResult] hits []string + hitsMu sync.Mutex beadStores map[string]*beads.Store advisoryIssues map[string]int @@ -70,8 +74,11 @@ type evalCycleFixture struct { func newEvalCycleFixture(t *testing.T) *evalCycleFixture { t.Helper() + // A prior rate-limit test can leave process-wide request pacing enabled. + github.ResetRateLimitPacingForTest() + t.Cleanup(github.ResetRateLimitPacingForTest) f := &evalCycleFixture{last: new(atomic.Pointer[github.ActionableResult])} - api := newEvalCycleAPI(t, &f.hits) + api := newEvalCycleAPI(t, &f.hitsMu, &f.hits) logger := restoreTestLogger() f.cfg = &config.Config{} @@ -92,7 +99,8 @@ func newEvalCycleFixture(t *testing.T) *evalCycleFixture { func (f *evalCycleFixture) run(t *testing.T) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + // Bound a stalled in-process cycle without imposing a tight CI deadline. + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() runEvalCycle(ctx, f.cfg, f.gh, f.gov, f.sched, f.agentMgr, f.dashSrv, nil, // notifier @@ -128,7 +136,10 @@ func TestRunEvalCycle_QuietOrgEnumeratesAndPublishesStatus(t *testing.T) { f.run(t) - if len(f.hits) == 0 { + f.hitsMu.Lock() + hitCount := len(f.hits) + f.hitsMu.Unlock() + if hitCount == 0 { t.Fatal("runEvalCycle made no GitHub calls: it bailed before enumeration") } if got := f.last.Load(); got == nil { @@ -176,14 +187,17 @@ func TestRunEvalCycle_PostsAdvisoryDigestAndHealsAppAuthFinding(t *testing.T) { f.run(t) + f.hitsMu.Lock() + hits := append([]string(nil), f.hits...) + f.hitsMu.Unlock() var posted bool - for _, h := range f.hits { + for _, h := range hits { if h == "POST /repos/testorg/widget/issues/4/comments" { posted = true } } if !posted { - t.Fatalf("digest was never posted to the pinned issue; hits=%v", f.hits) + t.Fatalf("digest was never posted to the pinned issue; hits=%v", hits) } healed, err := store.Get(finding.ID) if err != nil { From bd4dd2052bcb38b4e0660209f4e250af776d42e4 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:28:25 -0700 Subject: [PATCH 089/166] =?UTF-8?q?=F0=9F=90=9B=20gh-wrapper:=20let=20hive?= =?UTF-8?q?-baseline-check.sh's=20sibling=20lookup=20through=20the=20enume?= =?UTF-8?q?ration=20gate=20(#17838)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 gh-wrapper: let hive-baseline-check.sh's sibling lookup through the enumeration gate hive-baseline-check.sh is the kick's mandated triage step and its one narrow `gh pr list` projection is the sanctioned exception to the enumeration ban, but the wrapper refused it so the helper always exited 2. The helper now sets HIVE_BASELINE_CHECK=1 and the wrapper allows only --state open with --json statusCheckRollup and no --search; a policy block is reported distinctly from an API failure. Closes #17835 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 gh-wrapper: tighten _pr_list_baseline_check to an allowlist of the helper's exact projection Per maintainer review on #17838, the HIVE_BASELINE_CHECK=1 carve-out was wider than #17835's one projection: a marked `gh pr list --state open --label actionable --limit 500 --json number,title,labels,statusCheckRollup` passed. The predicate is now an allowlist that a forgotten flag cannot widen: only --repo/-R, --state/-s (open), --limit/-L (<=100) and --json are accepted, every --json field must be one of number,headRefOid,isCrossRepository,statusCheckRollup, and any other flag or positional refuses. The comment now states the marker is a caller-set hint, not a trust boundary. Tests: helper's exact call still allowed (plus -R/equals/short forms); --label, title field, --limit 500, missing --json and --state all blocked; existing --search and no-marker cases kept. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 gh-wrapper: validate every repeated --json in _pr_list_baseline_check gh accumulates repeated --json/--json= occurrences, but the baseline-check gate kept only the last value and validated that alone, so `--json title --json number` reached gh with `title` unchecked. Accumulate the fields across occurrences so the allowlist sees all of them, and add test rows for a blocked and an allowed repeated --json. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- bin/gh-wrapper.sh | 57 +++++++++++++++++++ bin/gh-wrapper.test.sh | 22 +++++++ bin/hive-baseline-check.sh | 12 +++- .../fixed-baseline-check-gh-wrapper.md | 1 + 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-baseline-check-gh-wrapper.md diff --git a/bin/gh-wrapper.sh b/bin/gh-wrapper.sh index 0284212fcc..b2c9d07624 100755 --- a/bin/gh-wrapper.sh +++ b/bin/gh-wrapper.sh @@ -592,6 +592,61 @@ _pr_list_merged_only() { [[ "$(_lower "$state")" = "merged" ]] } +# The one `gh pr list` that bin/hive-baseline-check.sh makes for its sibling-PR +# lookup (#17835): the kick mandates that helper before any check is retried, +# and names that lookup as the sanctioned exception to the enumeration ban. The +# helper marks its call with HIVE_BASELINE_CHECK=1. That marker is a caller-set +# hint, not a trust boundary — any agent can set it, exactly as with +# HIVE_CONTRIBUTOR_MODE above — so what makes the exception safe is that only +# the helper's exact projection is let through, checked as an allowlist that a +# forgotten flag cannot widen: the sole flags accepted are --repo/-R, --state/-s +# (must resolve to open), --limit/-L (at most 100, what the helper sends) and +# --json whose every field is one of number, headRefOid, isCrossRepository, +# statusCheckRollup. Any other flag or positional (--label, --search, --author, +# --base, -q, ...) refuses, since each would turn it into a general work queue. +_pr_list_baseline_check() { + [[ "${HIVE_BASELINE_CHECK:-}" = "1" ]] || return 1 + [[ "$subcmd" = "pr" && "$action" = "list" ]] || return 1 + local i a state="" json="" limit="" seen_sub=0 seen_act=0 + for ((i=0; i<${#args[@]}; i++)); do + a="${args[$i]}" + case "$a" in + --repo=*|-R=*|-R?*) ;; + --repo|-R) i=$((i+1)) ;; + --state=*) state="${a#--state=}" ;; + -s=*) state="${a#-s=}" ;; + -s?*) state="${a#-s}" ;; + --state|-s) i=$((i+1)); state="${args[$i]:-}" ;; + --limit=*) limit="${a#--limit=}" ;; + -L=*) limit="${a#-L=}" ;; + -L?*) limit="${a#-L}" ;; + --limit|-L) i=$((i+1)); limit="${args[$i]:-}" ;; + # gh accumulates repeated --json values, so every occurrence must be + # validated, not just the last one. + --json=*) json="${json:+${json},}${a#--json=}" ;; + --json) i=$((i+1)); json="${json:+${json},}${args[$i]:-}" ;; + -*) return 1 ;; + pr) [[ "$seen_sub" = 0 ]] || return 1; seen_sub=1 ;; + list) [[ "$seen_sub" = 1 && "$seen_act" = 0 ]] || return 1; seen_act=1 ;; + *) return 1 ;; + esac + done + [[ "$(_lower "$state")" = "open" ]] || return 1 + if [[ -n "$limit" ]]; then + [[ "$limit" =~ ^[0-9]+$ && "$limit" -le 100 ]] || return 1 + fi + [[ -n "$json" ]] || return 1 + local f _fields + IFS=',' read -ra _fields <<<"$json" + for f in "${_fields[@]}"; do + case "$f" in + number|headRefOid|isCrossRepository|statusCheckRollup) ;; + *) return 1 ;; + esac + done + return 0 +} + # Resolve the authenticated GitHub login. Initialize the cache internally so a # caller-controlled environment cannot seed a trusted identity (#3982). # @@ -726,6 +781,8 @@ if [ "$_gh_author_scoped" = true ]; then : elif _pr_list_merged_only; then : # Merged PRs only: history, not a work queue (see _pr_list_merged_only). + elif _pr_list_baseline_check; then + : # hive-baseline-check.sh's sibling lookup (see _pr_list_baseline_check). else # Root-caused in a live hive (2026-08-20): agents read this two-line # message as "all gh $subcmd commands are blocked" and silently skipped diff --git a/bin/gh-wrapper.test.sh b/bin/gh-wrapper.test.sh index 85d0a7175c..fd44b41572 100755 --- a/bin/gh-wrapper.test.sh +++ b/bin/gh-wrapper.test.sh @@ -495,6 +495,28 @@ _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged -S review (block pr list --repo test/repo --state merged -S review _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged --search=x (blocked: equals search flag)" \ pr list --repo test/repo --state merged --search=x +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --state open statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: baseline helper's sibling lookup)" \ + pr list --repo test/repo --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list -R before the subcommand, equals/short forms with HIVE_BASELINE_CHECK=1 (allowed: same projection)" \ + -R test/repo pr list --state=open -L50 --json=number,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --search with HIVE_BASELINE_CHECK=1 (blocked)" \ + pr list --repo test/repo --state open --search foo +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --label actionable with HIVE_BASELINE_CHECK=1 (blocked: flag outside the allowlist)" \ + pr list --repo test/repo --state open --label actionable --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json number,title,statusCheckRollup with HIVE_BASELINE_CHECK=1 (blocked: field outside the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number,title,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json title --json number with HIVE_BASELINE_CHECK=1 (blocked: gh accumulates repeated --json, every occurrence is checked)" \ + pr list --repo test/repo --state open --limit 100 --json title --json number +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --json number --json statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: repeated --json, all fields in the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number --json statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --limit 500 with HIVE_BASELINE_CHECK=1 (blocked: over the helper's 100)" \ + pr list --repo test/repo --state open --limit 500 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open without --json with HIVE_BASELINE_CHECK=1 (blocked: no projection)" \ + pr list --repo test/repo --state open --limit 100 +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state all with HIVE_BASELINE_CHECK=1 (blocked: --state open required)" \ + pr list --repo test/repo --state all --limit 100 --json number,statusCheckRollup +_run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --json statusCheckRollup without HIVE_BASELINE_CHECK (blocked)" \ + pr list --repo test/repo --state open --json statusCheckRollup _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state with no value (blocked)" \ pr list --repo test/repo --state _run_test_staff_gate 1 "$ENUM_DENY" "-" "issue list --state merged (blocked: issues are never merged)" \ diff --git a/bin/hive-baseline-check.sh b/bin/hive-baseline-check.sh index 83b930aa8f..07756ca84d 100755 --- a/bin/hive-baseline-check.sh +++ b/bin/hive-baseline-check.sh @@ -155,8 +155,16 @@ if (( BASE_AGE_HOURS > MAX_AGE_HOURS )); then BASE_STALE=true fi -if ! PRS_JSON="$(gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup)"; then - error "could not read open PR checks for $REPO" +# HIVE_BASELINE_CHECK=1 marks this call for the gh wrapper, which lets only this +# narrow projection through its `gh pr list` enumeration gate (#17835). +PRS_ERR_FILE="$(mktemp)" +trap 'rm -f "$PRS_ERR_FILE"' EXIT +if ! PRS_JSON="$(HIVE_BASELINE_CHECK=1 gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup 2>"$PRS_ERR_FILE")"; then + if grep -qE 'enumeration_disabled|BLOCKED' "$PRS_ERR_FILE"; then + error "gh wrapper policy blocked the open PR lookup for $REPO (not a GitHub API failure): $(head -c 300 "$PRS_ERR_FILE" | tr '\n' ' ')" + fi + cat "$PRS_ERR_FILE" >&2 + error "could not read open PR checks for $REPO (GitHub API failure)" fi if ! SIBLING_PRS="$(jq -cer --arg check "$CHECK_NAME" ' [ diff --git a/changelog.d/fixed-baseline-check-gh-wrapper.md b/changelog.d/fixed-baseline-check-gh-wrapper.md new file mode 100644 index 0000000000..4d3ccfe69b --- /dev/null +++ b/changelog.d/fixed-baseline-check-gh-wrapper.md @@ -0,0 +1 @@ +- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. From ac07475b00411d35d0633d50f78ea106c657a7b5 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:28:30 -0700 Subject: [PATCH 090/166] =?UTF-8?q?=E2=9C=A8=20[v5]=20claude=20auth:=20run?= =?UTF-8?q?=20Claude=20agents=20on=20an=20operator-supplied=20long-lived?= =?UTF-8?q?=20token=20(HIVE=5FCLAUDE=5FOAUTH=5FTOKEN)=20(#17890)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ claude auth: run Claude agents on an operator-supplied long-lived token (HIVE_CLAUDE_OAUTH_TOKEN) (#17762) * ✨ claude auth: inject operator-supplied long-lived token via HIVE_CLAUDE_OAUTH_TOKEN Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> * 🐛 claude auth: honor HIVE_CLAUDE_OAUTH_TOKEN in headless launches; resync api-reference citations HeadlessCredentialEnv now applies the same precedence as agentEnvPairs: the operator-supplied long-lived token wins over the short-lived credentials snapshot, so hub-launched claude CLIs no longer pin to a token that cannot refresh. Adds a unit test for the new path. api-reference.md citations for GET /api/logs/sources and /api/logs/read resynced via check-api-reference-citations.sh --fix after the import added in api_logs.go shifted the registrations by one line. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🔒 claude auth: strip HIVE_CLAUDE_OAUTH_TOKEN from every agent pane's inherited env The hive process holds HIVE_CLAUDE_OAUTH_TOKEN in its own environment, and each tmux server it starts inherits that into its GLOBAL environment, which tmux copies into every pane it forks. applySessionEnv never removed it, so copilot/codex/bob agent shells (and every tool shell they spawn) could read the year-long token under its original name — contradicting the claim that the token never appears in a pane. The claude backend was unaffected in practice only because it also receives the value as CLAUDE_CODE_OAUTH_TOKEN. Add claude.OperatorTokenEnvVar to inheritedCredentialEnvVars so the creation-time strip records a `set-environment -r` removal for it, hiding the global value from every process forked afterwards. The file form (HIVE_CLAUDE_OAUTH_TOKEN_FILE) carries only a path and is unaffected. The real-tmux leak test now plants the variable in the server env and asserts a fresh pane no longer sees it; a tmux-free unit test pins the entry in inheritedCredentialEnvVars. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🌱 chore: retrigger build after buildx GHA cache BlobNotFound Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 8c35975f2d2cf371656e9e2c64a79e0db31397da) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📖 docs: explain Claude setup-token operator flow (#17841) * 📖 docs: explain Claude setup-token operator flow Hive-Run: hivecommons/hive#17761 Hive-Plan: 17753 item 8 documentation Hive-Spec: 17753#docs Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> * chore: correct PR closing reference Remove the unrelated closing reference from the PR description; this signed follow-up requests a fresh review. Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> (cherry picked from commit 7f27bbaa3859cdeb22811537d69ce24644b729fc) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 claude auth: suspend the credential re-bridge and durable-credential watchdog while HIVE_CLAUDE_OAUTH_TOKEN is set (#17858) With the operator's long-lived token set, Claude agents authenticate through CLAUDE_CODE_OAUTH_TOKEN and the shared ~/.claude/.credentials.json is not in use. The per-agent credential re-bridge (launch, watchdog poll, post-sign-in) and the watchdog's Claude probe now do nothing in that mode: no files moved, no claude_token_missing audit or warning, no broken-bridge flag in status. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset nothing changes. Fixes #17760 Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Andrew Anderson <andy@clubanderson.com> (cherry picked from commit 9903fea75b994e36310fc97c1623f5d40e77c5cb) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix(claude): adapt long-lived token port to v5 v5 has no HeadlessCredentialEnv, no /api/logs read API and no CredentialBridgeBroken status field, so those parts of the v6 commits have nothing to attach to here: the headless and log-scrub hunks were dropped during the #17762 cherry-pick, and the manager-level status override (snapshotAgent) from #17858 is removed. The operator-token-mode tests now build on v5's bridgeTestFixture/simulateClaudeRefresh helpers. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-claude-long-lived-token.md | 1 + ...claude-operator-token-suspends-rebridge.md | 1 + src/crux/agent/claude_credential_bridge.go | 15 +- src/crux/agent/claude_operator_token_mode.go | 38 ++++ .../agent/claude_operator_token_mode_test.go | 127 +++++++++++++ src/crux/agent/claude_operator_token_test.go | 167 ++++++++++++++++++ src/crux/agent/credentials_watchdog.go | 11 +- src/crux/agent/manager.go | 17 +- src/crux/agent/manager_env.go | 10 +- src/crux/agent/manager_tmux.go | 8 + src/crux/agent/tmux_inherited_env_test.go | 3 +- src/crux/claude/operator_token.go | 40 +++++ src/crux/claude/operator_token_test.go | 41 +++++ src/docs/agent-configuration.md | 26 +++ src/docs/backend-auth-recovery-runbook.md | 18 +- src/docs/env-vars.md | 4 +- 16 files changed, 514 insertions(+), 13 deletions(-) create mode 100644 changelog.d/added-claude-long-lived-token.md create mode 100644 changelog.d/changed-claude-operator-token-suspends-rebridge.md create mode 100644 src/crux/agent/claude_operator_token_mode.go create mode 100644 src/crux/agent/claude_operator_token_mode_test.go create mode 100644 src/crux/agent/claude_operator_token_test.go create mode 100644 src/crux/claude/operator_token.go create mode 100644 src/crux/claude/operator_token_test.go diff --git a/changelog.d/added-claude-long-lived-token.md b/changelog.d/added-claude-long-lived-token.md new file mode 100644 index 0000000000..9c84edba5b --- /dev/null +++ b/changelog.d/added-claude-long-lived-token.md @@ -0,0 +1 @@ +- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. diff --git a/changelog.d/changed-claude-operator-token-suspends-rebridge.md b/changelog.d/changed-claude-operator-token-suspends-rebridge.md new file mode 100644 index 0000000000..22c625b98c --- /dev/null +++ b/changelog.d/changed-claude-operator-token-suspends-rebridge.md @@ -0,0 +1 @@ +- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go index b511e8d955..1f36cb2096 100644 --- a/src/crux/agent/claude_credential_bridge.go +++ b/src/crux/agent/claude_credential_bridge.go @@ -521,6 +521,14 @@ func preserveClaudeCredential(path string, now time.Time) (string, error) { // outcome. It is the single entry point the launch path, the watchdog poll and // the sign-in hook share, so all three log and audit identically. func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger string) *claudeBridgeRepair { + // Operator-token mode: agents do not authenticate with the shared + // credentials file, so a private copy left by a refresh is nothing to + // repair (claude_operator_token_mode.go). Covers launch, watchdog and + // sign-in alike. + if m.claudeOperatorTokenMode() { + m.logClaudeOperatorTokenModeOnce() + return nil + } rep, err := m.repairClaudeCredentialBridge(agentName, link, target, trigger) if errors.Is(err, errClaudeBridgeLinkIsTarget) { // The legacy layout: skipped, and said once per agent rather than on @@ -594,8 +602,13 @@ func (m *Manager) logClaudeBridgeUnreadable(agentName, link, trigger string, err // RepairClaudeCredentialBridges checks every per-agent Claude credential // bridge and repairs each one a regular file has replaced. It returns the // names of the agents it re-bridged, sorted. Agents are snapshotted under m.mu -// and repaired after it is released. +// and repaired after it is released. In operator-token mode it does nothing +// and returns nil (claude_operator_token_mode.go). func (m *Manager) RepairClaudeCredentialBridges(trigger string) []string { + if m.claudeOperatorTokenMode() { + m.logClaudeOperatorTokenModeOnce() + return nil + } type candidate struct { name string uid int diff --git a/src/crux/agent/claude_operator_token_mode.go b/src/crux/agent/claude_operator_token_mode.go new file mode 100644 index 0000000000..76898bd3ea --- /dev/null +++ b/src/crux/agent/claude_operator_token_mode.go @@ -0,0 +1,38 @@ +package agent + +// Operator-token mode for Claude (#17760). +// +// With HIVE_CLAUDE_OAUTH_TOKEN set, every claude-backend agent runs on the +// operator's long-lived `claude setup-token` token through +// CLAUDE_CODE_OAUTH_TOKEN (manager_env.go). That token is never refreshed, so +// the machinery built around the shared ~/.claude/.credentials.json — the +// per-agent credential re-bridge that publishes a refreshed token back into +// the shared login, and the durable-credential watchdog that alerts when that +// file is missing or spent — has nothing to act on: the file is not what the +// agents authenticate with. Acting anyway would move credential files around +// for no reason; alerting would send an operator to redo a dashboard sign-in +// that the fleet does not use. Both are therefore suspended for Claude while +// the token is set and behave exactly as before when it is unset. The token's +// own health (a 401 from the API) is surfaced separately (#17759). + +// claudeOperatorTokenMode reports whether Claude agents run on the operator's +// long-lived token rather than the shared credentials file. The token is read +// once at construction (claudeOperatorToken), so the mode is fixed for the +// life of the process. +func (m *Manager) claudeOperatorTokenMode() bool { + return m.claudeOperatorToken != "" +} + +// logClaudeOperatorTokenModeOnce says, once per process, that the Claude +// credential re-bridge and watchdog alerts are suspended, so an operator +// reading the log knows why neither will ever fire. Never logs the token. +func (m *Manager) logClaudeOperatorTokenModeOnce() { + if !m.claudeOperatorTokenMode() { + return + } + m.claudeOperatorModeLogOnce.Do(func() { + m.logger.Info("claude auth: operator-supplied long-lived token is set; Claude agents run on it, so the shared credentials file is not in use — credential re-bridge and durable-credential watchdog alerts are suspended for the claude backend", + "source", "HIVE_CLAUDE_OAUTH_TOKEN", + "suspended", "claude credential re-bridge, claude durable-credential watchdog") + }) +} diff --git a/src/crux/agent/claude_operator_token_mode_test.go b/src/crux/agent/claude_operator_token_mode_test.go new file mode 100644 index 0000000000..e34eb37464 --- /dev/null +++ b/src/crux/agent/claude_operator_token_mode_test.go @@ -0,0 +1,127 @@ +package agent + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// operatorModeFixture is a manager in operator-token mode (the token field set +// as NewManager does from HIVE_CLAUDE_OAUTH_TOKEN) with one claude agent whose +// credential link a refresh replaced, plus the log it writes. +func operatorModeFixture(t *testing.T) (*Manager, *fakeAuditSink, *AgentProcess, string, string, *bytes.Buffer) { + t.Helper() + m, sink, ap, link, target := bridgeTestFixture(t) + var logs bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + m.claudeOperatorToken = operatorTokenSentinel + now := time.Now() + if err := os.WriteFile(target, bridgeTestCreds(t, "shared", now.Add(10*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + simulateClaudeRefresh(t, link, bridgeTestCreds(t, "private", now.Add(8*time.Hour).UnixMilli())) + return m, sink, ap, link, target, &logs +} + +// While the operator token is set the re-bridge does nothing on any trigger: +// the private file stays a private file, the shared login is untouched, +// and nothing is audited. +func TestClaudeOperatorTokenMode_RebridgeIsSuspended(t *testing.T) { + m, sink, _, link, target, logs := operatorModeFixture(t) + sharedBefore, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + agentBefore, err := os.ReadFile(link) + if err != nil { + t.Fatal(err) + } + + for _, trigger := range []string{claudeBridgeTriggerWatchdog, claudeBridgeTriggerSignIn} { + if got := m.RepairClaudeCredentialBridges(trigger); len(got) != 0 { + t.Fatalf("trigger %s: RepairClaudeCredentialBridges = %v, want none in operator-token mode", trigger, got) + } + } + if rep := m.rebridgeClaudeCredential("scanner", link, target, claudeBridgeTriggerLaunch); rep != nil { + t.Fatalf("launch re-bridge ran in operator-token mode: %+v", rep) + } + m.rebridgeClaudeCredentialsAfterSignIn() + + if fi, err := os.Lstat(link); err != nil || !fi.Mode().IsRegular() { + t.Fatalf("agent's private credential must be left exactly where it is; Lstat: %v %v", fi, err) + } + if got, _ := os.ReadFile(link); !bytes.Equal(got, agentBefore) { + t.Error("agent's private credential was rewritten") + } + if got, _ := os.ReadFile(target); !bytes.Equal(got, sharedBefore) { + t.Error("shared credential was rewritten") + } + if stale, _ := filepath.Glob(filepath.Join(filepath.Dir(link), "*"+claudeCredentialStaleInfix+"*")); len(stale) != 0 { + t.Errorf("files were moved aside: %v", stale) + } + if n := sink.count(); n != 0 { + t.Errorf("re-bridge audited %d entries in operator-token mode, want 0", n) + } + if n := strings.Count(logs.String(), "watchdog alerts are suspended"); n != 1 { + t.Errorf("suspension logged %d times across several calls, want exactly once:\n%s", n, logs.String()) + } + if strings.Contains(logs.String(), operatorTokenSentinel) { + t.Fatal("operator token reached the log") + } +} + +// While the operator token is set the watchdog never alerts about the Claude +// credentials file — absent, spent or otherwise — but still watches Copilot. +func TestClaudeOperatorTokenMode_WatchdogClaudeProbeIsSuspended(t *testing.T) { + m, sink, _, _, _, logs := operatorModeFixture(t) + m.mu.Lock() + m.agents["reviewer"] = &AgentProcess{Name: "reviewer", UID: 2002, Config: config.AgentConfig{Backend: "copilot"}} + m.mu.Unlock() + + missing := filepath.Join(t.TempDir(), "missing") + last := map[string]bool{"claude": true} // a stale observation from before token mode is cleared, not re-alerted + m.evalCredentialWatch(credentialWatch{backend: "claude", path: missing, + auditAction: AuditClaudeTokenMissing, probe: claudeTokenUsable}, last) + if sink.count() != 0 { + t.Fatalf("claude watch audited %d entries in operator-token mode, want 0", sink.count()) + } + if _, tracked := last["claude"]; tracked { + t.Error("suspended claude watch must not keep a transition record") + } + if strings.Contains(logs.String(), "durable credential unusable") { + t.Errorf("claude watch warned in operator-token mode:\n%s", logs.String()) + } + + m.evalCredentialWatch(credentialWatch{backend: "copilot", path: missing, + auditAction: AuditCopilotTokenMissing, probe: copilotTokenUsable}, last) + if e := sink.find(AuditCopilotTokenMissing); e == nil { + t.Fatal("copilot watch must keep alerting in operator-token mode") + } +} + +// Unset, the same fixture behaves exactly as today: the re-bridge repairs the +// link and the watchdog probes Claude. +func TestClaudeOperatorTokenMode_UnsetKeepsTodaysBehaviour(t *testing.T) { + m, sink, _, link, target, logs := operatorModeFixture(t) + m.claudeOperatorToken = "" + + if got := m.RepairClaudeCredentialBridges(claudeBridgeTriggerWatchdog); len(got) != 1 || got[0] != "scanner" { + t.Fatalf("RepairClaudeCredentialBridges = %v, want [scanner]", got) + } + assertLinkRestored(t, link, target) + last := map[string]bool{} + m.evalCredentialWatch(credentialWatch{backend: "claude", path: filepath.Join(t.TempDir(), "missing"), + auditAction: AuditClaudeTokenMissing, probe: claudeTokenUsable}, last) + if e := sink.find(AuditClaudeTokenMissing); e == nil { + t.Fatal("claude watch must alert when no operator token is set") + } + if strings.Contains(logs.String(), "watchdog alerts are suspended") { + t.Error("suspension must not be logged when the token is unset") + } +} diff --git a/src/crux/agent/claude_operator_token_test.go b/src/crux/agent/claude_operator_token_test.go new file mode 100644 index 0000000000..8ad2b15647 --- /dev/null +++ b/src/crux/agent/claude_operator_token_test.go @@ -0,0 +1,167 @@ +package agent + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" +) + +const operatorTokenSentinel = "sk-ant-oat01-OPERATOR-SENTINEL-7f3c9a1e5b" + +// stageOperatorToken makes claude.OperatorToken hermetic: the env var holds +// token ("" for unset) and the default secret file points at a missing path. +func stageOperatorToken(t *testing.T, token string) { + t.Helper() + t.Setenv(claude.OperatorTokenEnvVar, token) + t.Setenv(claude.OperatorTokenFileEnvVar, "") + orig := claude.OperatorTokenFile + claude.OperatorTokenFile = filepath.Join(t.TempDir(), "missing") + t.Cleanup(func() { claude.OperatorTokenFile = orig }) +} + +// With a long-lived operator token set, an agent whose credential file is +// readable and fresh still runs on the operator token: that file is the +// 8-hour login that logs the fleet out (#17753). +func TestClaudeOperatorToken_InjectedOverAReadableCredential(t *testing.T) { + stageSharedClaudeCredential(t, map[string]any{ + "accessToken": "sk-ant-oat-live", + "expiresAt": time.Now().Add(6 * time.Hour).UnixMilli(), + }) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "sk-ant-oat-snapshot-from-boot") + + pair, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"] + if !ok || pair.Value != operatorTokenSentinel { + t.Fatalf("pair = %+v, %v; want the operator token over a readable credential", pair, ok) + } + if !pair.Secret { + t.Fatal("the operator token must be a secret pair (tmux set-environment, never a command line)") + } +} + +// With no credential file at all, the operator token is still injected, and it +// beats the dashboard snapshot the last-resort fallback would otherwise use. +func TestClaudeOperatorToken_InjectedWithNoCredentialFile(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "sk-ant-oat-from-dashboard-login") + + pair, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"] + if !ok || pair.Value != operatorTokenSentinel || !pair.Secret { + t.Fatalf("pair = %+v, %v; want the operator token as a secret pair", pair, ok) + } +} + +// A dashboard sign-in after boot only replaces claudeAuthToken; it must not +// switch agents back to the 8-hour token. +func TestClaudeOperatorToken_WinsOverALaterDashboardSignIn(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + m, agent := claudeAgentFixture(t, "") + m.mu.Lock() + m.claudeAuthToken = "sk-ant-oat-fresh-dashboard-login" + m.mu.Unlock() + + pairs := m.agentEnvPairs(agent) + var got []string + for _, p := range pairs { + if p.Key == "CLAUDE_CODE_OAUTH_TOKEN" { + got = append(got, p.Value) + } + } + if len(got) != 1 || got[0] != operatorTokenSentinel { + t.Fatalf("CLAUDE_CODE_OAUTH_TOKEN values = %q; want exactly the operator token", got) + } +} + +// Unset: behaviour is exactly today's — no override over a readable +// credential, the dashboard snapshot when nothing is readable. +func TestClaudeOperatorToken_UnsetKeepsTodaysBehaviour(t *testing.T) { + stageSharedClaudeCredential(t, map[string]any{ + "accessToken": "sk-ant-oat-live", + "expiresAt": time.Now().Add(6 * time.Hour).UnixMilli(), + }) + stageOperatorToken(t, "") + m, agent := claudeAgentFixture(t, "sk-ant-oat-snapshot-from-boot") + if _, ok := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; ok { + t.Fatal("with the operator token unset, nothing may be injected over a readable credential") + } + + stageSharedClaudeCredential(t, nil) + m, agent = claudeAgentFixture(t, "sk-ant-oat-from-dashboard-login") + if pair := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; pair.Value != "sk-ant-oat-from-dashboard-login" { + t.Fatalf("with the operator token unset, the last-resort fallback must be unchanged: %+v", pair) + } +} + +// The token can come from the secret-file form and is never sent to an agent +// on another backend. +func TestClaudeOperatorToken_FileFormAndOtherBackends(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, "") + path := filepath.Join(t.TempDir(), "claude-oauth-token") + if err := os.WriteFile(path, []byte(operatorTokenSentinel+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(claude.OperatorTokenFileEnvVar, path) + + m, agent := claudeAgentFixture(t, "") + if pair := envPairsByKey(m, agent)["CLAUDE_CODE_OAUTH_TOKEN"]; pair.Value != operatorTokenSentinel { + t.Fatalf("file form not injected: %+v", pair) + } + + other := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "copilot"}, + }, discardLogger(), ProjectContext{ACMMLevel: 5}) + other.mu.RLock() + otherAgent := other.agents["scanner"] + other.mu.RUnlock() + if _, ok := envPairsByKey(other, otherAgent)["CLAUDE_CODE_OAUTH_TOKEN"]; ok { + t.Fatal("a copilot agent must not receive the operator Claude token") + } +} + +// The manager never writes the operator token to its log while loading it +// and building an agent's environment. +func TestClaudeOperatorToken_NeverLogged(t *testing.T) { + stageSharedClaudeCredential(t, nil) + stageOperatorToken(t, operatorTokenSentinel) + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "claude", Model: "claude-sonnet-5"}, + }, logger, ProjectContext{ACMMLevel: 5}) + m.mu.RLock() + agent := m.agents["scanner"] + m.mu.RUnlock() + _ = m.agentEnvPairs(agent) + _ = m.buildEnvPrefix(agent) + + if strings.Contains(buf.String(), operatorTokenSentinel) { + t.Fatalf("operator token written to the hive log: %s", buf.String()) + } + if strings.Contains(m.buildEnvPrefix(agent), operatorTokenSentinel) { + t.Fatal("operator token reached the shell command-line env prefix") + } +} + +// The hive process holds HIVE_CLAUDE_OAUTH_TOKEN in its own environment, and +// every tmux server it starts inherits that globally. applySessionEnv must +// strip it (with -r, so the global value is hidden too) or every pane of every +// backend — copilot, codex, bob — could read the year-long token under its +// original name. The claude backend receives it only as CLAUDE_CODE_OAUTH_TOKEN. +func TestClaudeOperatorToken_StrippedFromInheritedPaneEnv(t *testing.T) { + for _, k := range inheritedCredentialEnvVars { + if k == claude.OperatorTokenEnvVar { + return + } + } + t.Fatalf("%s is not in inheritedCredentialEnvVars; it would reach every agent pane via the tmux global environment", claude.OperatorTokenEnvVar) +} diff --git a/src/crux/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go index 5a3c02d85e..7259d488e0 100644 --- a/src/crux/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -220,7 +220,13 @@ func credentialWatches() []credentialWatch { // Safe to start unconditionally at boot: like StartAgentTokenRefresh it tracks // live state each tick rather than a boot-time snapshot, so a hive that gains // or loses a backend via config reload is evaluated correctly. +// +// Operator-token mode (HIVE_CLAUDE_OAUTH_TOKEN set): the Claude agents do not +// use the shared credentials file, so the Claude re-bridge and the Claude +// probe are suspended and said so once at start; Copilot is watched as +// before. See claude_operator_token_mode.go. func (m *Manager) StartCredentialWatchdog(ctx context.Context) { + m.logClaudeOperatorTokenModeOnce() ticker := time.NewTicker(credentialWatchdogInterval()) defer ticker.Stop() bridgeTicker := time.NewTicker(claudeBridgeInterval()) @@ -254,8 +260,9 @@ func (m *Manager) StartCredentialWatchdog(ctx context.Context) { // use) and emits a transition-edge log + audit event. lastUnusable carries the // previous observation per backend across ticks. func (m *Manager) evalCredentialWatch(w credentialWatch, lastUnusable map[string]bool) { - if !m.backendInUse(w.backend) { - // Not in use: nothing to watch. Reset the tracker so a later config + if !m.backendInUse(w.backend) || (w.backend == "claude" && m.claudeOperatorTokenMode()) { + // Not in use (or, for Claude, the operator token is in use instead of + // the file): nothing to watch. Reset the tracker so a later config // change that adds this backend with a bad credential alerts on its // first miss rather than being masked as "no transition". delete(lastUnusable, w.backend) diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 92102c3b9d..9713d21661 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -594,12 +594,16 @@ type Manager struct { // precedence for whatever fresh token the operator supplied. copilotAuthTokenRejected bool claudeAuthToken string - uidMap *UIDMap - sharedPool SharedSessionPool - uidMapPath string - appAuth AppTokenMinter - appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged - agentMint AgentMintIssuer // optional, opt-in mint credential (nil ⇒ off) + claudeOperatorToken string // HIVE_CLAUDE_OAUTH_TOKEN, read once at construction; a dashboard sign-in never replaces it + // claudeOperatorModeLogOnce guards the one-time "re-bridge and watchdog + // suspended" log for operator-token mode (claude_operator_token_mode.go). + claudeOperatorModeLogOnce sync.Once + uidMap *UIDMap + sharedPool SharedSessionPool + uidMapPath string + appAuth AppTokenMinter + appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged + agentMint AgentMintIssuer // optional, opt-in mint credential (nil ⇒ off) // sharedPoolRenewInterval overrides the lease watcher tick (zero means // defaultSharedPoolRenewInterval). Set before any watcher starts; it is @@ -886,6 +890,7 @@ func NewManagerWithOptions(agents map[string]config.AgentConfig, logger *slog.Lo copilotAuthTokenAuthoritative: copilotTokenAuthoritative, copilotAuthTokenSource: copilotTokenSource, claudeAuthToken: claudeToken, + claudeOperatorToken: claude.OperatorToken(), uidMap: uidMap, sharedPool: opts.sharedPool, uidMapPath: selectedUIDMapPath, diff --git a/src/crux/agent/manager_env.go b/src/crux/agent/manager_env.go index c930d652aa..9e866de3d2 100644 --- a/src/crux/agent/manager_env.go +++ b/src/crux/agent/manager_env.go @@ -422,7 +422,15 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // keeps the variable doing the job it was added for (#c5648bc9: deliver a // dashboard-obtained token to an agent that cannot see the file) and stops // it overriding a credential that can still refresh itself. - if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { + // + // An operator-supplied long-lived token (HIVE_CLAUDE_OAUTH_TOKEN, from + // `claude setup-token`) is the exception: it lasts about a year and never + // needs a refresh, so pinning it is the point. It is injected whether or + // not a credentials file is reachable and wins over the dashboard + // sign-in, which only ever updates claudeAuthToken. + if m.claudeOperatorToken != "" && backend == "claude" { + vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeOperatorToken, true}) + } else if m.claudeAuthToken != "" && backend == "claude" && !claudeCredentialReachable(agent, backend) { vars = append(vars, agentEnvPair{"CLAUDE_CODE_OAUTH_TOKEN", m.claudeAuthToken, true}) } // bob reads its key from BOBSHELL_API_KEY in 1.x and BOB_API_KEY in 2.x. diff --git a/src/crux/agent/manager_tmux.go b/src/crux/agent/manager_tmux.go index 87932ded5e..4358f9b333 100644 --- a/src/crux/agent/manager_tmux.go +++ b/src/crux/agent/manager_tmux.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hivecommons/hive/crux/agent/tmux" + "github.com/hivecommons/hive/crux/claude" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/internal/textutil" "github.com/hivecommons/hive/crux/upstreamproxy" @@ -50,8 +51,15 @@ var defaultTmuxSocket string // global value from every process forked afterwards; the sanctioned // per-agent pair set afterwards overrides the removal for the one variable // the agent is meant to have. +// +// HIVE_CLAUDE_OAUTH_TOKEN is the operator's long-lived Claude token. It is +// delivered to claude-backend agents only, as CLAUDE_CODE_OAUTH_TOKEN via +// agentEnvPairs; under its own name it must reach no pane at all, or every +// copilot/codex/bob shell (and every tool shell they spawn) could read a +// year-long credential it has no use for. var inheritedCredentialEnvVars = []string{ "HIVE_GITHUB_TOKEN", + claude.OperatorTokenEnvVar, linearAPIKeyEnvVar, linearAccessTokenEnvVar, "LINEAR_CLIENT_ID", diff --git a/src/crux/agent/tmux_inherited_env_test.go b/src/crux/agent/tmux_inherited_env_test.go index 2b8a40b799..bf9821249a 100644 --- a/src/crux/agent/tmux_inherited_env_test.go +++ b/src/crux/agent/tmux_inherited_env_test.go @@ -62,6 +62,7 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { "HIVE_UPSTREAM_PROXY_HOSTS=ghe.corp.example", "HIVE_UPSTREAM_PROXY_NO_PROXY=10.0.0.0/8", "HIVE_NO_PROXY=10.0.0.0/8", + "HIVE_CLAUDE_OAUTH_TOKEN=sk-ant-oat01-operator_leaked", ) if out, err := start.CombinedOutput(); err != nil { t.Fatalf("starting tmux server: %v: %s", err, out) @@ -126,7 +127,7 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { "LINEAR_API_KEY", "LINEAR_CLIENT_SECRET", "LINEAR_WEBHOOK_SECRET", "HIVE_GITHUB_TOKEN", "GITHUB_TOKEN", "DASHBOARD_AUTH_TOKEN", "HIVE_DASHBOARD_TOKEN", "HIVE_TERMINAL_KEY", "HIVE_INVITE_KEY", "HIVE_HEARTBEAT_KEY", "HIVE_HUB_SECRET", "HIVE_TTYD_CREDENTIAL", "HIVE_UPSTREAM_PROXY_URL", "HIVE_UPSTREAM_PROXY_HOSTS", - "HIVE_UPSTREAM_PROXY_NO_PROXY", "HIVE_NO_PROXY", + "HIVE_UPSTREAM_PROXY_NO_PROXY", "HIVE_NO_PROXY", "HIVE_CLAUDE_OAUTH_TOKEN", } { if v, ok := after[k]; ok { t.Errorf("%s=%q still reaches a pane after applySessionEnv — the inherited credential was not removed", k, v) diff --git a/src/crux/claude/operator_token.go b/src/crux/claude/operator_token.go new file mode 100644 index 0000000000..f0f7bb3b00 --- /dev/null +++ b/src/crux/claude/operator_token.go @@ -0,0 +1,40 @@ +package claude + +import ( + "os" + "strings" +) + +const ( + // OperatorTokenEnvVar holds an operator-generated long-lived token + // (`claude setup-token`). When set, every claude-backend agent runs on it + // through CLAUDE_CODE_OAUTH_TOKEN instead of the dashboard sign-in. + OperatorTokenEnvVar = "HIVE_CLAUDE_OAUTH_TOKEN" + + // OperatorTokenFileEnvVar names a file holding the same token, for + // deployments that mount it as a secret rather than an env var. + OperatorTokenFileEnvVar = "HIVE_CLAUDE_OAUTH_TOKEN_FILE" +) + +// OperatorTokenFile is the mounted Secret key read when neither variable is +// set, mirroring the dashboard token's /secrets/dashboard-token. A var so tests +// can point it at a hermetic path. +var OperatorTokenFile = "/secrets/claude-oauth-token" + +// OperatorToken returns the operator-supplied long-lived Claude token, or "" +// when none is configured. The env var wins over the file. The value is a +// secret: callers must only hand it to an agent as a secret env pair. +func OperatorToken() string { + if v := strings.TrimSpace(os.Getenv(OperatorTokenEnvVar)); v != "" { + return v + } + path := strings.TrimSpace(os.Getenv(OperatorTokenFileEnvVar)) + if path == "" { + path = OperatorTokenFile + } + data, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} diff --git a/src/crux/claude/operator_token_test.go b/src/crux/claude/operator_token_test.go new file mode 100644 index 0000000000..ec3e5223ef --- /dev/null +++ b/src/crux/claude/operator_token_test.go @@ -0,0 +1,41 @@ +package claude + +import ( + "os" + "path/filepath" + "testing" +) + +func TestOperatorTokenSources(t *testing.T) { + dir := t.TempDir() + orig := OperatorTokenFile + OperatorTokenFile = filepath.Join(dir, "default-token") + t.Cleanup(func() { OperatorTokenFile = orig }) + t.Setenv(OperatorTokenEnvVar, "") + t.Setenv(OperatorTokenFileEnvVar, "") + + if got := OperatorToken(); got != "" { + t.Fatalf("unset: got %q, want empty", got) + } + + if err := os.WriteFile(OperatorTokenFile, []byte(" sk-ant-oat01-default \n"), 0o600); err != nil { + t.Fatal(err) + } + if got := OperatorToken(); got != "sk-ant-oat01-default" { + t.Fatalf("default secret file: got %q", got) + } + + named := filepath.Join(dir, "named-token") + if err := os.WriteFile(named, []byte("sk-ant-oat01-named\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(OperatorTokenFileEnvVar, named) + if got := OperatorToken(); got != "sk-ant-oat01-named" { + t.Fatalf("%s: got %q", OperatorTokenFileEnvVar, got) + } + + t.Setenv(OperatorTokenEnvVar, " sk-ant-oat01-env ") + if got := OperatorToken(); got != "sk-ant-oat01-env" { + t.Fatalf("env must win over the file: got %q", got) + } +} diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index e270605f8b..49a8f6bba0 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -157,6 +157,32 @@ The dashboard API can update this field through [`POST /api/effort/{agent}/{effo The dashboard shows a reasoning-effort dropdown next to the model picker for every method that has an effort control (`codex`, `agy`, `claude`, and `omp`). The value is validated at set time against the method's own set, so a `codex`-only level such as `minimal` is refused for a `claude` agent rather than stored and dropped at launch. For `omp`, the dropdown is per selected model: Hive reads `omp models --json` and uses that model's `thinking` levels, hiding the control for models that report no thinking support. Picking `default` clears the stored value; for `claude` and `omp` that means no effort flag at all and the CLI's own default. The dropdown persists through the same path as the model picker (`hive.yaml` plus the per-agent overlay under `agents_dir`), which is the durable path on a hosted spoke: a hand edit of the agent's file is reverted by the next sync, the dashboard write is not. Each dropdown change is recorded in the audit trail as `set_reasoning_effort` with the new value, like the other per-agent setting changes. The effort the agent was actually launched with is what the run rows and the `- hive:` PR trailer report, so a `claude` agent at `xhigh` shows `effort=xhigh` there and an agent with no stored effort shows none. +#### Claude setup-token + +For `backend: claude`, an operator can use a long-lived token instead of the +dashboard sign-in. Run `claude setup-token` interactively as the licensed +operator and complete the prompts. The resulting token lasts about a year and +is inference-only; it is not a general-purpose account credential. See +[Claude Code authentication](https://code.claude.com/docs/en/authentication). + +Supply it to the Hive process as the secret `HIVE_CLAUDE_OAUTH_TOKEN`, or mount +it in a secret file readable by Hive. Set `HIVE_CLAUDE_OAUTH_TOKEN_FILE` to +that file's path, or use the default `/secrets/claude-oauth-token`. The +non-empty environment variable wins over the file. Do not put the token in +`hive.yaml`, shell history, logs, issues, or agent panes. + +Hive passes the token to Claude agents as `CLAUDE_CODE_OAUTH_TOKEN` at launch, +even when a credentials file exists. It takes precedence over dashboard sign-in; +signing in again on the dashboard does not replace it. Each agent retains its +own Claude home and private state; sharing the token does not require sharing +the whole home. Without an operator token, dashboard sign-in works as before. + +To replace an expired or revoked token, run `claude setup-token` again, update +the configured secret source, then recreate or restart Hive to load the new +token and restart the affected Claude agents so their new sessions receive +the replacement. Verify an agent completes a turn. See the +[backend auth recovery runbook](backend-auth-recovery-runbook.md). + #### Oh My Pi (`omp`) hub agents `backend: omp` launches the OMP CLI as a first-class dashboard method. Hive gives each agent its own durable HOME under `/data/home/agents/<name>`, so `~/.omp/agent/agent.db` can hold several provider sign-ins for that one agent (for example `openai-codex` for the primary model and `anthropic` for an advisor) without sharing refresh-token writers across agents. Select models with their provider prefix (`provider/model`). The dashboard model probe runs `omp models --json` and uses each model's `thinking` list for the effort dropdown. At launch Hive passes `--approval-mode ${HIVE_OMP_APPROVAL_MODE:-yolo}`, `--model <id>`, and `--thinking <effort>` when set. diff --git a/src/docs/backend-auth-recovery-runbook.md b/src/docs/backend-auth-recovery-runbook.md index 5045b55ac1..76a710ea0c 100644 --- a/src/docs/backend-auth-recovery-runbook.md +++ b/src/docs/backend-auth-recovery-runbook.md @@ -51,11 +51,27 @@ recovered and no action is needed. | `backendAuthStatus` | First action | |---|---| | `unlicensed` | Check the seat at <https://github.com/settings/copilot>. If it is active, the hive is presenting a token the account no longer owns: follow the `/login` recovery in [troubleshooting.md](troubleshooting.md). | -| `token-expired` | If `backendAuthRefreshable` is `true` (provider `auth_state: refreshable`), restart the affected agents: Claude Code redeems the refresh grant on start and no login is needed. Otherwise run `/login` in any agent's Terminal with a licensed identity and wait about 30 seconds for the reconciler; affected agents relaunch onto the new token. | +| `token-expired` | For Claude with an operator setup token, replace it as described below; dashboard `/login` does not override it. Otherwise, if `backendAuthRefreshable` is `true` (provider `auth_state: refreshable`), restart the affected agents so Claude Code redeems the refresh grant. If not, run `/login` in any agent's Terminal with a licensed identity and wait about 30 seconds for the reconciler. | | `forbidden` | Cause is undetermined (bare 403). Read which credential was refused (dashboard model dropdown hover, or the `copilot model discovery rejected by upstream` log line's `credential` field) before re-logging in; an org policy denial is not fixed by a new login. | | `unreachable` | The inference backend is down or unroutable. Check the backend's own health and network path from the pod; re-login will not help. | | `quota` | The provider is refusing calls on quota. Wait for the window to reset, add quota, or switch the agents to another backend. Do not restart agents in a loop. | +### Replace a Claude operator token + +If `HIVE_CLAUDE_OAUTH_TOKEN` or its secret file is configured, a rejected +token may have expired or been revoked. Run `claude setup-token` interactively +as the licensed operator and complete its prompts. The replacement lasts about +a year and is inference-only. Store it securely in the same Hive environment +secret or in the file named by `HIVE_CLAUDE_OAUTH_TOKEN_FILE` (default +`/secrets/claude-oauth-token`); the non-empty environment variable takes +precedence over the file. Never paste the token into a pane, log, or report. + +Recreate or restart Hive to load the changed secret, then restart affected +Claude agents to give their new sessions the replacement. Follow the +verification steps below. A dashboard sign-in does not supersede the configured +operator token. See [Claude setup-token](agent-configuration.md#claude-setup-token) +for initial setup. + ## 3. Verify recovery 1. Within one reconciler tick (about 30 seconds) the affected agents drop diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index fd410f8a76..8f317217ad 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -295,6 +295,8 @@ and [network/credential controls](security-model.md#5-network-and-credentials). | `HIVE_BOB_API_URL` | No | `https://api.us-east.bob.ibm.com` | Bob key-test endpoint base URL override. | | `BOBSHELL_API_KEY` | Required by Bob CLI when Hive injects or contributor mode uses Bob | none | API key name read by bobshell itself. | | `COPILOT_GITHUB_TOKEN` | No | dashboard device-flow token file, if present | Copilot completion/model-discovery token and explicit agent injection. | +| `HIVE_CLAUDE_OAUTH_TOKEN` | No | none | Hive-side secret from `claude setup-token`; takes precedence over the secret file and dashboard sign-in for Claude agents. See [Claude setup-token](agent-configuration.md#claude-setup-token). | +| `HIVE_CLAUDE_OAUTH_TOKEN_FILE` | No | `/secrets/claude-oauth-token` | Path to a file containing the setup token, used when `HIVE_CLAUDE_OAUTH_TOKEN` is empty. Keep the file readable by Hive and restricted as a secret. | | `ANTHROPIC_API_KEY` | Required by `cmd/apiproxy` to inject an upstream key; agent inference backends receive a synthetic value | none | Anthropic-compatible **upstream** API key. Never used to authenticate callers of `cmd/apiproxy`. | | `PROXY_AUTH_TOKEN` | **Required** by `cmd/apiproxy`; the binary exits at startup when unset and the proxy handler returns `503` | none | **Client auth token** callers must present to `cmd/apiproxy` via `Authorization: Bearer` or `X-Api-Key`. Validated in constant time and stripped before the upstream request. Mandatory because an unauthenticated proxy would let any co-resident loopback caller spend the host `ANTHROPIC_API_KEY`. | | `CONTEXT7_API_KEY` | No | none | Optional key for Context7 knowledge API integration. | @@ -386,7 +388,7 @@ Set conditionally: | `COPILOT_GITHUB_TOKEN` | When the hive holds a Copilot auth token | Copilot OAuth token (authenticates the AI model, not GitHub writes). **Secret.** | | `GITHUB_TOKEN` | Only when App-authored PRs are enabled (`AppAuthoredPRs`), an App is configured, and the agent's mode can push | The per-agent tier-scoped App installation token, so the built-in GitHub MCP server writes as the App bot. Advisory agents have `GITHUB_TOKEN` deliberately stripped. **Secret.** | | `LINEAR_ACCESS_TOKEN` / `LINEAR_API_KEY` | ISSUES_ONLY+ agents on Linear-connected hives | See the [Linear agent integration](#linear-agent-integration) note above. | -| `CLAUDE_CODE_OAUTH_TOKEN` | Last resort: claude backend **and** no readable credential file | Dashboard-obtained access token; never injected when the agent can read (and refresh) `~/.claude/.credentials.json` itself. **Secret.** | +| `CLAUDE_CODE_OAUTH_TOKEN` | Claude backend with an operator token, or as a last resort when no credential file is readable | Hive injects the operator's setup token when configured, even if a credential file exists. Otherwise the fallback is the dashboard-obtained access token only when no credential file is readable. **Secret.** | | `BOBSHELL_API_KEY`, `BOB_API_KEY`, `BOBSHELL_DEFAULT_AUTH_TYPE` | bob backend only | The resolved Bob API key for Bob 1.x / 2.x (**secret**) and the literal `api-key` auth-type selector (non-secret by design - see the #2228 relaunch note in code). | | `HIVE_BOB_SESSION_LABEL` | bob backend only when `governor.bob.session_prefix` or `agents.<name>.bob.session_label` is set | The Bob session label also passed to Bob as `--instance-id`. A label only: `HIVE_AGENT` and all functional Hive identity are unchanged. See [agent-configuration.md](agent-configuration.md). | | `HIVE_BOB_DISPLAY_NAME` | bob backend only | Legacy alias for the Bob session label, falling back to the agent name for older scripts. Prefer `HIVE_BOB_SESSION_LABEL` when present. | From d46605288651df264a2256f8a087da1eaf55aaac Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:32:34 -0700 Subject: [PATCH 091/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hub-signed=20rela?= =?UTF-8?q?y=20lets=20hosted=20spokes=20verify=20relayed=20GitHub=20webhoo?= =?UTF-8?q?ks=20(#17914)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: hub-signed relay lets hosted spokes verify relayed GitHub webhooks Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> * 📝 docs: refresh api-reference citations shifted by relay-key change Running check-api-reference-citations.sh --fix after the hub relay-key code landed above the cited route registrations in server.go and api_review_event_dispatch.go. The api_review_event_dispatch.go:16 -> :15 citation uses the "GET "+constant shape the fixer cannot rewrite, so it is corrected by hand. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🔒 fix: store hive meta.json with the relay key rw------- like other hub secrets SaaSHive.WebhookRelayKey is serialised into saasHivesDir/<id>/meta.json, which saveSaaSHive wrote 0o644 under a 0o755 directory. The record carried no key material before this PR; the hub's other secret stores (cluster App keys, generation stores, session revocation) and the spoke-side relay key file all use 0o600/0o700. A locally leaked relay key would let an attacker forge deliveries to that hive's public /api/webhook/github. Write meta.json 0o600, create the hive directory 0o700, and best-effort chmod pre-existing directories so hives provisioned before this change are tightened on their next save. Add a regression test covering both modes. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 🐛 fix: tighten hive dir perms without widening a read-only dir saveSaaSHive forced the hive directory to 0700, which re-added owner-write to directories the reset-assignment tests lock to 0555 to simulate a failed persist, so TestHandleResetAssignmentReportsSaveFailure and TestSweepStuckAssignmentsSkipsWedgeWhoseSaveFails no longer saw the failure. Strip only group/other bits so legacy 0755 dirs still become 0700 while a deliberately read-only dir keeps failing the write. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * ♻️ test: reuse useTempSaaSHivesDir in saveSaaSHive perm tests TestSaveSaaSHiveKeepsRelayKeyPrivate and TestSaveSaaSHiveNeverWidensReadOnlyDir swapped saasHivesDir for t.TempDir() inline even though the file already defines useTempSaaSHivesDir for exactly that. Use the helper like the sibling tests do. Hive-Fix-Attempt: 3/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * chore: retrigger merge watcher after GitHub 503 exhausted the prior merge request Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-hosted-webhook-relay-signature.md | 1 + src/cmd/hive/main.go | 12 + src/docs/api-reference/agents-and-controls.md | 2 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/env-vars.md | 2 +- .../operator-reference/runtime-and-quota.md | 33 +- src/pkg/dashboard/api_github_webhook.go | 29 +- .../dashboard/api_review_event_dispatch.go | 8 +- src/pkg/dashboard/server.go | 8 +- src/pkg/dashboard/webhook_relay.go | 128 ++++++++ src/pkg/dashboard/webhook_relay_test.go | 268 ++++++++++++++++ src/pkg/github/webhook_cache.go | 6 +- src/pkg/github/webhook_relay_key.go | 134 ++++++++ src/pkg/github/webhook_relay_key_test.go | 70 +++++ src/pkg/hub/heartbeat.go | 5 + src/pkg/hub/lite_enrollment.go | 1 + src/pkg/hub/pool_replenisher.go | 7 +- src/pkg/hub/saas_hives_api.go | 11 +- src/pkg/hub/saas_provision.go | 26 +- src/pkg/hub/server.go | 15 + src/pkg/hub/spoke/heartbeat.go | 16 + src/pkg/hub/webhook.go | 16 +- src/pkg/hub/webhook_relay_key.go | 121 ++++++++ src/pkg/hub/webhook_relay_key_test.go | 286 ++++++++++++++++++ 27 files changed, 1178 insertions(+), 59 deletions(-) create mode 100644 changelog.d/fixed-hosted-webhook-relay-signature.md create mode 100644 src/pkg/dashboard/webhook_relay.go create mode 100644 src/pkg/dashboard/webhook_relay_test.go create mode 100644 src/pkg/github/webhook_relay_key.go create mode 100644 src/pkg/github/webhook_relay_key_test.go create mode 100644 src/pkg/hub/webhook_relay_key.go create mode 100644 src/pkg/hub/webhook_relay_key_test.go diff --git a/changelog.d/fixed-hosted-webhook-relay-signature.md b/changelog.d/fixed-hosted-webhook-relay-signature.md new file mode 100644 index 0000000000..436f4aa770 --- /dev/null +++ b/changelog.d/fixed-hosted-webhook-relay-signature.md @@ -0,0 +1 @@ +- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 2cbba7bd59..4e643473d5 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4852,6 +4852,9 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // nothing and a spoke holding the wrong one self-heals. GitHubAppKeyFingerprint: spoke.TimedSection("app_key_fingerprint", func() string { return appKeys.ReportedFingerprint(b.cfg.GitHub.KeyFile, b.cfg.GitHub.AppID) }), GitHubAppKeyPerHive: spoke.TimedSection("app_key_per_hive", func() bool { return appKeys.HasPerHiveKey(b.cfg.GitHub.KeyFile, b.cfg.GitHub.AppID) }), + // Fingerprint only (#17911): the hub re-delivers the webhook + // relay key until this matches the key on its record. + WebhookRelayKeyFingerprint: github.WebhookRelayKeyFingerprint(), // Report the App this hive believes it authenticates as. The hub // pairs it with the fingerprint above to tell a per-hive key that // is WRONG for this App from one that is deliberately for another. @@ -5228,6 +5231,15 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "has_key", ghCfg.PrivateKey != "", ) + // The webhook relay key (#17911) is independent of the App + // identity below, so it is adopted before the identity guard can + // refuse the rest of the delivery. Empty never blanks a held key. + if changed, err := github.AdoptWebhookRelayKey(ghCfg.WebhookRelayKey); err != nil { + b.logger.Error("webhook relay key adopted in memory but not persisted", "error", err) + } else if changed { + b.logger.Info("adopted webhook relay key from hub", "fingerprint", github.WebhookRelayKeyFingerprint()) + } + // WRITE-PATH GUARD. Compute the identity this push WOULD produce and // refuse the whole delivery if it is internally inconsistent. // diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index bf60205f3f..7f42dc9717 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -27,7 +27,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:518` | | `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:519` | | `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:523` | -| `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:16` | +| `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | | `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:527` | | `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:528` | | `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:530` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 8c5b76d503..1d3e6c8eff 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1687` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1688` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1689` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1688` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1689` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1690` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index aec304411e..8511ded0f0 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1694` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1695` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index a0737fea7e..85fa9acf7b 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1667` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1668` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1669` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1675` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1680` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1682` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1683` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1668` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1669` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1670` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1676` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1682` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1683` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1687` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 96caa5bfa3..c3da74dff5 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1757` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1758` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 8f317217ad..23d80031c7 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -104,7 +104,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret and rejects every delivery while it is unset. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index f85565eff8..bd85fb4553 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -262,18 +262,18 @@ accuracy guarantees of the ForgeState contract. A spoke accepts GitHub App webhooks on the public `POST /api/webhook/github`, either delivered directly by GitHub or relayed by the hub (the hub forwards the original signed body to the hive that manages the repository). The receiver -fails closed: it rejects every delivery until `GITHUB_WEBHOOK_SECRET` is set to -the App's webhook secret, and it verifies `X-Hub-Signature-256` over the raw -body exactly as the hub does. Deliveries for repositories the hive does not -manage are ignored. +fails closed: it accepts a delivery only when `X-Hub-Signature-256` verifies +against `GITHUB_WEBHOOK_SECRET` (exactly as the hub does), or when the hub's +relay signature verifies against this hive's own relay key (hosted spokes, +below). With neither configured it rejects every delivery with 503. +Deliveries for repositories the hive does not manage are ignored. #### What the operator configures -1. **Secret.** Set `GITHUB_WEBHOOK_SECRET` in the spoke's environment (a - Secret mount on Kubernetes; never in `hive.yaml`) to the App's webhook - secret. On a hosted spoke this is the same value as the hub's - `webhook-secret.key`. Without it the receiver answers 503 and the loop - stays on the polling cadence. +1. **Secret (self-hosted only).** Set `GITHUB_WEBHOOK_SECRET` in the spoke's + environment (a Secret mount on Kubernetes; never in `hive.yaml`) to the + App's webhook secret. A hosted spoke needs nothing here: it never holds the + App secret and verifies relays with its hub relay key instead (item 3). 2. **GitHub App webhook.** Point the App's webhook URL at the hub (`POST /api/github/webhook`) for hosted spokes, or at the spoke's own `/api/webhook/github` for a stand-alone hive (a repository or organization @@ -286,8 +286,14 @@ manage are ignored. (organization-scoped; see [Work sources](../work-sources.md#type-github_projects--github-projects-projects-v2)). 3. **Hub relay.** The hub verifies the HMAC and relays the body byte-for-byte, with the original signature and `X-GitHub-Delivery`, to the spoke whose - reported dashboard URL manages the repository. Nothing to configure on the - spoke beyond the shared secret. + reported dashboard URL manages the repository. For a hosted hive it also + adds `X-Hive-Relay-Timestamp` and `X-Hive-Relay-Signature-256` + (`sha256=` HMAC-SHA256 over `timestamp.delivery-id.body`) keyed with that + hive's own relay key, which the hub mints at provisioning and delivers over + the heartbeat (persisted on the spoke as `/data/webhook-relay.key`). The + spoke rejects a relay whose signature does not verify, whose timestamp is + more than 5 minutes off, or whose delivery ID it already accepted. + Nothing to configure. 4. **Pacing (optional).** `governor.eval_interval_webhook_s` (default `900`) is the base tick while webhooks are healthy; `governor.webhook_wake_debounce_s` (default `10`) and `governor.webhook_wake_min_gap_s` (default `30`) pace the @@ -675,9 +681,10 @@ starve the governor: `/api/status` and `/api/gh-rate-limits` expose `webhooks: {healthy, last_event_at, events_1h, invalidations_1h, targeted_refreshes_1h, tracker_invalidations_1h, mode, since, mode_reason, -healthy_windows}` (the middle two count the single-item re-reads and the +healthy_windows, verification}` (the middle two count the single-item re-reads and the tracker cache drops webhooks caused; `mode` is `webhook` | `polling` and -`since` when it began — see [webhooks.md](../webhooks.md)); +`since` when it began — see [webhooks.md](../webhooks.md); `verification` is +`github-signed`, `relay (hub-signed)` or `none`); `/api/status` adds `evalMode` (`webhook` | `poll`), `evalIntervalS`, `lastWakeReason`, `lastWakeAt` and `webhookWakes`; `graphql_pr_batch.webhook_skips` counts batch queries skipped because nothing diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index b8ad183a36..539de1084a 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -20,10 +20,11 @@ import ( const ( // githubWebhookPath is the PUBLIC path GitHub (or the hub relay) posts App // webhooks to. Not actually open: the handler fails closed without - // GITHUB_WEBHOOK_SECRET and verifies X-Hub-Signature-256 over the raw body. + // GITHUB_WEBHOOK_SECRET or a hub relay key, and verifies + // X-Hub-Signature-256 or the hub relay signature over the raw body. githubWebhookPath = "/api/webhook/github" - githubWebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + githubWebhookSecretEnvVar = github.WebhookSecretEnvVar githubWebhookMaxBodyBytes = 1 << 20 ) @@ -61,7 +62,10 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { return } secret := strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) - if secret == "" { + // A hosted spoke holds no App webhook secret; it verifies the hub's relay + // signature with this hive's own relay key instead (#17911). + relayKey := github.WebhookRelayKey() + if secret == "" && relayKey == "" { const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET to enable signed webhooks)" warned := false s.webhookNoSecretOnce.Do(func() { @@ -74,11 +78,20 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { jsonError(w, "webhooks disabled: no secret configured", http.StatusServiceUnavailable) return } - if !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { - s.webhookLogger().Warn("github webhook signature verification failed", "event", event) - github.RecordWebhookDeliveryFailure() - jsonError(w, "invalid signature", http.StatusUnauthorized) - return + if secret == "" || !verifyGitHubWebhookSignature(body, r.Header.Get("X-Hub-Signature-256"), secret) { + if relayKey == "" || r.Header.Get(webhookRelaySignatureHeader) == "" { + s.webhookLogger().Warn("github webhook signature verification failed", "event", event) + github.RecordWebhookDeliveryFailure() + jsonError(w, "invalid signature", http.StatusUnauthorized) + return + } + if reason := s.verifyWebhookRelay(r.Header, body, relayKey, time.Now()); reason != "" { + s.webhookLogger().Warn("hub-relayed github webhook rejected", "event", event, "reason", reason, + "delivery", r.Header.Get("X-GitHub-Delivery")) + github.RecordWebhookDeliveryFailure() + jsonError(w, "relay rejected: "+reason, http.StatusUnauthorized) + return + } } if event == githubWebhookEventProjectsV2Item { // Organization-scoped: no repository to resolve, so it never reaches diff --git a/src/pkg/dashboard/api_review_event_dispatch.go b/src/pkg/dashboard/api_review_event_dispatch.go index 1ce5817d56..6484e91e25 100644 --- a/src/pkg/dashboard/api_review_event_dispatch.go +++ b/src/pkg/dashboard/api_review_event_dispatch.go @@ -3,9 +3,8 @@ package dashboard import ( "encoding/json" "net/http" - "os" - "strings" + "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/review/eventdispatch" "github.com/hivecommons/hive/pkg/review/pipeline" ) @@ -24,8 +23,11 @@ type reviewEventDispatchResponse struct { eventdispatch.Status } +// githubWebhooksConfigured reports whether this spoke can verify webhook +// deliveries at all: with GITHUB_WEBHOOK_SECRET, or (hosted) with the hub +// relay key delivered over the heartbeat (#17911). func githubWebhooksConfigured() bool { - return strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) != "" + return github.WebhookVerification() != github.WebhookVerificationNone } func (s *Server) reviewEvents() *eventdispatch.Dispatcher { diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index e1db31882f..77b80d02e8 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -106,7 +106,8 @@ const staleDropStarvationThreshold = 5 type Server struct { nativeQueueTracker nativeQueueTrackerRuntime integration integrationRuntime - webhookNoSecretOnce sync.Once // warn once per process about a missing webhook secret (#13666) + webhookNoSecretOnce sync.Once // warn once per process about a missing webhook secret (#13666) + webhookRelaySeen webhookRelayReplayCache // hub-relayed delivery IDs already accepted (#17911) feedbackRefreshBackoff feedbackBackoff @@ -2494,8 +2495,9 @@ func (s *Server) isPublicPath(path string) bool { case path == githubWebhookPath: // GitHub App webhooks (direct or relayed by the hub): GitHub cannot // hold a dashboard session. NOT actually open — the handler fails - // closed without GITHUB_WEBHOOK_SECRET and verifies the HMAC - // X-Hub-Signature-256 over the raw body. + // closed without GITHUB_WEBHOOK_SECRET or a hub relay key and + // verifies the HMAC X-Hub-Signature-256 (or the hub's relay + // signature, with replay protection) over the raw body. return true case path == linearAgentWebhookPath: // Linear AgentSessionEvent webhooks: Linear's servers cannot hold a diff --git a/src/pkg/dashboard/webhook_relay.go b/src/pkg/dashboard/webhook_relay.go new file mode 100644 index 0000000000..24f01987c0 --- /dev/null +++ b/src/pkg/dashboard/webhook_relay.go @@ -0,0 +1,128 @@ +package dashboard + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// Hub relay signature on relayed GitHub webhooks (hivecommons/hive#17911). +// +// A hosted spoke never holds the GitHub App webhook secret, so the hub signs +// each delivery it relays with this hive's own relay key (delivered over the +// heartbeat; github.WebhookRelayKey). The scheme mirrors pkg/hub +// webhookRelaySignature: +// +// X-Hive-Relay-Timestamp: <unix seconds at relay time> +// X-Hive-Relay-Signature-256: sha256=<hex HMAC-SHA256(key, timestamp + "." + X-GitHub-Delivery + "." + raw body)> +// +// After the MAC verifies, a timestamp more than webhookRelayMaxSkew from now +// is rejected as stale, and a delivery ID already accepted inside the replay +// window is rejected as replayed. +const ( + webhookRelaySignatureHeader = "X-Hive-Relay-Signature-256" + webhookRelayTimestampHeader = "X-Hive-Relay-Timestamp" + + webhookRelayMaxSkew = 5 * time.Minute + // webhookRelayReplayTTL covers every timestamp still inside the skew + // window, so a captured relay cannot be replayed while it would verify. + webhookRelayReplayTTL = 2 * webhookRelayMaxSkew + // webhookRelayReplayMax bounds the remembered delivery IDs; the oldest is + // dropped when a burst exceeds it. + webhookRelayReplayMax = 4096 +) + +// Rejection reasons for hub-relayed deliveries, logged and returned in the +// 401 body; each is counted as a failed delivery like a bad GitHub signature. +const ( + webhookRelayReasonBadSignature = "relay_signature_invalid" + webhookRelayReasonNoDelivery = "relay_delivery_missing" + webhookRelayReasonStale = "relay_timestamp_stale" + webhookRelayReasonReplayed = "relay_delivery_replayed" +) + +// verifyWebhookRelaySignature is the constant-time HMAC-SHA256 check of the +// relay signature, in the same style as verifyGitHubWebhookSignature. +func verifyWebhookRelaySignature(body []byte, timestamp, delivery, signature, key string) bool { + if key == "" || !strings.HasPrefix(signature, "sha256=") { + return false + } + sig, err := hex.DecodeString(strings.TrimPrefix(signature, "sha256=")) + if err != nil { + return false + } + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(timestamp)) + mac.Write([]byte(".")) + mac.Write([]byte(delivery)) + mac.Write([]byte(".")) + mac.Write(body) + return hmac.Equal(sig, mac.Sum(nil)) +} + +// webhookRelayReplayCache remembers recently accepted relay delivery IDs. +// The zero value is ready to use. +type webhookRelayReplayCache struct { + mu sync.Mutex + seen map[string]time.Time +} + +// firstSeen records id at now and reports whether it was new. Expired +// entries are pruned on the way in. +func (c *webhookRelayReplayCache) firstSeen(id string, now time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.seen == nil { + c.seen = make(map[string]time.Time) + } + if at, ok := c.seen[id]; ok && now.Sub(at) < webhookRelayReplayTTL { + return false + } + for k, at := range c.seen { + if now.Sub(at) >= webhookRelayReplayTTL { + delete(c.seen, k) + } + } + if len(c.seen) >= webhookRelayReplayMax { + oldestID, oldestAt := "", now + for k, at := range c.seen { + if oldestID == "" || at.Before(oldestAt) { + oldestID, oldestAt = k, at + } + } + delete(c.seen, oldestID) + } + c.seen[id] = now + return true +} + +// verifyWebhookRelay checks a hub-relayed delivery and returns "" when it is +// accepted, otherwise the rejection reason. The delivery ID is remembered +// only once the signature and timestamp verify, so unsigned traffic cannot +// fill the replay cache. +func (s *Server) verifyWebhookRelay(h http.Header, body []byte, key string, now time.Time) string { + ts := h.Get(webhookRelayTimestampHeader) + delivery := h.Get("X-GitHub-Delivery") + if !verifyWebhookRelaySignature(body, ts, delivery, h.Get(webhookRelaySignatureHeader), key) { + return webhookRelayReasonBadSignature + } + if delivery == "" { + return webhookRelayReasonNoDelivery + } + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return webhookRelayReasonStale + } + if skew := now.Sub(time.Unix(sec, 0)); skew > webhookRelayMaxSkew || skew < -webhookRelayMaxSkew { + return webhookRelayReasonStale + } + if !s.webhookRelaySeen.firstSeen(delivery, now) { + return webhookRelayReasonReplayed + } + return "" +} diff --git a/src/pkg/dashboard/webhook_relay_test.go b/src/pkg/dashboard/webhook_relay_test.go new file mode 100644 index 0000000000..fca1551aa8 --- /dev/null +++ b/src/pkg/dashboard/webhook_relay_test.go @@ -0,0 +1,268 @@ +package dashboard + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +// signWebhookRelayForTest is the hub's relay signature (pkg/hub +// webhookRelaySignature), computed independently here. +func signWebhookRelayForTest(key, ts, delivery string, body []byte) string { + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(ts + "." + delivery + ".")) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// withWebhookRelayKey gives this spoke a hub relay key (as a heartbeat +// delivery would) in a temp store, and no GITHUB_WEBHOOK_SECRET. +func withWebhookRelayKey(t *testing.T, key string) { + t.Helper() + t.Setenv(githubWebhookSecretEnvVar, "") + t.Cleanup(github.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "webhook-relay.key"))) + if key == "" { + return + } + if changed, err := github.AdoptWebhookRelayKey(key); err != nil || !changed { + t.Fatalf("adopt relay key: changed=%v err=%v", changed, err) + } +} + +func postRelayedWebhook(s *Server, event, delivery, ts, relaySig string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, githubWebhookPath, bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", event) + if delivery != "" { + req.Header.Set("X-GitHub-Delivery", delivery) + } + // GitHub's own signature rides along unchanged; this spoke cannot verify it. + req.Header.Set("X-Hub-Signature-256", signGitHubWebhookForTest("app-secret-the-spoke-never-has", body)) + if ts != "" { + req.Header.Set(webhookRelayTimestampHeader, ts) + } + if relaySig != "" { + req.Header.Set(webhookRelaySignatureHeader, relaySig) + } + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + return rec +} + +func relayErr(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + var e struct { + Error string `json:"error"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &e) + return e.Error +} + +// TestHubRelayedWebhookAcceptedWithoutSecret: a hosted spoke with no +// GITHUB_WEBHOOK_SECRET applies a delivery its hub relayed and signed with +// this hive's relay key (#17911). +func TestHubRelayedWebhookAcceptedWithoutSecret(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + ts := strconv.FormatInt(time.Now().Unix(), 10) + rec := postRelayedWebhook(s, "pull_request", "relay-accept-1", ts, signWebhookRelayForTest(key, ts, "relay-accept-1", body), body) + if rec.Code != http.StatusOK { + t.Fatalf("relay-signed delivery: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + var got struct { + Status string `json:"status"` + Invalidated []int `json:"invalidated"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Status != "ok" || !reflect.DeepEqual(got.Invalidated, []int{5}) { + t.Fatalf("relay-signed delivery = %+v, want ok/[5]", got) + } +} + +// TestHubRelayedWebhookRejections: tampered bodies, wrong keys, stale +// timestamps, missing or replayed delivery IDs are each rejected with their +// own reason. +func TestHubRelayedWebhookRejections(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + now := strconv.FormatInt(time.Now().Unix(), 10) + + t.Run("tampered body", func(t *testing.T) { + tampered := []byte(strings.Replace(string(body), `"number":5`, `"number":6`, 1)) + rec := postRelayedWebhook(s, "pull_request", "tamper-1", now, signWebhookRelayForTest(key, now, "tamper-1", body), tampered) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("tampered body: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonBadSignature) + } + }) + t.Run("another hive's key", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "other-1", now, signWebhookRelayForTest("relay-key-h2", now, "other-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("other hive's key: %d %s, want 401", rec.Code, rec.Body.String()) + } + }) + t.Run("delivery ID swapped after signing", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "swapped-2", now, signWebhookRelayForTest(key, now, "swapped-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonBadSignature) { + t.Fatalf("swapped delivery: %d %s, want 401", rec.Code, rec.Body.String()) + } + }) + t.Run("missing relay signature", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "unsigned-1", now, "", body) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("unsigned relay: status = %d, want 401", rec.Code) + } + }) + t.Run("stale timestamp", func(t *testing.T) { + old := strconv.FormatInt(time.Now().Add(-10*time.Minute).Unix(), 10) + rec := postRelayedWebhook(s, "pull_request", "stale-1", old, signWebhookRelayForTest(key, old, "stale-1", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonStale) { + t.Fatalf("stale timestamp: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonStale) + } + future := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10) + rec = postRelayedWebhook(s, "pull_request", "stale-2", future, signWebhookRelayForTest(key, future, "stale-2", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonStale) { + t.Fatalf("future timestamp: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonStale) + } + }) + t.Run("missing delivery ID", func(t *testing.T) { + rec := postRelayedWebhook(s, "pull_request", "", now, signWebhookRelayForTest(key, now, "", body), body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonNoDelivery) { + t.Fatalf("missing delivery: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonNoDelivery) + } + }) + t.Run("replayed delivery ID", func(t *testing.T) { + sig := signWebhookRelayForTest(key, now, "replay-1", body) + if rec := postRelayedWebhook(s, "pull_request", "replay-1", now, sig, body); rec.Code != http.StatusOK { + t.Fatalf("first delivery: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + rec := postRelayedWebhook(s, "pull_request", "replay-1", now, sig, body) + if rec.Code != http.StatusUnauthorized || !strings.Contains(relayErr(t, rec), webhookRelayReasonReplayed) { + t.Fatalf("replay: %d %s, want 401 %s", rec.Code, rec.Body.String(), webhookRelayReasonReplayed) + } + }) +} + +// TestHubRelayedWebhookFailuresAreCounted: a rejected relay counts as a +// failed delivery, the same signal a bad GitHub signature feeds the cadence +// mode, so a burst of forged relays degrades to polling and is visible. +func TestHubRelayedWebhookFailuresAreCounted(t *testing.T) { + const key = "relay-key-h1" + withWebhookRelayKey(t, key) + t.Cleanup(github.ResetWebhookTrackerForTest()) + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + now := strconv.FormatInt(time.Now().Unix(), 10) + + if rec := postRelayedWebhook(s, "pull_request", "count-1", now, signWebhookRelayForTest(key, now, "count-1", body), body); rec.Code != http.StatusOK { + t.Fatalf("good relay: status = %d (%s)", rec.Code, rec.Body.String()) + } + if _, _, recovered := github.ObserveWebhookMode(); !recovered { + t.Fatal("a verified relay should engage webhook mode") + } + if rec := postRelayedWebhook(s, "pull_request", "count-2", now, signWebhookRelayForTest("wrong", now, "count-2", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("forged relay: status = %d, want 401", rec.Code) + } + if _, degraded, _ := github.ObserveWebhookMode(); !degraded { + t.Fatal("a rejected relay after the last good one must be counted as a failed delivery") + } +} + +// TestGitHubSignedWebhookUnchangedWithRelayKey: a spoke with +// GITHUB_WEBHOOK_SECRET keeps accepting GitHub-signed deliveries exactly as +// before, relay key or not — no relay headers needed and no replay check. +func TestGitHubSignedWebhookUnchangedWithRelayKey(t *testing.T) { + withWebhookRelayKey(t, "relay-key-h1") + const secret = "s3cr3t" + t.Setenv(githubWebhookSecretEnvVar, secret) + s := newGitHubWebhookTestServer(t) + + body := []byte(`{"number":5,"pull_request":{"number":5},"repository":{"full_name":"webhookorg/webhookrepo"}}`) + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodPost, githubWebhookPath, bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", "pull_request") + req.Header.Set("X-GitHub-Delivery", "gh-direct-1") + req.Header.Set("X-Hub-Signature-256", signGitHubWebhookForTest(secret, body)) + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GitHub-signed delivery %d: status = %d, want 200 (%s)", i+1, rec.Code, rec.Body.String()) + } + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("wrong", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("bad GitHub signature: status = %d, want 401", rec.Code) + } +} + +// TestWebhooksConfiguredByRelayKey: the relay key alone turns webhooks on +// for event-driven review dispatch and /api/status.webhooks.verification. +func TestWebhooksConfiguredByRelayKey(t *testing.T) { + withWebhookRelayKey(t, "") + if githubWebhooksConfigured() { + t.Fatal("no secret and no relay key must leave webhooks off") + } + s := newGitHubWebhookTestServer(t) + body := []byte(`{}`) + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("x", body), body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no secret, no relay key: status = %d, want 503", rec.Code) + } + + if _, err := github.AdoptWebhookRelayKey("relay-key-h1"); err != nil { + t.Fatal(err) + } + if !githubWebhooksConfigured() { + t.Fatal("a hub relay key must turn webhooks on") + } + + s.statusMu.Lock() + s.status = &StatusPayload{HiveID: "relay-test"} + s.statusMu.Unlock() + rec := httptest.NewRecorder() + s.handleStatus(rec, httptest.NewRequest(http.MethodGet, "/api/status?fields=webhooks", nil)) + var status struct { + Webhooks struct { + Verification string `json:"verification"` + } `json:"webhooks"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status.Webhooks.Verification != github.WebhookVerificationRelay { + t.Fatalf("webhooks.verification = %q, want %q", status.Webhooks.Verification, github.WebhookVerificationRelay) + } +} + +func TestWebhookRelayReplayCacheIsBounded(t *testing.T) { + var c webhookRelayReplayCache + now := time.Now() + for i := 0; i < webhookRelayReplayMax+10; i++ { + if !c.firstSeen(strconv.Itoa(i), now.Add(time.Duration(i)*time.Millisecond)) { + t.Fatalf("id %d reported as seen", i) + } + } + if len(c.seen) > webhookRelayReplayMax { + t.Fatalf("cache holds %d ids, want at most %d", len(c.seen), webhookRelayReplayMax) + } + later := now.Add(webhookRelayReplayTTL + time.Hour) + if !c.firstSeen(strconv.Itoa(webhookRelayReplayMax+5), later) { + t.Fatal("an id older than the replay TTL must be accepted again") + } +} diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 155572129d..aac6014c49 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -45,6 +45,10 @@ type WebhookHealth struct { // WebhookModeStatus is the reconcile cadence the health drives: // `mode` webhook|polling and `since` (hivecommons/hive#13381). WebhookModeStatus + // Verification is how this spoke authenticates deliveries: + // "github-signed", "relay (hub-signed)" or "none" (WebhookVerification, + // hivecommons/hive#17911). + Verification string `json:"verification,omitempty"` } // WebhookEventResult reports what one delivered GitHub webhook did. @@ -402,7 +406,7 @@ func (t *webhookTracker) snapshot() WebhookHealth { out := WebhookHealth{Healthy: healthy, Events1h: len(t.events), Invalidations1h: len(t.invalidations), TargetedRefreshes1h: len(t.refreshes), TrackerInvalidations1h: len(t.trackerDrops), CIEvents1h: len(t.ciEvents), CIEventsApplied1h: len(t.ciApplied), CIRefetches1h: len(t.ciRefetches), - WebhookModeStatus: t.mode.status(t.window, now)} + WebhookModeStatus: t.mode.status(t.window, now), Verification: WebhookVerification()} if !latest.IsZero() { at := latest.UTC() out.LastEventAt = &at diff --git a/src/pkg/github/webhook_relay_key.go b/src/pkg/github/webhook_relay_key.go new file mode 100644 index 0000000000..d850abdb66 --- /dev/null +++ b/src/pkg/github/webhook_relay_key.go @@ -0,0 +1,134 @@ +package github + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "sync" +) + +// Hub webhook relay key (hivecommons/hive#17911). +// +// A hosted spoke never holds the GitHub App webhook secret, so it cannot +// verify X-Hub-Signature-256 on a delivery the hub relays. The hub instead +// signs each relay with a key of this hive's own, delivered over the +// heartbeat (HeartbeatGitHubAppConfig.WebhookRelayKey); the receiver +// (pkg/dashboard api_github_webhook.go) verifies that signature. The key is +// persisted 0600 on the data volume so a restarted spoke accepts relays +// before its first heartbeat, and is only ever reported by fingerprint. + +const defaultWebhookRelayKeyPath = "/data/webhook-relay.key" + +// WebhookSecretEnvVar names the GitHub App webhook HMAC secret a self-hosted +// spoke verifies /api/webhook/github deliveries with. +const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" + +// Values of /api/status.webhooks.verification: how this spoke authenticates +// GitHub webhook deliveries. +const ( + // WebhookVerificationGitHub: GITHUB_WEBHOOK_SECRET is set and GitHub's + // X-Hub-Signature-256 is verified (self-hosted, or a hub relay to a spoke + // that holds the secret). + WebhookVerificationGitHub = "github-signed" + // WebhookVerificationRelay: no secret, but the hub delivered this hive's + // relay key, so hub-relayed deliveries are verified by the hub's signature. + WebhookVerificationRelay = "relay (hub-signed)" + // WebhookVerificationNone: neither; every delivery is rejected with 503. + WebhookVerificationNone = "none" +) + +var webhookRelayKeyState = struct { + mu sync.Mutex + path string + key string + loaded bool +}{path: defaultWebhookRelayKeyPath} + +func webhookRelayKeyLocked() string { + st := &webhookRelayKeyState + if !st.loaded { + st.loaded = true + if data, err := os.ReadFile(st.path); err == nil { + st.key = strings.TrimSpace(string(data)) + } + } + return st.key +} + +// WebhookRelayKey is the hub relay key this spoke holds, or "". +func WebhookRelayKey() string { + webhookRelayKeyState.mu.Lock() + defer webhookRelayKeyState.mu.Unlock() + return webhookRelayKeyLocked() +} + +// AdoptWebhookRelayKey stores a relay key delivered by the hub and reports +// whether it replaced the one held (a first delivery or a rotation). +// +// An empty key means "the hub is not speaking to this field", never "clear +// it", so a beat without the key can never blank a working one. A key that +// cannot be persisted is still adopted in memory (relays verify until the +// next restart) and the write error is returned for the caller to log. +func AdoptWebhookRelayKey(key string) (bool, error) { + key = strings.TrimSpace(key) + if key == "" { + return false, nil + } + st := &webhookRelayKeyState + st.mu.Lock() + defer st.mu.Unlock() + if webhookRelayKeyLocked() == key { + return false, nil + } + st.key = key + if err := os.MkdirAll(filepath.Dir(st.path), 0o700); err != nil { + return true, err + } + tmp := st.path + ".tmp" + if err := os.WriteFile(tmp, []byte(key+"\n"), 0o600); err != nil { + return true, err + } + return true, os.Rename(tmp, st.path) +} + +// WebhookRelayKeyFingerprint is the NON-SECRET "sha256:<hex>" of the relay key +// this spoke holds, reported on every heartbeat so the hub delivers the key +// only when the spoke lacks it or holds a rotated-out one. Empty when none. +func WebhookRelayKeyFingerprint() string { + key := WebhookRelayKey() + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(key)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// WebhookVerification reports how this spoke verifies webhook deliveries +// (WebhookVerification* above). Only the mode ever leaves this function, never +// the secret or the key. +func WebhookVerification() string { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return WebhookVerificationGitHub + } + if WebhookRelayKey() != "" { + return WebhookVerificationRelay + } + return WebhookVerificationNone +} + +// SetWebhookRelayKeyPathForTest points the relay key store at path, forgets +// any key held in memory, and returns a func that restores the previous state. +func SetWebhookRelayKeyPathForTest(path string) func() { + st := &webhookRelayKeyState + st.mu.Lock() + prevPath, prevKey, prevLoaded := st.path, st.key, st.loaded + st.path, st.key, st.loaded = path, "", false + st.mu.Unlock() + return func() { + st.mu.Lock() + st.path, st.key, st.loaded = prevPath, prevKey, prevLoaded + st.mu.Unlock() + } +} diff --git a/src/pkg/github/webhook_relay_key_test.go b/src/pkg/github/webhook_relay_key_test.go new file mode 100644 index 0000000000..4775181aab --- /dev/null +++ b/src/pkg/github/webhook_relay_key_test.go @@ -0,0 +1,70 @@ +package github + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +// TestAdoptWebhookRelayKey: the hub relay key delivered over the heartbeat is +// persisted 0600 and survives a restart, a rotation replaces it, and a beat +// that carries no key never blanks a working one (#17911). +func TestAdoptWebhookRelayKey(t *testing.T) { + path := filepath.Join(t.TempDir(), "webhook-relay.key") + t.Cleanup(SetWebhookRelayKeyPathForTest(path)) + t.Setenv(WebhookSecretEnvVar, "") + + if WebhookRelayKey() != "" || WebhookRelayKeyFingerprint() != "" || WebhookVerification() != WebhookVerificationNone { + t.Fatal("fresh spoke must hold no relay key") + } + + changed, err := AdoptWebhookRelayKey("key-1") + if err != nil || !changed { + t.Fatalf("first delivery: changed=%v err=%v, want adopted", changed, err) + } + if WebhookRelayKey() != "key-1" || WebhookVerification() != WebhookVerificationRelay { + t.Fatalf("after delivery: key=%q verification=%q", WebhookRelayKey(), WebhookVerification()) + } + fp := WebhookRelayKeyFingerprint() + // Same format the hub compares against (pkg/hub webhookRelayKeyFingerprint). + sum := sha256.Sum256([]byte("key-1")) + if want := "sha256:" + hex.EncodeToString(sum[:]); fp != want { + t.Fatalf("fingerprint = %q, want %q", fp, want) + } + if fi, err := os.Stat(path); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("relay key file: %v, want mode 0600", err) + } + + if changed, err := AdoptWebhookRelayKey("key-1"); err != nil || changed { + t.Fatalf("redelivery of the same key: changed=%v err=%v, want no change", changed, err) + } + for _, empty := range []string{"", " "} { + if changed, err := AdoptWebhookRelayKey(empty); err != nil || changed { + t.Fatalf("empty delivery %q: changed=%v err=%v", empty, changed, err) + } + if WebhookRelayKey() != "key-1" { + t.Fatal("an empty delivery blanked a working relay key") + } + } + + if changed, err := AdoptWebhookRelayKey("key-2"); err != nil || !changed { + t.Fatalf("rotation: changed=%v err=%v, want adopted", changed, err) + } + if WebhookRelayKeyFingerprint() == fp { + t.Fatal("fingerprint did not change after rotation") + } + + // A restarted spoke reads the persisted key before its first heartbeat. + restore := SetWebhookRelayKeyPathForTest(path) + defer restore() + if WebhookRelayKey() != "key-2" { + t.Fatalf("after restart key = %q, want the rotated key from disk", WebhookRelayKey()) + } + + t.Setenv(WebhookSecretEnvVar, "s3cr3t") + if WebhookVerification() != WebhookVerificationGitHub { + t.Fatalf("with GITHUB_WEBHOOK_SECRET set verification = %q, want %q", WebhookVerification(), WebhookVerificationGitHub) + } +} diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 242a9a3383..110e36b44c 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -343,6 +343,7 @@ type HeartbeatPayload struct { GatewayHealth []inferencehealth.GatewayStatus `json:"gateway_health,omitempty"` GitHubAppKeyFingerprint string `json:"github_app_key_fingerprint,omitempty"` GitHubAppKeyPerHive bool `json:"github_app_key_per_hive,omitempty"` + WebhookRelayKeyFingerprint string `json:"webhook_relay_key_fingerprint,omitempty"` GitHubAppID int64 `json:"github_app_id,omitempty"` GitHubAppSlug string `json:"github_app_slug,omitempty"` GitHubInstallationID int64 `json:"github_installation_id,omitempty"` @@ -411,6 +412,10 @@ type HeartbeatGitHubAppConfig struct { BaseURL string `json:"base_url,omitempty"` AdditionalKeys []HeartbeatAppKey `json:"additional_keys,omitempty"` SecondaryKey *HeartbeatAppKey `json:"secondary_key,omitempty"` + // WebhookRelayKey is this hive's own key for verifying webhooks the hub + // relays to it (webhook_relay_key.go). Never the App webhook secret. + // Empty means "leave the spoke's key unchanged". + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` } type HeartbeatAppKey struct { diff --git a/src/pkg/hub/lite_enrollment.go b/src/pkg/hub/lite_enrollment.go index 9e66e2207d..a1b5af3544 100644 --- a/src/pkg/hub/lite_enrollment.go +++ b/src/pkg/hub/lite_enrollment.go @@ -330,6 +330,7 @@ func (s *HubServer) requestHostedLiteSpoke(username, owner, repo, host string, i ClusterID: clusterID, IsPublic: false, AutoUpgrade: true, + WebhookRelayKey: newWebhookRelayKey(), } if host != "" && host != publicGitHubHost { h.GitHubHost = host diff --git a/src/pkg/hub/pool_replenisher.go b/src/pkg/hub/pool_replenisher.go index 19668a5ae8..7dd44ee912 100644 --- a/src/pkg/hub/pool_replenisher.go +++ b/src/pkg/hub/pool_replenisher.go @@ -145,9 +145,10 @@ func (s *HubServer) seedPlaceholder(cluster *ClusterConfig) error { // Pool slots track stable and auto-upgrade so inventory never rots on // an old image while waiting to be assigned — same posture as the // hand-seeded fleet. - TrackedChannel: "stable", - AutoUpgrade: true, - IsPublic: false, + TrackedChannel: "stable", + AutoUpgrade: true, + IsPublic: false, + WebhookRelayKey: newWebhookRelayKey(), } if err := saveSaaSHive(h); err != nil { return fmt.Errorf("save placeholder record: %w", err) diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index 71e215330d..a6713566dc 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -1552,8 +1552,9 @@ func (s *HubServer) handleCreateHive(w http.ResponseWriter, r *http.Request) { IsPublic: req.IsPublic == nil || *req.IsPublic, // Per-hive GitHub host override (empty = cluster default). Lets a // public-GitHub org provision correctly on a GHE-defaulted cluster. - GitHubBaseURL: req.GitHubBaseURL, - GitHubAPIURL: req.GitHubAPIURL, + GitHubBaseURL: req.GitHubBaseURL, + GitHubAPIURL: req.GitHubAPIURL, + WebhookRelayKey: newWebhookRelayKey(), } if err := saveSaaSHive(h); err != nil { @@ -1627,8 +1628,12 @@ func (s *HubServer) handleHiveStatus(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"access denied"}`, http.StatusForbidden) return } + // The relay key authenticates webhooks to this hive's spoke; it is + // delivered only over the spoke's own heartbeat, never to a browser. + out := *h + out.WebhookRelayKey = "" w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(h) + _ = json.NewEncoder(w).Encode(&out) } // handleOpenHive is the SSO handoff entry point: a hub-authenticated user hits diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index b5de4a99ca..75939a069a 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -797,6 +797,15 @@ type SaaSHive struct { // delivery (see AppIdentityDelivered), so a lost beat is retried. PendingAppConfig *PendingAppIdentity `json:"pending_app_config,omitempty"` + // WebhookRelayKey is this hive's own HMAC key for webhooks the hub relays + // to its spoke (webhook_relay_key.go, hivecommons/hive#17911). Minted at + // provisioning (or on the first heartbeat of a hive provisioned before + // relay keys existed) and delivered over the heartbeat. Per hive so one + // spoke's key can never sign a relay another spoke accepts, and so no + // spoke ever needs the GitHub App webhook secret. Replacing or clearing it + // rotates the key: the next beat delivers the new one. + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` + // RequestedRestartAt (RFC3339) arms a rolling restart of this hive's spoke, // delivered on every heartbeat inside spokeRestartDeliveryWindow and then // cleared. Durable on the hive record for the same reason as @@ -2081,15 +2090,26 @@ func saveSaaSHive(h *SaaSHive) error { return fmt.Errorf("invalid hive ID for save: %q", h.ID) } dir := filepath.Join(saasHivesDir, h.ID) - // Best-effort: a failed mkdir surfaces via the WriteFile error below. - _ = os.MkdirAll(dir, 0o755) + // meta.json carries WebhookRelayKey, so it is written rw------- under an + // rwx------ directory like the other hub secret stores (cluster App keys, + // generation stores). Best-effort: a failed mkdir surfaces via the + // WriteFile error below. + _ = os.MkdirAll(dir, 0o700) + // MkdirAll leaves pre-existing directories untouched, so tighten hives + // provisioned before meta.json carried key material. Only group/other + // bits are stripped: owner bits are left as found so a directory an + // operator (or a test) made read-only is never silently widened back to + // writable, and the WriteFile below still reports the failure. + if fi, err := os.Stat(dir); err == nil && fi.Mode().Perm()&0o077 != 0 { + _ = os.Chmod(dir, fi.Mode().Perm()&^0o077) + } data, err := json.MarshalIndent(h, "", " ") if err != nil { return err } path := filepath.Join(dir, "meta.json") tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, data, 0o644); err != nil { + if err := os.WriteFile(tmpPath, data, 0o600); err != nil { return err } if err := os.Rename(tmpPath, path); err != nil { diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index aedd7a7e1f..5638807292 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -3359,6 +3359,21 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { resp.GitHubAppConfig.SecondaryKey = secCfg } + // Per-hive webhook relay key (#17911), attached the same independent way as + // the secondary key: it rides whatever config the branches above produced, + // or an otherwise-empty one, and only until the spoke reports holding it. + if relayKey := s.webhookRelayKeyForHeartbeat(&payload); relayKey != "" { + if resp.GitHubAppConfig == nil { + resp.GitHubAppConfig = &HeartbeatGitHubAppConfig{} + } + resp.GitHubAppConfig.WebhookRelayKey = relayKey + s.logger.Info("heartbeat: delivering webhook relay key to spoke", + "hive_id", payload.HiveID, + "fingerprint", webhookRelayKeyFingerprint(relayKey), + "spoke_fingerprint", payload.WebhookRelayKeyFingerprint, + ) + } + // SECURITY (C1/N3, CWE-200/639): the fleet-wide additional-key broadcast that // once ran here was removed. It attached every OTHER tenant's App private key // to every heartbeat, and — because the handler trusts the body-supplied diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 86c71807b5..2ca00c37cd 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -1056,6 +1056,13 @@ type HeartbeatPayload struct { // its cluster. The hub honours it as an override: a deliberate per-hive // credential is never overwritten by the cluster default. GitHubAppKeyPerHive bool `json:"github_app_key_per_hive,omitempty"` + // WebhookRelayKeyFingerprint is the NON-SECRET "sha256:<hex>" of the hub + // webhook relay key this spoke holds (github.WebhookRelayKeyFingerprint). + // The hub delivers the key only while this does not match its record, so + // a missing or rotated-out key is repaired on the next beat. Empty means + // no key (or a spoke too old to report one); the key itself never travels + // spoke → hub. + WebhookRelayKeyFingerprint string `json:"webhook_relay_key_fingerprint,omitempty"` // GitHubAppID is the App ID this spoke believes it is authenticating AS — // cfg.GitHub.AppID, non-secret. It exists to make one specific distinction // decidable on the hub: a per-hive key that is WRONG versus a per-hive key @@ -2752,6 +2759,15 @@ type HeartbeatGitHubAppConfig struct { // nil for every hive without a SecondaryAppID, which is the entire fleet // today, so the heartbeat payload is byte-identical for them. SecondaryKey *HeartbeatAppKey `json:"secondary_key,omitempty"` + // WebhookRelayKey is this hive's OWN key for verifying the signature the + // hub adds to every GitHub webhook it relays here (hivecommons/hive#17911). + // It is per hive and is never the GitHub App webhook secret, so it cannot + // sign anything another spoke would accept. Secret — never logged; the + // spoke persists it 0600 (github.AdoptWebhookRelayKey). + // + // Empty means "leave the spoke's key unchanged" — never a way to blank a + // working key. + WebhookRelayKey string `json:"webhook_relay_key,omitempty"` } // HeartbeatAppKey is one (app_id, private key) pair the hub delivers alongside diff --git a/src/pkg/hub/webhook.go b/src/pkg/hub/webhook.go index b6dbe65dbd..d177a66280 100644 --- a/src/pkg/hub/webhook.go +++ b/src/pkg/hub/webhook.go @@ -129,8 +129,11 @@ var spokeWebhookRelayClient = &http.Client{ // relayWebhookToSpoke forwards a verified PR-shaped delivery, byte for byte // and with its original signature, to the hive that manages the repository so -// the spoke can invalidate its cached PR state (hivecommons/hive#11177). The -// spoke re-verifies X-Hub-Signature-256 with the same App webhook secret. +// the spoke can invalidate its cached PR state (hivecommons/hive#11177). A +// self-hosted spoke re-verifies X-Hub-Signature-256 with its own copy of the +// App webhook secret; a hosted spoke never holds that secret, so the hub also +// signs each relay with the target hive's own relay key +// (signWebhookRelayHeaders) and the spoke verifies that instead. // Best effort and asynchronous: GitHub's delivery must not wait on a spoke. func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body []byte) { var evt struct { @@ -184,8 +187,13 @@ func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body [ for _, t := range targets { hiveID, dashboardURL := t.hiveID, t.dashboardURL spokeURL := strings.TrimRight(dashboardURL, "/") + spokeWebhookPath - s.logger.Info("relaying webhook to spoke", "hive_id", hiveID, "spoke_url", spokeURL, "event", event, "delivery", delivery) + relayKey := webhookRelayKeyForHive(hiveID) + s.logger.Info("relaying webhook to spoke", "hive_id", hiveID, "spoke_url", spokeURL, "event", event, "delivery", delivery, "relay_signed", relayKey != "") payload := append([]byte(nil), body...) + hdr := forward.Clone() + if relayKey != "" { + signWebhookRelayHeaders(hdr, relayKey, time.Now(), delivery, payload) + } go func() { ctx, cancel := context.WithTimeout(context.Background(), webhookPushTimeout) defer cancel() @@ -198,7 +206,7 @@ func (s *HubServer) relayWebhookToSpoke(event string, header http.Header, body [ s.logger.Warn("failed to build spoke webhook relay request", "hive_id", hiveID, "error", err) return } - req.Header = forward.Clone() + req.Header = hdr resp, err := spokeWebhookRelayClient.Do(req) if err != nil { s.logger.Warn("spoke webhook relay failed", "hive_id", hiveID, "event", event, "error", err) diff --git a/src/pkg/hub/webhook_relay_key.go b/src/pkg/hub/webhook_relay_key.go new file mode 100644 index 0000000000..e0c9b9479c --- /dev/null +++ b/src/pkg/hub/webhook_relay_key.go @@ -0,0 +1,121 @@ +package hub + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "net/http" + "strconv" + "time" +) + +// Hub-signed webhook relay (hivecommons/hive#17911). +// +// A hosted spoke must be able to verify a delivery the hub relays to it, but +// it must never hold the GitHub App webhook secret: that one secret +// authenticates GitHub deliveries for the whole App, so any spoke holding it +// could forge deliveries to every other spoke's public /api/webhook/github. +// Instead every hosted hive gets its OWN random relay key, stored on its hub +// record (SaaSHive.WebhookRelayKey) and delivered over its per-hive +// authenticated heartbeat. The hub signs each relay with the target hive's +// key; a key held by one spoke cannot sign anything another spoke accepts. +// +// Signature scheme (mirrored by pkg/dashboard verifyWebhookRelaySignature): +// +// X-Hive-Relay-Timestamp: <unix seconds at relay time> +// X-Hive-Relay-Signature-256: sha256=<hex HMAC-SHA256(key, timestamp + "." + X-GitHub-Delivery + "." + raw body)> +// +// The timestamp and delivery ID are inside the MAC so the spoke can reject +// stale relays and replayed delivery IDs without trusting unsigned headers. +// GitHub's own headers (X-Hub-Signature-256 included) are forwarded +// unchanged next to these, so a self-hosted spoke is unaffected. +const ( + webhookRelaySignatureHeader = "X-Hive-Relay-Signature-256" + webhookRelayTimestampHeader = "X-Hive-Relay-Timestamp" +) + +// newWebhookRelayKey mints a 256-bit relay key, hex encoded. +func newWebhookRelayKey() string { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "" + } + return hex.EncodeToString(b) +} + +// webhookRelayKeyFingerprint is the NON-SECRET identifier of a relay key the +// spoke reports back on every heartbeat (webhook_relay_key_fingerprint), so the +// hub delivers the key only when the spoke does not already hold it. Same +// format as pkg/github WebhookRelayKeyFingerprint. +func webhookRelayKeyFingerprint(key string) string { + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(key)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// webhookRelaySignature computes the relay MAC described above. +func webhookRelaySignature(key, timestamp, delivery string, body []byte) string { + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(timestamp)) + mac.Write([]byte(".")) + mac.Write([]byte(delivery)) + mac.Write([]byte(".")) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// signWebhookRelayHeaders adds the relay timestamp and signature for one +// target hive's key to an outgoing relay. +func signWebhookRelayHeaders(h http.Header, key string, now time.Time, delivery string, body []byte) { + ts := strconv.FormatInt(now.Unix(), 10) + h.Set(webhookRelayTimestampHeader, ts) + h.Set(webhookRelaySignatureHeader, webhookRelaySignature(key, ts, delivery, body)) +} + +// webhookRelayKeyForHive is the relay key on a hive's hub record, or "" for a +// hive the hub did not provision (a self-hosted spoke keeps verifying GitHub's +// signature with its own secret). +func webhookRelayKeyForHive(hiveID string) string { + if h := loadSaaSHive(hiveID); h != nil { + return h.WebhookRelayKey + } + return "" +} + +// webhookRelayKeyForHeartbeat returns the relay key to deliver on this beat, +// or "" when the spoke already reports holding it. +// +// A hosted hive provisioned before relay keys existed has none on its record; +// one is minted here on its first beat so the fix reaches the existing fleet. +// Rotation is the same path: replace (or clear) SaaSHive.WebhookRelayKey and +// the spoke's fingerprint stops matching, so the next beat delivers the new +// key. The beat is authenticated with the per-hive bearer before this runs +// (verifyHeartbeatBearer), so a spoke can only ever be handed its own key. +func (s *HubServer) webhookRelayKeyForHeartbeat(p *HeartbeatPayload) string { + if p == nil || p.HiveID == "" { + return "" + } + h := loadSaaSHive(p.HiveID) + if h == nil { + return "" + } + if h.WebhookRelayKey == "" { + key := newWebhookRelayKey() + if key == "" { + return "" + } + h.WebhookRelayKey = key + if err := saveSaaSHive(h); err != nil { + s.logger.Warn("webhook relay: could not store a new relay key on the hive record", "hive_id", p.HiveID, "error", err) + return "" + } + s.logger.Info("webhook relay: minted relay key for hive", "hive_id", p.HiveID, "fingerprint", webhookRelayKeyFingerprint(key)) + } + if p.WebhookRelayKeyFingerprint == webhookRelayKeyFingerprint(h.WebhookRelayKey) { + return "" + } + return h.WebhookRelayKey +} diff --git a/src/pkg/hub/webhook_relay_key_test.go b/src/pkg/hub/webhook_relay_key_test.go new file mode 100644 index 0000000000..4e07fe1859 --- /dev/null +++ b/src/pkg/hub/webhook_relay_key_test.go @@ -0,0 +1,286 @@ +package hub + +import ( + "bytes" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +type relayedSignedWebhook struct { + header http.Header + body []byte +} + +func captureRelaySpoke(t *testing.T) (*httptest.Server, chan relayedSignedWebhook) { + t.Helper() + got := make(chan relayedSignedWebhook, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got <- relayedSignedWebhook{header: r.Header.Clone(), body: body} + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv, got +} + +func useTempSaaSHivesDir(t *testing.T) { + t.Helper() + old := saasHivesDir + saasHivesDir = t.TempDir() + t.Cleanup(func() { saasHivesDir = old }) +} + +// TestRelayWebhookSignsWithTargetHiveRelayKey: the relay to a hosted hive +// carries a hub signature that verifies with THAT hive's relay key and not +// with another hive's, binds the delivery ID and a fresh timestamp, and still +// forwards GitHub's own signature unchanged (#17911). +func TestRelayWebhookSignsWithTargetHiveRelayKey(t *testing.T) { + const secret = "s3cr3t" + t.Setenv(webhookSecretEnvVar, secret) + allowLoopbackRelay(t) + useTempSaaSHivesDir(t) + for id, key := range map[string]string{"h1": "relay-key-h1", "h2": "relay-key-h2"} { + if err := saveSaaSHive(&SaaSHive{ID: id, WebhookRelayKey: key}); err != nil { + t.Fatal(err) + } + } + spokeSrv, got := captureRelaySpoke(t) + + s := newWebhookHub() + s.registry.Hives = []RegistryEntry{{ID: "h1", Org: "acme", Repos: []string{"widgets"}, DashboardURL: spokeSrv.URL}} + + body := []byte(`{"action":"synchronize","number":7,"repository":{"name":"widgets","full_name":"acme/widgets","owner":{"login":"acme"}}}`) + ghSig := signWebhook(secret, body) + req := httptest.NewRequest(http.MethodPost, "/api/github/webhook", bytes.NewReader(body)) + req.Header.Set("X-GitHub-Event", "pull_request") + req.Header.Set("X-GitHub-Delivery", "delivery-1") + req.Header.Set("X-Hub-Signature-256", ghSig) + before := time.Now().Unix() + rec := httptest.NewRecorder() + s.handleGitHubWebhook(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + + var r relayedSignedWebhook + select { + case r = <-got: + case <-time.After(5 * time.Second): + t.Fatal("delivery was not relayed to the managing spoke") + } + if r.header.Get("X-Hub-Signature-256") != ghSig || !bytes.Equal(r.body, body) { + t.Fatal("GitHub's signature and body must still be forwarded unchanged") + } + ts := r.header.Get(webhookRelayTimestampHeader) + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil || sec < before || sec > time.Now().Unix() { + t.Fatalf("relay timestamp = %q, want unix seconds at relay time", ts) + } + sig := r.header.Get(webhookRelaySignatureHeader) + if sig == "" || sig != webhookRelaySignature("relay-key-h1", ts, "delivery-1", body) { + t.Fatalf("relay signature %q does not verify with the target hive's relay key", sig) + } + if sig == webhookRelaySignature("relay-key-h2", ts, "delivery-1", body) { + t.Fatal("relay signature verifies with another hive's relay key") + } + if sig == webhookRelaySignature("relay-key-h1", ts, "delivery-2", body) { + t.Fatal("relay signature must bind the delivery ID") + } +} + +// TestRelayWebhookUnsignedForHiveWithoutRelayKey: a hive the hub did not +// provision (no record, so no relay key) gets the relay exactly as before. +func TestRelayWebhookUnsignedForHiveWithoutRelayKey(t *testing.T) { + allowLoopbackRelay(t) + useTempSaaSHivesDir(t) + spokeSrv, got := captureRelaySpoke(t) + + s := newWebhookHub() + s.registry.Hives = []RegistryEntry{{ID: "selfhosted", Org: "acme", Repos: []string{"widgets"}, DashboardURL: spokeSrv.URL}} + hdr := http.Header{} + hdr.Set("X-GitHub-Delivery", "d") + s.relayWebhookToSpoke("push", hdr, []byte(`{"repository":{"name":"widgets","full_name":"acme/widgets","owner":{"login":"acme"}}}`)) + select { + case r := <-got: + if r.header.Get(webhookRelaySignatureHeader) != "" || r.header.Get(webhookRelayTimestampHeader) != "" { + t.Fatalf("hive without a relay key got relay headers: %v", r.header) + } + case <-time.After(5 * time.Second): + t.Fatal("delivery was not relayed") + } +} + +// TestWebhookRelayKeyForHeartbeat: the relay key is delivered until the spoke +// reports holding it, minted for a hosted hive provisioned before relay keys +// existed, re-delivered after a rotation, and never offered to a hive the hub +// has no record for. +func TestWebhookRelayKeyForHeartbeat(t *testing.T) { + useTempSaaSHivesDir(t) + s := &HubServer{logger: slog.Default()} + + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "unknown"}); got != "" { + t.Fatalf("hive without a record got a relay key") + } + + if err := saveSaaSHive(&SaaSHive{ID: "legacy"}); err != nil { + t.Fatal(err) + } + minted := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy"}) + if len(minted) != 64 { + t.Fatalf("legacy hive: delivered %q, want a minted 256-bit hex key", minted) + } + if h := loadSaaSHive("legacy"); h == nil || h.WebhookRelayKey != minted { + t.Fatal("minted relay key was not stored on the hive record") + } + if again := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy"}); again != minted { + t.Fatal("a spoke that has not confirmed the key must get the same key again, not a new one") + } + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint(minted)}); got != "" { + t.Fatal("key re-delivered to a spoke that already holds it") + } + + // Rotation: a new key on the record no longer matches the spoke's + // fingerprint, so the next beat delivers it. + h := loadSaaSHive("legacy") + h.WebhookRelayKey = "rotated-key" + if err := saveSaaSHive(h); err != nil { + t.Fatal(err) + } + if got := s.webhookRelayKeyForHeartbeat(&HeartbeatPayload{HiveID: "legacy", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint(minted)}); got != "rotated-key" { + t.Fatalf("after rotation delivered %q, want the rotated key", got) + } +} + +// TestProvisionedHiveRecordsHaveDistinctRelayKeys: provisioning mints a +// fresh random key per hive. +func TestProvisionedHiveRecordsHaveDistinctRelayKeys(t *testing.T) { + a, b := newWebhookRelayKey(), newWebhookRelayKey() + if len(a) != 64 || len(b) != 64 || a == b { + t.Fatalf("relay keys %q / %q, want two distinct 256-bit hex keys", a, b) + } +} + +// TestHiveStatusNeverExposesRelayKey: the owner-facing hive status returns +// the hive record, but never the relay key, which only the spoke's own +// heartbeat may carry. +func TestHiveStatusNeverExposesRelayKey(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + s := &HubServer{logger: slog.Default(), hubSecret: testHubSecret} + mkUser(t, "alice") + if err := saveSaaSHive(&SaaSHive{ID: "h1", Owner: "alice", WebhookRelayKey: "relay-key-h1"}); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.handleHiveStatus(rec, setPathValue(reqWithUser(http.MethodGet, "/st", "", "alice"), "id", "h1")) + if rec.Code != http.StatusOK { + t.Fatalf("owner hive-status = %d, want 200", rec.Code) + } + var got map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if _, ok := got["webhook_relay_key"]; ok || bytes.Contains(rec.Body.Bytes(), []byte("relay-key-h1")) { + t.Fatalf("hive status leaked the relay key: %s", rec.Body.String()) + } + if h := loadSaaSHive("h1"); h == nil || h.WebhookRelayKey != "relay-key-h1" { + t.Fatal("redacting the response must not touch the stored record") + } +} + +// TestWebhookRelayKeyHeartbeatWire: the key the hub attaches reaches the +// spoke's decoder, and the fingerprint the spoke reports reaches the hub's, +// with the same fingerprint format on both sides. +func TestWebhookRelayKeyHeartbeatWire(t *testing.T) { + raw, err := json.Marshal(HeartbeatResponse{GitHubAppConfig: &HeartbeatGitHubAppConfig{WebhookRelayKey: "relay-key-h1"}}) + if err != nil { + t.Fatal(err) + } + var spokeResp spoke.HeartbeatResponse + if err := json.Unmarshal(raw, &spokeResp); err != nil { + t.Fatal(err) + } + if spokeResp.GitHubAppConfig == nil || spokeResp.GitHubAppConfig.WebhookRelayKey != "relay-key-h1" { + t.Fatalf("spoke decoded %+v, want the relay key", spokeResp.GitHubAppConfig) + } + + raw, err = json.Marshal(spoke.HeartbeatPayload{HiveID: "h1", WebhookRelayKeyFingerprint: webhookRelayKeyFingerprint("relay-key-h1")}) + if err != nil { + t.Fatal(err) + } + var hubPayload HeartbeatPayload + if err := json.Unmarshal(raw, &hubPayload); err != nil { + t.Fatal(err) + } + if hubPayload.WebhookRelayKeyFingerprint != webhookRelayKeyFingerprint("relay-key-h1") { + t.Fatalf("hub decoded fingerprint %q", hubPayload.WebhookRelayKeyFingerprint) + } +} + +// meta.json carries the relay key, so it must be unreadable to other local +// users like the hub's other secret stores. A pre-existing world-readable hive +// directory must be tightened on save too. +func TestSaveSaaSHiveKeepsRelayKeyPrivate(t *testing.T) { + useTempSaaSHivesDir(t) + + dir := filepath.Join(saasHivesDir, "legacy") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := saveSaaSHive(&SaaSHive{ID: "legacy", WebhookRelayKey: newWebhookRelayKey()}); err != nil { + t.Fatal(err) + } + + fi, err := os.Stat(filepath.Join(dir, "meta.json")) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != 0o600 { + t.Fatalf("meta.json mode = %o, want 600", got) + } + di, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if got := di.Mode().Perm(); got != 0o700 { + t.Fatalf("hive dir mode = %o, want 700", got) + } +} + +// Tightening must only strip group/other bits. A hive directory that has +// been made read-only must stay read-only so the save fails loudly instead +// of being silently widened back to writable (which also masked the forced +// save failures in reset_assignment_sweep_branches_test.go). +func TestSaveSaaSHiveNeverWidensReadOnlyDir(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; read-only dir cannot force a write failure") + } + useTempSaaSHivesDir(t) + + dir := filepath.Join(saasHivesDir, "frozen") + if err := os.MkdirAll(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(dir, 0o755) }) + + if err := saveSaaSHive(&SaaSHive{ID: "frozen", WebhookRelayKey: newWebhookRelayKey()}); err == nil { + t.Fatal("saveSaaSHive succeeded into a read-only hive dir; want write failure") + } + di, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if got := di.Mode().Perm(); got != 0o500 { + t.Fatalf("hive dir mode = %o, want 500 (group/other stripped, owner bits untouched)", got) + } +} From d0a6c0673e97ace7cdf2a4940c440a9a1abb3e5b Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 08:41:05 -0700 Subject: [PATCH 092/166] =?UTF-8?q?=F0=9F=90=9B=20Inbox:=20render=20the=20?= =?UTF-8?q?empty=20state=20instead=20of=20"Loading=20inbox=E2=80=A6"=20for?= =?UTF-8?q?ever=20(#17920)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Inbox: render the empty state instead of "Loading inbox…" forever With no voice knowledge connector configured bootInbox never ran, so GET /api/inbox answered 404 and the loader returned early, leaving the placeholder up. The Inbox service now always boots (zero sources is a valid, empty inbox), the list endpoint returns 200 with an empty list even without a service, and a non-OK response renders a visible error. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 Inbox: reference #17918 in comments Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-inbox-empty-state.md | 1 + src/cmd/hive/boot_inbox.go | 10 +++--- src/cmd/hive/boot_inbox_test.go | 27 ++++++++++++++ src/cmd/hive/boot_knowledge_connectors.go | 5 ++- src/docs/inbox-voice-sources.md | 6 ++++ src/pkg/dashboard/api_inbox.go | 17 ++++++--- src/pkg/dashboard/api_inbox_test.go | 13 +++++-- src/pkg/dashboard/inbox_loader_static_test.go | 36 +++++++++++++++++++ src/pkg/dashboard/static-src/js/inbox.js | 10 ++++-- src/pkg/dashboard/static/index.html | 10 ++++-- 10 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 changelog.d/fixed-inbox-empty-state.md create mode 100644 src/cmd/hive/boot_inbox_test.go create mode 100644 src/pkg/dashboard/inbox_loader_static_test.go diff --git a/changelog.d/fixed-inbox-empty-state.md b/changelog.d/fixed-inbox-empty-state.md new file mode 100644 index 0000000000..d18badbc1f --- /dev/null +++ b/changelog.d/fixed-inbox-empty-state.md @@ -0,0 +1 @@ +- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index b30f16ec2b..d4a3e0808b 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -14,9 +14,10 @@ import ( // inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. const inboxSweepTimeout = 2 * time.Minute -// bootInbox builds the Inbox service when at least one configured knowledge -// connector is a voice source (inbox.sources, default plaud/wisprflow). -// Items are read from the connector vaults under <baseDir>/connectors; +// bootInbox builds the Inbox service. Sources are the configured knowledge +// connectors whose type is a voice source (inbox.sources, default +// plaud/wisprflow); with none configured the service still exists and lists +// nothing, so the dashboard shows the empty state. Items are read from the connector vaults under <baseDir>/connectors; // route state persists at <baseDir>/connector-state/_inbox.json (connector // names cannot start with "_", so it never collides with a connector). func (b *boot) bootInbox(baseDir string) { @@ -31,9 +32,6 @@ func (b *boot) bootInbox(baseDir string) { srcs = append(srcs, b.inboxSource(baseDir, typ)) } } - if len(srcs) == 0 { - return - } token := func(ctx context.Context, _ string) (string, error) { if b.appAuth == nil { return "", nil diff --git a/src/cmd/hive/boot_inbox_test.go b/src/cmd/hive/boot_inbox_test.go new file mode 100644 index 0000000000..f1c38a836f --- /dev/null +++ b/src/cmd/hive/boot_inbox_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// With no knowledge connectors configured the Inbox service must still boot +// so GET /api/inbox returns an empty list, not a 404 (#17918). +func TestBootInboxWithoutConnectors(t *testing.T) { + b := &boot{cfg: &config.Config{}, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + b.bootKnowledgeConnectors(bootKnowledgeDeps{}) + if b.inboxSvc == nil { + t.Fatal("inbox service not built without connectors") + } + items, err := b.inboxSvc.Items(context.Background()) + if err != nil || len(items) != 0 { + t.Fatalf("items = %v %v", items, err) + } + if b.knowledgeConnectors != nil { + t.Fatal("connector syncer should stay nil with no connectors") + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 3d9cbdc64f..070f0f9b5e 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -89,6 +89,10 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne // bootKnowledgeConnectors builds and starts the connector syncer. A failure // is logged and leaves connectors off; it never blocks the rest of boot. func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { + // The Inbox always boots, even with no connectors, so GET /api/inbox + // answers 200 with an empty list and the panel renders its empty state + // (#17918) instead of waiting forever on a 404. + b.bootInbox(knowledge.BaseDir()) s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) @@ -102,7 +106,6 @@ func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { deps.startConnectorSyncer(b.ctx, s) } b.logger.Info("knowledge connectors started", "count", len(b.cfg.Knowledge.Connectors)) - b.bootInbox(knowledge.BaseDir()) } // knowledgeConnectorRuntime exposes the syncer to the dashboard. A nil diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md index 4f71b60ba2..9947215654 100644 --- a/src/docs/inbox-voice-sources.md +++ b/src/docs/inbox-voice-sources.md @@ -95,3 +95,9 @@ See [api-reference/knowledge.md](api-reference/knowledge.md) for `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings` and `POST /api/knowledge/connectors/{name}/auth/complete`. + +`GET /api/inbox` always answers `200`: on a hive with no voice connector +configured it returns an empty `items` list and every source as +`configured: false`, and the Inbox panel shows its empty state with a +"Connect sources" button. A connector added under Knowledge → Integrations +starts syncing after the next hive restart. diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go index 1ca6dc14bd..67dfba6281 100644 --- a/src/pkg/dashboard/api_inbox.go +++ b/src/pkg/dashboard/api_inbox.go @@ -60,12 +60,19 @@ func (s *Server) inboxService(w http.ResponseWriter) (*inbox.Service, bool) { } func (s *Server) handleInboxList(w http.ResponseWriter, r *http.Request) { - svc, ok := s.inboxService(w) - if !ok { - return - } ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) defer cancel() + if s == nil || s.deps == nil || s.deps.Inbox == nil { + // No service (older boot path / tests): still a valid, empty inbox — + // the panel must render its empty state, not spin on a 404. + resp := InboxListResponse{Items: []inbox.Item{}, Sources: []InboxSourceStatus{}, Repos: s.inboxRepos(), Settings: inbox.Settings{Rules: []config.InboxRule{}}} + if s != nil && s.deps != nil && s.deps.Config != nil { + resp.Sources = s.inboxSources(ctx, s.deps.Config.Inbox.EffectiveSources()) + } + jsonResponse(w, resp) + return + } + svc := s.deps.Inbox items, err := svc.Items(ctx) resp := InboxListResponse{ Items: items, @@ -115,7 +122,7 @@ func (s *Server) inboxSources(ctx context.Context, types []string) []InboxSource } func (s *Server) inboxRepos() []string { - if s.deps == nil || s.deps.Config == nil { + if s == nil || s.deps == nil || s.deps.Config == nil { return []string{} } repos := append([]string(nil), s.deps.Config.Project.Repos...) diff --git a/src/pkg/dashboard/api_inbox_test.go b/src/pkg/dashboard/api_inbox_test.go index 150f36c188..72f407b2b9 100644 --- a/src/pkg/dashboard/api_inbox_test.go +++ b/src/pkg/dashboard/api_inbox_test.go @@ -79,8 +79,17 @@ func decodeInboxList(t *testing.T, body []byte) InboxListResponse { func TestInboxEndpoints(t *testing.T) { s, deps := apiServer(t) - if rec := doGet(s, "/api/inbox"); rec.Code != http.StatusNotFound { - t.Fatalf("inbox without service = %d, want 404", rec.Code) + // No service: still 200 with an empty list so the panel renders its + // empty state instead of "Loading inbox…" forever (#17918). + rec0 := doGet(s, "/api/inbox") + if rec0.Code != http.StatusOK { + t.Fatalf("inbox without service = %d, want 200: %s", rec0.Code, rec0.Body.String()) + } + if empty := decodeInboxList(t, rec0.Body.Bytes()); empty.Items == nil || len(empty.Items) != 0 || empty.Sources == nil || empty.Settings.Rules == nil { + t.Fatalf("empty inbox = %s", rec0.Body.String()) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusNotFound { + t.Fatalf("settings without service = %d, want 404", rec.Code) } spek := &inboxTestDest{name: inbox.DestSpekRFC} dibs := &inboxTestDest{name: inbox.DestDibs, err: errors.New("dibs down")} diff --git a/src/pkg/dashboard/inbox_loader_static_test.go b/src/pkg/dashboard/inbox_loader_static_test.go new file mode 100644 index 0000000000..1e4e75185a --- /dev/null +++ b/src/pkg/dashboard/inbox_loader_static_test.go @@ -0,0 +1,36 @@ +package dashboard + +import ( + "os" + "strings" + "testing" +) + +// The Inbox loader must never leave the "Loading inbox…" placeholder up: a +// non-OK /api/inbox response renders an error state, and an OK response +// with zero items renders the empty-state text (#17918). +func TestInboxLoaderHandlesNonOKAndEmpty(t *testing.T) { + src, err := os.ReadFile("static-src/js/inbox.js") + if err != nil { + t.Fatal(err) + } + js := string(src) + start := strings.Index(js, "async function fetchInbox()") + if start < 0 { + t.Fatal("fetchInbox missing") + } + body := js[start : strings.Index(js[start:], "dashboardScheduleWork('inbox'")+start] + if strings.Contains(body, "if (!r.ok) return;") { + t.Fatal("fetchInbox bails out on a non-OK response, leaving the loading placeholder in place") + } + for _, want := range []string{"if (!r.ok) {", "_inboxCache = { items: []", "renderInbox();"} { + if !strings.Contains(body, want) { + t.Fatalf("fetchInbox non-OK branch missing %q:\n%s", want, body) + } + } + for _, want := range []string{"if (data.error) html +=", "Nothing yet. Connect Plaud or another voice source"} { + if !strings.Contains(js, want) { + t.Fatalf("renderInbox missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 9d2233d788..c638c57712 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -26,7 +26,13 @@ async function fetchInbox() { try { const r = await fetch('/api/inbox'); - if (!r.ok) return; + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } _inboxCache = await r.json(); renderInbox(); clearTimeout(_inboxBusyTimer); @@ -113,7 +119,7 @@ html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; } - if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; panel.innerHTML = html; } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 517974a26c..8a6db5eb16 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46222,7 +46222,13 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> async function fetchInbox() { try { const r = await fetch('/api/inbox'); - if (!r.ok) return; + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } _inboxCache = await r.json(); renderInbox(); clearTimeout(_inboxBusyTimer); @@ -46309,7 +46315,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; } - if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync.') + '</div>'; + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; panel.innerHTML = html; } From 307bcec20b2f0e51ee5ccad3c6dfcffb456daf63 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 08:55:46 -0700 Subject: [PATCH 093/166] =?UTF-8?q?=F0=9F=93=96=20releases:=20candidate=20?= =?UTF-8?q?hives=20follow=20their=20stored=20auto-upgrade=20cadence=20(#17?= =?UTF-8?q?922)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since #11816 the hub respects a candidate hive's daily/weekly mode by default; the docs still said the opposite, which read as a stuck upgrade on a daily hive. Document the actual behaviour and how to switch a hive to instant. Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-docs-candidate-cadence.md | 1 + src/docs/releases.md | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-docs-candidate-cadence.md diff --git a/changelog.d/fixed-docs-candidate-cadence.md b/changelog.d/fixed-docs-candidate-cadence.md new file mode 100644 index 0000000000..9989234640 --- /dev/null +++ b/changelog.d/fixed-docs-candidate-cadence.md @@ -0,0 +1 @@ +- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. diff --git a/src/docs/releases.md b/src/docs/releases.md index 0dc22efe02..704bc9de42 100644 --- a/src/docs/releases.md +++ b/src/docs/releases.md @@ -212,13 +212,18 @@ separate, deliberate policy described in silently couples to it, on purpose — the operator explicitly asked for these to stay decoupled. -Hub-managed hives on the `candidate` channel are canaries: by default the hub -does not hold them behind the daily/weekly auto-upgrade cadence. Once the -candidate image for a newer commit is published, the hub may arm the hive on the -next poll as long as the existing idle, health and wave gates allow it. Stable -channel hives keep the slower soak/cadence rules. Operators who deliberately -want candidate hives to obey their stored cadence can set -`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=true` on the hub. +Hub-managed hives on the `candidate` channel follow their stored auto-upgrade +cadence (`auto_upgrade_mode`: `daily` by default, `weekly`, or `instant`) just +like stable hives (#11816). A daily candidate hive therefore picks up the newest +published candidate image once a day, in the first poll at or after 1pm ET; the +hub reports the hold as `upgradeWhyNotNow: "before the daily window"` and the +spoke's `/api/version` shows `state: behind` until then. To track every +published candidate image, set the hive's mode to `instant` +(`PUT /api/saas/hives/{id}/auto-upgrade` with +`{"auto_upgrade": true, "auto_upgrade_mode": "instant"}`, or the Manage → +Auto-upgrade control on the hub). Setting +`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=false` on the hub restores the +old behaviour where every candidate hive bypasses its cadence. The hub itself does **not** follow the hosted-hive daily/weekly cadence: when hub auto-upgrade is enabled and not paused, the hub rolls itself **instantly** From 35d01e27f797c45dc2bda139c3ce3e3059736903 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 16:00:30 +0000 Subject: [PATCH 094/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.235.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.235.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-baseline-check-gh-wrapper.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-baseline-check-gh-wrapper.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0337292185..bd5e2eb51d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.235.1) + +### Fixed + +- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. + ## 2026-11-12 (v5.235.0) ### Added diff --git a/changelog.d/fixed-baseline-check-gh-wrapper.md b/changelog.d/fixed-baseline-check-gh-wrapper.md deleted file mode 100644 index 4d3ccfe69b..0000000000 --- a/changelog.d/fixed-baseline-check-gh-wrapper.md +++ /dev/null @@ -1 +0,0 @@ -- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. From 5849f6811590efe08bacd2ee51ce30843b41f956 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 16:30:20 +0000 Subject: [PATCH 095/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 16 ++++++++++++++++ changelog.d/added-claude-long-lived-token.md | 1 - ...ed-claude-operator-token-suspends-rebridge.md | 1 - changelog.d/fixed-docs-candidate-cadence.md | 1 - .../fixed-hosted-webhook-relay-signature.md | 1 - changelog.d/fixed-inbox-empty-state.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-claude-long-lived-token.md delete mode 100644 changelog.d/changed-claude-operator-token-suspends-rebridge.md delete mode 100644 changelog.d/fixed-docs-candidate-cadence.md delete mode 100644 changelog.d/fixed-hosted-webhook-relay-signature.md delete mode 100644 changelog.d/fixed-inbox-empty-state.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bd5e2eb51d..c1159587c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.0) + +### Added + +- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. + +### Changed + +- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) + +### Fixed + +- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. +- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. +- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. + ## 2026-11-12 (v5.235.1) ### Fixed diff --git a/changelog.d/added-claude-long-lived-token.md b/changelog.d/added-claude-long-lived-token.md deleted file mode 100644 index 9c84edba5b..0000000000 --- a/changelog.d/added-claude-long-lived-token.md +++ /dev/null @@ -1 +0,0 @@ -- Claude agents can run on an operator-supplied long-lived token: set `HIVE_CLAUDE_OAUTH_TOKEN` (or mount it at `/secrets/claude-oauth-token`, or point `HIVE_CLAUDE_OAUTH_TOKEN_FILE` at it) to a `claude setup-token` token and every `claude`-backend agent is launched with it as `CLAUDE_CODE_OAUTH_TOKEN`, ahead of the 8-hour dashboard sign-in; unset, nothing changes. diff --git a/changelog.d/changed-claude-operator-token-suspends-rebridge.md b/changelog.d/changed-claude-operator-token-suspends-rebridge.md deleted file mode 100644 index 22c625b98c..0000000000 --- a/changelog.d/changed-claude-operator-token-suspends-rebridge.md +++ /dev/null @@ -1 +0,0 @@ -- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) diff --git a/changelog.d/fixed-docs-candidate-cadence.md b/changelog.d/fixed-docs-candidate-cadence.md deleted file mode 100644 index 9989234640..0000000000 --- a/changelog.d/fixed-docs-candidate-cadence.md +++ /dev/null @@ -1 +0,0 @@ -- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. diff --git a/changelog.d/fixed-hosted-webhook-relay-signature.md b/changelog.d/fixed-hosted-webhook-relay-signature.md deleted file mode 100644 index 436f4aa770..0000000000 --- a/changelog.d/fixed-hosted-webhook-relay-signature.md +++ /dev/null @@ -1 +0,0 @@ -- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. diff --git a/changelog.d/fixed-inbox-empty-state.md b/changelog.d/fixed-inbox-empty-state.md deleted file mode 100644 index d18badbc1f..0000000000 --- a/changelog.d/fixed-inbox-empty-state.md +++ /dev/null @@ -1 +0,0 @@ -- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. From 054e2eabdd5e66efbc64b2c922e63de7b28cb9e3 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:56:34 +0000 Subject: [PATCH 096/166] =?UTF-8?q?=F0=9F=93=96=20docs:=20ADR-0026=20?= =?UTF-8?q?=E2=80=94=20Substrate=20actors=20run=20credential-free;=20hive?= =?UTF-8?q?=20brokers=20pushes=20post-kick=20(#17934)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: ADR-0026 — Substrate actors run credential-free; hive brokers pushes post-kick Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> * changelog: fragment for #17934 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-17934.md | 1 + src/deploy/substrate.md | 4 ++-- .../adr/0026-sandbox-runtime-selection.md | 21 ++++++++++++++++++- src/docs/sandbox-isolation.md | 2 +- 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 changelog.d/changed-17934.md diff --git a/changelog.d/changed-17934.md b/changelog.d/changed-17934.md new file mode 100644 index 0000000000..f0e9d94f58 --- /dev/null +++ b/changelog.d/changed-17934.md @@ -0,0 +1 @@ +- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) diff --git a/src/deploy/substrate.md b/src/deploy/substrate.md index 941d32d2c3..707db2e458 100644 --- a/src/deploy/substrate.md +++ b/src/deploy/substrate.md @@ -84,8 +84,8 @@ Then create the pieces hive's runtime depends on, in Substrate terms: it is tracked in [#17697](https://github.com/hivecommons/hive/issues/17697). - **Egress**: hive writes its own `EgressPolicy` per actor (`github.com`, `api.github.com`, `allow_hosts`). Do **not** also configure Substrate's - credential injection for GitHub — hive's MITM proxy brokers credentials and - the two would conflict. + credential injection for GitHub — actors run credential-free and hive + brokers pushes after the kick (ADR-0026, #17695). ## Verify diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index d5f27798c2..aae4f451f1 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -242,7 +242,7 @@ opt-in and inert unless selected. |---|---|---| | Kernel-enforced confinement | gVisor user-space kernel or a microVM; worker pods are `privileged: false` but keep `SYS_ADMIN`/`NET_ADMIN` | selects the class through `worker_selector`; no change to hive | | Default-deny egress | `atenet-egress` `EgressPolicy` per actor; rules are TLS-passthrough or terminate per host:port, no per-method read-only mode | `SubstrateEgressPolicy`: `github.com` and `api.github.com` on 443 plus `sandbox.substrate.allow_hosts`, TLS passthrough; everything else (and all UDP) denied | -| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | unchanged hive brokering — the agent still gets `HTTPS_PROXY` to the hive MITM proxy (ADR-0002); Substrate's injection is **not** used, the two are complementary | +| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | actors run credential-free: no `HTTPS_PROXY`/MITM broker inside the actor and no GitHub credential enters it; hive does every push and GitHub write after the kick; Substrate's injection is **not** used (see the amendment below) | | Workspace in, commits out | `durableDir` volumes and object-storage snapshots; no exec/attach API | an in-actor HTTP **bridge** (`crux substrate-bridge`) reached through the router's mTLS `CONNECT` tunnel: push tar → `POST /run` → pull tar; the hive-side workspace stays authoritative | | Cleanup and timeouts | `DeleteActor` / `SuspendActor` / `DeleteActorTemplate` | one template + actor per kick, both deleted on success, failure and timeout (fresh bounded context, `ExitCode -1`, 16 MiB cap, as `job`/`openshell`) | @@ -257,6 +257,25 @@ opt-in and inert unless selected. | snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; only the latest suspended actor per agent per hive process is retained (the previous one is deleted on the next suspend; actors from before a restart are not reclaimed); resume-on-next-kick is not implemented (#17697) | | `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | +### Amendment (2026-11-12, #17695): credential-free actors + +Decision by the hive owner on #17695, option A: + +- Substrate actors run **credential-free**. There is no `HTTPS_PROXY` or MITM + broker inside the actor and no GitHub credential enters it; the actor's + egress is governed only by Substrate's `EgressPolicy`. +- Hive brokers pushes **post-kick**: it performs git pushes and GitHub writes + itself, from the workspace the bridge returns. +- **Phase 3 (future)**: SVID-authenticated broker delegation over the atenet + mTLS channel, so hive can delegate scoped GitHub writes to the actor without + a credential entering it. It is gated on hosted clusters serving the podcert + feature gates (`PodCertificateRequest` / `ClusterTrustBundle`, + `certificates.k8s.io/v1beta1`) and is tracked as a child issue of #17695. + +This supersedes the earlier statement that hive's credential brokering is +unchanged for Substrate. The podman, `job` and `openshell` runtimes keep +their proxy model (ADR-0002). + ### Cluster prerequisites and which hive clusters qualify Substrate's real requirement is **not** a kernel floor: the gVisor class runs diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index a1e1779719..17cdbca242 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -319,7 +319,7 @@ agents: **One actor per kick.** Hive creates an `ActorTemplate` (one container running the bridge command, the workspace as a `durableDir`, the operator's limits and `worker_selector`) and an `Actor` from it, both named `hive-kick-<agent>-<unix>`, attaches the egress policy, waits for the actor to report `RUNNING`, and then talks to a small HTTP **bridge** inside the actor (`crux substrate-bridge`, overridable with `bridge_command`, port `bridge_port`) through the router's mTLS `CONNECT` tunnel: it pushes the workspace as a tar, runs the kick with the sanitized environment, and pulls the workspace back. The hive-side workspace remains the source of truth and feeds the push broker exactly as for Podman. Substrate has no exec API and hive shares no PVC with the worker nodes, which is why the workspace is copied rather than mounted; the copy is bounded by the kick timeout. -**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Credential brokering is unchanged: the agent inside the actor still receives `HTTPS_PROXY` pointing at the hive MITM proxy ([ADR-0002](adr/0002-mitm-proxy-network-enforcement.md)); Substrate's own zero-credential injection is complementary and not used by hive. +**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Actors run credential-free: there is no `HTTPS_PROXY` or MITM broker inside the actor and no credential enters it, and hive brokers pushes after the kick from the returned workspace (decision on [#17695](https://github.com/hivecommons/hive/issues/17695), [ADR-0026](adr/0026-sandbox-runtime-selection.md)). Substrate's own credential injection is not used; SVID-authenticated broker delegation over the atenet mTLS channel is a future phase 3. **Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). At most one suspended actor is retained per agent per hive process: the previous one is deleted after the next successful suspend (skipped with a warning if the agent's `atespace` changed in between), and actors left by an earlier hive process are not reclaimed because the Control API has no `ListActors`. `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. From fc0c0a9f23cea4de2953429e8ea112464d80e0c7 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:13:05 +0100 Subject: [PATCH 097/166] =?UTF-8?q?=F0=9F=90=9B=20Classify=20missing=20Dep?= =?UTF-8?q?endabot=20alerts=20permission=20(#17929)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Classify missing Dependabot alerts permission Hive-Run: hivecommons/hive#17925 Hive-Plan: classify missing Dependabot alerts permission Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * 🐛 Fix Dependabot report changelog category Hive-Run: hivecommons/hive#17925 Hive-Plan: classify missing Dependabot alerts permission Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * test: cover Dependabot permission runtime thresholds Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- .../fixed-dependabot-permission-selfreport.md | 1 + src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 31 ++++++++++--------- .../delegatedecision/issue_judgment_test.go | 9 ++++++ src/pkg/fleetreport/runtime.go | 2 ++ src/pkg/fleetreport/runtime_test.go | 2 ++ src/pkg/fleetreport/slogfeed/slogfeed_test.go | 20 ++++++++++++ 7 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-dependabot-permission-selfreport.md diff --git a/changelog.d/fixed-dependabot-permission-selfreport.md b/changelog.d/fixed-dependabot-permission-selfreport.md new file mode 100644 index 0000000000..2438213269 --- /dev/null +++ b/changelog.d/fixed-dependabot-permission-selfreport.md @@ -0,0 +1 @@ +- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index fcd9a05adc..fdc732371d 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -145,6 +145,7 @@ title is `[selfreport] <class>: <short message>`. | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 377adbc3da..4d1ea1a596 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -94,21 +94,22 @@ func ParseSelfReport(title, body string) (SelfReport, bool) { // constants (pinned by TestSelfreportOpsClassesMatchFleetReport) without // importing that package's dependency tree into this leaf. var selfreportOpsRemedies = map[string]string{ - "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", - "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", - "data_disk_pressure": "free or grow the data disk on the spoke", - "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", - "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", - "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", - "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", - "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", - "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", - "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", - "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", - "inference_upstream_error": "check the inference endpoint the spoke's provider points at", - "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", - "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", - "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", + "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", + "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", + "data_disk_pressure": "free or grow the data disk on the spoke", + "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", + "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", + "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", + "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", + "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", + "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", + "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", + "inference_upstream_error": "check the inference endpoint the spoke's provider points at", + "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", + "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", + "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", } // selfreportOpsClass reports whether a self-report class is an operator's to diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 9f0bdc0ae3..6e17ebf4ca 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -27,6 +27,14 @@ func TestDecideIssueLikeTheOwner(t *testing.T) { text: []string{"@clubanderson", "re-authenticate the agent provider", "hosted-ibm-alchemy-logg-ptoo"}, actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, }, + { + name: "missing Dependabot alerts permission is operator action", + in: Input{Title: "[selfreport] github_app_alerts_permission_missing: scoped token mint", + Body: "<!-- hive:selfreport fp=x hive=spoke -->\nclass `github_app_alerts_permission_missing`\n| owner(s) | @clubanderson |\n| spoke | spoke |"}, + rule: "selfreport-operator-action", verdict: VerdictOperatorAction, + text: []string{"@clubanderson", "Dependabot alerts read-only", "re-accept", "spoke"}, + actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, + }, { name: "hive#15329 RFC on hive-filed proposals (human author) → recommend only, with a real recommendation", in: fixtureHive15329, rule: "hive-policy", verdict: VerdictRecommendOnly, @@ -160,6 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 3946d8adb4..5e56884fd9 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -77,6 +77,7 @@ const ( ClassBeadFactIngestFailed = "bead_fact_ingest_failed" ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" ClassProxyConnectDialFailed = "proxy_connect_dial_failed" ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" @@ -201,6 +202,7 @@ var Classes = map[string]ClassSpec{ ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, + ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index bf9c2dafbf..ec5791dc89 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -453,6 +453,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, + {class: ClassGitHubAppAlertsPermission, message: "scoped token mint with vulnerability_alerts:read failed; retrying without it"}, {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, @@ -1240,6 +1241,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "fleet report: cleared comment failed": ClassSelfReportClearedFailed, "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 4d2757e831..b92640d6c3 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -335,6 +335,26 @@ func TestRequiredChecksOverrideMismatchLogsWithoutFleetReport(t *testing.T) { } } +func TestMissingAlertsPermissionReportsOperatorCondition(t *testing.T) { + now := time.Date(2026, 11, 12, 15, 40, 0, 0, time.UTC) + tr := fleetreport.NewRuntimeTracker(func() time.Time { return now }) + var out bytes.Buffer + logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe)) + const msg = "scoped token mint with vulnerability_alerts:read failed; retrying without it — the sec-check Dependabot alerts call will 403 until the App grants Dependabot alerts read-only and the installation re-accepts it; see docs/github-app-setup.md" + for range 5 { + logger.Warn(msg, "tier", "sec-check", "error", "permissions requested are not granted") + now = now.Add(time.Minute) + } + ev := tr.Evidence() + if len(ev) != 1 || ev[0].Class != fleetreport.ClassGitHubAppAlertsPermission || ev[0].Count != 5 { + t.Fatalf("missing permission evidence = %+v", ev) + } + now = now.Add(21 * time.Minute) + if ev := tr.Evidence(); len(ev) != 0 { + t.Fatalf("quiet condition did not clear: %+v", ev) + } +} + func TestFeedLogFedPath(t *testing.T) { tests := []struct { name string From 23084fe7bbebda40baefc21350c6ad14b34d025a Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 17:29:59 +0000 Subject: [PATCH 098/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/changed-17934.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/changed-17934.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c1159587c9..73d850f252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.1) + +### Changed + +- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) + ## 2026-11-12 (v5.236.0) ### Added diff --git a/changelog.d/changed-17934.md b/changelog.d/changed-17934.md deleted file mode 100644 index f0e9d94f58..0000000000 --- a/changelog.d/changed-17934.md +++ /dev/null @@ -1 +0,0 @@ -- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) From 82634f754c8be1c89aa5364fba9ee09f3a23bc63 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:49:29 +0100 Subject: [PATCH 099/166] =?UTF-8?q?=F0=9F=90=9B=20Handle=20unsupported=20s?= =?UTF-8?q?igned-commit=20rewrites=20explicitly=20(#17927)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive-Run: hivecommons/hive#17923 Hive-Plan: signed-commit-reconciler-refusal Hive-Spec: hivecommons/hive#17923 Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-signed-reconcile-unsupported.md | 1 + src/docs/github-app-setup.md | 4 +- src/pkg/github/pr_signed_reconcile.go | 32 ++++++++------ src/pkg/github/pr_signed_reconcile_test.go | 42 +++++++++++++++++++ 4 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 changelog.d/fixed-signed-reconcile-unsupported.md diff --git a/changelog.d/fixed-signed-reconcile-unsupported.md b/changelog.d/fixed-signed-reconcile-unsupported.md new file mode 100644 index 0000000000..ad1001d019 --- /dev/null +++ b/changelog.d/fixed-signed-reconcile-unsupported.md @@ -0,0 +1 @@ +- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 2976b95c7b..0bf7f5d43a 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -257,7 +257,9 @@ What it does not do, and falls back on (the PR still opens on the agent's own co **Follow-up commits.** An agent's later pushes to its open PR (a CI fix, review follow-ups, a rebase) are plain git and unsigned, and so is a PR that opened on a `signed_skipped` fallback. With `app_signed_commits` on, the PR-request watcher also runs a reconcile pass, at most once a minute (#9364). A PR needs signing when *any* of its commits is not **Verified**, not only its head: a person can sign their own commit on top of an agent's unsigned one, which leaves the head Verified while the PR is still blocked (#9531). For every open PR authored by the App bot, the pass finds the first unverified commit (oldest to newest) and re-authors everything from there through the head as one GitHub-signed commit on top of the commit right before it. The tree is the same, and the message carries the range's messages and DCO trailers, re-addressed as above. When the PR's first commit is the unverified one, it signs against the PR's base branch, as at open time. A fully Verified PR is left alone. The open path's guards all still apply. A head that moves mid-rewrite is retried on the next pass, and so is a head GitHub could not be reached for (a 5xx or network error). A head GitHub *refuses* to sign (a 4xx answer, such as the 422 on the scratch ref in #16193) is retried on three consecutive passes and then settled, with the `hive:signed-commits-blocked` comment quoting the error; there is no person's commit to squash in that case, so the remedy is to push a new head (which restarts signing) or fix the repository-side cause the error names. A PR whose head hasn't changed since the last pass costs no calls beyond the open-PR list. -The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. So does any change the mutation can't express. The PR then gets one comment (marked `<!-- hive:signed-commits-blocked -->`, posted once per PR) naming the unsigned commit and the commit that blocks the rewrite, so it can't merge under `required_signatures` until someone acts. The manual fix is to squash the unsigned commit into a commit the person signs, or to drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. +The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. The PR gets one comment (marked `<!-- hive:signed-commits-blocked -->`, posted once per PR) naming the blocker. To unblock it, squash the unsigned commit into a commit the person signs, or drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. + +Changes the signing mutation cannot safely express also leave the branch untouched and get a single comment with the reason. For example, it cannot preserve executable files, symlinks or submodule pointers. A contributor can re-author the unsigned commits with their own verified signature, preserving their contents and modes, and push the signed branch. These deliberate skips are logged at INFO rather than as API failures; a new head is examined again. **Agents after a push.** The rewrite replaces the agent's commits on the remote, so its local branch no longer matches, even though the tree is identical. Before committing again on a branch with an open PR, run `git fetch && git rebase origin/<branch>`. The rebase drops the now-empty local commits and keeps any new work. Never force-push over the signed commit: that puts the unsigned commits back, and the next pass has to sign them again. diff --git a/src/pkg/github/pr_signed_reconcile.go b/src/pkg/github/pr_signed_reconcile.go index e2a999337f..b8770a30bd 100644 --- a/src/pkg/github/pr_signed_reconcile.go +++ b/src/pkg/github/pr_signed_reconcile.go @@ -69,6 +69,12 @@ const signedSkipRemedyPerson = "The hive re-signs only its own agents' commits ( "or drop your own commit from the branch so the hive can re-sign the agent's work on a later pass, " + "then push your change again, signed, on top." +// signedSkipRemedyUnsupported applies when the change cannot safely be +// re-authored, rather than a person's commit blocking the rewrite. +const signedSkipRemedyUnsupported = "The hive could not safely represent this change with GitHub's signing API. " + + "Have a contributor re-author the unsigned commits with their own verified signature and push the signed branch, " + + "preserving the file modes and contents. A new head also lets the hive try again." + // signedSkipRemedyAPIRefused tells the PR owner how to unblock a head GitHub // kept refusing to sign: there is no person's commit to squash, only a new // head (or a repository-side fix) re-arms the reconciler. @@ -336,23 +342,25 @@ func (c *Client) reconcileSignedPR(ctx context.Context, owner, repo string, pr * oid, replaced, err := c.signBranch(ctx, owner, repo, base, head) if err != nil { var moved signedHeadMovedError - c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) if errors.As(err, &moved) { + c.info("signed-commit reconciler: head moved while signing; will retry", append(logArgs, "base", base, "reason", err.Error())...) return "" // transient: retry next pass } - if !errors.Is(err, errSignedCommitSkip) { - if !signedAPIRefusal(err) { - return "" // outage (5xx, network): retry next pass, uncounted - } - if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { - return "" // retry next pass - } - c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) - reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) - c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) + if errors.Is(err, errSignedCommitSkip) { + c.info("signed-commit reconciler: cannot safely re-author unsigned tail; leaving branch unchanged", append(logArgs, "base", base, "reason", err.Error())...) + c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyUnsupported) return headSHA } - c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyPerson) + c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) + if !signedAPIRefusal(err) { + return "" // outage (5xx, network): retry next pass, uncounted + } + if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { + return "" // retry next pass + } + c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) + reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) + c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) return headSHA } c.signedReconcile.clearFailures(key) diff --git a/src/pkg/github/pr_signed_reconcile_test.go b/src/pkg/github/pr_signed_reconcile_test.go index 4245f5bd57..9b65a47495 100644 --- a/src/pkg/github/pr_signed_reconcile_test.go +++ b/src/pkg/github/pr_signed_reconcile_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" "strings" "testing" @@ -396,6 +397,47 @@ func TestSignedReconcile_UnchangedHeadIsNotRelisted(t *testing.T) { } } +// Unsupported file modes are a deliberate safety refusal, not an API outage. +// Keep the branch intact, explain how to unblock it, and settle the head. +func TestSignedReconcile_UnsupportedModeSettlesWithActionableNote(t *testing.T) { + for _, mode := range []string{"100755", "120000", "160000"} { + t.Run(mode, func(t *testing.T) { + m, c := reconcileFixture(t, true) + m.treeMode["tests/new_test.py"] = mode + m.prCommits = []map[string]any{prCommitJSON(m.headSHA, false, "", "quality@hive.kubestellar.io")} + var logs strings.Builder + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + + c.reconcileSignedCommits(context.Background()) + c.reconcileSignedCommits(context.Background()) + + if m.refPatch != nil || m.graphql != nil || m.callIndex("POST /repos/o/r/git/refs") >= 0 { + t.Fatalf("unsupported mode must leave the branch untouched: %v", m.calls) + } + if m.countCalls(prCommitsCall) != 1 { + t.Errorf("unchanged unsupported head must be settled: %v", m.calls) + } + if len(m.posted) != 1 || !strings.Contains(m.posted[0], mode) || + !strings.Contains(m.posted[0], "their own verified signature") || + strings.Contains(m.posted[0], "drop your own commit") { + t.Fatalf("expected one actionable mode-specific comment, got %q", m.posted) + } + if strings.Contains(logs.String(), "could not re-author the unsigned tail") { + t.Errorf("expected skip must not be reported as an API failure: %s", logs.String()) + } + + newHead := "dddd000000000000000000000000000000000030" + m.headSHA = newHead + m.openPRs = []map[string]any{reconcilePR(77, reconcileBot, newHead)} + m.prCommits = []map[string]any{prCommitJSON(newHead, false, "", "quality@hive.kubestellar.io")} + c.reconcileSignedCommits(context.Background()) + if m.countCalls(prCommitsCall) != 2 || len(m.posted) != 1 { + t.Errorf("new head must be retried without another comment: calls=%v posted=%q", m.calls, m.posted) + } + }) + } +} + // A head that moved while the signed commit was built is transient: no // comment, and the next pass looks again (the open-time skip is retried). func TestSignedReconcile_MovedHeadRetriesNextPass(t *testing.T) { From 8c6b3798371760c54320e4dc5b1f468b27c4c29c Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:02:36 +0100 Subject: [PATCH 100/166] =?UTF-8?q?=F0=9F=90=9B=20Avoid=20fleet=20reports?= =?UTF-8?q?=20for=20canceled=20adoption=20reads=20(#17931)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive-Run: hivecommons/hive#17926 Hive-Plan: classify canceled adoption sweep reads Hive-Spec: fleet-report#runtime-error-classes Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- changelog.d/fixed-canceled-adoption-report.md | 1 + src/docs/fleet-report.md | 4 ++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 21 ++++++++++++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 18 ++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-canceled-adoption-report.md diff --git a/changelog.d/fixed-canceled-adoption-report.md b/changelog.d/fixed-canceled-adoption-report.md new file mode 100644 index 0000000000..803255f554 --- /dev/null +++ b/changelog.d/fixed-canceled-adoption-report.md @@ -0,0 +1 @@ +- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index fdc732371d..8a3f887968 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -175,6 +175,10 @@ when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. Issue activity keeps its last rollup and is retried by the collector. +Canceled maintainer-edit access reads during an in-place adoption sweep stay in +local logs without triggering the catch-all; adoption is preserved and the +read is retried on the next sweep. Other access-read errors remain reportable. + Callers that fail closed on a GitHub HTTP 500 from the combined-status endpoint — the merge-eligible lane gate, the trusted-author and contributor-lane automerge sweeps, and the stage-label check digest — also stay in the logs diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index a5ee890b10..1550bafe9b 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -14,6 +14,7 @@ package slogfeed import ( "context" + "errors" "log/slog" "strings" @@ -114,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledAdoptionRead(r) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -135,6 +136,24 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } +// A canceled sweep leaves the existing adoption intact and retries later. +// Other failures reading push access still warrant a repeated-error report. +func canceledAdoptionRead(r slog.Record) bool { + if strings.TrimSpace(r.Message) != "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" { + return false + } + canceled := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "error" { + err, ok := a.Value.Any().(error) + canceled = (ok && errors.Is(err, context.Canceled)) || a.Value.String() == context.Canceled.Error() + return !canceled + } + return true + }) + return canceled +} + // expectedKickHold reports whether a "failed to send kick" record carries the // deliberate post-restart kick cooldown refusal (#7363). The hold is the // breaker working as designed, not a repeated runtime defect (#14422); other diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index b92640d6c3..ce7c2d09ab 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -64,6 +64,24 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestCanceledAdoptionReadStaysInLog(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" + for range 8 { + logger.Warn(msg, "repo", "server", "pr", 649, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + } + logger.Warn(msg, "error", "context canceled") + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 9 { + t.Fatalf("canceled reads: observations=%+v, log=%s", *seen, out.String()) + } + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn("another operation failed", "error", context.Canceled) + if len(*seen) != 3 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + // #16510: polling an exhausted rebase episode must keep warning locally // without turning the intentional handoff into a repeated runtime defect. func TestRebaseAttemptsExhaustedLogsWithoutFleetReport(t *testing.T) { From 5b191ff2929addc98257636e576a630635f32037 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 11:12:02 -0700 Subject: [PATCH 101/166] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20transpor?= =?UTF-8?q?t-level=20App=20token=20mint=20failures=20as=20forge=5Funreacha?= =?UTF-8?q?ble,=20not=20a=20mint=20failure=20(#17948)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix: classify transport-level token mint failures as forge_unreachable A hosted GHE hive on a cluster whose egress to github.ibm.com was blocked filed github_app_token_mint_failed (high, github-client) for every agent, and the Diagnostics remedy told the owner to check the App key and installation — which were fine. The mint never reached GitHub. mintAgentTokenUnlocked now checks the mint error with the same test the dashboard forge probe uses (#17427): context deadline, *url.Error or net.Error → observe crux.ClassForgeUnreachable (component network, medium) and clear the mint-failed condition; other errors keep the mint-failed class and clear forge_unreachable; success clears all three. Documented in docs/fleet-report.md; selfreport remedy points at egress, DNS and proxy/hostAliases. Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 fix: move mint classification to manager_mint.go (crux size ratchet) and add forge_unreachable threshold case Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 test: register forge_unreachable in the selfreport ops-class mirror Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-forge-unreachable-mint-class.md | 1 + src/crux/agent/manager_mint.go | 98 +++++++++++++++++++ src/crux/agent/manager_tokens.go | 55 ----------- src/crux/agent/mint_forge_unreachable_test.go | 93 ++++++++++++++++++ src/crux/conditions.go | 13 ++- src/docs/fleet-report.md | 1 + src/pkg/delegatedecision/issue_judgment.go | 1 + .../delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 2 + src/pkg/fleetreport/runtime_test.go | 1 + 10 files changed, 208 insertions(+), 59 deletions(-) create mode 100644 changelog.d/fixed-forge-unreachable-mint-class.md create mode 100644 src/crux/agent/manager_mint.go create mode 100644 src/crux/agent/mint_forge_unreachable_test.go diff --git a/changelog.d/fixed-forge-unreachable-mint-class.md b/changelog.d/fixed-forge-unreachable-mint-class.md new file mode 100644 index 0000000000..65050c8d68 --- /dev/null +++ b/changelog.d/fixed-forge-unreachable-mint-class.md @@ -0,0 +1 @@ +- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. diff --git a/src/crux/agent/manager_mint.go b/src/crux/agent/manager_mint.go new file mode 100644 index 0000000000..fe8896f327 --- /dev/null +++ b/src/crux/agent/manager_mint.go @@ -0,0 +1,98 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + + "github.com/hivecommons/hive/crux" +) + +// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token +// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is +// the single mint step every (re)launch path — Start, Resume, Restart, +// RestartWithBootstrap — runs before launchInTmux (#3962: previously only +// Start minted, so an agent relaunched via Resume/Restart never got a token +// until the whole process restarted; its cache file stayed 0 bytes and gh/git +// push failed silently). +// +// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub +// API call (through the MITM egress proxy, which the production incident +// showed can hang), and issueAgentMintToken can do the same. Holding m.mu +// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the +// exact liveness flap Start's phase split was built to avoid. The function +// reads only immutable agent identity (Name, UID) and Config via agentMode, +// and writes no m.mu-guarded AgentProcess field, so running it lock-free is +// race-free. m.appAuth is read without the lock exactly as Start's unlocked +// phase always did: it is injected once at wiring time, before agents start. +func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { + if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { + return + } + tier := m.agentTokenTier(agent) + if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { + // Static misconfiguration: every tick would fail identically, so log + // once per agent until a mint succeeds instead of every supervisor tick. + if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { + m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", + "agent", agent.Name, "tier", tier, "error", err) + } + m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") + } else if err != nil && isForgeTransportErr(err) { + // The GitHub host did not answer (dial timeout, connection refused, + // deadline exceeded). Nothing about the App key or installation can + // be concluded from that, so report it as a forge-reachability + // condition rather than a mint failure — the latter tells the + // operator to re-check App permissions, which is wrong during an + // egress outage. The mint-failed condition is cleared so a hive that + // slid from a real mint error into an outage does not hold both open. + m.logger.Warn("per-agent scoped token NOT delivered — GitHub host unreachable from the pod; this agent's GitHub writes (gh + git push) will FAIL until egress is restored", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.observeCondition(crux.ClassForgeUnreachable, agent.Name, "github host unreachable during token mint: "+err.Error(), "") + } else if err != nil { + // Be precise about the blast radius. Since audit H3 the shared-cache + // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no + // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the + // FULL installation token, now owner-only 0600). They FAIL LOUD when + // the per-agent scoped token is absent rather than silently + // escalating this agent to full privilege. So a delivery failure here + // means this agent's GitHub writes (gh + git push) will fail until + // token delivery is repaired — not a silent privilege escalation. + m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") + } else { + m.appNotInstalledWarned.Delete(agent.Name) + m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + } + // Additionally issue an opt-in short-lived mint token (no-op when the mint + // is disabled). This is additive and fail-safe — it never blocks launch. + if agent.UID > 0 { + m.issueAgentMintToken(agent.Name, tier, agent.UID) + } +} + +// isForgeTransportErr reports whether a mint error never got an answer from +// the GitHub host: a deadline, a *url.Error (dial/TLS/timeout from the HTTP +// client) or any net.Error. An HTTP status from GitHub (401/404/422) is not +// a transport error and keeps its mint-failed classification. Mirrors the +// dashboard's forge probe (#17427) so both report the same outage. +func isForgeTransportErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var urlErr *url.Error + if errors.As(err, &urlErr) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) +} diff --git a/src/crux/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go index 44cc4933dc..93affc50fc 100644 --- a/src/crux/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -2,7 +2,6 @@ package agent import ( "context" - "errors" "fmt" "os" "strings" @@ -903,60 +902,6 @@ func (m *Manager) refreshAgentTokens(ctx context.Context) { func truncateStr(s string, maxRunes int) string { return textutil.Head(s, maxRunes) } -// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token -// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is -// the single mint step every (re)launch path — Start, Resume, Restart, -// RestartWithBootstrap — runs before launchInTmux (#3962: previously only -// Start minted, so an agent relaunched via Resume/Restart never got a token -// until the whole process restarted; its cache file stayed 0 bytes and gh/git -// push failed silently). -// -// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub -// API call (through the MITM egress proxy, which the production incident -// showed can hang), and issueAgentMintToken can do the same. Holding m.mu -// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the -// exact liveness flap Start's phase split was built to avoid. The function -// reads only immutable agent identity (Name, UID) and Config via agentMode, -// and writes no m.mu-guarded AgentProcess field, so running it lock-free is -// race-free. m.appAuth is read without the lock exactly as Start's unlocked -// phase always did: it is injected once at wiring time, before agents start. -func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { - if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { - return - } - tier := m.agentTokenTier(agent) - if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { - // Static misconfiguration: every tick would fail identically, so log - // once per agent until a mint succeeds instead of every supervisor tick. - if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { - m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", - "agent", agent.Name, "tier", tier, "error", err) - } - m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") - } else if err != nil { - // Be precise about the blast radius. Since audit H3 the shared-cache - // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no - // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the - // FULL installation token, now owner-only 0600). They FAIL LOUD when - // the per-agent scoped token is absent rather than silently - // escalating this agent to full privilege. So a delivery failure here - // means this agent's GitHub writes (gh + git push) will fail until - // token delivery is repaired — not a silent privilege escalation. - m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", - "agent", agent.Name, "tier", tier, "error", err) - m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") - } else { - m.appNotInstalledWarned.Delete(agent.Name) - m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) - m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) - } - // Additionally issue an opt-in short-lived mint token (no-op when the mint - // is disabled). This is additive and fail-safe — it never blocks launch. - if agent.UID > 0 { - m.issueAgentMintToken(agent.Name, tier, agent.UID) - } -} - // agentTokenCacheEnabled keeps the shared-UID read path opt-in and App-only. func (m *Manager) agentTokenCacheEnabled(a *AgentProcess) bool { return a.UID > 0 || (a.UID == 0 && m.project.SharedUIDReadToken && m.appAuth != nil) diff --git a/src/crux/agent/mint_forge_unreachable_test.go b/src/crux/agent/mint_forge_unreachable_test.go new file mode 100644 index 0000000000..69e8cdd055 --- /dev/null +++ b/src/crux/agent/mint_forge_unreachable_test.go @@ -0,0 +1,93 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// failingMinter returns the configured error from WriteAgentToken. +type failingMinter struct{ err error } + +func (f *failingMinter) WriteAgentToken(context.Context, string, string, int) error { return f.err } + +type fakeNetErr struct{} + +func (fakeNetErr) Error() string { return "dial tcp 129.34.104.21:443: connect: connection refused" } +func (fakeNetErr) Timeout() bool { return false } +func (fakeNetErr) Temporary() bool { return false } + +var _ net.Error = fakeNetErr{} + +func mintClassManager(t *testing.T, minter AppTokenMinter) (*Manager, *AgentProcess) { + t.Helper() + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + m := NewManager(map[string]config.AgentConfig{ + "worker": makeAgentConfig("claude", "sonnet"), + }, discardLogger(), ProjectContext{}) + m.mu.Lock() + a := m.agents["worker"] + a.UID = 2101 + m.mu.Unlock() + m.SetAppAuth(minter) + return m, a +} + +// A mint that never reaches the forge is a forge_unreachable condition, not a +// github_app_token_mint_failed one — the App is not implicated by an egress +// outage, and the mint-failed condition is cleared so both are never open. +func TestMintTransportErrorReportsForgeUnreachable(t *testing.T) { + cases := map[string]error{ + "deadline": context.DeadlineExceeded, + "url_error": &url.Error{Op: "Post", URL: "https://github.ibm.com/api/v3/app/installations/45218/access_tokens", Err: context.DeadlineExceeded}, + "net_error": errors.Join(errors.New("creating installation token"), fakeNetErr{}), + } + for name, err := range cases { + t.Run(name, func(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: err}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassForgeUnreachable); len(got) != 1 || got[0].Key != "worker" { + t.Fatalf("forge_unreachable active = %+v, want one for worker", got) + } + if got := rec.observations(crux.ClassGitHubAppTokenMintFailed); len(got) != 0 { + t.Fatalf("github_app_token_mint_failed observed on a transport error: %+v", got) + } + if got := rec.clears(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("mint-failed clears = %d, want 1", len(got)) + } + }) + } +} + +// An answer from the forge that refuses the mint keeps the mint-failed class +// and clears any forge_unreachable left over from an outage. +func TestMintNonTransportErrorKeepsMintFailed(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: errors.New("creating installation token: POST .../access_tokens: 401 Bad credentials")}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("github_app_token_mint_failed active = %+v, want one", got) + } + if got := rec.observations(crux.ClassForgeUnreachable); len(got) != 0 { + t.Fatalf("forge_unreachable observed on an HTTP error: %+v", got) + } + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} + +// A successful mint clears every mint-related condition, including the new one. +func TestMintSuccessClearsForgeUnreachable(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: nil}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} diff --git a/src/crux/conditions.go b/src/crux/conditions.go index 3ee1942289..0175629224 100644 --- a/src/crux/conditions.go +++ b/src/crux/conditions.go @@ -37,9 +37,16 @@ const ( ClassDataDiskPressure = "data_disk_pressure" ClassGitHubAppTokenMintFailed = "github_app_token_mint_failed" ClassGitHubAppNotInstalled = "github_app_not_installed" - ClassAgentStall = "agent_stall" - ClassAgentCrashLoop = "agent_crash_loop" - ClassCIPollNudge = "ci_poll_nudge" + // ClassForgeUnreachable: a per-agent token mint failed at the transport + // layer (dial timeout, connection refused, context deadline) — the GitHub + // host cannot be reached from the pod. The App key and installation are + // not implicated, so this is kept apart from + // ClassGitHubAppTokenMintFailed, which would otherwise send the operator + // off to re-check App permissions during a network outage. + ClassForgeUnreachable = "forge_unreachable" + ClassAgentStall = "agent_stall" + ClassAgentCrashLoop = "agent_crash_loop" + ClassCIPollNudge = "ci_poll_nudge" // ClassRuntimeUnavailable is raised by the HOST, not the runtime: a // sidecar host whose runtime API stopped answering (hivecommons/hive#14138 // stage 4). The runtime cannot report its own absence. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 8a3f887968..88fcc6b764 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -148,6 +148,7 @@ title is `[selfreport] <class>: <short message>`. | `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | +| `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 4d1ea1a596..25678d8042 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -98,6 +98,7 @@ var selfreportOpsRemedies = map[string]string{ "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", "data_disk_pressure": "free or grow the data disk on the spoke", "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 6e17ebf4ca..b73565886e 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -168,7 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassGitHubAppAlertsPermission, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassForgeUnreachable, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 5e56884fd9..62362f055e 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -39,6 +39,7 @@ const ( ClassDataDiskPressure = crux.ClassDataDiskPressure ClassGitHubAppTokenMintFailed = crux.ClassGitHubAppTokenMintFailed ClassGitHubAppNotInstalled = crux.ClassGitHubAppNotInstalled + ClassForgeUnreachable = crux.ClassForgeUnreachable ClassUnexpectedHub5xx = "unexpected_http_5xx_from_hub" ClassUnclassifiedRepeated = "unclassified_error_repeated" // ClassAgentStall, ClassAgentCrashLoop and ClassCIPollNudge replace the @@ -183,6 +184,7 @@ var Classes = map[string]ClassSpec{ ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, + ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index ec5791dc89..8ff311c552 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -433,6 +433,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassUpgradeStalled, message: "standalone upgrade helper did not finish within 10m0s"}, {class: ClassDataDiskPressure, message: "data disk pressure: disk-full: /data is 97% used (threshold 95%)"}, {class: ClassGitHubAppTokenMintFailed, message: "github app token mint failed: POST /app/installations/123/access_tokens: 401"}, + {class: ClassForgeUnreachable, message: "github host unreachable during token mint: creating installation token: context deadline exceeded"}, {class: ClassUnexpectedHub5xx, message: "hub heartbeat returned HTTP 503"}, {class: ClassUnclassifiedRepeated, message: "weird error from somewhere"}, {class: ClassAgentStall, message: "agent stalled after kick: 3 continue nudges sent without the pane moving"}, From 06cfb1ac8b4ff485ed81ce9bbec9bc857c1f6aad Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 18:25:42 +0000 Subject: [PATCH 102/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-dependabot-permission-selfreport.md | 1 - changelog.d/fixed-signed-reconcile-unsupported.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-dependabot-permission-selfreport.md delete mode 100644 changelog.d/fixed-signed-reconcile-unsupported.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 73d850f252..8d46ba2981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.2) + +### Fixed + +- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. +- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. + ## 2026-11-12 (v5.236.1) ### Changed diff --git a/changelog.d/fixed-dependabot-permission-selfreport.md b/changelog.d/fixed-dependabot-permission-selfreport.md deleted file mode 100644 index 2438213269..0000000000 --- a/changelog.d/fixed-dependabot-permission-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. diff --git a/changelog.d/fixed-signed-reconcile-unsupported.md b/changelog.d/fixed-signed-reconcile-unsupported.md deleted file mode 100644 index ad1001d019..0000000000 --- a/changelog.d/fixed-signed-reconcile-unsupported.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. From d0e1f62675995b80467677faf7485773fffe62d1 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 18:57:28 +0000 Subject: [PATCH 103/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.3 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-canceled-adoption-report.md | 1 - changelog.d/fixed-forge-unreachable-mint-class.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-canceled-adoption-report.md delete mode 100644 changelog.d/fixed-forge-unreachable-mint-class.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d46ba2981..d45b2467b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.3) + +### Fixed + +- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. +- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. + ## 2026-11-12 (v5.236.2) ### Fixed diff --git a/changelog.d/fixed-canceled-adoption-report.md b/changelog.d/fixed-canceled-adoption-report.md deleted file mode 100644 index 803255f554..0000000000 --- a/changelog.d/fixed-canceled-adoption-report.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. diff --git a/changelog.d/fixed-forge-unreachable-mint-class.md b/changelog.d/fixed-forge-unreachable-mint-class.md deleted file mode 100644 index 65050c8d68..0000000000 --- a/changelog.d/fixed-forge-unreachable-mint-class.md +++ /dev/null @@ -1 +0,0 @@ -- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. From 49c5e1e478f3e3dc4c95e52eed6876eefc91c71a Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:13:05 +0000 Subject: [PATCH 104/166] =?UTF-8?q?=F0=9F=90=9B=20fix(crux):=20prepare=20p?= =?UTF-8?q?luk=20session=20log=20without=20following=20planted=20symlinks?= =?UTF-8?q?=20(#17967)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensurePlukLogFile ran os.Chmod (and, as root, os.Chown) on a path inside /var/run/pluk/logs, a 0770+setgid directory every agent uid can write to. A symlink planted there by an agent at its own predictable session-log name was followed on the next relaunch, widening the link's target to the shared group. Open with O_NOFOLLOW|O_NONBLOCK, require a regular file on the descriptor, and apply owner and mode through the descriptor; a symlink or FIFO found at the path is replaced like an unopenable stale log already was. Fixes #17966 Signed-off-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: sec-check <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../security-17966-pluk-log-nofollow.md | 1 + src/crux/agent/pluk_dirs.go | 64 +++++++++++--- src/crux/agent/pluk_dirs_branches_test.go | 88 +++++++++++++++++++ 3 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 changelog.d/security-17966-pluk-log-nofollow.md diff --git a/changelog.d/security-17966-pluk-log-nofollow.md b/changelog.d/security-17966-pluk-log-nofollow.md new file mode 100644 index 0000000000..338e015b68 --- /dev/null +++ b/changelog.d/security-17966-pluk-log-nofollow.md @@ -0,0 +1 @@ +- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). diff --git a/src/crux/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go index a8bd56c0c2..6273fb36ba 100644 --- a/src/crux/agent/pluk_dirs.go +++ b/src/crux/agent/pluk_dirs.go @@ -6,6 +6,9 @@ import ( "io/fs" "os" "path/filepath" + "syscall" + + "github.com/hivecommons/hive/crux/sharedfile" ) // plukRunDir is where pluk's run state and per-session logs live. @@ -45,42 +48,79 @@ func plukSessionLogPath(runDir, session string) string { // ensurePlukLogFile creates (or leaves alone) the session's JSONL log with a // mode peer agents can read, and returns its path. Both the O_CREATE mode and a -// following Chmod are needed: O_CREATE is umask-filtered, and an existing file +// following chmod are needed: O_CREATE is umask-filtered, and an existing file // from an earlier run under a tighter umask has to be widened back to the shared // mode rather than inherited as-is. +// +// The logs dir is group-writable by every agent uid (0770+setgid, no sticky +// bit), so anything found at path may have been planted by an agent: the open +// never follows a symlink, the descriptor is checked to be a regular file, +// and the owner and mode are applied through that descriptor, never by path. +// A path-based chmod/chown here would let one agent widen any dev-owned file +// to the shared group by linking its own session log at it (#17966). func ensurePlukLogFile(runDir, session string) (string, error) { path := plukSessionLogPath(runDir, session) - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) - if errors.Is(err, fs.ErrPermission) { + f, err := openPlukLogNoFollow(path) + if errors.Is(err, fs.ErrPermission) || errors.Is(err, sharedfile.ErrNotRegular) { // A log left behind by another uid (a pane shell's `>>` under a tight // umask, or an earlier run as a different user) cannot be opened or - // chmodded by us, but the shared setgid logs dir is group-writable, so - // the stale file can be replaced. Replacing beats leaving the session's - // log unreadable to peers. + // chmodded by us, and a symlink or other non-regular entry at the path + // is not a log at all. The shared setgid logs dir is group-writable, so + // the entry can be replaced. Replacing beats leaving the session's log + // unreadable to peers, or following a planted link. if rmErr := os.Remove(path); rmErr == nil { - f, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + f, err = openPlukLogNoFollow(path) } } if err != nil { return "", fmt.Errorf("creating %s: %w", path, err) } - if err := f.Close(); err != nil { - return "", fmt.Errorf("closing %s: %w", path, err) - } + defer f.Close() if os.Geteuid() == 0 { - if err := os.Chown(path, DevUID, NodeGID); err != nil { + if err := f.Chown(DevUID, NodeGID); err != nil { return "", fmt.Errorf("chowning %s to %d:%d: %w", path, DevUID, NodeGID, err) } } - if err := os.Chmod(path, plukLogFileMode); err != nil { + if err := f.Chmod(plukLogFileMode); err != nil { return "", fmt.Errorf("chmoding %s to %s: %w", path, plukLogFileMode, err) } + if err := f.Close(); err != nil { + return "", fmt.Errorf("closing %s: %w", path, err) + } return path, nil } +// openPlukLogNoFollow opens (creating if absent) the regular file at path for +// append without following a symlink there. A symlink is reported as +// sharedfile.ErrNotRegular (the kernel answers ELOOP), as is any other +// non-regular file found behind the descriptor. O_NONBLOCK keeps a planted +// FIFO from parking the launch path on open (it fails with ENXIO instead); +// the kernel ignores it for a regular file. +func openPlukLogNoFollow(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, plukLogFileMode) + if err != nil { + // ELOOP: a symlink. ENXIO: a FIFO with no reader (O_NONBLOCK). Neither + // is a log; both are reported as not-regular so the caller replaces them. + if errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.ENXIO) { + return nil, fmt.Errorf("%w: %s (%v)", sharedfile.ErrNotRegular, path, err) + } + return nil, err + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + if !info.Mode().IsRegular() { + f.Close() + return nil, fmt.Errorf("%w: %s is %s", sharedfile.ErrNotRegular, path, info.Mode().Type()) + } + return f, nil +} + func ensurePlukRunDirs(runDir string) error { for _, name := range []string{"logs", "commands"} { dir := filepath.Join(runDir, name) diff --git a/src/crux/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go index 5cac61004b..0d0affa096 100644 --- a/src/crux/agent/pluk_dirs_branches_test.go +++ b/src/crux/agent/pluk_dirs_branches_test.go @@ -6,7 +6,9 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" + "time" ) // pluk_dirs_branches_test.go covers the pluk_dirs.go branches the setgid-mode @@ -192,3 +194,89 @@ func TestEnsurePlukRunDirsRewidensTightenedDirs(t *testing.T) { } } } + +// TestEnsurePlukLogFileDoesNotFollowPlantedSymlink pins #17966: the logs dir is +// group-writable by every agent uid, so a symlink at the session log path may +// have been planted by an agent. The chmod (and, as root, the chown) must never +// reach the link's target; the link is replaced by a regular log instead. +func TestEnsurePlukLogFileDoesNotFollowPlantedSymlink(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + secret := filepath.Join(t.TempDir(), "proxy-ca-key.pem") + if err := os.WriteFile(secret, []byte("private"), 0o600); err != nil { + t.Fatalf("seed secret: %v", err) + } + path := plukSessionLogPath(runDir, "hive-attacker") + if err := os.Symlink(secret, path); err != nil { + t.Fatalf("plant symlink: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-attacker") + if err != nil { + t.Fatalf("ensure pluk log file over a planted symlink: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + + info, err := os.Stat(secret) + if err != nil { + t.Fatalf("stat secret: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("symlink target mode = %04o, want 0600 untouched (chmod followed the planted link)", perm) + } + data, err := os.ReadFile(secret) + if err != nil || string(data) != "private" { + t.Fatalf("symlink target changed: %q, %v", data, err) + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the planted link", li.Mode().Type()) + } + if perm := li.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + +// TestEnsurePlukLogFileReplacesPlantedFIFO covers the other non-regular entry +// an agent can create in the shared dir: a FIFO with no reader must neither +// block the launch path on open nor be kept; it is replaced like a symlink. +func TestEnsurePlukLogFileReplacesPlantedFIFO(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + path := plukSessionLogPath(runDir, "hive-fifo") + if err := syscall.Mkfifo(path, 0o660); err != nil { + t.Fatalf("plant fifo: %v", err) + } + + done := make(chan error, 1) + go func() { + _, err := ensurePlukLogFile(runDir, "hive-fifo") + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatalf("ensure pluk log file over a planted fifo: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("ensure pluk log file blocked on a planted fifo") + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the fifo", li.Mode().Type()) + } +} From 4efd08ed575786d1b5b99b9a4f1699cb5370397b Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 19:43:43 +0000 Subject: [PATCH 105/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.4 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/security-17966-pluk-log-nofollow.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/security-17966-pluk-log-nofollow.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d45b2467b5..f7f0019c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.4) + +### Security + +- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). + ## 2026-11-12 (v5.236.3) ### Fixed diff --git a/changelog.d/security-17966-pluk-log-nofollow.md b/changelog.d/security-17966-pluk-log-nofollow.md deleted file mode 100644 index 338e015b68..0000000000 --- a/changelog.d/security-17966-pluk-log-nofollow.md +++ /dev/null @@ -1 +0,0 @@ -- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). From 6a7514bd3d64c2d746841f6574873af90a655018 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:47:15 +0100 Subject: [PATCH 106/166] =?UTF-8?q?=F0=9F=90=9B=20fix(fleetreport):=20reco?= =?UTF-8?q?gnize=20structured=20context=20cancellation=20(#17937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(fleetreport): recognize structured context cancellation Apply the existing cancellation exclusion to structured error attributes while keeping local logs and other errors visible. Hive-Run: hivecommons/hive#17935 Hive-Plan: structured-cancellation-selfreport Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * fix(fleetreport): recognize wrapped cancellations Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ixed-structured-cancellation-selfreport.md | 1 + src/docs/fleet-report.md | 6 ++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 35 +++++++++++-------- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 28 ++++++++++++++- 4 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-structured-cancellation-selfreport.md diff --git a/changelog.d/fixed-structured-cancellation-selfreport.md b/changelog.d/fixed-structured-cancellation-selfreport.md new file mode 100644 index 0000000000..5d9018befc --- /dev/null +++ b/changelog.d/fixed-structured-cancellation-selfreport.md @@ -0,0 +1 @@ +- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 88fcc6b764..aab1ac0c3f 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -171,6 +171,12 @@ title is `[selfreport] <class>: <short message>`. | `models_endpoint_empty` | the dashboard's inference model discovery (`/api/inference/{backend}/models`) found no models from any configured endpoint (`no models found from any endpoint`, `pkg/dashboard/api_models.go`) and fell back to the static aliases, flagged unverified in the UI. The source logs it at WARN once per backend per hour (DEBUG on repeats, re-armed when discovery succeeds), so this fires only when the condition flaps ≥5 times in 10 min (gateway key rejected, endpoint down). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | +The cancellation exclusion includes structured `error` attributes, including +logger-bound attributes: typed errors wrapping `context.Canceled`, exact +`context canceled` strings, and strings ending in `: context canceled`. +Warnings remain in local logs; deadlines and other errors remain reportable, +including when another `error` attribute accompanies a cancellation. + The GitHub rate-limit exclusion also covers the local REST reserve guard: when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 1550bafe9b..e35e8781fd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledAdoptionRead(r) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -136,22 +136,29 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } -// A canceled sweep leaves the existing adoption intact and retries later. -// Other failures reading push access still warrant a repeated-error report. -func canceledAdoptionRead(r slog.Record) bool { - if strings.TrimSpace(r.Message) != "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" { - return false - } - canceled := false - r.Attrs(func(a slog.Attr) bool { +// canceledError applies the existing cancellation exclusion to structured +// errors too. An accompanying different error must remain reportable. +func canceledError(r slog.Record, bound []slog.Attr) bool { + found, other := false, false + check := func(a slog.Attr) bool { if a.Key == "error" { - err, ok := a.Value.Any().(error) - canceled = (ok && errors.Is(err, context.Canceled)) || a.Value.String() == context.Canceled.Error() - return !canceled + value := a.Value.Resolve() + err, ok := value.Any().(error) + if ok && errors.Is(err, context.Canceled) { + found = true + } else if text := value.String(); text == context.Canceled.Error() || strings.HasSuffix(text, ": "+context.Canceled.Error()) { + found = true + } else { + other = true + } } return true - }) - return canceled + } + for _, a := range bound { + check(a) + } + r.Attrs(check) + return found && !other } // expectedKickHold reports whether a "failed to send kick" record carries the diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index ce7c2d09ab..838050edbd 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "log/slog" "net/url" "strings" @@ -77,7 +78,32 @@ func TestCanceledAdoptionReadStaysInLog(t *testing.T) { logger.Warn(msg, "error", errors.New("permission denied")) logger.Warn(msg, "error", context.DeadlineExceeded) logger.Warn("another operation failed", "error", context.Canceled) - if len(*seen) != 3 { + if len(*seen) != 2 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + +func TestStructuredCancellationDoesNotCreateFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels reconcile failed" + for range 10 { + logger.Warn(msg, "error", context.Canceled, "items", 6, "writes", 0) + } + logger.With("error", context.Canceled).Warn(msg) + logger.Warn("stage labels: check digest failed", "error", fmt.Errorf("GET checks: %w", context.Canceled)) + logger.Warn("stage labels: check digest failed", "error", "GET checks: context canceled") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", context.Canceled) + logger.Warn(msg, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 15 { + t.Fatalf("observed=%+v logs=%s", *seen, out.String()) + } + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn(msg, "error", "permission denied") + logger.With("error", context.Canceled).Warn(msg, "error", "permission denied") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", "permission denied") + logger.Warn(msg, "error", "context canceled: permission denied") + logger.Warn(msg, "error", fmt.Errorf("request: %w", context.DeadlineExceeded)) + if len(*seen) != 6 { t.Fatalf("other failures must remain reportable: %+v", *seen) } } From bd2ee3a4549c1fee51d8c49cddc6c36d95d6431c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:56:27 +0000 Subject: [PATCH 107/166] =?UTF-8?q?=E2=9C=A8=20crux=20proxy:=20add=20OTel?= =?UTF-8?q?=20metrics=20and=20span=20to=20inference=20translator=20(#17974?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inference hop was only observable via slog. Follow the serve/metrics.go pattern with bounded attributes (status_class, outcome enum) so nothing exports unless OTEL_* is configured. Refs hivecommons/crux#153 (re-homed from hivecommons/crux#155: crux Go code is synced from src/crux here) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-crux-proxy-inference-otel.md | 1 + src/crux/proxy/github_proxy_inference.go | 4 +- src/crux/proxy/metrics.go | 164 ++++++++++++++++++ src/crux/proxy/metrics_test.go | 89 ++++++++++ 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added-crux-proxy-inference-otel.md create mode 100644 src/crux/proxy/metrics.go create mode 100644 src/crux/proxy/metrics_test.go diff --git a/changelog.d/added-crux-proxy-inference-otel.md b/changelog.d/added-crux-proxy-inference-otel.md new file mode 100644 index 0000000000..2ed23a192b --- /dev/null +++ b/changelog.d/added-crux-proxy-inference-otel.md @@ -0,0 +1 @@ +- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 4d1e6b4d35..154abf41f1 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -397,6 +397,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { if !ok { p.logger.Warn("inference request from unidentified caller refused", p.translatorRefusalAttrs(r)...) + setTranslatorOutcome(r.Context(), outcomeUnidentifiedCaller) http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) return } @@ -412,6 +413,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { route := p.inference.Get(agentName) if route == nil { p.logger.Warn("inference no route", "agent", agentName, "reason", p.inference.MissReason(agentName)) + setTranslatorOutcome(r.Context(), outcomeNoRoute) http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) return } @@ -585,7 +587,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { _, _ = w.Write(translated) }) - return mux + return instrumentTranslator(mux) } // StartInferenceTranslator runs the Anthropic-compatible inference translator diff --git a/src/crux/proxy/metrics.go b/src/crux/proxy/metrics.go new file mode 100644 index 0000000000..bc7c4e0311 --- /dev/null +++ b/src/crux/proxy/metrics.go @@ -0,0 +1,164 @@ +package proxy + +import ( + "context" + "net/http" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +// Inference translator instruments (hivecommons/crux#153). Like serve's they +// sit on the global providers, which are OTel's delegating no-ops until a host +// registers an SDK provider, so recording costs nothing by default. +// +// Attributes are bounded to the status class and a fixed outcome enum. Never an +// agent name, path, model name, token or body. +const ( + instrumentationName = "github.com/hivecommons/hive/crux/proxy" + + translatorDurationMetric = "crux.proxy.inference.duration" + translatorCountMetric = "crux.proxy.inference.requests" +) + +// Outcome of one translator request: a fixed enum, so the attribute stays bounded. +const ( + outcomeOK = "ok" + outcomeUnidentifiedCaller = "unidentified_caller" + outcomeNoRoute = "no_route" + outcomeUpstreamError = "upstream_error" +) + +var ( + attrStatusClass = attribute.Key("status_class") + attrOutcome = attribute.Key("outcome") +) + +type translatorMetrics struct { + duration metric.Float64Histogram + requests metric.Int64Counter +} + +func newTranslatorMetrics(meter metric.Meter) translatorMetrics { + // Instrument constructors only fail on an invalid name or unit; both are + // constants here, and the returned instrument is a usable no-op either way. + duration, _ := meter.Float64Histogram(translatorDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time the inference translator took to answer a request.")) + requests, _ := meter.Int64Counter(translatorCountMetric, + metric.WithDescription("Inference translator requests, by status class and outcome.")) + return translatorMetrics{duration: duration, requests: requests} +} + +var globalTranslatorMetrics = sync.OnceValue(func() translatorMetrics { + return newTranslatorMetrics(otel.Meter(instrumentationName)) +}) + +func (m translatorMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { + opt := metric.WithAttributes(attrs...) + m.duration.Record(ctx, d.Seconds(), opt) + m.requests.Add(ctx, 1, opt) +} + +type outcomeKey struct{} + +// outcomeHolder lets the handler name an outcome the status alone cannot tell. +type outcomeHolder struct{ outcome string } + +func setTranslatorOutcome(ctx context.Context, outcome string) { + if h, ok := ctx.Value(outcomeKey{}).(*outcomeHolder); ok { + h.outcome = outcome + } +} + +type translatorStatusRecorder struct { + http.ResponseWriter + status int +} + +func (w *translatorStatusRecorder) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *translatorStatusRecorder) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.ResponseWriter.Write(b) +} + +func (w *translatorStatusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +// translatorFlushRecorder offers Flush only when the wrapped writer does; the +// SSE path type-asserts http.Flusher. +type translatorFlushRecorder struct { + *translatorStatusRecorder + http.Flusher +} + +func translatorStatusClass(status int) string { + switch { + case status >= 500: + return "5xx" + case status >= 400: + return "4xx" + case status >= 300: + return "3xx" + case status >= 200: + return "2xx" + default: + return "1xx" + } +} + +// instrumentTranslator records one request's server span, duration and count +// around next. +func instrumentTranslator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + holder := &outcomeHolder{} + ctx, span := otel.Tracer(instrumentationName).Start( + context.WithValue(r.Context(), outcomeKey{}, holder), + "inference_translator", trace.WithSpanKind(trace.SpanKindServer)) + rec := &translatorStatusRecorder{ResponseWriter: w} + var out http.ResponseWriter = rec + if f, ok := w.(http.Flusher); ok { + out = translatorFlushRecorder{rec, f} + } + returned := false + defer func() { + status := rec.status + if status == 0 { + status = http.StatusOK + } + if !returned { + status = http.StatusInternalServerError + span.SetStatus(codes.Error, "handler panic") + } + outcome := holder.outcome + if outcome == "" { + outcome = outcomeOK + if status >= 400 { + outcome = outcomeUpstreamError + } + } + attrs := []attribute.KeyValue{ + attrStatusClass.String(translatorStatusClass(status)), + attrOutcome.String(outcome), + } + span.SetAttributes(attrs...) + span.End() + globalTranslatorMetrics().record(ctx, time.Since(start), attrs) + }() + next.ServeHTTP(out, r.WithContext(ctx)) + returned = true + }) +} diff --git a/src/crux/proxy/metrics_test.go b/src/crux/proxy/metrics_test.go new file mode 100644 index 0000000000..0fcb9b8df4 --- /dev/null +++ b/src/crux/proxy/metrics_test.go @@ -0,0 +1,89 @@ +package proxy + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/hivecommons/hive/crux/agent" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestTranslatorMetricsBoundedAttributes(t *testing.T) { + fakeProcNetTCP(t) + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + m := newTranslatorMetrics(mp.Meter(instrumentationName)) + prev := globalTranslatorMetrics + globalTranslatorMetrics = func() translatorMetrics { return m } + t.Cleanup(func() { + globalTranslatorMetrics = prev + _ = mp.Shutdown(context.Background()) + }) + + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"quality": 2002}} + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "quality")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", rec.Code) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + var counts, durations int + for _, sm := range rm.ScopeMetrics { + for _, mt := range sm.Metrics { + got := map[string]string{} + collect := func(len int, kv func(string) (string, bool)) { + if len != 2 { + t.Errorf("%s: %d attributes, want exactly status_class and outcome", mt.Name, len) + } + for _, k := range []string{"status_class", "outcome", "agent", "path", "model"} { + if v, ok := kv(k); ok { + got[k] = v + } + } + } + switch data := mt.Data.(type) { + case metricdata.Sum[int64]: + counts++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + case metricdata.Histogram[float64]: + durations++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + } + if got["status_class"] != "4xx" || got["outcome"] != outcomeUnidentifiedCaller { + t.Errorf("%s attributes = %v, want 4xx/unidentified_caller", mt.Name, got) + } + for _, bad := range []string{"agent", "path", "model"} { + if _, ok := got[bad]; ok { + t.Errorf("%s carries forbidden attribute %q", mt.Name, bad) + } + } + } + } + if counts != 1 || durations != 1 { + t.Fatalf("counters = %d, histograms = %d, want 1 each", counts, durations) + } +} + +func attrKey(k string) attribute.Key { return attribute.Key(k) } From 66482ec3d077dcf194cd72a4d6f472eb1f682d2c Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:17:05 +0000 Subject: [PATCH 108/166] =?UTF-8?q?=F0=9F=90=9B=20fix(crux/proxy):=20make?= =?UTF-8?q?=20CONNECT-refusal=20test=20hermetic=20(#17981)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestDialUpstreamTLSSurfacesProxyRefusal relied on the test CONNECT proxy failing to resolve ghe.corp.invalid within 3s; on CI the resolver hung and the client timed out instead of seeing the 502. The test proxy now refuses on demand so the test does not depend on DNS. Closes #17979 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/proxy/upstream_chain_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crux/proxy/upstream_chain_test.go b/src/crux/proxy/upstream_chain_test.go index 4c902d3483..9c68b4b5bf 100644 --- a/src/crux/proxy/upstream_chain_test.go +++ b/src/crux/proxy/upstream_chain_test.go @@ -23,6 +23,7 @@ type testConnectProxy struct { ln net.Listener targets atomic.Int32 last atomic.Value + refuse atomic.Bool } func startTestConnectProxy(t *testing.T) *testConnectProxy { @@ -55,6 +56,10 @@ func (cp *testConnectProxy) serve(c net.Conn) { } cp.targets.Add(1) cp.last.Store(req.Host) + if cp.refuse.Load() { + _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } up, err := net.DialTimeout("tcp", req.Host, 5*time.Second) if err != nil { _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") @@ -240,7 +245,8 @@ func TestDialUpstreamRawChainsAndGuardsPrivateLiterals(t *testing.T) { func TestDialUpstreamTLSSurfacesProxyRefusal(t *testing.T) { cp := startTestConnectProxy(t) p := newChainedTestProxy(t, cp, []string{"ghe.corp.invalid"}) - // The test proxy dials the target itself; an unresolvable name yields 502. + // The test proxy is told to refuse so the test does not depend on DNS. + cp.refuse.Store(true) _, err := p.dialUpstreamTLS("ghe.corp.invalid:443", "ghe.corp.invalid", 3*time.Second) if err == nil || !strings.Contains(err.Error(), "refused") { t.Fatalf("err = %v, want CONNECT refusal", err) From d57b78568d614c24e3f0b8d720876e541c67c856 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 20:21:57 +0000 Subject: [PATCH 109/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.236.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.236.5 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-structured-cancellation-selfreport.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-structured-cancellation-selfreport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f0019c95..5d9d84ce6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.236.5) + +### Fixed + +- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). + ## 2026-11-12 (v5.236.4) ### Security diff --git a/changelog.d/fixed-structured-cancellation-selfreport.md b/changelog.d/fixed-structured-cancellation-selfreport.md deleted file mode 100644 index 5d9018befc..0000000000 --- a/changelog.d/fixed-structured-cancellation-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). From 64e1073420901601dd1f628741483d8bfdec13eb Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 20:53:12 +0000 Subject: [PATCH 110/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-crux-proxy-inference-otel.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-crux-proxy-inference-otel.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9d84ce6d..b12cfc3707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.0) + +### Added + +- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). + ## 2026-11-12 (v5.236.5) ### Fixed diff --git a/changelog.d/added-crux-proxy-inference-otel.md b/changelog.d/added-crux-proxy-inference-otel.md deleted file mode 100644 index 2ed23a192b..0000000000 --- a/changelog.d/added-crux-proxy-inference-otel.md +++ /dev/null @@ -1 +0,0 @@ -- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). From c2f2687d1b1c65c010f4506f95eea96e1cbe7a5b Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:54:02 +0000 Subject: [PATCH 111/166] =?UTF-8?q?=F0=9F=90=9B=20fix(fleetreport):=20name?= =?UTF-8?q?=20the=20prow/lgtm=20statuses-permission=20403=20as=20its=20own?= =?UTF-8?q?=20selfreport=20class=20(#17980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(fleetreport): name the prow/lgtm statuses-permission 403 as its own selfreport class Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * 📝 fix(fleetreport): document the statuses-permission class and gate it on the 403 Review follow-up for #17980: - fleet-report.md: add the github_app_statuses_permission_missing row beside its sibling github_app_alerts_permission_missing (component, log-fed gate, Commit statuses read & write remedy, github-app-setup.md link). - bindProwLgtm: emit the "lacks the statuses: write permission" wording only when the CreateStatus error is a 403 *gh.ErrorResponse; 5xx, 422 and transport failures log a neutral line, so a transient outage is no longer filed as an installation-permission fault with a grant-permission remedy. - fleetreport keyword now matches the 403-only line; tests pin both the positive match and that the neutral line stays out of the class. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs: re-anchor RankActionableIssues citation after client.go shift Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ed-selfreport-statuses-permission-class.md | 1 + src/docs/fleet-report.md | 1 + src/docs/scale-envelope.md | 2 +- src/pkg/delegatedecision/issue_judgment.go | 35 ++-- .../delegatedecision/issue_judgment_test.go | 2 +- src/pkg/fleetreport/runtime.go | 166 +++++++++--------- src/pkg/fleetreport/runtime_test.go | 111 ++++++------ src/pkg/github/automerge_queue_prow_test.go | 38 ++++ src/pkg/github/client.go | 27 ++- 9 files changed, 228 insertions(+), 155 deletions(-) create mode 100644 changelog.d/fixed-selfreport-statuses-permission-class.md diff --git a/changelog.d/fixed-selfreport-statuses-permission-class.md b/changelog.d/fixed-selfreport-statuses-permission-class.md new file mode 100644 index 0000000000..84c964db2a --- /dev/null +++ b/changelog.d/fixed-selfreport-statuses-permission-class.md @@ -0,0 +1 @@ +- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index aab1ac0c3f..bd63e9662b 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -146,6 +146,7 @@ title is `[selfreport] <class>: <short message>`. | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | | `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | +| `github_app_statuses_permission_missing` | GitHub answered the queue action's `prow/lgtm` commit-status write with 403 (`could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission`, `pkg/github/client.go`): the App installation lacks Commit statuses read & write, so Prow may strip the `lgtm` label and the auto-merge queue may not hold. The App owner must grant Commit statuses read & write and re-accept it on the installation. Only the 403 carries this wording — any other status failure (5xx, 422, transport) logs the neutral `could not set the prow/lgtm commit status` and is not this class. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index ea6cfeadf6..af8d9a3c73 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5263-5273`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5286-5296`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index 25678d8042..16718210f7 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -94,23 +94,24 @@ func ParseSelfReport(title, body string) (SelfReport, bool) { // constants (pinned by TestSelfreportOpsClassesMatchFleetReport) without // importing that package's dependency tree into this leaf. var selfreportOpsRemedies = map[string]string{ - "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", - "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", - "data_disk_pressure": "free or grow the data disk on the spoke", - "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", - "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", - "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", - "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", - "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", - "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", - "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", - "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", - "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", - "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", - "inference_upstream_error": "check the inference endpoint the spoke's provider points at", - "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", - "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", - "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", + "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", + "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", + "data_disk_pressure": "free or grow the data disk on the spoke", + "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", + "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", + "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", + "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", + "github_app_statuses_permission_missing": "grant Commit statuses read & write in the GitHub App settings and re-accept the permission on its installation", + "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", + "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", + "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", + "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", + "inference_upstream_error": "check the inference endpoint the spoke's provider points at", + "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", + "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", + "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", } // selfreportOpsClass reports whether a self-report class is an operator's to diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index b73565886e..603bafd44f 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -168,7 +168,7 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassForgeUnreachable, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassGitHubAppStatusesPermission, fleetreport.ClassForgeUnreachable, fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 62362f055e..90e653690b 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -69,37 +69,38 @@ const ( // Log-fed classes: faults that have no dedicated hook site and are recognised // from their WARN+ log line by the slog feed (see LogFedClass). const ( - ClassTmuxSessionCheckFailed = "tmux_session_check_failed" - ClassAgentPaneTimeout = "agent_pane_timeout" - ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" - ClassAgentDirSetupFailed = "agent_dir_setup_failed" - ClassReviewEvidenceWriteFail = "review_evidence_write_failed" - ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" - ClassBeadFactIngestFailed = "bead_fact_ingest_failed" - ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" - ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" - ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" - ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" - ClassProxyConnectDialFailed = "proxy_connect_dial_failed" - ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" - ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" - ClassInferenceUpstreamError = "inference_upstream_error" - ClassInferenceCallerUnknown = "inference_caller_unidentified" - ClassInferenceNoRoute = "inference_no_route" - ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" - ClassKickFreshnessWithheld = "kick_freshness_withheld" - ClassModelsEndpointEmpty = "models_endpoint_empty" - ClassAgentLoginRequired = "agent_login_required" - ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" - ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" - ClassGitHubFetchTransient = "github_fetch_transient" - ClassLabelProvisioningFailed = "label_provisioning_failed" - ClassAgentCLIRestartLoop = "agent_cli_restart_loop" - ClassCopilotCLIHung = "copilot_cli_hung" - ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" - ClassTranscriptScanSlow = "transcript_scan_slow" - ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" - ClassIssueClaimEffectFailed = "issue_claim_effect_failed" + ClassTmuxSessionCheckFailed = "tmux_session_check_failed" + ClassAgentPaneTimeout = "agent_pane_timeout" + ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" + ClassAgentDirSetupFailed = "agent_dir_setup_failed" + ClassReviewEvidenceWriteFail = "review_evidence_write_failed" + ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" + ClassBeadFactIngestFailed = "bead_fact_ingest_failed" + ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" + ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" + ClassGitHubAppStatusesPermission = "github_app_statuses_permission_missing" + ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" + ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" + ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" + ClassInferenceUpstreamError = "inference_upstream_error" + ClassInferenceCallerUnknown = "inference_caller_unidentified" + ClassInferenceNoRoute = "inference_no_route" + ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" + ClassKickFreshnessWithheld = "kick_freshness_withheld" + ClassModelsEndpointEmpty = "models_endpoint_empty" + ClassAgentLoginRequired = "agent_login_required" + ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" + ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" + ClassGitHubFetchTransient = "github_fetch_transient" + ClassLabelProvisioningFailed = "label_provisioning_failed" + ClassAgentCLIRestartLoop = "agent_cli_restart_loop" + ClassCopilotCLIHung = "copilot_cli_hung" + ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" + ClassTranscriptScanSlow = "transcript_scan_slow" + ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" + ClassIssueClaimEffectFailed = "issue_claim_effect_failed" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -174,57 +175,58 @@ type ClassSpec struct { // Classes is the runtime error allow-list. var Classes = map[string]ClassSpec{ - ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, - ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, - ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, - ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, - ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, - ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, - ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, - ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, - ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, - ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, - ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, - ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, - ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, - ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, - ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, - ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, - ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, - ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, - ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, - ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, - ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, - ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt"}}, - ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, - ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, - ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, - ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, - ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, - ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, - ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, - ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, - ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, - ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, - ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, - ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, - ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, - ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, - ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, - ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, - ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, - ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, - ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, - ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, - ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, - ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, - ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, - ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, - ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, - ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, - ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, - ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, - ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, + ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, + ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, + ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, + ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, + ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, + ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, + ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, + ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, + ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, + ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, + ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, + ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, + ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, + ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, + ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, + ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, + ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, + ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, + ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, + ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, + ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, + ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt"}}, + ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, + ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, + ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, + ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, + ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, + ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, + ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, + ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, + ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, + ClassGitHubAppStatusesPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"could not set the prow/lgtm commit status: the github app lacks the statuses: write permission"}}, + ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, + ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, + ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, + ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, + ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, + ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, + ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, + ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, + ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, + ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, + ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, + ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, + ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, + ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, + ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, + ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, + ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, + ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, + ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, + ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, } // LogFedClass returns the LogFed class whose Keyword msg contains, or "". diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 8ff311c552..7c7a024afe 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -455,6 +455,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, {class: ClassGitHubAppAlertsPermission, message: "scoped token mint with vulnerability_alerts:read failed; retrying without it"}, + {class: ClassGitHubAppStatusesPermission, message: "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission"}, {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, @@ -1200,58 +1201,59 @@ func TestUnclassifiedRepeatedHonoursWindowAndIdleClear(t *testing.T) { func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, - "tmux has-session failed": ClassTmuxSessionCheckFailed, - "failed to send kick": ClassAgentLifecycleRepeated, - "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, - "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, - "agent tmux session missing": ClassAgentLifecycleRepeated, - "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, - "pane content at timeout": ClassAgentPaneTimeout, - "prompt timeout — dumping pane": ClassAgentPaneTimeout, - "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, - "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, - "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, - "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, - "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, - "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, - "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, - "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, - "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, - "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, - "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, - "no models found from any endpoint": ClassModelsEndpointEmpty, - "login required detected": ClassAgentLoginRequired, - "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, - "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, - "watchdog: agent credential failure": ClassAgentLoginRequired, - "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, - "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, - "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, - "failed to restore interactive home mode": ClassAgentDirSetupFailed, - "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, - "failed to tighten interactive home mode": ClassAgentDirSetupFailed, - "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, - "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, - "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, - "failed to restore inference home mode": ClassAgentDirSetupFailed, - "failed to tighten inference home mode": ClassAgentDirSetupFailed, - "failed to ingest bead fact": ClassBeadFactIngestFailed, - "fleet report: cleared comment failed": ClassSelfReportClearedFailed, - "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, - "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, - "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, - "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, - "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, - "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, - "inference no route": ClassInferenceNoRoute, - "inference upstream error": ClassInferenceUpstreamError, - "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, - "inference upstream failed": ClassInferenceUpstreamError, - "inference translate request failed": ClassInferenceUpstreamError, + "tmux has-session failed": ClassTmuxSessionCheckFailed, + "failed to send kick": ClassAgentLifecycleRepeated, + "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, + "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, + "agent tmux session missing": ClassAgentLifecycleRepeated, + "watchdog: would restart dead agent (observe mode; no action taken)": ClassAgentLifecycleRepeated, + "pane content at timeout": ClassAgentPaneTimeout, + "prompt timeout — dumping pane": ClassAgentPaneTimeout, + "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, + "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, + "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, + "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, + "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, + "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, + "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, + "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, + "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, + "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, + "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, + "no models found from any endpoint": ClassModelsEndpointEmpty, + "login required detected": ClassAgentLoginRequired, + "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, + "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, + "watchdog: agent credential failure": ClassAgentLoginRequired, + "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, + "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, + "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, + "failed to restore interactive home mode": ClassAgentDirSetupFailed, + "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, + "failed to tighten interactive home mode": ClassAgentDirSetupFailed, + "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, + "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, + "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, + "failed to restore inference home mode": ClassAgentDirSetupFailed, + "failed to tighten inference home mode": ClassAgentDirSetupFailed, + "failed to ingest bead fact": ClassBeadFactIngestFailed, + "fleet report: cleared comment failed": ClassSelfReportClearedFailed, + "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, + "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission": ClassGitHubAppStatusesPermission, + "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, + "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "proxy: plain HTTP forward failed": ClassProxyPlainForwardFailed, + "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, + "inference no route": ClassInferenceNoRoute, + "inference upstream error": ClassInferenceUpstreamError, + "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, + "inference upstream failed": ClassInferenceUpstreamError, + "inference translate request failed": ClassInferenceUpstreamError, } for msg, want := range fed { if got := LogFedClass(msg); got != want { @@ -1261,6 +1263,11 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { t.Errorf("%q not matched by an explicit class", msg) } } + // The neutral prow/lgtm line (5xx, 422, transport) must not file as the + // operator-only permission class; only the 403 wording names it. + if got := LogFedClass("could not set the prow/lgtm commit status"); got == ClassGitHubAppStatusesPermission { + t.Errorf("neutral prow/lgtm status failure classified as %q", got) + } for _, m := range []string{ "github REST 404 cached; suppressing repeated fetches", "copilot token validation rate-limited by GitHub; relaunching after cooldown", diff --git a/src/pkg/github/automerge_queue_prow_test.go b/src/pkg/github/automerge_queue_prow_test.go index 701ea76755..5896aef6aa 100644 --- a/src/pkg/github/automerge_queue_prow_test.go +++ b/src/pkg/github/automerge_queue_prow_test.go @@ -1,9 +1,11 @@ package github import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -186,3 +188,39 @@ func TestBindProwLgtmWarningOutlivesCallerDeadline(t *testing.T) { t.Fatalf("warning comments = %d, want 1 despite the caller's deadline expiring", rec.comments) } } + +// The permission wording is the github_app_statuses_permission_missing +// self-report keyword, so it must name only the 403; any other failure logs +// the neutral line and stays out of the operator-only permission class. +func TestBindProwLgtmWarningNamesPermissionOnlyOn403(t *testing.T) { + for _, tc := range []struct { + status int + wantPerm bool + wantPrefix string + }{ + {status: http.StatusForbidden, wantPerm: true}, + {status: http.StatusBadGateway, wantPerm: false}, + {status: http.StatusUnprocessableEntity, wantPerm: false}, + } { + t.Run(http.StatusText(tc.status), func(t *testing.T) { + rec := &prowQueueRecorder{} + api := prowQueueServer(t, "lgtm", tc.status, rec) + defer api.Close() + var logBuf bytes.Buffer + c := NewClient("token", "acme", []string{"widget"}, slog.New(slog.NewTextHandler(&logBuf, nil)), api.URL) + if err := c.bindProwLgtm(context.Background(), "acme", "widget", 7, "0123456789abcdef", "alice"); err == nil { + t.Fatal("want the status error returned") + } + logs := logBuf.String() + if !strings.Contains(logs, "could not set the prow/lgtm commit status") { + t.Fatalf("status failure not logged: %q", logs) + } + if got := strings.Contains(logs, prowLgtmStatusForbiddenMsg); got != tc.wantPerm { + t.Fatalf("permission wording present = %v, want %v: %q", got, tc.wantPerm, logs) + } + if rec.comments != 1 { + t.Fatalf("warning comments = %d, want 1", rec.comments) + } + }) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 5e655aadd1..da0fa04b23 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -3367,8 +3367,22 @@ const ( prowLgtmWarningMarker = "<!-- hive-prow-lgtm-status-warning -->" prowWarningTimeout = 15 * time.Second + + // prowLgtmStatusForbiddenMsg is the WARN logged when GitHub answers the + // prow/lgtm status write with 403. Its text is the keyword behind the + // github_app_statuses_permission_missing self-report class + // (pkg/fleetreport/runtime.go), so it is emitted only for that case. + prowLgtmStatusForbiddenMsg = "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission" ) +// isStatusesPermissionDenied reports whether err is GitHub refusing the +// status write with 403 ("Resource not accessible by integration"), the +// signature of an App installation without Commit statuses read & write. +func isStatusesPermissionDenied(err error) bool { + var ghErr *gh.ErrorResponse + return errors.As(err, &ghErr) && ghErr.Response != nil && ghErr.Response.StatusCode == http.StatusForbidden +} + // bindProwLgtm records the `prow/lgtm` success status on headSHA, the commit // the queue review approved. On failure it returns the error, logs it when a // logger is available, and attempts to surface a warning on the PR. The caller @@ -3399,8 +3413,17 @@ func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, numbe return nil } if c.logger != nil { - c.logger.Warn("could not set the prow/lgtm commit status; the GitHub App likely lacks the statuses: write permission", - "repo", owner+"/"+repoName, "number", number, "error", err) + // Only a 403 names the permission gap: that line is the + // github_app_statuses_permission_missing self-report keyword. Any + // other failure (5xx, 422, transport, deadline) stays neutral so a + // transient outage is not filed as an installation-permission fault. + if isStatusesPermissionDenied(err) { + c.logger.Warn(prowLgtmStatusForbiddenMsg, + "repo", owner+"/"+repoName, "number", number, "error", err) + } else { + c.logger.Warn("could not set the prow/lgtm commit status", + "repo", owner+"/"+repoName, "number", number, "error", err) + } } note := "Hive queued this PR but could not set the `prow/lgtm` commit status, so Prow may strip the `lgtm` label and the auto-merge queue may not hold." // A caller that withdrew has nothing left to warn about; a caller whose From 17e3165adfbe96a7a745092f6b52bef8093fbd22 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:01:04 +0100 Subject: [PATCH 112/166] fix: avoid fleet reports for canceled stage digest reads (#17946) Hive-Run: hivecommons/hive#17944 Hive-Plan: stage digest cancellation Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <Karibhou@users.noreply.github.com> --- .../fixed-canceled-stage-digest-report.md | 1 + src/pkg/fleetreport/slogfeed/slogfeed_test.go | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 changelog.d/fixed-canceled-stage-digest-report.md diff --git a/changelog.d/fixed-canceled-stage-digest-report.md b/changelog.d/fixed-canceled-stage-digest-report.md new file mode 100644 index 0000000000..b2c11c298e --- /dev/null +++ b/changelog.d/fixed-canceled-stage-digest-report.md @@ -0,0 +1 @@ +- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 838050edbd..1474ff0dd9 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -65,6 +65,27 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestCanceledStageDigestStaysLocal(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: check digest failed" + for range 6 { + logger.Warn(msg, "repo", "utah", "pr", 1194, "error", context.Canceled) + } + logger.Warn(msg, "error", errors.New("GET checks: context canceled")) + if len(*seen) != 0 { + t.Fatalf("canceled reads produced fleet observations: %+v", *seen) + } + if n := strings.Count(out.String(), "\n"); n != 7 { + t.Fatalf("local log records = %d, want 7", n) + } + logger.Warn(msg, "error", errors.New("context deadline exceeded")) + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn("another failure", "error", context.Canceled) + if len(*seen) != 2 { + t.Fatalf("unexpected errors must remain reportable: %+v", *seen) + } +} + func TestCanceledAdoptionReadStaysInLog(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" From d013e138ffe4f9dfe22d7e75ed7bb8eb6672b9e0 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 21:30:20 +0000 Subject: [PATCH 113/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/fixed-canceled-stage-digest-report.md | 1 - changelog.d/fixed-selfreport-statuses-permission-class.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/fixed-canceled-stage-digest-report.md delete mode 100644 changelog.d/fixed-selfreport-statuses-permission-class.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b12cfc3707..fc028bac11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.1) + +### Fixed + +- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). +- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). + ## 2026-11-12 (v5.237.0) ### Added diff --git a/changelog.d/fixed-canceled-stage-digest-report.md b/changelog.d/fixed-canceled-stage-digest-report.md deleted file mode 100644 index b2c11c298e..0000000000 --- a/changelog.d/fixed-canceled-stage-digest-report.md +++ /dev/null @@ -1 +0,0 @@ -- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). diff --git a/changelog.d/fixed-selfreport-statuses-permission-class.md b/changelog.d/fixed-selfreport-statuses-permission-class.md deleted file mode 100644 index 84c964db2a..0000000000 --- a/changelog.d/fixed-selfreport-statuses-permission-class.md +++ /dev/null @@ -1 +0,0 @@ -- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). From d58ab72481aa92d3749d493168e1529aeb9eba23 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:30:45 +0000 Subject: [PATCH 114/166] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20don't=20WA?= =?UTF-8?q?RN-dump=20the=20pane=20when=20a=20prompt=20wait=20outlasts=20a?= =?UTF-8?q?=20live=20turn=20(#17996)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../fixed-17993-mid-turn-prompt-timeout.md | 1 + src/crux/agent/agent_working_test.go | 47 +++++++++++++++++++ src/crux/agent/manager.go | 16 +++++++ 3 files changed, 64 insertions(+) create mode 100644 changelog.d/fixed-17993-mid-turn-prompt-timeout.md diff --git a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md new file mode 100644 index 0000000000..7e2b338b5b --- /dev/null +++ b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md @@ -0,0 +1 @@ +- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). diff --git a/src/crux/agent/agent_working_test.go b/src/crux/agent/agent_working_test.go index 902143114e..8e9df60c8c 100644 --- a/src/crux/agent/agent_working_test.go +++ b/src/crux/agent/agent_working_test.go @@ -1,6 +1,9 @@ package agent import ( + "bytes" + "log/slog" + "strings" "testing" "time" @@ -67,6 +70,50 @@ func TestWaitForInputPrompt_WorkingGate(t *testing.T) { } } +// A prompt wait that times out because the agent is still mid-turn must not +// log the WARN pane dump: the slog feed files that line as agent_pane_timeout, +// so every re-kick that outwaited a long turn self-reported (#17993). A pane +// that never reaches a prompt for any other reason still dumps at WARN. +func TestWaitForInputPrompt_TimeoutWhileWorkingIsNotWarned(t *testing.T) { + origTimeout, origPoll := inputPromptTimeout, inputPromptPollInterval + inputPromptTimeout = 200 * time.Millisecond + inputPromptPollInterval = 10 * time.Millisecond + defer func() { + inputPromptTimeout, inputPromptPollInterval = origTimeout, origPoll + }() + + run := func(pane string) string { + t.Helper() + var logBuf bytes.Buffer + m := NewManager( + map[string]config.AgentConfig{"quality": {Backend: "claude"}}, + slog.New(slog.NewTextHandler(&logBuf, nil)), ProjectContext{}) + m.mu.Lock() + agent := m.agents["quality"] + m.mu.Unlock() + ft := termSeams(m) + ft.captureVisiblePane = func(*AgentProcess) string { return pane } + ft.capturePane = func(*AgentProcess) string { return pane } + if m.waitForInputPromptForAgent(agent) { + t.Fatalf("pane %q: returned ready; want not ready", pane) + } + return logBuf.String() + } + + busy := run(" ◉ Working · 42.6 KiB esc interrupt Claude Fable 5\n❯") + if strings.Contains(busy, "prompt timeout — dumping pane") { + t.Errorf("mid-turn timeout logged a WARN pane dump; want an INFO line only:\n%s", busy) + } + if !strings.Contains(busy, "prompt wait ended: agent still mid-turn") { + t.Errorf("mid-turn timeout did not log the mid-turn INFO line:\n%s", busy) + } + + stuck := run("some unrecognised screen with no prompt") + if !strings.Contains(stuck, "prompt timeout — dumping pane") { + t.Errorf("non-working timeout must still dump the pane at WARN:\n%s", stuck) + } +} + // Bob-Shell 2.x keeps the "❯" input box painted while a response streams and // signals busy only with "⠇ Processing… (Enter to steer, Tab to queue)". The // busy pane must read as working (delivery gate) and NOT as a finished turn diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 9713d21661..0d25429396 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -1242,6 +1242,8 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu defer ticker.Stop() menuEscapes := 0 dialogAnswers := 0 + // lastWorking records whether the most recent poll saw the agent mid-turn. + lastWorking := false // Inference-routed agents run Claude Code too, so a permission dialog in // their pane is answered on the same terms as a plain claude agent's. dialogBackend := backend @@ -1255,6 +1257,18 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu } select { case <-deadline: + // An agent still mid-turn at the deadline is busy, not stuck: the + // kick paths already report it as busy and leave a genuinely stalled + // turn to the busy-no-activity watchdog (#12268). Dumping the pane at + // WARN here fed the agent_pane_timeout self-report every time a + // re-kick outwaited a long turn (#17993). + if lastWorking { + m.logger.Info("prompt wait ended: agent still mid-turn", + "agent", agent.Name, + "session", agent.tmuxSession, + "waited", inputPromptTimeout) + return false, kickPromptNotReadyReason + } m.logger.Warn("prompt timeout — dumping pane", "agent", agent.Name, "session", agent.tmuxSession) @@ -1281,6 +1295,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Check the visible pane only: a dismissed consent screen // lingers in the scrollback that captureTmuxPaneForAgent sees. visible := m.captureVisiblePaneForAgent(agent) + lastWorking = false if paneShowsCopilotQuestionForm(visible) { m.dismissCopilotQuestionFormForKick(agent) continue @@ -1313,6 +1328,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Use the visible pane only for the same reason as above: a // finished task's working marker lingers in scrollback. if paneShowsAgentWorking(visible) { + lastWorking = true continue } if paneShowsInputPrompt(visible) { From 845bf9e3eefecb9f2029f27a975a117fe7519373 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:46:22 +0000 Subject: [PATCH 115/166] =?UTF-8?q?=F0=9F=90=9B=20fix(review):=20add=20rev?= =?UTF-8?q?iew.perspectives=5Fsource=20config=20and=20ReviewPerspectives?= =?UTF-8?q?=20parser=20(#17994)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...17986-review-perspectives-source-config.md | 1 + src/pkg/config/config.go | 38 ++++++ .../config/review_perspectives_source_test.go | 109 ++++++++++++++++ .../dashboard/settings_parity_allowlist.txt | 6 + src/pkg/perspsrc/perspsrc.go | 90 ++++++++++++++ src/pkg/perspsrc/perspsrc_test.go | 116 ++++++++++++++++++ 6 files changed, 360 insertions(+) create mode 100644 changelog.d/added-17986-review-perspectives-source-config.md create mode 100644 src/pkg/config/review_perspectives_source_test.go create mode 100644 src/pkg/perspsrc/perspsrc.go create mode 100644 src/pkg/perspsrc/perspsrc_test.go diff --git a/changelog.d/added-17986-review-perspectives-source-config.md b/changelog.d/added-17986-review-perspectives-source-config.md new file mode 100644 index 0000000000..5965ebd30b --- /dev/null +++ b/changelog.d/added-17986-review-perspectives-source-config.md @@ -0,0 +1 @@ +- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 6c80ab373b..ecf4cb63f0 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -8332,6 +8332,20 @@ type ReviewConfig struct { // rather than left to infer them from the tree; a hive can only get that // by writing it down. An entry that is blank means "use the built-in". PerspectivePrompts map[string]string `yaml:"perspective_prompts,omitempty" json:"perspective_prompts,omitempty"` + // PerspectivesSource live-links the perspective names and focus text to a + // kind: ReviewPerspectives file in a GitHub repo (hivecommons/hive#17975), + // so maintainers can change the set through ordinary PRs. When it is set + // and resolved it wins over Perspectives/PerspectivePrompts, which remain + // the fallback. A partially filled source (no owner, repo or path) is + // treated as unset, matching DefinitionSourceConfig.IsSet. Only names and + // focus are sourced: combined_perspectives, plan_match and + // max_perspectives_per_pr stay operator-owned. + PerspectivesSource *DefinitionSourceConfig `yaml:"perspectives_source,omitempty" json:"perspectives_source,omitempty"` + // LinkedPerspectives is the set most recently resolved from + // PerspectivesSource. It is runtime-only and never persisted, so a fetched + // document can never be written back into the overlay. Read the effective + // set through EffectivePerspectives, never this field directly. + LinkedPerspectives *LinkedPerspectives `yaml:"-" json:"-"` // PerspectiveModels pins the backend/model one perspective's swarm review // runs with, keyed by perspective name. An entry replaces the reviewer // agent's review_models pool for that perspective only; the agent's @@ -8579,6 +8593,30 @@ type ReviewConfig struct { Reviewers ReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitzero"` } +// LinkedPerspectives is a review perspective set resolved from +// ReviewConfig.PerspectivesSource. It is already validated by the resolver; +// pkg/config deliberately does not import pkg/review to re-check it. +type LinkedPerspectives struct { + Names []string + Focus map[string]string + // Source is the provenance shown to operators, e.g. owner/repo/path@ref. + Source string + // Warning carries the last resolve failure while an older, still-valid set + // stays in effect. Empty when the latest document was applied. + Warning string +} + +// EffectivePerspectives returns the perspective names and focus text every +// consumer should use: the linked set when PerspectivesSource is set and has +// resolved, otherwise the inline Perspectives and PerspectivePrompts. With no +// source configured the result is exactly the inline values, unchanged. +func (r ReviewConfig) EffectivePerspectives() (names []string, focus map[string]string) { + if r.PerspectivesSource.IsSet() && r.LinkedPerspectives != nil { + return r.LinkedPerspectives.Names, r.LinkedPerspectives.Focus + } + return r.Perspectives, r.PerspectivePrompts +} + // PlanMatchConfig is the switch for the plan_match review perspective // (hivecommons/hive#8317). Off by default: the perspective only earns its cost // on a hive whose implementation PRs carry run trailers and whose plans live diff --git a/src/pkg/config/review_perspectives_source_test.go b/src/pkg/config/review_perspectives_source_test.go new file mode 100644 index 0000000000..85a7c66211 --- /dev/null +++ b/src/pkg/config/review_perspectives_source_test.go @@ -0,0 +1,109 @@ +package config + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestEffectivePerspectivesPrecedence(t *testing.T) { + source := &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"} + linked := &LinkedPerspectives{ + Names: []string{"correctness", "spyre-numerics"}, + Focus: map[string]string{"spyre-numerics": "fp16 accumulation order"}, + Source: "acme/widgets/.hive/review-perspectives.yaml", + } + inlineNames := []string{"security"} + inlineFocus := map[string]string{"security": "secrets in logs"} + + tests := []struct { + name string + cfg ReviewConfig + wantNames []string + wantFocus map[string]string + }{ + { + name: "source set and resolved uses linked set", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source, LinkedPerspectives: linked}, + wantNames: linked.Names, + wantFocus: linked.Focus, + }, + { + name: "source set but unresolved falls back to inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "partial source is unset", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets"}, LinkedPerspectives: linked}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "source unset uses inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "nothing configured", + cfg: ReviewConfig{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus := tc.cfg.EffectivePerspectives() + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} + +func TestLinkedPerspectivesNotPersisted(t *testing.T) { + cfg := ReviewConfig{ + PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"}, + LinkedPerspectives: &LinkedPerspectives{ + Names: []string{"linkedonlyname"}, + Focus: map[string]string{"linkedonlyname": "linkedonlyfocus"}, + Source: "linkedonlyprovenance", + Warning: "linkedonlywarning", + }, + } + y, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + j, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + for format, out := range map[string]string{"yaml": string(y), "json": string(j)} { + if !strings.Contains(out, "perspectives_source") { + t.Errorf("%s output lost perspectives_source:\n%s", format, out) + } + for _, leaked := range []string{"linked", "Linked"} { + if strings.Contains(out, leaked) { + t.Errorf("%s output persists linked data (%q):\n%s", format, leaked, out) + } + } + } + + var round ReviewConfig + if err := yaml.Unmarshal(y, &round); err != nil { + t.Fatalf("yaml.Unmarshal: %v", err) + } + if round.LinkedPerspectives != nil { + t.Errorf("LinkedPerspectives survived a yaml round-trip: %+v", round.LinkedPerspectives) + } + if !round.PerspectivesSource.IsSet() { + t.Errorf("PerspectivesSource lost in yaml round-trip: %+v", round.PerspectivesSource) + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 0c0c2c8f71..3bf0e75a3c 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -516,6 +516,12 @@ review.perspective_models.<key>.backend # TODO #13243 review.perspective_models.<key>.model # TODO #13243 review.perspective_prompts.<key> # TODO #11854 review.perspectives[] # TODO #11854 +review.perspectives_source.owner # TODO #17975 +review.perspectives_source.path # TODO #17975 +review.perspectives_source.ref # TODO #17975 +review.perspectives_source.repo # TODO #17975 +review.perspectives_source.type # TODO #17975 +review.perspectives_source.url # TODO #17975 review.recommendations.labels[] # TODO #11854 review.recommendations.repos[] # TODO #11854 review.reviewer_agents[] # TODO #11854 diff --git a/src/pkg/perspsrc/perspsrc.go b/src/pkg/perspsrc/perspsrc.go new file mode 100644 index 0000000000..0d03c4bc68 --- /dev/null +++ b/src/pkg/perspsrc/perspsrc.go @@ -0,0 +1,90 @@ +// Package perspsrc parses a live-linked review perspective set: a +// kind: ReviewPerspectives YAML document kept in a GitHub repo and named by +// review.perspectives_source (hivecommons/hive#17975), so maintainers can add, +// remove and re-describe perspectives through ordinary PRs. +// +// It is the review-set analogue of pkg/defsrc (which live-sources a whole +// agent definition) and follows the same design principles: +// +// - Validate before use: a document is only accepted when it would load as +// inline config would, via review.NewPerspectiveSet. A document that would +// not is an error, never a silent fall back to the built-in set. +// - Bounded input: a document over maxDocumentBytes is rejected outright. +// +// The document covers perspective names and focus text only. Cost and +// dispatch settings (combined_perspectives, plan_match, +// max_perspectives_per_pr) stay operator-owned in the hive config. +// +// This package, not pkg/config, owns the dependency on pkg/review, so +// pkg/config never has to import pkg/review. +package perspsrc + +import ( + "fmt" + "strings" + + "github.com/hivecommons/hive/pkg/review" + "gopkg.in/yaml.v3" +) + +// maxDocumentBytes caps how large a ReviewPerspectives document may be. A real +// one is a short list (well under a kilobyte per perspective); this bounds +// memory/log blast radius if a source path points at something huge. +const maxDocumentBytes = 512 * 1024 // 512 KiB + +// Kind is the required `kind` value of a review perspective document. +const Kind = "ReviewPerspectives" + +// Document is the YAML format of a live-linked review perspective set. +type Document struct { + Kind string `yaml:"kind" json:"kind"` + Perspectives []Entry `yaml:"perspectives" json:"perspectives"` +} + +// Entry is one perspective. Focus is optional for a built-in perspective +// (empty keeps the built-in text) and required for a hive-defined one. +type Entry struct { + Name string `yaml:"name" json:"name"` + Focus string `yaml:"focus,omitempty" json:"focus,omitempty"` +} + +// ParseDocument parses and validates a ReviewPerspectives document, returning +// the perspective names in dispatch order and the focus text keyed by name, in +// the same shape as review.perspectives and review.perspective_prompts. +func ParseDocument(doc string) (names []string, focus map[string]string, err error) { + if len(doc) > maxDocumentBytes { + return nil, nil, fmt.Errorf("perspectives document is %d bytes, over the limit of %d", len(doc), maxDocumentBytes) + } + var d Document + if err := yaml.Unmarshal([]byte(doc), &d); err != nil { + return nil, nil, fmt.Errorf("invalid perspectives YAML: %w", err) + } + if d.Kind != Kind { + return nil, nil, fmt.Errorf("expected kind %q, got %q", Kind, d.Kind) + } + if len(d.Perspectives) == 0 { + return nil, nil, fmt.Errorf("perspectives list is empty") + } + names = make([]string, 0, len(d.Perspectives)) + focus = map[string]string{} + seen := map[string]bool{} + for i, e := range d.Perspectives { + name := strings.TrimSpace(e.Name) + if name == "" { + return nil, nil, fmt.Errorf("perspectives[%d].name is required", i) + } + key := strings.ToLower(name) + if seen[key] { + return nil, nil, fmt.Errorf("duplicate perspective %q", name) + } + seen[key] = true + names = append(names, name) + if f := strings.TrimSpace(e.Focus); f != "" { + focus[name] = f + } + } + if _, err := review.NewPerspectiveSet(names, focus); err != nil { + return nil, nil, err + } + return names, focus, nil +} diff --git a/src/pkg/perspsrc/perspsrc_test.go b/src/pkg/perspsrc/perspsrc_test.go new file mode 100644 index 0000000000..ae352b834a --- /dev/null +++ b/src/pkg/perspsrc/perspsrc_test.go @@ -0,0 +1,116 @@ +package perspsrc + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseDocument(t *testing.T) { + tests := []struct { + name string + doc string + wantNames []string + wantFocus map[string]string + wantErr string + }{ + { + name: "built-in only", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: security +`, + wantNames: []string{"correctness", "security"}, + wantFocus: map[string]string{}, + }, + { + name: "custom with focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: spyre-constraints + focus: "Head size a multiple of 64; fp16 only." +`, + wantNames: []string{"correctness", "spyre-constraints"}, + wantFocus: map[string]string{"spyre-constraints": "Head size a multiple of 64; fp16 only."}, + }, + { + name: "built-in with focus override", + doc: `kind: ReviewPerspectives +perspectives: + - name: style + focus: "Follow the repo's CONTRIBUTING.md conventions." +`, + wantNames: []string{"style"}, + wantFocus: map[string]string{"style": "Follow the repo's CONTRIBUTING.md conventions."}, + }, + { + name: "wrong kind", + doc: `kind: AgentDefinition +perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives"`, + }, + { + name: "missing kind", + doc: `perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives", got ""`, + }, + { + name: "empty list", + doc: "kind: ReviewPerspectives\nperspectives: []\n", + wantErr: "perspectives list is empty", + }, + { + name: "unknown name without focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: sekurity +`, + wantErr: "has no focus text", + }, + { + name: "duplicate name", + doc: `kind: ReviewPerspectives +perspectives: + - name: security + - name: Security +`, + wantErr: "duplicate perspective", + }, + { + name: "oversize", + doc: "kind: ReviewPerspectives\n# " + strings.Repeat("x", maxDocumentBytes) + "\nperspectives:\n - name: correctness\n", + wantErr: "over the limit", + }, + { + name: "invalid yaml", + doc: "kind: [unterminated", + wantErr: "invalid perspectives YAML", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus, err := ParseDocument(tc.doc) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ParseDocument() error = %v, want containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("ParseDocument() unexpected error: %v", err) + } + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} From 19cc8e78aa719a5b490dc8feb3a20d491f2c5015 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:04:48 +0100 Subject: [PATCH 116/166] =?UTF-8?q?=F0=9F=90=9B=20Avoid=20self-reporting?= =?UTF-8?q?=20intentional=20signing=20skips=20(#17928)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Avoid self-reporting intentional signing skips Hive-Run: hivecommons/hive#17924 Hive-Plan: signed-commit skip self-report Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * docs: explain signed-commit ownership suppression Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-signed-commit-skip-selfreport.md | 1 + src/docs/fleet-report.md | 10 ++++++++++ src/pkg/fleetreport/slogfeed/slogfeed.go | 9 ++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-signed-commit-skip-selfreport.md diff --git a/changelog.d/fixed-signed-commit-skip-selfreport.md b/changelog.d/fixed-signed-commit-skip-selfreport.md new file mode 100644 index 0000000000..cbf6f24ade --- /dev/null +++ b/changelog.d/fixed-signed-commit-skip-selfreport.md @@ -0,0 +1 @@ +- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index bd63e9662b..db5eb42f78 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -246,6 +246,16 @@ nudge-delivery failures remain eligible for reporting when repeated. It also skips `stage transition suppressed by dampener` (`stageDampenerSuppression`, `pkg/fleetreport/slogfeed/slogfeed.go`): the stage-label store suppresses flapping transitions during its dampener hold window (hivecommons/hive#17103). That is the dampener working as designed, not a runtime error. The same function skips `stage flap suppressed` (`suppressStageFlap`, `pkg/stageengine/writer.go`): the stage engine refuses a same-head, same-evidence reversal of its last label write inside its flap window (hivecommons/hive#17142). +It also skips the exact WARN `signed-commit reconciler: skipped; the PR keeps an +unsigned commit` (`signedCommitOwnershipSkip`, `pkg/fleetreport/slogfeed/slogfeed.go`; +emitted by `pkg/github/pr_signed_reconcile.go`): the signed-commit reconciler +deliberately refuses to rewrite a merge commit or a commit authored or committed +by anyone other than a hive agent, and records the remedy on the PR +(`noteSignedSkip`). Repeated refusals are the ownership guard working as +designed, not a repeated runtime defect (hivecommons/hive#17924). The WARN and +its `reason` stay in local logs; other reconciler failures and the same message +at ERROR remain reportable. + The re-engagement sweep logs `re-engagement kick undeliverable; budget not charged` at INFO rather than WARN when the cause is `shared session pool is busy` (`deliverReEngagement`, `src/cmd/hive/main_sweeps.go`): a saturated shared CLI diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index e35e8781fd..5a9507e578 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -136,6 +136,13 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } +// signedCommitOwnershipSkip is the reconciler's deliberate refusal to rewrite +// a foreign or merge commit. The warning and PR remedy remain visible; repeated +// skips across PRs are not a runtime defect. +func signedCommitOwnershipSkip(r slog.Record) bool { + return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == "signed-commit reconciler: skipped; the PR keeps an unsigned commit" +} + // canceledError applies the existing cancellation exclusion to structured // errors too. An accompanying different error must remain reportable. func canceledError(r slog.Record, bound []slog.Attr) bool { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index 1474ff0dd9..d21d195f56 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -65,6 +65,25 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestSignedCommitOwnershipSkipStaysInLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "signed-commit reconciler: skipped; the PR keeps an unsigned commit" + for range 6 { + logger.Warn(msg, "repo", "o/r", "pr", 1403, "reason", "commit is authored by another account") + } + if len(*seen) != 0 { + t.Fatalf("expected ownership skips produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), msg); n != 6 || !strings.Contains(out.String(), "another account") { + t.Fatalf("local logs lost the warning or its reason: %s", out.String()) + } + logger.Warn("signed-commit reconciler: listing PR commits failed", "err", "unexpected failure") + logger.Error(msg, "reason", "unexpected failure") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + func TestCanceledStageDigestStaysLocal(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: check digest failed" From 1512c022bc20251b9898609c806c438e4136f9ac Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 22:07:53 +0000 Subject: [PATCH 117/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.237.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.237.2 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/fixed-17993-mid-turn-prompt-timeout.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fixed-17993-mid-turn-prompt-timeout.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fc028bac11..f12805c14c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.237.2) + +### Fixed + +- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). + ## 2026-11-12 (v5.237.1) ### Fixed diff --git a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md b/changelog.d/fixed-17993-mid-turn-prompt-timeout.md deleted file mode 100644 index 7e2b338b5b..0000000000 --- a/changelog.d/fixed-17993-mid-turn-prompt-timeout.md +++ /dev/null @@ -1 +0,0 @@ -- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). From 8a14a6e509da8b5aa73a919c8e220db418670c98 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:20:13 +0000 Subject: [PATCH 118/166] fix(dashboard): summarize dropped SSE events instead of warning per event (#17998) * fix(dashboard): summarize dropped SSE events instead of warning per event Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> * docs(api-reference): refresh server.go citations after SSE drop summary Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --------- Signed-off-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: operations <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-sse-drop-warn-flood.md | 1 + src/docs/api-reference/auth.md | 6 ++--- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++++++++---------- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/dashboard/server.go | 21 +++++++++++++++++- src/pkg/dashboard/server_test.go | 21 ++++++++++++++++++ 7 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 changelog.d/fixed-sse-drop-warn-flood.md diff --git a/changelog.d/fixed-sse-drop-warn-flood.md b/changelog.d/fixed-sse-drop-warn-flood.md new file mode 100644 index 0000000000..a5494ff99b --- /dev/null +++ b/changelog.d/fixed-sse-drop-warn-flood.md @@ -0,0 +1 @@ +- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 1d3e6c8eff..83820127f5 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1688` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1689` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1690` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1693` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1694` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1695` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 8511ded0f0..96f6d9c44f 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1695` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1700` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 85fa9acf7b..acefbb9f3c 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1668` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1669` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1670` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1676` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1682` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1683` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1686` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1687` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1673` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1674` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1675` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1681` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1686` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1687` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1688` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1689` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1690` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1691` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1692` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index c3da74dff5..d6909a3512 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1758` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1763` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 77b80d02e8..fd00f40ddb 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -64,6 +64,9 @@ const agentSkipAfterFullBroadcastS = 5 * time.Second const maxSSEClients = 100 +// sseDropWarnInterval bounds how often dropped SSE events are logged at WARN. +const sseDropWarnInterval = 5 * time.Minute + const sessionCookieName = "hive_session" const sessionCookieMaxAge = 30 * 24 * 60 * 60 @@ -139,6 +142,8 @@ type Server struct { status *StatusPayload sseClients map[chan []byte]struct{} sseMu sync.Mutex + sseDropped int // events dropped since the last summary warning; guarded by sseMu + sseDropWarn time.Time // last summary warning; guarded by sseMu logger *slog.Logger mux *http.ServeMux deps *Dependencies @@ -4049,9 +4054,23 @@ func (s *Server) broadcastFrame(frame string) { select { case ch <- raw: default: - s.logger.Warn("SSE client too slow, dropping event") + s.sseDropped++ + s.logger.Debug("SSE client too slow, dropping event") } } + s.warnSSEDropsLocked(time.Now()) +} + +// warnSSEDropsLocked emits one summary warning per sseDropWarnInterval instead +// of one per dropped event, so a single stalled browser tab does not flood logs +// or trip repeated-error fleet reporting. Caller holds sseMu. +func (s *Server) warnSSEDropsLocked(now time.Time) { + if s.sseDropped == 0 || now.Sub(s.sseDropWarn) < sseDropWarnInterval { + return + } + s.logger.Warn("SSE clients too slow, dropped events", "dropped", s.sseDropped, "interval", sseDropWarnInterval.String()) + s.sseDropped = 0 + s.sseDropWarn = now } // initHistories lazily builds the three sparkline ring buffers. Called (once) diff --git a/src/pkg/dashboard/server_test.go b/src/pkg/dashboard/server_test.go index 522f7bfa06..a0fe9a0704 100644 --- a/src/pkg/dashboard/server_test.go +++ b/src/pkg/dashboard/server_test.go @@ -2,6 +2,7 @@ package dashboard import ( "bufio" + "bytes" "encoding/json" "fmt" "log/slog" @@ -905,3 +906,23 @@ func TestNoAuth_AllEndpointsAccessible(t *testing.T) { t.Errorf("expected 200 when auth is not configured, got %d", resp.StatusCode) } } + +func TestBroadcast_SlowClientDropsWarnOncePerInterval(t *testing.T) { + var buf bytes.Buffer + s := newTestServer() + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + full := make(chan []byte, 1) + full <- []byte("old-data") + s.sseMu.Lock() + s.sseClients[full] = struct{}{} + s.sseMu.Unlock() + + for i := 0; i < 20; i++ { + s.broadcastFrame("data: {}\n\n") + } + + if got := strings.Count(buf.String(), "level=WARN"); got != 1 { + t.Errorf("WARN lines: want 1, got %d\n%s", got, buf.String()) + } +} From a041003628f3d4cdba0dbb7d9e73291616e85258 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:37:24 -0700 Subject: [PATCH 119/166] =?UTF-8?q?=F0=9F=90=9B=20Claim=20onto=20a=20non-d?= =?UTF-8?q?efault=20forge:=20show=20the=20selected=20forge=20while=20pendi?= =?UTF-8?q?ng=20and=20discover=20the=20App=20installation=20on=20it=20(#18?= =?UTF-8?q?007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Show operator-selected forge on claim-pending fleet rows While a placeholder is assigned-but-unclaimed the spoke still heartbeats the pool default github_host (github.ibm.com on vllm-d), which the hub treated as authoritative and rendered over the explicit forge the operator picked in the assign dialog. Prefer the hive record's github_host for AssignedUnclaimed rows; the spoke report resumes authority once the claim is delivered. Fixes #18006 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 Move App installation discovery to the hub-delivered App on forge switch A claim that switches forge (GHE pool default → github.com) delivers the github.com App with installation_id 0. HasUsableApp is false for that identity so the client rebuild was skipped — correctly — but installation discovery kept the boot-time AppAuth (App 5686 @ github.ibm.com) and polled the wrong forge forever. Build a discovery-only AppAuth for the new App and hand it to the dashboard so the owner's installation is adopted. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- .../fixed-claim-forge-switch-discovery.md | 1 + changelog.d/fixed-claim-pending-forge-pill.md | 1 + .../discovery_appauth_forge_switch_test.go | 54 ++++++++++++++++ src/cmd/hive/main.go | 10 +++ src/cmd/hive/main_github_app.go | 54 ++++++++++++++++ src/pkg/dashboard/server.go | 10 +++ src/pkg/hub/claim_pending_forge_pill_test.go | 62 +++++++++++++++++++ src/pkg/hub/saas_hives_api.go | 25 ++++++++ 8 files changed, 217 insertions(+) create mode 100644 changelog.d/fixed-claim-forge-switch-discovery.md create mode 100644 changelog.d/fixed-claim-pending-forge-pill.md create mode 100644 src/cmd/hive/discovery_appauth_forge_switch_test.go create mode 100644 src/pkg/hub/claim_pending_forge_pill_test.go diff --git a/changelog.d/fixed-claim-forge-switch-discovery.md b/changelog.d/fixed-claim-forge-switch-discovery.md new file mode 100644 index 0000000000..788c262af6 --- /dev/null +++ b/changelog.d/fixed-claim-forge-switch-discovery.md @@ -0,0 +1 @@ +- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. diff --git a/changelog.d/fixed-claim-pending-forge-pill.md b/changelog.d/fixed-claim-pending-forge-pill.md new file mode 100644 index 0000000000..c5b31e94d6 --- /dev/null +++ b/changelog.d/fixed-claim-pending-forge-pill.md @@ -0,0 +1 @@ +- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). diff --git a/src/cmd/hive/discovery_appauth_forge_switch_test.go b/src/cmd/hive/discovery_appauth_forge_switch_test.go new file mode 100644 index 0000000000..c9a946d56f --- /dev/null +++ b/src/cmd/hive/discovery_appauth_forge_switch_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// A placeholder booted against its pool's default forge (GHE App 5686) and +// then claimed onto github.com receives App 3568013 with installation_id 0. +// HasUsableApp is false for that identity, so the full client rebuild is +// (correctly) skipped — but installation discovery must still move to the new +// App/forge, or it polls the old one forever and the hive never adopts the +// installation its owner makes (live: `ash` → thinkscientist on vllm-d). +func TestDiscoveryAppAuthStaleOnForgeSwitchWithoutInstallation(t *testing.T) { + gheCfg := &config.Config{} + gheCfg.GitHub.AppID = 5686 + gheCfg.GitHub.APIURL = "https://github.ibm.com/api/v3" + booted := testAppAuth(t, gheCfg) + + public := config.GitHubConfig{AppID: 3568013, InstallationID: 0, APIURL: "https://api.github.com"} + + if !discoveryAppAuthStale(booted, public, "/data/gh-app-key-3568013.pem") { + t.Fatal("GHE-booted AppAuth must be replaced when the hub delivers the github.com App with no installation") + } + if !discoveryAppAuthStale(nil, public, "/data/gh-app-key-3568013.pem") { + t.Fatal("a spoke that booted with no AppAuth must get one for discovery") + } +} + +func TestDiscoveryAppAuthStaleNoOpCases(t *testing.T) { + cfg := &config.Config{} + cfg.GitHub.AppID = 3568013 + cfg.GitHub.APIURL = "https://api.github.com" + current := testAppAuth(t, cfg) + + cases := []struct { + name string + gh config.GitHubConfig + key string + }{ + {"same App and forge already held", config.GitHubConfig{AppID: 3568013, APIURL: "https://api.github.com/"}, "/k.pem"}, + {"installation known — the usable-App rebuild owns this", config.GitHubConfig{AppID: 5686, InstallationID: 42, APIURL: "https://github.ibm.com/api/v3"}, "/k.pem"}, + {"no key resolved", config.GitHubConfig{AppID: 5686, APIURL: "https://github.ibm.com/api/v3"}, ""}, + {"placeholder App is not a real App", config.GitHubConfig{AppID: config.PlaceholderAppID}, "/k.pem"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if discoveryAppAuthStale(current, tc.gh, tc.key) { + t.Errorf("expected no discovery AppAuth swap") + } + }) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 4e643473d5..ea1821f5e2 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5495,6 +5495,16 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "app_id", b.cfg.GitHub.AppID, "installation_id", b.cfg.GitHub.InstallationID, ) + } else if discoveryAppAuthStale(b.appAuth, b.cfg.GitHub, rebuildKeyFile) { + // A real App with no installation yet (forge switch on claim): + // give installation discovery the NEW App's JWT so it polls + // the forge the repos actually live on. + newAppAuth, err := github.NewAppAuth(b.cfg.GitHub.AppID, 0, rebuildKeyFile, b.logger, b.cfg.GitHub.ResolvedAPIURL()) + if err != nil { + b.logger.Error("github app auth init for installation discovery failed", "error", err) + return + } + b.adoptDiscoveryAppAuth(newAppAuth) } }), diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go index b747bdca20..a3a89f0f26 100644 --- a/src/cmd/hive/main_github_app.go +++ b/src/cmd/hive/main_github_app.go @@ -558,3 +558,57 @@ func advisoryIssueMissingError(repo string, cause error) string { } return base } + +// discoveryAppAuthStale reports whether the AppAuth the installation-discovery +// loop is polling with no longer names the App the config now claims. +// +// A hub-delivered identity with installation_id 0 (a placeholder claimed onto +// a DIFFERENT forge than its pool default: GHE slot → github.com repo) is not +// "usable" — HasUsableApp is false — so the heartbeat rebuild skipped it and +// the boot-time AppAuth (old App, old api_url) kept serving auto-discovery. +// Discovery then polled the wrong forge forever: the installation the owner +// made on the right one could never be found, and the hive sat at "App <old +// id> has no installation" until a pod restart. Boot builds an AppAuth the +// moment a real app_id and key exist precisely so discovery has a JWT to use; +// this is the same rule applied to a hub delivery. +func discoveryAppAuthStale(current *github.AppAuth, gh config.GitHubConfig, keyFile string) bool { + if !gh.HasApp() || gh.InstallationID != 0 || keyFile == "" { + return false + } + if current == nil { + return true + } + return current.AppID() != gh.AppID || !sameAPIURL(current.APIURL(), gh.ResolvedAPIURL()) +} + +// sameAPIURL compares two GitHub API base URLs ignoring scheme-case and a +// trailing slash, the two spellings the config and the client disagree on. +func sameAPIURL(a, b string) bool { + norm := func(s string) string { return strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/") } + return norm(a) == norm(b) +} + +// adoptDiscoveryAppAuth hands a not-yet-installed App's AppAuth to the +// installation-discovery loop and flips the banner/hub state to +// not-installed for the App it now names, then forces one discovery pass so +// an installation already made on the new forge is adopted without waiting +// for the next tick. No client is built: there is no installation to mint +// against yet — AutoDiscoverGitHubInstallationID builds it on success. +func (b *boot) adoptDiscoveryAppAuth(newAppAuth *github.AppAuth) { + if b == nil || newAppAuth == nil { + return + } + b.appAuth = newAppAuth + diag := "GitHub App " + strconv.FormatInt(b.cfg.GitHub.AppID, 10) + + " has no installation for this org — install it (the spoke adopts the installation automatically)" + b.appAuthFailure = "The GitHub App is configured but has no installation. Install the app on your org; this hive will discover the installation automatically." + if b.dashSrv != nil { + b.dashSrv.UpdateGitHubAppAuth(newAppAuth) + b.dashSrv.SetGitHubAppRequired(true) + b.dashSrv.SetGitHubAppState(github.AppStateNotInstalled.String()) + b.dashSrv.SetGitHubAppPermIssue(diag) + go func() { _, _ = b.dashSrv.AutoDiscoverGitHubInstallationID(b.ctx, true) }() + } + b.logger.Info("github app identity adopted for installation discovery", + "app_id", newAppAuth.AppID(), "api_url", newAppAuth.APIURL()) +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index fd00f40ddb..d53f4cc24b 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -3176,6 +3176,16 @@ func (s *Server) UpdateGitHubClient(client *github.Client, auth *github.AppAuth) } } +// UpdateGitHubAppAuth swaps only the App credential the installation-discovery +// loop polls with, leaving the client untouched. Used when a hub delivery names +// a new App (or forge) whose installation is not known yet, so discovery asks +// the right forge instead of the one the pod booted against. +func (s *Server) UpdateGitHubAppAuth(auth *github.AppAuth) { + if s.deps != nil { + s.deps.GHAppAuth = auth + } +} + // SetGoodFirstIssueStats installs the provider of good-first-issue curation // counters; nil removes it. func (s *Server) SetGoodFirstIssueStats(fn func() any) { diff --git a/src/pkg/hub/claim_pending_forge_pill_test.go b/src/pkg/hub/claim_pending_forge_pill_test.go new file mode 100644 index 0000000000..2f96d13ee1 --- /dev/null +++ b/src/pkg/hub/claim_pending_forge_pill_test.go @@ -0,0 +1,62 @@ +package hub + +import "testing" + +// A placeholder assigned with an explicit forge must show THAT forge in the +// Location pill while the claim is pending, not the cluster default the spoke +// still heartbeats from its pre-claim config (vllm-d: github.ibm.com). Live +// example: "ash" assigned to thinkscientist with github.com selected read +// github.ibm.com for the whole "claim pending" window. +func TestClaimPendingGitHubHostOverridesSpokeReport(t *testing.T) { + cases := []struct { + name string + entry MyHiveEntry + sh *SaaSHive + wantHost string + wantApply bool + }{ + { + name: "pending claim with explicit github.com beats GHE heartbeat", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned, GitHubHost: "github.com"}, + wantHost: "github.com", + wantApply: true, + }, + { + name: "pending claim with explicit GHE beats public heartbeat", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned, GitHubHost: "https://github.ibm.com/"}, + wantHost: "github.ibm.com", + wantApply: true, + }, + { + name: "delivered claim leaves the spoke report authoritative", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: false}, + sh: &SaaSHive{Status: statusAssigned, ClaimDelivered: true, GitHubHost: "github.com"}, + wantApply: false, + }, + { + name: "pending claim without a recorded host does not override", + entry: MyHiveEntry{RegistryEntry: RegistryEntry{GitHubHost: "github.ibm.com"}, AssignedUnclaimed: true}, + sh: &SaaSHive{Status: statusAssigned}, + wantApply: false, + }, + { + name: "no hive record does not override", + entry: MyHiveEntry{AssignedUnclaimed: true}, + sh: nil, + wantApply: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := claimPendingGitHubHost(&tc.entry, tc.sh) + if ok != tc.wantApply { + t.Fatalf("apply = %v, want %v", ok, tc.wantApply) + } + if ok && got != tc.wantHost { + t.Errorf("host = %q, want %q", got, tc.wantHost) + } + }) + } +} diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index a6713566dc..f71da247f5 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -340,6 +340,27 @@ type MyHiveEntry struct { Quadrant *Quadrant `json:"quadrant,omitempty"` } +// claimPendingGitHubHost returns the GitHub host the Location pill should show +// for an assigned-but-unclaimed row, and whether that override applies. +// +// While a placeholder is "claim pending" the spoke still runs the pool's +// default config, so its heartbeat reports the CLUSTER default (github.ibm.com +// on vllm-d) even though the operator just picked github.com in the assign +// dialog. For that window the hive record's github_host — the operator's +// explicit choice, stamped by handleAssignHive — is the truth the row must +// show; the spoke's value is simply the stale pre-claim config, not evidence +// of where the assigned repos live. Once the claim is delivered the spoke +// report resumes its authority (see resolveGitHubHost). +func claimPendingGitHubHost(entry *MyHiveEntry, sh *SaaSHive) (string, bool) { + if entry == nil || sh == nil || !entry.AssignedUnclaimed { + return "", false + } + if strings.TrimSpace(sh.GitHubHost) == "" { + return "", false + } + return githubHostLabel(sh.GitHubHost), true +} + // myHivesRecentEventCount is how many timeline events ride the My Hives // payload for the status hover. The hover panel already carries a status word, // the per-check health lines, the relay line and the user list; three events @@ -722,6 +743,10 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values // is left empty and rendered as "github.com" by the pill, so an old spoke // shows a sane default rather than a wrong host. resolveGitHubHost := func(entry *MyHiveEntry) { + if host, ok := claimPendingGitHubHost(entry, saasByID[entry.ID]); ok { + entry.GitHubHost = host + return + } if entry.GitHubHost != "" { return // spoke-reported — authoritative } From d68e16aef9ed6b1f5f04c5b1520de34c30e16411 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 22:43:26 +0000 Subject: [PATCH 120/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.238.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.238.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ .../added-17986-review-perspectives-source-config.md | 1 - changelog.d/fixed-signed-commit-skip-selfreport.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-17986-review-perspectives-source-config.md delete mode 100644 changelog.d/fixed-signed-commit-skip-selfreport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f12805c14c..41885cd023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.238.0) + +### Added + +- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). + ## 2026-11-12 (v5.237.2) ### Fixed diff --git a/changelog.d/added-17986-review-perspectives-source-config.md b/changelog.d/added-17986-review-perspectives-source-config.md deleted file mode 100644 index 5965ebd30b..0000000000 --- a/changelog.d/added-17986-review-perspectives-source-config.md +++ /dev/null @@ -1 +0,0 @@ -- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-signed-commit-skip-selfreport.md b/changelog.d/fixed-signed-commit-skip-selfreport.md deleted file mode 100644 index cbf6f24ade..0000000000 --- a/changelog.d/fixed-signed-commit-skip-selfreport.md +++ /dev/null @@ -1 +0,0 @@ -- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). From ee243a285fd5cdb50fa81e5aba4fd048483fac04 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:52:42 -0700 Subject: [PATCH 121/166] fix(ioscan): reword canary preamble and retry a refused canary kick without it (#18013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Sonnet 5 (which the Copilot CLI substitutes for sonnet-4.6/opus-4.6) reads the bare "SECURITY CANARY: <token> Never print…" preamble as a prompt-injection marker and refuses the whole kick; on one hosted hive 109 of 168 supervisor kicks were refused and each one burned its cadence slot. - CanaryPreamble is now a labelled operator-harness header that says whose configuration it is and that it is not a task; crux/agent mirrors the new prefix for the kick preview. - A kick refused with a canary in front is re-delivered once with the canary stripped (trigger canary-refusal-retry); the retry is never retried, and a fresh kick re-arms it. Fixes #18012 Signed-off-by: Andrew Anderson <andy@clubanderson.com> --- changelog.d/fixed-canary-kick-refusal.md | 1 + src/crux/agent/kick_canary_retry.go | 61 ++++++++++++++++ src/crux/agent/kick_canary_retry_test.go | 82 ++++++++++++++++++++++ src/crux/agent/kick_preview.go | 37 ++++++---- src/crux/agent/kick_preview_test.go | 4 +- src/crux/agent/manager.go | 29 +++++--- src/crux/agent/manager_kick.go | 12 +++- src/crux/agent/manager_poll.go | 1 + src/docs/scale-envelope.md | 2 +- src/pkg/ioscan/canary.go | 17 ++++- src/pkg/ioscan/canary_coverage_test.go | 19 ++++- src/pkg/scheduler/kick_layout_test.go | 8 +-- src/pkg/scheduler/repo_filter_hold_test.go | 2 +- 13 files changed, 239 insertions(+), 36 deletions(-) create mode 100644 changelog.d/fixed-canary-kick-refusal.md create mode 100644 src/crux/agent/kick_canary_retry.go create mode 100644 src/crux/agent/kick_canary_retry_test.go diff --git a/changelog.d/fixed-canary-kick-refusal.md b/changelog.d/fixed-canary-kick-refusal.md new file mode 100644 index 0000000000..5c1235ffe5 --- /dev/null +++ b/changelog.d/fixed-canary-kick-refusal.md @@ -0,0 +1 @@ +- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). diff --git a/src/crux/agent/kick_canary_retry.go b/src/crux/agent/kick_canary_retry.go new file mode 100644 index 0000000000..99eb8be642 --- /dev/null +++ b/src/crux/agent/kick_canary_retry.go @@ -0,0 +1,61 @@ +package agent + +// A kick that opens with the ioscan exfiltration canary can be refused as a +// whole by a model that reads the canary as a prompt-injection marker +// (Claude Sonnet 5 via the Copilot CLI: "this request includes a 'SECURITY +// CANARY' marker and asks me to act as a scanner agent …"). The refusal used +// to cost the entire cadence slot — a 4h scanner kick, or ~150 five-minute +// supervisor kicks a day on one hosted hive. The refused kick is instead +// re-delivered once with the canary preamble stripped; the retry is a plain +// kick, so a second refusal is the model's verdict on the work itself and +// stands. + +// kickTriggerCanaryRetry is the delivery trigger of the one canary-free +// re-delivery a refused canary kick gets. recordDeliveredKickLocked uses it +// to mark the retry so it is never itself retried. +const kickTriggerCanaryRetry = "canary-refusal-retry" + +// planCanaryRetry decides from the refused kick alone whether it earns a +// canary-free re-delivery: the kick as handed to delivery must have carried +// a canary preamble with a body after it, and must not already be the retry. +func planCanaryRetry(lastKickInput string, alreadyRetried bool) (plain string, ok bool) { + if alreadyRetried { + return "", false + } + plain, had := stripCanaryPreamble(lastKickInput) + if !had { + return "", false + } + return plain, true +} + +// retryRefusedCanaryKick is the checkKickRefusal hook: when the kick just +// refused led with a canary, re-deliver it once without one. Delivery goes +// through the full SendKick readiness chain on its own goroutine (the caller +// is the pane poller, and SendKick blocks on the input prompt), so the retry +// lands once the refusing turn has returned to the prompt. Must NOT be +// called with m.mu held. +func (m *Manager) retryRefusedCanaryKick(agent *AgentProcess) { + m.mu.RLock() + plain, ok := planCanaryRetry(agent.lastKickInput, agent.canaryRetryDone) + m.mu.RUnlock() + if !ok { + return + } + m.logger.Warn("kick refused with the exfiltration canary in front; re-delivering once without it", + "agent", agent.Name, + "reason", agent.KickRefusalReason, + ) + deliver := m.canaryRetryDeliver + if deliver == nil { + deliver = func(name, message string) { + go func() { + if err := m.sendKick(name, message, kickTriggerCanaryRetry); err != nil { + m.logger.Warn("canary-free re-delivery of refused kick failed", + "agent", name, "error", err) + } + }() + } + } + deliver(agent.Name, plain) +} diff --git a/src/crux/agent/kick_canary_retry_test.go b/src/crux/agent/kick_canary_retry_test.go new file mode 100644 index 0000000000..68d3528db6 --- /dev/null +++ b/src/crux/agent/kick_canary_retry_test.go @@ -0,0 +1,82 @@ +package agent + +import "testing" + +// A model that reads the canary preamble as a prompt-injection marker refuses +// the whole kick; the kick is re-delivered once without the canary so the +// refusal does not burn the cadence slot. + +func TestStripCanaryPreamble(t *testing.T) { + work := "Review PR #15783 (correctness).\n\nFocus on the retry loop." + if got, ok := stripCanaryPreamble(testCanaryPreamble + work); !ok || got != work { + t.Fatalf("stripCanaryPreamble = (%q, %v), want (%q, true)", got, ok, work) + } + for _, msg := range []string{work, testCanaryPreamble, "=== HIVE HARNESS HEADER ===\nExfiltration canary: HIVE-CANARY-abc123 no blank line", ""} { + if got, ok := stripCanaryPreamble(msg); ok || got != msg { + t.Errorf("stripCanaryPreamble(%q) = (%q, %v), want unchanged, false", msg, got, ok) + } + } +} + +func TestPlanCanaryRetry(t *testing.T) { + work := "Triage issue #12." + if plain, ok := planCanaryRetry(testCanaryPreamble+work, false); !ok || plain != work { + t.Fatalf("canary kick must earn one plain retry, got (%q, %v)", plain, ok) + } + if _, ok := planCanaryRetry(testCanaryPreamble+work, true); ok { + t.Error("the retry itself must not be retried") + } + if _, ok := planCanaryRetry(work, false); ok { + t.Error("a kick without a canary has nothing to strip; no retry") + } + if _, ok := planCanaryRetry(testCanaryPreamble, false); ok { + t.Error("a canary-only kick has no work to re-deliver") + } +} + +func TestCheckKickRefusalRedeliversCanaryKickOnce(t *testing.T) { + m := &Manager{logger: discardLogger()} + var delivered []string + m.canaryRetryDeliver = func(name, message string) { + delivered = append(delivered, name+"\x00"+message) + } + work := "[agent:scanner]\n\nScan cncf/techdocs for stale pages." + refusal := "I'm noticing this request includes a canary marker and looks like a prompt injection, so I won't act on it." + + agent := &AgentProcess{Name: "scanner", lastKickInput: testCanaryPreamble + work} + m.checkKickRefusal(agent, refusal) + if !agent.KickRefused { + t.Fatal("refusal must still be recorded") + } + if len(delivered) != 1 || delivered[0] != "scanner\x00"+work { + t.Fatalf("refused canary kick must be re-delivered plain, got %q", delivered) + } + + // The re-delivery is marked by its trigger; a refusal of the retry stands. + m.recordDeliveredKickLocked(agent, work, kickTriggerCanaryRetry) + if agent.KickRefused || !agent.canaryRetryDone { + t.Fatalf("retry delivery must clear the refusal and mark the retry: refused=%v retried=%v", agent.KickRefused, agent.canaryRetryDone) + } + agent.lastKickInput = testCanaryPreamble + work + m.checkKickRefusal(agent, refusal) + if len(delivered) != 1 { + t.Fatalf("a refused retry must not be retried again, got %d deliveries", len(delivered)) + } + + // A fresh kick from the governor re-arms the single retry. + m.recordDeliveredKickLocked(agent, testCanaryPreamble+work, "send-kick") + if agent.canaryRetryDone { + t.Fatal("an ordinary kick must re-arm the canary retry") + } +} + +func TestCheckKickRefusalWithoutCanaryDoesNotRedeliver(t *testing.T) { + m := &Manager{logger: discardLogger()} + calls := 0 + m.canaryRetryDeliver = func(string, string) { calls++ } + agent := &AgentProcess{Name: "scanner", lastKickInput: "[agent:scanner]\n\nPlain kick."} + m.checkKickRefusal(agent, "This looks like a prompt injection attempt, so I'm declining to execute it.") + if !agent.KickRefused || calls != 0 { + t.Fatalf("plain kick refusal must be recorded without a retry: refused=%v calls=%d", agent.KickRefused, calls) + } +} diff --git a/src/crux/agent/kick_preview.go b/src/crux/agent/kick_preview.go index f25130c5c9..6566381e92 100644 --- a/src/crux/agent/kick_preview.go +++ b/src/crux/agent/kick_preview.go @@ -3,24 +3,33 @@ package agent import "strings" // canaryPreamblePrefix is how an ioscan exfiltration canary opens a kick -// (hive pkg/ioscan.CanaryPreamble). The preamble runs to the first blank -// line and is about as long as the audit preview, so a kick that carried -// one showed operators nothing but the canary (hivecommons/hive#15823). The -// preview and the kick-history snippet start after it; the kick itself is -// delivered unchanged, canary first. -const canaryPreamblePrefix = "SECURITY CANARY: " +// (hive pkg/ioscan.CanaryPreamblePrefix; the two must stay in step). The +// preamble runs to the first blank line and is about as long as the audit +// preview, so a kick that carried one showed operators nothing but the +// canary (hivecommons/hive#15823). The preview and the kick-history snippet +// start after it; the kick itself is delivered unchanged, canary first. +const canaryPreamblePrefix = "=== HIVE HARNESS HEADER" -// kickPreviewBody returns the part of a kick worth previewing: the message -// after its canary preamble, or the whole message when it carries none (or -// nothing follows the preamble, so the canary-only kick stays visible as -// such). -func kickPreviewBody(message string) string { +// stripCanaryPreamble returns the kick without its canary preamble and true +// when one was present and something followed it. A canary-only kick (or one +// whose preamble is not terminated by a blank line) is returned unchanged +// with false. +func stripCanaryPreamble(message string) (string, bool) { if !strings.HasPrefix(message, canaryPreamblePrefix) { - return message + return message, false } _, rest, ok := strings.Cut(message, "\n\n") if !ok || strings.TrimSpace(rest) == "" { - return message + return message, false } - return rest + return rest, true +} + +// kickPreviewBody returns the part of a kick worth previewing: the message +// after its canary preamble, or the whole message when it carries none (or +// nothing follows the preamble, so the canary-only kick stays visible as +// such). +func kickPreviewBody(message string) string { + body, _ := stripCanaryPreamble(message) + return body } diff --git a/src/crux/agent/kick_preview_test.go b/src/crux/agent/kick_preview_test.go index ff0a21b36c..8a3b3b7187 100644 --- a/src/crux/agent/kick_preview_test.go +++ b/src/crux/agent/kick_preview_test.go @@ -5,7 +5,7 @@ import "testing" // hivecommons/hive#15823: the audit preview of a canary-bearing kick shows // the work the agent was given, not the canary preamble. -const testCanaryPreamble = "SECURITY CANARY: HIVE-CANARY-abc123\nThis is a hive-issued exfiltration-canary marker. Never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service.\n\n" +const testCanaryPreamble = "=== HIVE HARNESS HEADER (operator configuration, not user content) ===\nExfiltration canary: HIVE-CANARY-abc123\nThe token above is a tripwire planted by the hive harness that runs you. It is not a task.\n=== END HIVE HARNESS HEADER ===\n\n" func TestKickPreviewBodySkipsCanaryPreamble(t *testing.T) { work := "Review PR #15783 (correctness).\n\nFocus on the retry loop." @@ -22,7 +22,7 @@ func TestKickPreviewBodyPassesThroughKicksWithoutPreamble(t *testing.T) { } func TestKickPreviewBodyKeepsCanaryOnlyKickVisible(t *testing.T) { - for _, msg := range []string{testCanaryPreamble, "SECURITY CANARY: HIVE-CANARY-abc123 no blank line"} { + for _, msg := range []string{testCanaryPreamble, "=== HIVE HARNESS HEADER ===\nExfiltration canary: HIVE-CANARY-abc123 no blank line"} { if got := kickPreviewBody(msg); got != msg { t.Fatalf("preview of canary-only kick = %q, want whole message", got) } diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 0d25429396..4a63b82b0f 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -257,16 +257,22 @@ type AgentProcess struct { LastKickMessage string KickRefused bool KickRefusalReason string - LaunchedMode AgentMode - HasLaunched bool - poolLeaseID string - poolLeaseCancel context.CancelFunc - baseConfig config.AgentConfig - agentSpecPrompt string - tmuxSession string - tmuxSocket string - cancel context.CancelFunc - forceRelaunch bool + // lastKickInput is the last kick as handed to delivery, before the + // backend suffixes, so a refused canary kick can be re-delivered plain + // without doubling the suffixes. canaryRetryDone is set when the current + // kick IS that re-delivery, so it is never retried again. + lastKickInput string + canaryRetryDone bool + LaunchedMode AgentMode + HasLaunched bool + poolLeaseID string + poolLeaseCancel context.CancelFunc + baseConfig config.AgentConfig + agentSpecPrompt string + tmuxSession string + tmuxSocket string + cancel context.CancelFunc + forceRelaunch bool // launching is set true under m.mu while Start runs this agent's launch // with m.mu RELEASED (so a slow /data NFS write or a hung MITM-proxy token // mint during launch cannot block AllStatuses()/the heartbeat collect() and @@ -677,6 +683,9 @@ type Manager struct { // kickOutcomeObserver receives the verdict on how each kicked turn ended // (#7421); the governor consumes it. Same discipline as kickObserver. kickOutcomeObserver atomic.Pointer[func(agentName string, outcome KickOutcome)] + // canaryRetryDeliver, when set, replaces the goroutine+sendKick path a + // refused canary kick is re-delivered through (tests inject a recorder). + canaryRetryDeliver func(name, message string) // kickDispatches tracks asynchronous kick dispatches (#5325): the in-flight // guard that makes delivery exactly-once, and the latest outcome per agent diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index f3a877382b..838d6a783d 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -98,7 +98,13 @@ func notRunningReason(agent *AgentProcess) string { } } -func (m *Manager) SendKick(name string, message string) (err error) { +func (m *Manager) SendKick(name string, message string) error { + return m.sendKick(name, message, "send-kick") +} + +// sendKick is SendKick with the delivery trigger spelled out; the trigger is +// what the audit log, prompt history and the canary-retry bookkeeping see. +func (m *Manager) sendKick(name, message, trigger string) (err error) { // Agent-kick span. No-op with zero export cost when tracing is disabled // (the default). SendKick has no context parameter, so this span roots at // Background; it still captures the kick leg of the governor→agent @@ -229,7 +235,7 @@ func (m *Manager) SendKick(name string, message string) (err error) { return err } - return m.deliverKickLocked(agent, message, "send-kick") + return m.deliverKickLocked(agent, message, trigger) } // deliverKickLocked types a message into the agent's CLI and records the @@ -257,6 +263,7 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string return fmt.Errorf("%w: agent %s kick delivery already in progress", crux.ErrAgentBusy, agent.Name) } defer agent.kickDelivering.Store(false) + agent.lastKickInput = message visible := m.captureVisiblePaneForAgent(agent) if err := m.checkKickPromptReadyLocked(agent, visible, trigger); err != nil { @@ -420,6 +427,7 @@ func (m *Manager) recordDeliveredKickLocked(agent *AgentProcess, message, trigge agent.LastKickMessage = message agent.KickRefused = false agent.KickRefusalReason = "" + agent.canaryRetryDone = trigger == kickTriggerCanaryRetry // The session now holds this kick's output; the next rotation point // (kick, restart, shutdown) must archive it before it is destroyed. agent.kickLogPending = true diff --git a/src/crux/agent/manager_poll.go b/src/crux/agent/manager_poll.go index a620e31f25..e02a2907c0 100644 --- a/src/crux/agent/manager_poll.go +++ b/src/crux/agent/manager_poll.go @@ -586,6 +586,7 @@ func (m *Manager) checkKickRefusal(agent *AgentProcess, line string) { "pattern", pattern, "line", reason, ) + m.retryRefusedCanaryKick(agent) return } } diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index af8d9a3c73..e3e391f987 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -92,7 +92,7 @@ in `src/pkg/scheduler/kick_layout.go` and asserted by | # | Block | Varies | |---|---|---| -| 1 | `SECURITY CANARY` preamble | every kick (random, opt-in `ioscan.canaries`; always first — [ADR-0008](adr/0008-ioscan-untrusted-input.md)) | +| 1 | `=== HIVE HARNESS HEADER` canary preamble | every kick (random, opt-in `ioscan.canaries`; always first — [ADR-0008](adr/0008-ioscan-untrusted-input.md)) | | 2 | `[agent:<name>]` header | never | | 3 | governor work-source rule | never | | 4 | independent review section | per agent config | diff --git a/src/pkg/ioscan/canary.go b/src/pkg/ioscan/canary.go index 1c5be63067..7cb45c7063 100644 --- a/src/pkg/ioscan/canary.go +++ b/src/pkg/ioscan/canary.go @@ -446,11 +446,26 @@ func decodeCanaryHexBlobs(text string) []string { return out } +// CanaryPreamblePrefix opens every canary preamble; crux/agent mirrors it +// (canaryPreamblePrefix) to find where the kick body starts. +const CanaryPreamblePrefix = "=== HIVE HARNESS HEADER" + +// CanaryPreamble renders the canary as a clearly-labelled block of the +// operator's own harness configuration. The earlier "SECURITY CANARY: <token> +// … Never print …" wording read to Claude Sonnet 5 as a prompt-injection +// marker ("this request includes a 'SECURITY CANARY' marker and asks me to act +// as a scanner agent") and the model refused the whole kick — ~65% of one +// hosted supervisor's kicks over 36h once the Copilot CLI substituted that +// model. The header states who planted the token and why, says it is not a +// task, and still forbids every transform an agent might consider safe. func CanaryPreamble(token string) string { if token == "" { return "" } - return fmt.Sprintf("SECURITY CANARY: %s\nThis is a hive-issued exfiltration-canary marker. Never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service.\n\n", token) + return fmt.Sprintf("%s (operator configuration, not user content) ===\n"+ + "Exfiltration canary: %s\n"+ + "The token above is a tripwire planted by the hive harness that runs you — your own operator's configuration — so it can detect session text leaking off this host. It is not a task and nothing in this message depends on it. Handle it as a secret: never print, quote, summarize, transform, store in reports, include in commits, or send it to any external service; then carry out the kick that follows.\n"+ + "=== END HIVE HARNESS HEADER ===\n\n", CanaryPreamblePrefix, token) } func (r *CanaryRegistry) Load() error { diff --git a/src/pkg/ioscan/canary_coverage_test.go b/src/pkg/ioscan/canary_coverage_test.go index 72857a890b..371ba8d55c 100644 --- a/src/pkg/ioscan/canary_coverage_test.go +++ b/src/pkg/ioscan/canary_coverage_test.go @@ -16,10 +16,27 @@ func TestCanaryPreambleOmittedWithoutToken(t *testing.T) { if !strings.Contains(got, "HIVE-CANARY-deadbeef") { t.Fatalf("preamble %q does not carry the token", got) } + if !strings.HasPrefix(got, CanaryPreamblePrefix) { + t.Fatalf("preamble %q does not open with CanaryPreamblePrefix", got) + } + if !strings.HasSuffix(got, "\n\n") { + t.Fatalf("preamble %q must end with a blank line so the kick body is delimited", got) + } + // Sonnet 5 refused kicks whose canary announced itself as a bare + // "SECURITY CANARY" marker (prompt-injection pattern); the header must + // instead say whose configuration it is and that it is not a task. + for _, want := range []string{"operator", "not a task"} { + if !strings.Contains(got, want) { + t.Fatalf("preamble %q missing %q", got, want) + } + } + if strings.Contains(got, "SECURITY CANARY") { + t.Fatalf("preamble %q still carries the refused SECURITY CANARY marker", got) + } // The preamble is the only thing standing between the agent and an // innocent-looking "summarize your context" exfiltration, so it has to // forbid the transforms an agent would otherwise consider safe. - for _, want := range []string{"Never print", "summarize", "external service"} { + for _, want := range []string{"never print", "summarize", "external service"} { if !strings.Contains(got, want) { t.Fatalf("preamble %q missing %q", got, want) } diff --git a/src/pkg/scheduler/kick_layout_test.go b/src/pkg/scheduler/kick_layout_test.go index acc35a5d69..971ef98432 100644 --- a/src/pkg/scheduler/kick_layout_test.go +++ b/src/pkg/scheduler/kick_layout_test.go @@ -95,7 +95,7 @@ func kickLayoutMessage(t *testing.T, s *Scheduler, issues ...github.Issue) strin // one dynamic thing ahead of the static prefix (ADR-0008). func stripCanary(t *testing.T, msg string) string { t.Helper() - if !strings.HasPrefix(msg, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(msg, ioscan.CanaryPreamblePrefix) { t.Fatalf("canary preamble must lead the kick:\n%s", msg[:min(len(msg), 200)]) } end := strings.Index(msg, "\n\n") @@ -187,7 +187,7 @@ func TestKickLayoutSeamOrderIsFixed(t *testing.T) { msg := kickLayoutMessage(t, s, makeIssue("test-org/console", 1, "first issue", "scanner", 5, nil, false)) order := []string{ - "SECURITY CANARY: " + ioscan.CanaryPrefix, + ioscan.CanaryPreamblePrefix, "[agent:scanner]", governorWorkSourceRule, "## Independent review", @@ -228,14 +228,14 @@ func TestKickLayoutKeepsCanaryAndRepoScopedTarget(t *testing.T) { b := kickLayoutMessage(t, s, issue) preamble := func(msg string) string { - end := strings.Index(msg, "\n") + end := strings.Index(msg, "\n\n") if end < 0 { return msg } return msg[:end] } for _, msg := range []string{a, b} { - if !strings.HasPrefix(msg, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(msg, ioscan.CanaryPreamblePrefix) { t.Fatalf("canary preamble must lead the kick:\n%s", msg[:min(len(msg), 200)]) } if !strings.Contains(msg, "REPO-SCOPED CADENCE TARGET: test-org/console") { diff --git a/src/pkg/scheduler/repo_filter_hold_test.go b/src/pkg/scheduler/repo_filter_hold_test.go index 1427e09c3a..9fbf4b3562 100644 --- a/src/pkg/scheduler/repo_filter_hold_test.go +++ b/src/pkg/scheduler/repo_filter_hold_test.go @@ -172,7 +172,7 @@ func TestAddCanaryPreambleEnabledPrependsRegisteredToken(t *testing.T) { const msg = "kick body" got := s.addCanaryPreamble("scanner", msg) - if !strings.HasPrefix(got, "SECURITY CANARY: "+ioscan.CanaryPrefix) { + if !strings.HasPrefix(got, ioscan.CanaryPreamblePrefix) { t.Fatalf("preamble missing canary header: %q", got) } if !strings.HasSuffix(got, msg) { From 72f60ee5dc491769ef7cfe421fdb46156443962e Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:53:41 +0000 Subject: [PATCH 122/166] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20stop?= =?UTF-8?q?=20canceled=20contributor=20sweep=20passes=20(#18003)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(automerge): stop canceled contributor sweep passes Stop a contributor pass interrupted by its canceled generation context instead of reporting a failure and reconciling stages on that context. Preserve warnings for other errors. Hive-Run: hivecommons/hive#17949 Hive-Plan: contributor-sweep-cancellation Hive-Spec: fleet-report#unclassified-error-repeated Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> * docs: clarify cancellation exclusion context Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --------- Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: Karibhou <157208370+Karibhou@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-contributor-sweep-cancellation.md | 1 + src/docs/fleet-report.md | 4 ++ .../automerge_sweep_cancellation_test.go | 57 +++++++++++++++++++ .../github/automerge/automerge_sweep_loop.go | 9 ++- 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-contributor-sweep-cancellation.md create mode 100644 src/pkg/github/automerge/automerge_sweep_cancellation_test.go diff --git a/changelog.d/fixed-contributor-sweep-cancellation.md b/changelog.d/fixed-contributor-sweep-cancellation.md new file mode 100644 index 0000000000..be00f6d940 --- /dev/null +++ b/changelog.d/fixed-contributor-sweep-cancellation.md @@ -0,0 +1 @@ +- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index db5eb42f78..7c0b9a20dc 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -178,6 +178,10 @@ logger-bound attributes: typed errors wrapping `context.Canceled`, exact Warnings remain in local logs; deadlines and other errors remain reportable, including when another `error` attribute accompanies a cancellation. +When a client hand-over or shutdown cancels an in-flight contributor-lane +automerge pass, the pass stops without warning or stage reconciliation. Other +contributor sweep failures remain reportable. + The GitHub rate-limit exclusion also covers the local REST reserve guard: when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. diff --git a/src/pkg/github/automerge/automerge_sweep_cancellation_test.go b/src/pkg/github/automerge/automerge_sweep_cancellation_test.go new file mode 100644 index 0000000000..8de5fd12c3 --- /dev/null +++ b/src/pkg/github/automerge/automerge_sweep_cancellation_test.go @@ -0,0 +1,57 @@ +package automerge + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestContributorSweepPassCancellation(t *testing.T) { + for _, tc := range []struct { + name string + cancel bool + }{ + {name: "generation canceled", cancel: true}, + {name: "github failure"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var requests atomic.Int32 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if tc.cancel { + cancel() + <-r.Context().Done() + return + } + http.Error(w, "upstream failure", http.StatusInternalServerError) + })) + defer api.Close() + + var logs bytes.Buffer + clock := &contributorLaneClock{at: time.Date(2026, 10, 15, 13, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngine(api.URL, defaultContributorLanePolicy(), clock, Options{ + Logger: slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + result := engine.sweepPass(ctx, 1, false, nil, "tick") + if requests.Load() == 0 { + t.Fatal("contributor lane did not attempt to list pull requests") + } + failed := strings.Contains(logs.String(), "contributor-lane automerge sweep failed") + if tc.cancel { + if result != nil || failed || strings.Contains(logs.String(), "stage labels reconcile failed") { + t.Fatalf("canceled pass: result=%+v logs=%s", result, logs.String()) + } + } else if result == nil || !failed { + t.Fatalf("GitHub failure must remain visible: result=%+v logs=%s", result, logs.String()) + } + }) + } +} diff --git a/src/pkg/github/automerge/automerge_sweep_loop.go b/src/pkg/github/automerge/automerge_sweep_loop.go index 636a529c9b..a5e9740c5a 100644 --- a/src/pkg/github/automerge/automerge_sweep_loop.go +++ b/src/pkg/github/automerge/automerge_sweep_loop.go @@ -735,7 +735,8 @@ func (c *Engine) StartSelfAuthoredAutoMergeSweep(ctx context.Context, maxMerges // sweepPass runs the three lanes once, in order, sharing one merge budget, // then reconciles stage labels. targets narrows an event-driven pass to the // PRs a webhook named (nil: everything); cause names what started the pass -// for the log. It returns nil when the self-authored lane failed outright +// for the log. It returns nil when the self-authored lane failed outright or +// the contributor lane was interrupted by cancellation of the sweep context // (its callers then keep the current cadence). func (c *Engine) sweepPass(ctx context.Context, maxMerges int, acmmAllowed bool, targets SweepTargets, cause string) *AutoMergeSweepResult { c.sweepClock.begin(cause, targets, c.now()) @@ -784,6 +785,12 @@ func (c *Engine) sweepPass(ctx context.Context, maxMerges int, acmmAllowed bool, if remaining > 0 && c.currentContributorLanePolicy().Enabled { laneResult, laneErr := c.SweepContributorLaneAutoMerges(ctx, AutoMergeSweepOptions{MaxMerges: remaining, Targets: targets}) if laneErr != nil { + // A client hand-over or shutdown cancels the whole sweep. Do not + // report that expected interruption as a failed contributor pass, + // or start stage reconciliation on the canceled context. + if ctx.Err() == context.Canceled && errors.Is(laneErr, context.Canceled) { + return nil + } c.warn("contributor-lane automerge sweep failed", "error", laneErr, "cause", cause) if isRateLimited(laneErr) { c.refreshRateLimitCache(ctx) From 296783b71e2188008a260dfc0417475cdc6aff93 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:54:42 +0000 Subject: [PATCH 123/166] =?UTF-8?q?=E2=9C=A8=20review:=20add=20perspsrc=20?= =?UTF-8?q?resolver=20with=20last-known-good=20fallback=20(#18011)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-17987-review-perspsrc-resolver.md | 1 + src/cmd/hive/main_sweeps.go | 6 +- src/cmd/hive/review_config_parsers_test.go | 20 ++ src/pkg/perspsrc/resolver.go | 184 ++++++++++++++++++ src/pkg/perspsrc/resolver_test.go | 176 +++++++++++++++++ src/pkg/scheduler/policy_overlays.go | 2 +- 6 files changed, 386 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-17987-review-perspsrc-resolver.md create mode 100644 src/pkg/perspsrc/resolver.go create mode 100644 src/pkg/perspsrc/resolver_test.go diff --git a/changelog.d/added-17987-review-perspsrc-resolver.md b/changelog.d/added-17987-review-perspsrc-resolver.md new file mode 100644 index 0000000000..f3bbd4c8eb --- /dev/null +++ b/changelog.d/added-17987-review-perspsrc-resolver.md @@ -0,0 +1 @@ +- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index ab9064d0ab..7487b7007d 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -2263,7 +2263,9 @@ func applyHumanDecisionLabels(ctx context.Context, cfg *config.Config, ghClient // silently becomes the current time would re-open every verdict in the // artifact at once — the opposite of the narrow, deliberate correction this // setting exists for. -// reviewPerspectiveSet resolves the hive's configured review perspectives. +// reviewPerspectiveSet resolves the hive's configured review perspectives: the +// set linked from review.perspectives_source when one has resolved, otherwise +// the inline review.perspectives (ReviewConfig.EffectivePerspectives). // // A bad configuration falls back to the built-in set and says so, rather than // disabling review. Silently reviewing nothing would look identical to a healthy @@ -2273,7 +2275,7 @@ func reviewPerspectiveSet(cfg *config.Config, logger *slog.Logger) review.Perspe if cfg == nil { return review.PerspectiveSet{} } - set, err := review.NewPerspectiveSet(cfg.Review.Perspectives, cfg.Review.PerspectivePrompts) + set, err := review.NewPerspectiveSet(cfg.Review.EffectivePerspectives()) if err != nil { if logger != nil { logger.Warn("review.perspectives is invalid; reviewing with the built-in set until it is fixed", diff --git a/src/cmd/hive/review_config_parsers_test.go b/src/cmd/hive/review_config_parsers_test.go index 1c0dc67eab..ad5a24d3cd 100644 --- a/src/cmd/hive/review_config_parsers_test.go +++ b/src/cmd/hive/review_config_parsers_test.go @@ -94,6 +94,26 @@ func TestReviewPerspectiveSet(t *testing.T) { } }) + t.Run("linked perspectives win over inline", func(t *testing.T) { + cfg := &config.Config{} + cfg.Review.Perspectives = []string{"correctness"} + cfg.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "torch-spyre", Repo: "spyre-inference", Path: ".hive/review-perspectives.yaml"} + cfg.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: []string{"security", "spyre-numerics"}, + Focus: map[string]string{"spyre-numerics": "Silent CPU fallbacks and fp16 accumulation-order changes."}, + } + got := reviewPerspectiveSet(cfg, nil).List() + want := []review.Perspective{review.PerspectiveSecurity, review.Perspective("spyre-numerics")} + if len(got) != len(want) { + t.Fatalf("List() = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("List()[%d] = %q, want %q", i, got[i], want[i]) + } + } + }) + t.Run("invalid selection with nil logger does not panic", func(t *testing.T) { cfg := &config.Config{} cfg.Review.Perspectives = []string{"sekurity"} diff --git a/src/pkg/perspsrc/resolver.go b/src/pkg/perspsrc/resolver.go new file mode 100644 index 0000000000..747ce35c4c --- /dev/null +++ b/src/pkg/perspsrc/resolver.go @@ -0,0 +1,184 @@ +package perspsrc + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/defsrc" +) + +// defaultFetchTimeout bounds a single live fetch so a slow/hung GitHub API call +// cannot stall a reload. On timeout the resolver keeps the last-known-good set. +const defaultFetchTimeout = 8 * time.Second + +// Resolver fetches review perspective documents live and remembers the last +// good document per source so a later failure keeps that set in effect. It is +// modelled on defsrc.Resolver and shares its Fetcher, AllowFunc and Logger. +// Safe for concurrent use. +type Resolver struct { + fetcher defsrc.Fetcher + allow defsrc.AllowFunc + logger defsrc.Logger + timeout time.Duration + + mu sync.RWMutex + cache map[string]string // source key -> last-known-good raw YAML +} + +// NewResolver builds a Resolver. fetcher may be nil (token-mode boot without an +// App client), in which case only a cached document can resolve. Callers wire +// allow to the seed-only gate (config.Config.GitHubPromptAllowed / +// GitHubDefinitionAllowed); a nil allow denies everything (fail closed). +// logger may be nil. +func NewResolver(fetcher defsrc.Fetcher, allow defsrc.AllowFunc, logger defsrc.Logger) *Resolver { + if allow == nil { + allow = func(string) bool { return false } + } + return &Resolver{ + fetcher: fetcher, + allow: allow, + logger: logger, + timeout: defaultFetchTimeout, + cache: map[string]string{}, + } +} + +// Result carries a resolved perspective set plus provenance. Ok is false when +// nothing could be resolved (not set, denied, no fetcher, unreachable or +// malformed with no last-known-good); the caller then falls back to inline +// review.perspectives. Warning is set when the latest fetch or document failed, +// whether or not a last-known-good set is still in effect. +type Result struct { + Names []string + Focus map[string]string + Ok bool + Source string // "github:owner/repo@ref", "github-cache:owner/repo", "denied", "unset", "no-client", "error" + Warning string +} + +// Resolve fetches the perspective document for src. It never returns an error: +// a reload must proceed even when the source is unreachable or malformed. A +// document that fails validation keeps the last-known-good set (with a +// Warning) rather than silently falling back to the built-in set. +func (r *Resolver) Resolve(ctx context.Context, src config.DefinitionSourceConfig) Result { + if !src.IsSet() { + return Result{Source: "unset"} + } + slug := src.Slug() + if !r.allow(slug) { + if r.logger != nil { + r.logger.Warn("review perspectives source repo not allowlisted, ignoring (keeping inline review.perspectives)", "repo", slug) + } + return Result{Source: "denied"} + } + if r.fetcher == nil { + if res, ok := r.fromCache(src, ""); ok { + return res + } + return Result{Source: "no-client"} + } + + fetchCtx := ctx + if ctx == nil { + fetchCtx = context.Background() + } + fetchCtx, cancel := context.WithTimeout(fetchCtx, r.timeout) + defer cancel() + + raw, err := r.fetcher.GetFileContentRef(fetchCtx, src.Owner, src.Repo, src.Path, src.Ref) + if err != nil { + warning := fmt.Sprintf("fetching review perspectives from %s/%s failed: %v", slug, src.Path, err) + if r.logger != nil { + r.logger.Warn("failed to fetch review perspectives from GitHub, keeping last-known-good", "repo", slug, "path", src.Path, "error", err) + } + if res, ok := r.fromCache(src, warning); ok { + return res + } + return Result{Source: "error", Warning: warning} + } + names, focus, err := ParseDocument(raw) + if err != nil { + warning := fmt.Sprintf("review perspectives document %s/%s is invalid: %v", slug, src.Path, err) + if r.logger != nil { + r.logger.Warn("review perspectives from GitHub are malformed, keeping last-known-good", "repo", slug, "path", src.Path, "error", err) + } + if res, ok := r.fromCache(src, warning); ok { + return res + } + return Result{Source: "error", Warning: warning} + } + // Only a document that parsed cleanly is cached, so a later failure keeps + // a known-good set, never a corrupt one. + r.store(src, raw) + return Result{Names: names, Focus: focus, Ok: true, Source: provenance(src)} +} + +// fromCache resolves the last-known-good document for src, if any. It was +// validated when cached, so a parse failure here is not expected; it is still +// treated as a miss rather than trusted. +func (r *Resolver) fromCache(src config.DefinitionSourceConfig, warning string) (Result, bool) { + raw, ok := r.cached(src) + if !ok { + return Result{}, false + } + names, focus, err := ParseDocument(raw) + if err != nil { + return Result{}, false + } + return Result{Names: names, Focus: focus, Ok: true, Source: "github-cache:" + src.Slug(), Warning: warning}, true +} + +func provenance(src config.DefinitionSourceConfig) string { + ref := src.Ref + if ref == "" { + ref = "default" + } + return fmt.Sprintf("github:%s@%s", src.Slug(), ref) +} + +func sourceKey(src config.DefinitionSourceConfig) string { + return src.Owner + "\x00" + src.Repo + "\x00" + src.Path + "\x00" + src.Ref +} + +func (r *Resolver) cached(src config.DefinitionSourceConfig) (string, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + raw, ok := r.cache[sourceKey(src)] + return raw, ok +} + +func (r *Resolver) store(src config.DefinitionSourceConfig, raw string) { + r.mu.Lock() + defer r.mu.Unlock() + r.cache[sourceKey(src)] = raw +} + +// ApplyToConfig resolves cfg.Review.PerspectivesSource and records the result +// on cfg.Review.LinkedPerspectives, with its provenance and any warning. When +// nothing resolves, LinkedPerspectives is left nil so EffectivePerspectives +// falls back to inline review.perspectives. A nil cfg or resolver is a no-op. +func ApplyToConfig(ctx context.Context, cfg *config.Config, resolver *Resolver, logger defsrc.Logger) { + if cfg == nil || resolver == nil { + return + } + cfg.Review.LinkedPerspectives = nil + if !cfg.Review.PerspectivesSource.IsSet() { + return + } + res := resolver.Resolve(ctx, *cfg.Review.PerspectivesSource) + if !res.Ok { + return + } + cfg.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: res.Names, + Focus: res.Focus, + Source: res.Source, + Warning: res.Warning, + } + if logger != nil { + logger.Info("applied live review perspectives", "source", res.Source, "perspectives", len(res.Names)) + } +} diff --git a/src/pkg/perspsrc/resolver_test.go b/src/pkg/perspsrc/resolver_test.go new file mode 100644 index 0000000000..451856888f --- /dev/null +++ b/src/pkg/perspsrc/resolver_test.go @@ -0,0 +1,176 @@ +package perspsrc + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +type stubFetcher struct { + docs []string + errs []error + calls int +} + +func (f *stubFetcher) GetFileContentRef(_ context.Context, _, _, _, _ string) (string, error) { + i := f.calls + f.calls++ + var err error + if i < len(f.errs) { + err = f.errs[i] + } + if err != nil { + return "", err + } + if i < len(f.docs) { + return f.docs[i], nil + } + return "", errors.New("no document") +} + +type recLogger struct{ warns int } + +func (l *recLogger) Warn(string, ...any) { l.warns++ } +func (l *recLogger) Info(string, ...any) {} + +func allowAll(string) bool { return true } + +var testSource = config.DefinitionSourceConfig{Owner: "torch-spyre", Repo: "spyre-inference", Path: ".hive/review-perspectives.yaml"} + +const goodDoc = `kind: ReviewPerspectives +perspectives: + - name: security + - name: spyre-numerics + focus: "Silent CPU fallbacks." +` + +// A custom perspective without focus text fails NewPerspectiveSet validation. +const invalidDoc = `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: undescribed-custom +` + +func TestResolverDeniedNotAllowlisted(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc}} + log := &recLogger{} + r := NewResolver(f, func(string) bool { return false }, log) + res := r.Resolve(context.Background(), testSource) + if res.Ok || res.Source != "denied" { + t.Fatalf("Resolve = %+v, want Ok=false Source=denied", res) + } + if f.calls != 0 { + t.Errorf("fetcher called %d times for a denied repo, want 0", f.calls) + } + if log.warns == 0 { + t.Error("expected a Warn log for a denied repo") + } + + // A nil allow fails closed. + if res := NewResolver(f, nil, nil).Resolve(context.Background(), testSource); res.Ok || res.Source != "denied" { + t.Errorf("nil allow Resolve = %+v, want denied", res) + } +} + +func TestResolverNoClientUsesCache(t *testing.T) { + r := NewResolver(nil, allowAll, nil) + if res := r.Resolve(context.Background(), testSource); res.Ok || res.Source != "no-client" { + t.Fatalf("empty-cache Resolve = %+v, want Ok=false Source=no-client", res) + } + + r.store(testSource, goodDoc) + res := r.Resolve(context.Background(), testSource) + if !res.Ok || res.Source != "github-cache:torch-spyre/spyre-inference" { + t.Fatalf("cached Resolve = %+v, want Ok=true from github-cache", res) + } + if want := []string{"security", "spyre-numerics"}; !reflect.DeepEqual(res.Names, want) { + t.Errorf("Names = %v, want %v", res.Names, want) + } +} + +func TestResolverFetchErrorKeepsLastKnownGood(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc}, errs: []error{nil, errors.New("boom")}} + r := NewResolver(f, allowAll, nil) + first := r.Resolve(context.Background(), testSource) + if !first.Ok || first.Source != "github:torch-spyre/spyre-inference@default" { + t.Fatalf("first Resolve = %+v, want Ok from github", first) + } + second := r.Resolve(context.Background(), testSource) + if !second.Ok || second.Source != "github-cache:torch-spyre/spyre-inference" { + t.Fatalf("second Resolve = %+v, want Ok from github-cache", second) + } + if !reflect.DeepEqual(second.Names, first.Names) || !reflect.DeepEqual(second.Focus, first.Focus) { + t.Errorf("second = %v %v, want last-known-good %v %v", second.Names, second.Focus, first.Names, first.Focus) + } + + noCache := NewResolver(&stubFetcher{errs: []error{errors.New("boom")}}, allowAll, nil) + if res := noCache.Resolve(context.Background(), testSource); res.Ok || res.Source != "error" { + t.Errorf("no-cache fetch error Resolve = %+v, want Ok=false Source=error", res) + } +} + +func TestResolverMalformedKeepsLastKnownGoodAndWarns(t *testing.T) { + f := &stubFetcher{docs: []string{goodDoc, invalidDoc}} + log := &recLogger{} + r := NewResolver(f, allowAll, log) + first := r.Resolve(context.Background(), testSource) + if !first.Ok || first.Warning != "" { + t.Fatalf("first Resolve = %+v, want Ok with no warning", first) + } + second := r.Resolve(context.Background(), testSource) + if !second.Ok { + t.Fatalf("second Resolve = %+v, want last-known-good kept", second) + } + if !reflect.DeepEqual(second.Names, first.Names) { + t.Errorf("Names = %v, want last-known-good %v", second.Names, first.Names) + } + if second.Warning == "" { + t.Error("expected a Warning for the malformed document") + } + if log.warns == 0 { + t.Error("expected a Warn log for the malformed document") + } +} + +func TestResolverMalformedNoCacheFallsBackToInline(t *testing.T) { + r := NewResolver(&stubFetcher{docs: []string{invalidDoc}}, allowAll, nil) + res := r.Resolve(context.Background(), testSource) + if res.Ok { + t.Fatalf("Resolve = %+v, want Ok=false", res) + } + if res.Warning == "" { + t.Error("expected a Warning for the malformed document") + } + + cfg := &config.Config{} + cfg.Review.Perspectives = []string{"correctness"} + src := testSource + cfg.Review.PerspectivesSource = &src + ApplyToConfig(context.Background(), cfg, NewResolver(&stubFetcher{docs: []string{invalidDoc}}, allowAll, nil), nil) + if cfg.Review.LinkedPerspectives != nil { + t.Fatalf("LinkedPerspectives = %+v, want nil", cfg.Review.LinkedPerspectives) + } + if names, _ := cfg.Review.EffectivePerspectives(); !reflect.DeepEqual(names, []string{"correctness"}) { + t.Errorf("EffectivePerspectives names = %v, want inline [correctness]", names) + } +} + +func TestApplyToConfigRecordsProvenance(t *testing.T) { + cfg := &config.Config{} + src := testSource + cfg.Review.PerspectivesSource = &src + ApplyToConfig(context.Background(), cfg, NewResolver(&stubFetcher{docs: []string{goodDoc}}, allowAll, nil), nil) + lp := cfg.Review.LinkedPerspectives + if lp == nil { + t.Fatal("LinkedPerspectives = nil, want resolved set") + } + if lp.Source != "github:torch-spyre/spyre-inference@default" || lp.Warning != "" { + t.Errorf("LinkedPerspectives = %+v, want github provenance and no warning", lp) + } + if names, _ := cfg.Review.EffectivePerspectives(); !reflect.DeepEqual(names, []string{"security", "spyre-numerics"}) { + t.Errorf("EffectivePerspectives names = %v, want linked set", names) + } +} diff --git a/src/pkg/scheduler/policy_overlays.go b/src/pkg/scheduler/policy_overlays.go index d551ac96aa..21b9dbb712 100644 --- a/src/pkg/scheduler/policy_overlays.go +++ b/src/pkg/scheduler/policy_overlays.go @@ -770,7 +770,7 @@ func (s *Scheduler) addIndependentReviewSection(agentName, msg string) string { // reported"). Falls back to the built-in set when the configured one does not // validate, the same fallback the relay applies. func (s *Scheduler) reviewPerspectivesSection() string { - set, err := review.NewPerspectiveSet(s.cfg.Review.Perspectives, s.cfg.Review.PerspectivePrompts) + set, err := review.NewPerspectiveSet(s.cfg.Review.EffectivePerspectives()) if err != nil { set = review.PerspectiveSet{} } From c0a5dd622a397e420da3284e79e2e1fe8f3ee658 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 15:58:41 -0700 Subject: [PATCH 124/166] =?UTF-8?q?=F0=9F=90=9B=20Activity=20monitor:=20co?= =?UTF-8?q?lumn=20header=20follows=20the=20rows'=20workflow=20model=20(#18?= =?UTF-8?q?005)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Activity monitor: column header follows the rows' workflow model The sticky column header was always hive-default's names, even when every row's chevron strip wore a bound workflow model's columns. Derive the header from the same model the strips use: the sticky header is the lead row's model, an ungrouped page is partitioned (stably) by model with a header per run, and a hive-default group under a model's sticky header draws hive-default's names. Hives with no binding render exactly as before. Fixes #18002 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 tests: pin the sticky header's headHtml wiring Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...activity-monitor-workflow-column-header.md | 1 + src/pkg/dashboard/stage_tracker_cues_test.go | 2 +- .../stage_tracker_workflows_ui_test.go | 41 ++++++++++- .../static-src/js/tracker-filter-render.js | 15 +++- .../static-src/js/tracker-row-render.js | 2 +- .../static-src/js/tracker-workflows.js | 56 ++++++++++++++ src/pkg/dashboard/static/index.html | 73 ++++++++++++++++++- 7 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-activity-monitor-workflow-column-header.md diff --git a/changelog.d/fixed-activity-monitor-workflow-column-header.md b/changelog.d/fixed-activity-monitor-workflow-column-header.md new file mode 100644 index 0000000000..e868548c15 --- /dev/null +++ b/changelog.d/fixed-activity-monitor-workflow-column-header.md @@ -0,0 +1 @@ +- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). diff --git a/src/pkg/dashboard/stage_tracker_cues_test.go b/src/pkg/dashboard/stage_tracker_cues_test.go index a691119203..0aeee73a5a 100644 --- a/src/pkg/dashboard/stage_tracker_cues_test.go +++ b/src/pkg/dashboard/stage_tracker_cues_test.go @@ -106,7 +106,7 @@ func TestStageTrackerCuesWiring(t *testing.T) { `stageTrackerNoteEvents(data); stageTrackerResultToastsSync(data); stageTrackerPendingExpire(data);`, `stageTrackerCueAgeSync(el, nowMs);`, `function celebrateRow(rowEl, opts) {`, - `const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }]`, + `const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }]`, `if (!visible.length && !Object.keys(_stageTrackerCelebrating).length) {`, `data-change-action="stageTrackerCelebrateChange"`, } { diff --git a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go index 1e2d2a1ce9..3b9437a5f1 100644 --- a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go +++ b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go @@ -22,7 +22,13 @@ func TestStageTrackerWorkflowColumnsWiring(t *testing.T) { `const rowWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, item.repo) : null;`, `${rowWorkflow ? stageTrackerWorkflowStripItems(shown, rowWorkflow, stages) : stageTrackerStripItems(`, `const groupWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, g.repo) : null;`, - `${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`, + `stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx)`, + // The sticky header and the ungrouped run headers follow the model + // the rows wear, never hive-default's names over model chevrons. + `const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null;`, + `ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null;`, + `stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages)`, + `stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages)`, `.ham-wf-badge {`, `.ham-seg.ham-wf-uncovered {`, // Settings → Repos binding select. @@ -53,7 +59,9 @@ const STAGE_TRACKER_TERMINAL = 110; function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } ` + jsFunc(t, html, "stageTrackerWorkflowOf") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + jsFunc(t, html, "stageTrackerWorkflowBadgeHtml") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + - jsFunc(t, html, "stageTrackerWorkflowStripItems") + ` + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + + jsFunc(t, html, "stageTrackerColumnsHeaderHtml") + jsFunc(t, html, "stageTrackerWorkflowRuns") + + jsFunc(t, html, "stageTrackerWorkflowRunHeaderRows") + jsFunc(t, html, "stageTrackerGroupColumnsHtml") + ` const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; const def = { name: 'hive-default', version: '1', columns: [{ id: 'open', label: 'Open', states: ['open'], numbers: [] }] }; const td = { name: 'cncf-techdocs', version: '1', columns: [ @@ -94,6 +102,35 @@ const head = stageTrackerWorkflowHeaderHtml(td); assert.ok(head.includes('data-ham-workflow="cncf-techdocs@1"') && head.includes('title="Asked for"') && li(head).length === 4, head); assert.ok(stageTrackerWorkflowBadgeHtml(td).includes('>cncf-techdocs@1<')); assert.strictEqual(stageTrackerWorkflowBadgeHtml(null), ''); + +// The header over a strip is the strip's own model: the sticky header is +// the lead row's, an ungrouped page is partitioned by model with a header +// per run, and a hive-default group under a model's sticky header draws +// hive-default's names (a hive with no binding: one run, no headers added). +const STAGE_TRACKER_GROUP_KEY_PREFIX = 'group:'; +const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; +function stageTrackerHeaderHtml() { return '<div class="ham-cols default"></div>'; } +function stageTrackerItemKey(it) { return (it.repo || '') + '/' + (it.kind || '') + '#' + (Number(it.number) || 0); } +const rows = [ + { repo: 'cncf/plain', kind: 'issue', number: 1 }, { repo: 'cncf/techdocs', kind: 'issue', number: 2 }, + { repo: 'cncf/plain', kind: 'issue', number: 3 }, { repo: 'cncf/techdocs', kind: 'pr', number: 4 }, +]; +assert.strictEqual(stageTrackerColumnsHeaderHtml(null, []), stageTrackerHeaderHtml()); +assert.strictEqual(stageTrackerColumnsHeaderHtml(td, []), head); +let runs = stageTrackerWorkflowRuns(rows, ctx); +assert.deepStrictEqual(runs.map(r => [r.key, r.rows.map(x => x.number)]), [['', [1, 3]], ['cncf-techdocs@1', [2, 4]]], 'stable, first-seen partition'); +assert.strictEqual(runs[1].wf, td); +assert.strictEqual(stageTrackerWorkflowRuns(rows, {}).length, 1, 'no binding → one run'); +assert.strictEqual(stageTrackerWorkflowRuns(rows.filter(r => r.repo === 'cncf/techdocs'), ctx).length, 1, 'one model → one run'); +const keyed = runs.reduce((all, r) => all.concat(r.rows), []).map(it => ({ key: stageTrackerItemKey(it), html: 'row' })); +const withHeads = stageTrackerWorkflowRunHeaderRows(keyed, runs, []); +assert.deepStrictEqual(withHeads.map(r => r.key), ['cncf/plain/issue#1', 'cncf/plain/issue#3', 'group:workflow:cncf-techdocs@1', 'cncf/techdocs/issue#2', 'cncf/techdocs/pr#4']); +assert.strictEqual(withHeads[2].html, head, 'the run header is the model header'); +assert.deepStrictEqual(stageTrackerWorkflowRunHeaderRows(keyed, runs.slice(0, 1), []), keyed, 'one run → untouched'); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, td, [], {}), head); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: true }, td, [], {}), ''); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, null, [], { leadWorkflow: null }), '', 'hive-default group under the hive-default sticky header: nothing to repeat'); +assert.strictEqual(stageTrackerGroupColumnsHtml({ collapsed: false }, null, [], { leadWorkflow: td }), stageTrackerHeaderHtml(), 'hive-default group under a model sticky header draws its own names'); ` if out, err := runNodeStdin(node, script); err != nil { t.Fatalf("node workflow columns fixture failed: %v\n%s", err, out) diff --git a/src/pkg/dashboard/static-src/js/tracker-filter-render.js b/src/pkg/dashboard/static-src/js/tracker-filter-render.js index d90bdceaa9..b2d3e8619f 100644 --- a/src/pkg/dashboard/static-src/js/tracker-filter-render.js +++ b/src/pkg/dashboard/static-src/js/tracker-filter-render.js @@ -117,6 +117,17 @@ ? stageTrackerGroupPage(stageTrackerGroups(ordered, { primary: stageTrackerPrimaryRepo(), collapsed: _stageTrackerCollapsed, stages }), f) : stageTrackerPage(ordered, f); if (!tv && page.page !== f.page) f.page = page.page; + // Rows bound to a workflow model wear its columns (tracker-workflows.js), + // so the header over them must be that model's too (hivecommons/hive#18002: + // hive-default names over cncf-techdocs chevrons). Ungrouped, the + // page is partitioned by model (stable, first-seen order) with a + // header per run; grouped, a group header carries its own columns + // when the sticky header is another model's. The sticky header is + // the lead row's. A hive with no binding resolves one run, no model. + const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null; + if (wfRuns && wfRuns.length > 1) page.rows = wfRuns.reduce((all, run) => all.concat(run.rows), []); + const leadRepo = page.rows.length ? (grouping ? page.rows[0].group.repo : page.rows[0].repo) : ''; + ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null; // A folded secondary is not filtered out: it counts toward the total. if (foldedShared > 0) page.total += foldedShared; // The rows this render draws (this page, group headers aside), for @@ -192,7 +203,9 @@ : itemRow(pageItems[i], i); const trainRows = stageTrackerTrainRows(page.rows, rowOf, typeof stageTrackerTrainWhere === 'function' ? stageTrackerTrainWhere(ordered, d.items) : null); const inlineRows = typeof stageTrackerPreparationInlineRows === 'function' ? stageTrackerPreparationInlineRows(trainRows, pageItems, stages) : trainRows; - const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }].concat(inlineRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); + const runRows = wfRuns && wfRuns.length > 1 ? stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages) : inlineRows; + const headHtml = typeof stageTrackerColumnsHeaderHtml === 'function' ? stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages) : stageTrackerHeaderHtml(stages); + const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }].concat(runRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); const rec = stageTrackerReconcile(el, rows, { effect, hidden: document.hidden }); if (rec.deferred) stageTrackerScheduleSettle(); // Selection is keyed on the row, never baked into its HTML, so a kept diff --git a/src/pkg/dashboard/static-src/js/tracker-row-render.js b/src/pkg/dashboard/static-src/js/tracker-row-render.js index 98c6f556f9..5c394ade3f 100644 --- a/src/pkg/dashboard/static-src/js/tracker-row-render.js +++ b/src/pkg/dashboard/static-src/js/tracker-row-render.js @@ -74,7 +74,7 @@ ${stageTrackerRepoChipHtml(g.repo, chipCtx)}<span class="ham-group-name">${esc(g.repo)}</span>${groupWorkflow ? stageTrackerWorkflowBadgeHtml(groupWorkflow) : ''} <span class="ham-group-n">${g.count} ${g.count === 1 ? 'row' : 'rows'}</span>${attn} ${groupWorkflow ? '' : `<span class="ham-group-dist" role="img" aria-label="${esc(g.cols.map((c, i) => stageTrackerColName(i, stages) + ' ' + c).join(', '))}">${dist}</span>`} - </div>${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`; + </div>${typeof stageTrackerGroupColumnsHtml === 'function' ? stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) : ''}`; } // Row = title line (number, title, chips, since) + thin strip. Non-active diff --git a/src/pkg/dashboard/static-src/js/tracker-workflows.js b/src/pkg/dashboard/static-src/js/tracker-workflows.js index 03b8e6d24a..fa24eacf3d 100644 --- a/src/pkg/dashboard/static-src/js/tracker-workflows.js +++ b/src/pkg/dashboard/static-src/js/tracker-workflows.js @@ -71,6 +71,62 @@ return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } + // Pure: the key a row's model resolves to ('' for hive-default). + function stageTrackerWorkflowKey(wf) { return wf ? wf.name + '@' + wf.version : ''; } + + // Pure: the column header the rows under it follow — the model's own + // when they are bound to one, else hive-default's. The one path for + // the sticky header and the run headers, so the names above a strip + // are always the names its chevrons wear (hivecommons/hive#18002: + // hive-default's "PR in progress / Merge queue" over model chevrons). + function stageTrackerColumnsHeaderHtml(wf, stages) { + return wf ? stageTrackerWorkflowHeaderHtml(wf) : stageTrackerHeaderHtml(stages); + } + + // Pure: an ungrouped page's rows partitioned, stably and in first-seen + // order, by the model each row's repository follows: [{ wf, key, rows }]. + // One run when every row follows the same model (or none does), so a + // hive-default page is left exactly as sorted. + function stageTrackerWorkflowRuns(rows, ctx) { + const runs = []; + const by = {}; + for (const it of rows || []) { + const wf = stageTrackerWorkflowOf(ctx, it.repo); + const key = stageTrackerWorkflowKey(wf); + if (!by[key]) { by[key] = { wf, key, rows: [] }; runs.push(by[key]); } + by[key].rows.push(it); + } + return runs; + } + + // Run headers are keyed under the group prefix so the selection and + // the reconcile treat them as headers, never as item rows. + const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; + + // Pure: the page's keyed rows with a column header spliced in before + // each run after the first (the first run sits under the sticky + // header). A run whose lead row the train/preparation wrappers did not + // keep by key gets no header rather than a misplaced one. + function stageTrackerWorkflowRunHeaderRows(rows, runs, stages) { + const out = (rows || []).slice(); + (runs || []).slice(1).forEach(run => { + const lead = run.rows[0] ? stageTrackerItemKey(run.rows[0]) : null; + const at = out.findIndex(r => r.key === lead); + if (at >= 0) out.splice(at, 0, { key: STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX + (run.key || 'hive-default'), html: stageTrackerColumnsHeaderHtml(run.wf, stages) }); + }); + return out; + } + + // Under a group header: a bound repository's own column header; a + // hive-default repository's only when the sticky header above is a + // model's (ctx.leadWorkflow), so its rows are not read against that + // model's names. Collapsed groups draw none. + function stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) { + if (!g || g.collapsed) return ''; + if (groupWorkflow) return stageTrackerWorkflowHeaderHtml(groupWorkflow); + return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 8a6db5eb16..22024fe971 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -19932,6 +19932,62 @@ <h3 class="faq-group-title">Where to get help</h3> return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } + // Pure: the key a row's model resolves to ('' for hive-default). + function stageTrackerWorkflowKey(wf) { return wf ? wf.name + '@' + wf.version : ''; } + + // Pure: the column header the rows under it follow — the model's own + // when they are bound to one, else hive-default's. The one path for + // the sticky header and the run headers, so the names above a strip + // are always the names its chevrons wear (hivecommons/hive#18002: + // hive-default's "PR in progress / Merge queue" over model chevrons). + function stageTrackerColumnsHeaderHtml(wf, stages) { + return wf ? stageTrackerWorkflowHeaderHtml(wf) : stageTrackerHeaderHtml(stages); + } + + // Pure: an ungrouped page's rows partitioned, stably and in first-seen + // order, by the model each row's repository follows: [{ wf, key, rows }]. + // One run when every row follows the same model (or none does), so a + // hive-default page is left exactly as sorted. + function stageTrackerWorkflowRuns(rows, ctx) { + const runs = []; + const by = {}; + for (const it of rows || []) { + const wf = stageTrackerWorkflowOf(ctx, it.repo); + const key = stageTrackerWorkflowKey(wf); + if (!by[key]) { by[key] = { wf, key, rows: [] }; runs.push(by[key]); } + by[key].rows.push(it); + } + return runs; + } + + // Run headers are keyed under the group prefix so the selection and + // the reconcile treat them as headers, never as item rows. + const STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX = STAGE_TRACKER_GROUP_KEY_PREFIX + 'workflow:'; + + // Pure: the page's keyed rows with a column header spliced in before + // each run after the first (the first run sits under the sticky + // header). A run whose lead row the train/preparation wrappers did not + // keep by key gets no header rather than a misplaced one. + function stageTrackerWorkflowRunHeaderRows(rows, runs, stages) { + const out = (rows || []).slice(); + (runs || []).slice(1).forEach(run => { + const lead = run.rows[0] ? stageTrackerItemKey(run.rows[0]) : null; + const at = out.findIndex(r => r.key === lead); + if (at >= 0) out.splice(at, 0, { key: STAGE_TRACKER_WORKFLOW_HEADER_KEY_PREFIX + (run.key || 'hive-default'), html: stageTrackerColumnsHeaderHtml(run.wf, stages) }); + }); + return out; + } + + // Under a group header: a bound repository's own column header; a + // hive-default repository's only when the sticky header above is a + // model's (ctx.leadWorkflow), so its rows are not read against that + // model's names. Collapsed groups draw none. + function stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) { + if (!g || g.collapsed) return ''; + if (groupWorkflow) return stageTrackerWorkflowHeaderHtml(groupWorkflow); + return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber @@ -20912,7 +20968,7 @@ <h3 class="faq-group-title">Where to get help</h3> ${stageTrackerRepoChipHtml(g.repo, chipCtx)}<span class="ham-group-name">${esc(g.repo)}</span>${groupWorkflow ? stageTrackerWorkflowBadgeHtml(groupWorkflow) : ''} <span class="ham-group-n">${g.count} ${g.count === 1 ? 'row' : 'rows'}</span>${attn} ${groupWorkflow ? '' : `<span class="ham-group-dist" role="img" aria-label="${esc(g.cols.map((c, i) => stageTrackerColName(i, stages) + ' ' + c).join(', '))}">${dist}</span>`} - </div>${groupWorkflow && !g.collapsed ? stageTrackerWorkflowHeaderHtml(groupWorkflow) : ''}`; + </div>${typeof stageTrackerGroupColumnsHtml === 'function' ? stageTrackerGroupColumnsHtml(g, groupWorkflow, stages, ctx) : ''}`; } // Row = title line (number, title, chips, since) + thin strip. Non-active @@ -23545,6 +23601,17 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> ? stageTrackerGroupPage(stageTrackerGroups(ordered, { primary: stageTrackerPrimaryRepo(), collapsed: _stageTrackerCollapsed, stages }), f) : stageTrackerPage(ordered, f); if (!tv && page.page !== f.page) f.page = page.page; + // Rows bound to a workflow model wear its columns (tracker-workflows.js), + // so the header over them must be that model's too (hivecommons/hive#18002: + // hive-default names over cncf-techdocs chevrons). Ungrouped, the + // page is partitioned by model (stable, first-seen order) with a + // header per run; grouped, a group header carries its own columns + // when the sticky header is another model's. The sticky header is + // the lead row's. A hive with no binding resolves one run, no model. + const wfRuns = !grouping && ctx.workflows && typeof stageTrackerWorkflowRuns === 'function' ? stageTrackerWorkflowRuns(page.rows, ctx) : null; + if (wfRuns && wfRuns.length > 1) page.rows = wfRuns.reduce((all, run) => all.concat(run.rows), []); + const leadRepo = page.rows.length ? (grouping ? page.rows[0].group.repo : page.rows[0].repo) : ''; + ctx.leadWorkflow = typeof stageTrackerWorkflowOf === 'function' ? stageTrackerWorkflowOf(ctx, leadRepo) : null; // A folded secondary is not filtered out: it counts toward the total. if (foldedShared > 0) page.total += foldedShared; // The rows this render draws (this page, group headers aside), for @@ -23620,7 +23687,9 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> : itemRow(pageItems[i], i); const trainRows = stageTrackerTrainRows(page.rows, rowOf, typeof stageTrackerTrainWhere === 'function' ? stageTrackerTrainWhere(ordered, d.items) : null); const inlineRows = typeof stageTrackerPreparationInlineRows === 'function' ? stageTrackerPreparationInlineRows(trainRows, pageItems, stages) : trainRows; - const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: stageTrackerHeaderHtml(stages) }].concat(inlineRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); + const runRows = wfRuns && wfRuns.length > 1 ? stageTrackerWorkflowRunHeaderRows(inlineRows, wfRuns, stages) : inlineRows; + const headHtml = typeof stageTrackerColumnsHeaderHtml === 'function' ? stageTrackerColumnsHeaderHtml(ctx.leadWorkflow, stages) : stageTrackerHeaderHtml(stages); + const rows = stageTrackerPinnedRows(el, [{ key: STAGE_TRACKER_HEADER_KEY, html: headHtml }].concat(runRows), nowMs, it => stageTrackerRowHtml(it, stages, ctx)); const rec = stageTrackerReconcile(el, rows, { effect, hidden: document.hidden }); if (rec.deferred) stageTrackerScheduleSettle(); // Selection is keyed on the row, never baked into its HTML, so a kept From d72b70dfdee535f556d6efe280f4a3d351fbb973 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:06:40 -0700 Subject: [PATCH 125/166] =?UTF-8?q?=F0=9F=90=9B=20spoke:=20install=20missi?= =?UTF-8?q?ng=20startupProbe=20on=20self-upgrade=20so=20forge-blocked=20bo?= =?UTF-8?q?ots=20are=20not=20liveness-killed=20(#18010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 spoke: install missing startupProbe on self-upgrade so forge-blocked boots are not liveness-killed Pre-#9692 hosted Deployments on pull-only clusters (hub has no kubectl reach) still have no startupProbe. With github.ibm.com unreachable the spoke's boot (App token mint, advisory ensure, coverage classify) takes longer than the 3x30s livenessProbe, so kubelet SIGTERMs it before the dashboard binds and the pod crash-loops for the whole outage — the hub then shows the row offline (grey dot). The spoke-side backfill explicitly skipped containers with no probe; it now installs the template's full probe (port taken from the existing readiness/liveness probe, else 3002). Also: the fleet status-dot hovers gain a colour legend and the grey dot says 'no heartbeat received in the last 5 min'. Signed-off-by: Andy Anderson <andy@clubanderson.com> * test: pin new offline-dot title and give the no-restamp fixture the template probe Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...ed-spoke-installs-missing-startup-probe.md | 1 + src/pkg/hub/assets/dashboard.html | 11 +++- src/pkg/hub/offline_dot_tooltip_test.go | 3 +- src/pkg/hub/spoke/self_startup_probe.go | 64 +++++++++++++++++-- src/pkg/hub/spoke/self_startup_probe_test.go | 20 +++++- .../hub/spoke/self_upgrade_coverage_test.go | 6 +- 6 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 changelog.d/fixed-spoke-installs-missing-startup-probe.md diff --git a/changelog.d/fixed-spoke-installs-missing-startup-probe.md b/changelog.d/fixed-spoke-installs-missing-startup-probe.md new file mode 100644 index 0000000000..3680f325a5 --- /dev/null +++ b/changelog.d/fixed-spoke-installs-missing-startup-probe.md @@ -0,0 +1 @@ +- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 220d0efbad..2e910c1e5e 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -4127,6 +4127,9 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) rows + viewAll; } + /* One-line colour key appended to the status-dot hovers so the dot is + self-explaining. Mirrors the healthBadge() palette + the offline grey. */ + var STATUS_DOT_LEGEND = 'Dot colours: green ok \u00b7 amber warning/stale \u00b7 red degraded/critical \u00b7 blue upgrading \u00b7 grey offline (no heartbeat >5 min)'; function healthBadge(h) { var hp = h.health || {}; var st = hp.status || 'unknown'; @@ -4227,6 +4230,7 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) if (hns) { lines.push('ns: ' + hns); } + lines.push(STATUS_DOT_LEGEND); var access = h.access || []; var dotMarkup = isUpgrading && !isDeleting ? '<span class="online-dot upgrading" style="margin-right:var(--sp-0)"></span>' @@ -8919,13 +8923,18 @@ <h2 style="font-size:1.3rem;color:var(--hub-dashboard-accent);margin:var(--sp-0) // cluster. Built defensively \u2014 every part is skipped when its field is // empty \u2014 and escaped with escAttr since it lands inside a quoted title. var offlineDotTitle = (function() { - var parts = ['Offline']; + /* Say WHAT grey means, not just "Offline": the hub flips a row + offline when no heartbeat arrives within maxHeartbeatAge (5 min, + pkg/hub/server.go), which operators kept reading as "unknown + health" rather than "the spoke stopped talking". */ + var parts = ['Offline \u2014 no heartbeat received in the last 5 min (grey dot)']; var ns = hiveNamespace(h); // 'hive-hosted-<id>' or '' for local if (ns) parts.push('ns: ' + ns); var cluster = h.clusterName || h.clusterId || ''; if (cluster) parts.push('cluster: ' + cluster); if (h.lastHeartbeat) parts.push('last seen: ' + fmtUserTS(h.lastHeartbeat)); else if (h.offlineSince) parts.push('offline since: ' + fmtUserTS(h.offlineSince)); + parts.push(STATUS_DOT_LEGEND); return parts.join('\n'); })(); var dot = h.upgrading diff --git a/src/pkg/hub/offline_dot_tooltip_test.go b/src/pkg/hub/offline_dot_tooltip_test.go index f0c0ae68f3..786e97d31b 100644 --- a/src/pkg/hub/offline_dot_tooltip_test.go +++ b/src/pkg/hub/offline_dot_tooltip_test.go @@ -30,7 +30,8 @@ func TestOfflineStatusDotCarriesTooltip(t *testing.T) { // cluster and last-seen lines, each skipped when its field is empty. for _, part := range []string{ `var offlineDotTitle = (function() {`, - `var parts = ['Offline'];`, + `var parts = ['Offline \u2014 no heartbeat received in the last 5 min (grey dot)'];`, + `parts.push(STATUS_DOT_LEGEND);`, `var ns = hiveNamespace(h);`, `if (ns) parts.push('ns: ' + ns);`, `var cluster = h.clusterName || h.clusterId || '';`, diff --git a/src/pkg/hub/spoke/self_startup_probe.go b/src/pkg/hub/spoke/self_startup_probe.go index 980d7cf95c..88d9cca758 100644 --- a/src/pkg/hub/spoke/self_startup_probe.go +++ b/src/pkg/hub/spoke/self_startup_probe.go @@ -13,12 +13,24 @@ const ( HostedStartupProbeInitialDelaySeconds = 10 HostedStartupProbePeriodSeconds = 5 HostedStartupProbeFailureThreshold = (int(HostedStartupBudget/time.Second) - HostedStartupProbeInitialDelaySeconds) / HostedStartupProbePeriodSeconds + + // HostedStartupProbePath and HostedDashboardPort are the hosted template's + // startupProbe handler, used when a hive container has NO startupProbe at + // all and the whole probe has to be installed (not just its threshold). + HostedStartupProbePath = "/api/health" + HostedDashboardPort = 3002 ) // k8sUpgradePatch backfills the startup budget in the SAME template update as -// the image/restart annotation, avoiding a second rollout. Only raise an existing -// hive probe: sidecars, init containers and deployments without a probe keep -// their configuration. Strategic merge preserves the handler and other timings. +// the image/restart annotation, avoiding a second rollout. Only the hive +// container is touched: sidecars and init containers keep their configuration. +// An existing hive probe is raised to the template budget (strategic merge +// preserves its handler and other timings). A hive container with NO +// startupProbe gets the template's full probe: pre-#9692 deployments on +// pull-only clusters (where the hub cannot reach the Deployment itself) were +// left with only the 3×30s livenessProbe, so a boot stalled >90s behind forge +// timeouts was SIGTERMed before the dashboard bound and the spoke crash-looped +// for the length of the forge outage. // A nil body requests only a probe backfill and skips PATCH when none is needed. func k8sUpgradePatch(path string, body []byte) error { current, err := k8sAPIGet(path) @@ -41,6 +53,8 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { StartupProbe *struct { FailureThreshold int `json:"failureThreshold"` } `json:"startupProbe"` + ReadinessProbe *probeHandler `json:"readinessProbe"` + LivenessProbe *probeHandler `json:"livenessProbe"` } `json:"containers"` } `json:"spec"` } `json:"template"` @@ -51,7 +65,7 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { } probeOnly := body == nil for _, c := range dep.Spec.Template.Spec.Containers { - if c.Name != selfUpgradeDeployName || c.StartupProbe == nil || c.StartupProbe.FailureThreshold >= HostedStartupProbeFailureThreshold { + if c.Name != selfUpgradeDeployName || (c.StartupProbe != nil && c.StartupProbe.FailureThreshold >= HostedStartupProbeFailureThreshold) { continue } if probeOnly { @@ -82,7 +96,11 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { target = map[string]any{"name": c.Name} containers = append(containers, target) } - target["startupProbe"] = map[string]any{"failureThreshold": HostedStartupProbeFailureThreshold} + if c.StartupProbe == nil { + target["startupProbe"] = fullStartupProbe(startupProbePort(c.ReadinessProbe, c.LivenessProbe)) + } else { + target["startupProbe"] = map[string]any{"failureThreshold": HostedStartupProbeFailureThreshold} + } spec["containers"] = containers body, err = json.Marshal(patch) if err != nil { @@ -95,3 +113,39 @@ func k8sUpgradePatchWithDeployment(path string, body, current []byte) error { } return k8sAPIPatch(path, body) } + +// probeHandler is the slice of a Kubernetes probe the backfill needs: the +// httpGet port the dashboard already answers on. +type probeHandler struct { + HTTPGet *struct { + Port json.Number `json:"port"` + } `json:"httpGet"` +} + +// startupProbePort picks the port for a newly installed startupProbe: the +// port the existing readiness or liveness probe already targets (the dashboard +// is proven to answer there), else the hosted template default. +func startupProbePort(probes ...*probeHandler) any { + for _, p := range probes { + if p == nil || p.HTTPGet == nil || p.HTTPGet.Port == "" { + continue + } + if n, err := p.HTTPGet.Port.Int64(); err == nil && n > 0 { + return n + } + return p.HTTPGet.Port.String() + } + return HostedDashboardPort +} + +// fullStartupProbe renders the hosted template's startupProbe for a container +// that has none, so the self-upgrade path converges exactly what provisioning +// would have written. +func fullStartupProbe(port any) map[string]any { + return map[string]any{ + "httpGet": map[string]any{"path": HostedStartupProbePath, "port": port}, + "initialDelaySeconds": HostedStartupProbeInitialDelaySeconds, + "periodSeconds": HostedStartupProbePeriodSeconds, + "failureThreshold": HostedStartupProbeFailureThreshold, + } +} diff --git a/src/pkg/hub/spoke/self_startup_probe_test.go b/src/pkg/hub/spoke/self_startup_probe_test.go index 1b9a11d052..c189db572a 100644 --- a/src/pkg/hub/spoke/self_startup_probe_test.go +++ b/src/pkg/hub/spoke/self_startup_probe_test.go @@ -20,7 +20,10 @@ func TestSelfUpgradeBackfillsStartupProbe(t *testing.T) { {"stale", `,"startupProbe":{"httpGet":{"path":"/api/health","port":3002},"failureThreshold":30,"periodSeconds":5}`, true}, {"current", `,"startupProbe":{"failureThreshold":118}`, false}, {"larger", `,"startupProbe":{"failureThreshold":200}`, false}, - {"absent", ``, false}, + // No probe at all: install the template's full probe. The port + // follows the existing readiness probe when one is present. + {"absent", ``, true}, + {"absent-with-readiness", `,"readinessProbe":{"httpGet":{"path":"/api/health","port":4100}}`, true}, } { t.Run(operation+"/"+probe.name, func(t *testing.T) { probeJSON := probe.json @@ -105,9 +108,22 @@ func TestSelfUpgradeBackfillsStartupProbe(t *testing.T) { for _, c := range patch.Spec.Template.Spec.Containers { if c.Name == "hive" { found = c.Probe != nil - if found && (len(c.Probe) != 1 || c.Probe["failureThreshold"] != float64(118)) { + if found && c.Probe["failureThreshold"] != float64(118) { t.Errorf("unexpected probe patch: %v", c.Probe) } + if found && strings.HasPrefix(probe.name, "absent") { + wantPort := float64(3002) + if probe.name == "absent-with-readiness" { + wantPort = 4100 + } + hg, _ := c.Probe["httpGet"].(map[string]any) + if hg == nil || hg["path"] != "/api/health" || hg["port"] != wantPort || + c.Probe["periodSeconds"] != float64(5) || c.Probe["initialDelaySeconds"] != float64(10) { + t.Errorf("missing probe must be installed in full (port %v): %v", wantPort, c.Probe) + } + } else if found && len(c.Probe) != 1 { + t.Errorf("existing probe must only have its threshold raised: %v", c.Probe) + } if operation == "image" && c.Image != "ghcr.io/hivecommons/hive:candidate" { t.Error("lost image update") } diff --git a/src/pkg/hub/spoke/self_upgrade_coverage_test.go b/src/pkg/hub/spoke/self_upgrade_coverage_test.go index f64e521120..1ca7beb9e3 100644 --- a/src/pkg/hub/spoke/self_upgrade_coverage_test.go +++ b/src/pkg/hub/spoke/self_upgrade_coverage_test.go @@ -269,12 +269,14 @@ func TestSwitchImageSelfSuccess(t *testing.T) { // template each time — so the mid-init pod was killed and replaced on every // beat (observed live: 18 ReplicaSets in 8 minutes, none reaching Ready). A // Deployment whose containers all already carry the target must not be -// PATCHed again. +// PATCHed again. The fixture carries the template startupProbe: a hive +// container with NO probe is the one case that still warrants a (one-time, +// annotation-free) PATCH mid-flight — see TestSelfUpgradeBackfillsStartupProbe. func TestSwitchImageSelfDoesNotRestampWhenAlreadyAtImage(t *testing.T) { const target = "ghcr.io/hivecommons/hive:candidate" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet { - w.Write([]byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive","image":"` + target + `"}],"initContainers":[{"name":"init","image":"` + target + `"}]}}}}`)) + w.Write([]byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive","image":"` + target + `","startupProbe":{"failureThreshold":118}}],"initContainers":[{"name":"init","image":"` + target + `"}]}}}}`)) return } t.Errorf("deployment already at %s, but a %s was issued — that re-rolls the pod mid-init", target, r.Method) From 1f486b92e11f1afbe25a80739fd7bbaecb51323d Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 23:19:37 +0000 Subject: [PATCH 126/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.238.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.238.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-claim-forge-switch-discovery.md | 1 - changelog.d/fixed-claim-pending-forge-pill.md | 1 - changelog.d/fixed-sse-drop-warn-flood.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-claim-forge-switch-discovery.md delete mode 100644 changelog.d/fixed-claim-pending-forge-pill.md delete mode 100644 changelog.d/fixed-sse-drop-warn-flood.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 41885cd023..bf271ee566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.238.1) + +### Fixed + +- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. +- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). +- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). + ## 2026-11-12 (v5.238.0) ### Added diff --git a/changelog.d/fixed-claim-forge-switch-discovery.md b/changelog.d/fixed-claim-forge-switch-discovery.md deleted file mode 100644 index 788c262af6..0000000000 --- a/changelog.d/fixed-claim-forge-switch-discovery.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke: a placeholder claimed onto a different forge than its pool default (GHE slot → github.com repo) now moves GitHub App installation discovery to the delivered App and API URL even while installation_id is still 0, instead of polling the boot-time App/forge forever and reporting "App 5686 has no installation" until a pod restart. diff --git a/changelog.d/fixed-claim-pending-forge-pill.md b/changelog.d/fixed-claim-pending-forge-pill.md deleted file mode 100644 index c5b31e94d6..0000000000 --- a/changelog.d/fixed-claim-pending-forge-pill.md +++ /dev/null @@ -1 +0,0 @@ -- Hub fleet view: a placeholder assigned with an explicit forge now shows that forge (e.g. github.com) in the Location pill while the claim is pending, instead of the cluster default the spoke still heartbeats from its pre-claim config (github.ibm.com on vllm-d). diff --git a/changelog.d/fixed-sse-drop-warn-flood.md b/changelog.d/fixed-sse-drop-warn-flood.md deleted file mode 100644 index a5494ff99b..0000000000 --- a/changelog.d/fixed-sse-drop-warn-flood.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). From f191315499ddc9de6576169380391d32bf2fb8b8 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:33:43 -0700 Subject: [PATCH 127/166] =?UTF-8?q?=E2=9C=A8=20project.reference=5Frepos:?= =?UTF-8?q?=20study-only=20repos=20with=20optional=20issues/prs=20opt-in?= =?UTF-8?q?=20(#18015)=20(#18023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ project.reference_repos: study-only repos with optional issues/prs opt-in (#18015) A hive whose deliverable is about another project (CNCF TechDocs assessing hivecommons/* into cncf/techdocs) can list owner/name repos it may clone unauthenticated and GET. The proxy strips the hive token on those repos and refuses every write by default with a message naming reference_repos; a rare per-entry write: [issues, prs] opt-in allows only open-issue, open-PR/topic branch push and comments on own items, sent with a credential valid for that repo (App installed on the target org, or write_token_file). Boot refuses write-mode without either (exit 22). The kick lists reference repos so the scanner stops flagging such items out-of-scope/critical; dashboard shows a minimal reference · <tier> card. Fixes #18015 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🌱 reference repos: settings-parity allowlist, no new inline styles Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-reference-repos.md | 1 + src/cmd/hive/main.go | 4 + src/cmd/hive/reference_repos.go | 87 ++++ src/cmd/hive/reference_repos_test.go | 46 ++ src/crux/proxy/github_proxy.go | 10 + src/crux/proxy/github_proxy_mitm.go | 28 ++ src/crux/proxy/github_proxy_policy.go | 36 ++ src/crux/proxy/reference_repos.go | 406 ++++++++++++++++++ src/crux/proxy/reference_repos_test.go | 237 ++++++++++ src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/docs/github-write-surface.md | 15 + src/docs/operator-reference/README.md | 3 + src/pkg/config/config.go | 7 + src/pkg/config/reference_repos.go | 231 ++++++++++ src/pkg/config/reference_repos_test.go | 101 +++++ src/pkg/config/validate.go | 3 + src/pkg/dashboard/server.go | 21 +- .../dashboard/settings_parity_allowlist.txt | 3 + src/pkg/dashboard/static-src/css/repos.css | 7 + .../dashboard/static-src/js/repos-render.js | 27 +- src/pkg/dashboard/static/index.html | 34 +- src/pkg/dashboard/status_builder.go | 14 + src/pkg/github/reference_repo_token.go | 190 ++++++++ src/pkg/github/reference_repo_token_test.go | 120 ++++++ src/pkg/scheduler/agent_repo_scope_test.go | 27 ++ src/pkg/scheduler/kickmessage.go | 12 + 29 files changed, 1679 insertions(+), 23 deletions(-) create mode 100644 changelog.d/added-reference-repos.md create mode 100644 src/cmd/hive/reference_repos.go create mode 100644 src/cmd/hive/reference_repos_test.go create mode 100644 src/crux/proxy/reference_repos.go create mode 100644 src/crux/proxy/reference_repos_test.go create mode 100644 src/pkg/config/reference_repos.go create mode 100644 src/pkg/config/reference_repos_test.go create mode 100644 src/pkg/github/reference_repo_token.go create mode 100644 src/pkg/github/reference_repo_token_test.go diff --git a/changelog.d/added-reference-repos.md b/changelog.d/added-reference-repos.md new file mode 100644 index 0000000000..0105e3a710 --- /dev/null +++ b/changelog.d/added-reference-repos.md @@ -0,0 +1 @@ +- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index ea1821f5e2..e58fe9c42a 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4143,6 +4143,10 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { // unconditionally: with HIVE_PROXY_INJECT_GH_AUTH unset (the default) // the proxy never consults the source and the registry stays empty. b.githubProxy.SetAgentTokenSource(github.AgentProxyToken) + // project.reference_repos (#18015): study-only repos outside the + // project, with a rare per-entry issues/prs opt-in served by a + // credential verified here (boot refuses otherwise). + b.wireReferenceRepos() // Durable attempted-vs-blocked write counters (#12069); observation only. if err := proxy.MigrateWriteCounters(proxy.LegacyWriteCountersPath, proxy.WriteCountersPath, time.Now()); err != nil { b.logger.Warn("write counters: legacy file not migrated", "error", err) diff --git a/src/cmd/hive/reference_repos.go b/src/cmd/hive/reference_repos.go new file mode 100644 index 0000000000..f87c367245 --- /dev/null +++ b/src/cmd/hive/reference_repos.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// referenceRepoCredentialExitCode marks an exit caused by a +// project.reference_repos entry that opted in to writes without a credential +// GitHub would honour for that repo (hivecommons/hive#18015). Its own code so +// the termination state says "install the App on the target org or supply +// write_token_file", not "bug". +const referenceRepoCredentialExitCode = 22 + +// referenceRepoDeps are the seams buildReferenceRepoPolicies needs: App +// installation discovery on a foreign org and token-identity resolution. +// Either may be nil when the hive has no GitHub App auth, in which case a +// write-mode entry can only be served by write_token_file. +type referenceRepoDeps struct { + botLogin string + discover func(ctx context.Context, org string) (int64, error) + whoAmI func(ctx context.Context, token string) (string, error) + auth func() *github.AppAuth +} + +// buildReferenceRepoPolicies turns the config list into proxy policies. A +// read-only entry needs nothing; a write-mode entry is verified now so the +// failure is a boot-time config error naming the repo. +func buildReferenceRepoPolicies(ctx context.Context, refs []config.ReferenceRepo, deps referenceRepoDeps, logger *slog.Logger) ([]proxy.ReferenceRepoPolicy, error) { + out := make([]proxy.ReferenceRepoPolicy, 0, len(refs)) + for _, rr := range refs { + pol := proxy.ReferenceRepoPolicy{Repo: rr.Repo, Issues: rr.WritesIssues(), PRs: rr.WritesPRs()} + if !rr.ReadOnly() { + login, err := github.VerifyReferenceRepoWriteCredential(ctx, rr, deps.botLogin, deps.discover, deps.whoAmI) + if err != nil { + return nil, err + } + pol.Login = login + pol.Token = github.ReferenceRepoTokenSource(rr, deps.auth, logger) + } + out = append(out, pol) + } + return out, nil +} + +// wireReferenceRepos installs project.reference_repos on the proxy, refusing +// to boot when a write-mode entry has no usable credential. +func (b *boot) wireReferenceRepos() { + refs := b.cfg.Project.ReferenceRepos + if len(refs) == 0 || b.githubProxy == nil { + return + } + deps := referenceRepoDeps{auth: func() *github.AppAuth { return nil }} + if b.ghClient != nil { + if auth := b.ghClient.AppAuth(); auth != nil { + apiURL := auth.APIURL() + deps.botLogin = auth.BotLogin() + deps.discover = auth.DiscoverInstallationID + deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, apiURL) + } + // Re-fetched per call: the AppAuth is replaced on key rotation. + deps.auth = func() *github.AppAuth { return b.ghClient.AppAuth() } + } else { + deps.whoAmI = func(ctx context.Context, token string) (string, error) { + return github.TokenUserLogin(ctx, token, "") + } + } + } + policies, err := buildReferenceRepoPolicies(b.ctx, refs, deps, b.logger) + if err != nil { + b.logger.Error("refusing to start: project.reference_repos write-mode entry has no usable credential (#18015)", "error", err.Error()) + if errors.Is(err, github.ErrReferenceRepoNoCredential) { + fmt.Fprintln(os.Stderr, err.Error()) + } + os.Exit(referenceRepoCredentialExitCode) + } + b.githubProxy.SetReferenceRepos(policies) + b.logger.Info("reference repos wired", "count", len(policies), "repos", b.cfg.ReferenceRepoSlugs()) +} diff --git a/src/cmd/hive/reference_repos_test.go b/src/cmd/hive/reference_repos_test.go new file mode 100644 index 0000000000..b10d70646a --- /dev/null +++ b/src/cmd/hive/reference_repos_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +func TestBuildReferenceRepoPolicies(t *testing.T) { + ctx := context.Background() + deps := referenceRepoDeps{ + botLogin: "hive[bot]", + discover: func(_ context.Context, org string) (int64, error) { + if org == "hivecommons" { + return 1, nil + } + return 0, github.ErrNoInstallationForOrg + }, + auth: func() *github.AppAuth { return nil }, + } + refs := []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + pols, err := buildReferenceRepoPolicies(ctx, refs, deps, testLogger()) + if err != nil { + t.Fatal(err) + } + if len(pols) != 2 { + t.Fatalf("got %d policies", len(pols)) + } + if pols[0].Token != nil || pols[0].Issues || pols[0].PRs || pols[0].Login != "" { + t.Errorf("read-only policy carries write state: %+v", pols[0]) + } + if pols[1].Token == nil || !pols[1].Issues || !pols[1].PRs || pols[1].Login != "hive[bot]" { + t.Errorf("write policy incomplete: %+v", pols[1]) + } + + _, err = buildReferenceRepoPolicies(ctx, []config.ReferenceRepo{{Repo: "kubernetes/website", Write: []string{"issues"}}}, deps, testLogger()) + if !errors.Is(err, github.ErrReferenceRepoNoCredential) { + t.Fatalf("uninstalled org must refuse with ErrReferenceRepoNoCredential, got %v", err) + } +} diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index c47abfedf8..5b616e62b2 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -86,6 +86,16 @@ type GitHubProxy struct { logger *slog.Logger uidMap *agent.UIDMap allowedRepos map[string]bool + // Reference repos (#18015): study-only repositories outside allowedRepos, + // read-only by default with a per-entry issues/prs opt-in. Reads to them + // are forwarded without the hive's credential; writes are gated by + // referenceRepoRefusal. referenceDefaults caches each repo's default + // branch for the push guard; referenceLookup is the GET those checks use + // (nil = defaultReferenceLookup; tests inject one). + referenceMu sync.RWMutex + referenceRepos map[string]ReferenceRepoPolicy + referenceDefaults map[string]referenceDefaultBranch + referenceLookup referenceLookupFunc // repoPaused reports whether a repo ("owner/repo", as ExtractRepo yields it) // is under an operator pause (#6203). Guarded because the dashboard can diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go index da81467ffe..d750bd976e 100644 --- a/src/crux/proxy/github_proxy_mitm.go +++ b/src/crux/proxy/github_proxy_mitm.go @@ -77,7 +77,29 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // refused and audited as agent_write_refused. Checked after pause and // scope so their more specific reasons win when both apply. writeSurfaceKind := "" + // Reference repos (#18015) sit outside every list below: not in + // allowedRepos (so the mode chain's repo filter would refuse the + // write with the wrong reason), not pausable, not agent-scoped. The + // gate runs first so its refusal — naming reference_repos and the + // tier — is what the agent reads, and an allowed opt-in write is + // remembered so the repo filter and the PR hard-deny below do not + // undo it (the credential it will carry is the reference repo's + // own, so App-bot authorship is already guaranteed). + referenceWrite := false if agentName != internalCallerName { + if pol, isRef := p.referenceRepoFor(repoScopePath(req)); isRef { + if reason, refused := p.referenceRepoRefusal(req); refused { + blocked = true + blockReason = reason + blockCode = DenyReasonReferenceRepo + } else { + referenceWrite = referenceWriteCredential(pol, req.Method, req.URL.Path, req.URL.RawQuery) + } + } + } + if blocked { + // Refused as a reference-repo write; nothing below can un-block it. + } else if agentName != internalCallerName { if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { blocked = true blockReason = reason @@ -165,6 +187,12 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // autonomy. Its control-plane calls (App token mint, heartbeat) // must not be gated as if they were agent writes. Repo-filter and // canary-egress checks below still apply. + } else if referenceWrite && !referenceWriteAllowedByMode(mode, req.Method, req.URL.Path) { + blocked = true + blockCode = DenyReasonMode + } else if referenceWrite { + // Opt-in reference write at a sufficient mode: skip the PR + // hard-deny and the repo filter, both of which assume a managed repo. } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { blocked = true blockCode = DenyReasonMode diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index b61b8d03ab..2d60d115e4 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -180,6 +180,17 @@ const loginPathPrefix = "/login/" func (p *GitHubProxy) rewriteGitHubAuth(req *http.Request, agentName string) { // NOTE: callers must gate on injectsGitHubAuthForHost first; this function // does not know which host the request is bound for. + if agentName != internalCallerName { + if pol, isRef := p.referenceRepoFor(repoScopePath(req)); isRef { + // Reference repo (#18015): the agent's own credential belongs to + // the hive's installation and is never valid here, so it is + // stripped whatever the injection flag says. A read goes out + // unauthenticated, as the public internet would send it; an + // opt-in write carries the repo's own write credential. + p.rewriteReferenceRepoAuth(req, agentName, pol) + return + } + } if !p.injectGHAuth { // Flag off (the default): behavior byte-identical to before #1861 — // no header is touched on any path. @@ -372,3 +383,28 @@ func isGitPath(path string) bool { strings.HasSuffix(path, "/git-upload-pack") || strings.HasSuffix(path, "/info/refs") } + +func (p *GitHubProxy) rewriteReferenceRepoAuth(req *http.Request, agentName string, pol ReferenceRepoPolicy) { + req.Header.Del("Authorization") + if isGitPath(req.URL.Path) { + // Same reason as the managed-repo git branch above: each git request + // must re-enter this rewrite on its own connection. + req.Close = true + } + if !referenceWriteCredential(pol, req.Method, req.URL.Path, req.URL.RawQuery) { + p.logger.Debug("proxy auth: reference repo read forwarded without credentials", "agent", agentName, "repo", pol.Repo, "injected", false) + return + } + token := p.referenceToken(pol) + if token == "" { + p.logger.Warn("proxy auth: reference repo write has no credential — forwarding without one (check the write_token_file or the App installation on the target org)", "agent", agentName, "repo", pol.Repo, "injected", false) + return + } + if isGitPath(req.URL.Path) { + basic := base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser + ":" + token)) + req.Header.Set("Authorization", "Basic "+basic) + } else { + req.Header.Set("Authorization", "token "+token) + } + p.logger.Debug("proxy auth: reference repo write credential attached", "agent", agentName, "repo", pol.Repo, "injected", true) +} diff --git a/src/crux/proxy/reference_repos.go b/src/crux/proxy/reference_repos.go new file mode 100644 index 0000000000..be69c0c531 --- /dev/null +++ b/src/crux/proxy/reference_repos.go @@ -0,0 +1,406 @@ +package proxy + +import ( + "bufio" + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/agent" +) + +// Reference repositories (hivecommons/hive#18015). +// +// A reference repo is one the hive's agents may STUDY without managing it: +// project.reference_repos. Reads — git clone/fetch and GitHub GETs — pass, and +// pass WITHOUT the hive's credential: the agent's injected token belongs to the +// hive's own installation and is not valid for a foreign repo, so it is +// stripped and the request goes out as the public internet would send it. +// +// Writes are refused unless the entry opted in, and even then only to the +// narrow surface an assessor needs: open an issue, open a PR from a topic +// branch, comment on the hive's OWN issue or PR. Merging, pushing the default +// branch, labels/assignees/closing on anyone's items and reviews are never +// allowed — there is no mode, capability or relay that widens this. The write +// credential is the policy's own Token source (an installation token minted +// on the TARGET org, or an operator-supplied file), never the hive's token. + +// DenyReasonReferenceRepo is the 403 reason code for a refused reference-repo +// write (see denial_message.go for the vocabulary). +const DenyReasonReferenceRepo = "reference_repo" + +// ReferenceRepoPolicy is the proxy's view of one project.reference_repos entry. +type ReferenceRepoPolicy struct { + // Repo is the "owner/name" slug. + Repo string + // Issues and PRs are the write opt-ins. + Issues, PRs bool + // Token yields the write credential for Repo. nil for a read-only entry. + Token func() (string, bool) + // Login is the GitHub login the Token acts as ("<app>[bot]" or the PAT's + // user); own-item checks compare an item's author against it. + Login string +} + +func (r ReferenceRepoPolicy) readOnly() bool { return !r.Issues && !r.PRs } + +func (r ReferenceRepoPolicy) tier() string { + switch { + case r.Issues && r.PRs: + return "issues+prs" + case r.Issues: + return "issues" + case r.PRs: + return "prs" + } + return "read-only" +} + +// referenceDecision is the outcome of the path-only part of the write check. +type referenceDecision int + +const ( + referenceNotReference referenceDecision = iota // not a reference repo, or a read: nothing to do + referenceAllow + referenceDeny + referenceOwnItem // allowed only if item #N was authored by the policy's Login + referencePush // git push: allowed only when no ref targets the default branch +) + +var ( + refIssuesPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) + refPullsPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`) + refItemCommentsPath = regexp.MustCompile(`^/repos/[^/]+/[^/]+/(?:issues|pulls)/(\d+)/comments$`) + refGitReceivePackRe = regexp.MustCompile(`\.git/git-receive-pack$`) + refGitInfoRefsRe = regexp.MustCompile(`\.git/info/refs$`) + referenceLookupLimit = int64(1 << 20) +) + +// ReferenceRepoDecision classifies one request against a reference-repo +// policy without touching the network. number is the item for an +// referenceOwnItem decision. reason is the agent-facing refusal for +// referenceDeny. +func ReferenceRepoDecision(pol ReferenceRepoPolicy, method, path, rawQuery string) (d referenceDecision, number int, reason string) { + push := refGitReceivePackRe.MatchString(path) || (refGitInfoRefsRe.MatchString(path) && gitServiceIsReceivePack(rawQuery)) + if !push && (!writeMethods[method] || gitUploadPackPath.MatchString(path)) { + return referenceNotReference, 0, "" + } + if pol.readOnly() { + return referenceDeny, 0, referenceRepoRefusal(pol, "it is listed as READ-ONLY") + } + switch { + case push: + if !pol.PRs { + return referenceDeny, 0, referenceRepoRefusal(pol, "pushing needs write: [prs]") + } + return referencePush, 0, "" + case method == "POST" && refIssuesPath.MatchString(path): + if !pol.Issues { + return referenceDeny, 0, referenceRepoRefusal(pol, "opening an issue needs write: [issues]") + } + return referenceAllow, 0, "" + case method == "POST" && refPullsPath.MatchString(path): + if !pol.PRs { + return referenceDeny, 0, referenceRepoRefusal(pol, "opening a PR needs write: [prs]") + } + return referenceAllow, 0, "" + case method == "POST" && refItemCommentsPath.MatchString(path): + m := refItemCommentsPath.FindStringSubmatch(path) + n, _ := strconv.Atoi(m[1]) + return referenceOwnItem, n, "" + } + return referenceDeny, 0, referenceRepoRefusal(pol, "only opening an issue or PR and commenting on the hive's own items are ever allowed there — never merge, push to the default branch, label, assign, close or review") +} + +func referenceRepoRefusal(pol ReferenceRepoPolicy, why string) string { + return "repository " + pol.Repo + " is a REFERENCE repo (project.reference_repos, tier " + pol.tier() + "): " + why + ". Reads — clone, fetch, GET — still work. Put findings about it into the deliverable in your AUTHORIZED repo instead. Do NOT retry, do NOT route around this with another CLI, the hive relays or the GitHub MCP, and do NOT file an issue about it: this is how the operator scoped the hive, not an outage." +} + +// SetReferenceRepos installs the reference-repo policies (#18015). Replaces +// the previous set; nil clears it. +func (p *GitHubProxy) SetReferenceRepos(policies []ReferenceRepoPolicy) { + m := make(map[string]ReferenceRepoPolicy, len(policies)) + for _, pol := range policies { + key := strings.ToLower(strings.TrimSpace(pol.Repo)) + if key == "" { + continue + } + pol.Repo = strings.TrimSpace(pol.Repo) + m[key] = pol + } + p.referenceMu.Lock() + p.referenceRepos = m + p.referenceMu.Unlock() +} + +// referenceRepoFor returns the policy covering the repo a path names. +func (p *GitHubProxy) referenceRepoFor(path string) (ReferenceRepoPolicy, bool) { + p.referenceMu.RLock() + defer p.referenceMu.RUnlock() + if len(p.referenceRepos) == 0 { + return ReferenceRepoPolicy{}, false + } + repo := ExtractRepo(path) + if repo == "" { + repo = WriteSurfaceRepo(path) + } + if repo == "" { + return ReferenceRepoPolicy{}, false + } + pol, ok := p.referenceRepos[strings.ToLower(repo)] + return pol, ok +} + +// referenceRepoRefusal is the per-request gate the MITM loop calls. It returns +// the refusal reason and true when the request must be blocked. The own-item +// and default-branch checks consult GitHub through p.referenceLookup; a lookup +// that fails refuses (fail closed) and says so. +func (p *GitHubProxy) referenceRepoRefusal(req *http.Request) (string, bool) { + pol, ok := p.referenceRepoFor(repoScopePath(req)) + if !ok { + return "", false + } + d, number, reason := ReferenceRepoDecision(pol, req.Method, req.URL.Path, req.URL.RawQuery) + switch d { + case referenceNotReference, referenceAllow: + return "", false + case referenceDeny: + return reason, true + case referenceOwnItem: + author, err := p.referenceItemAuthor(req, pol, number) + if err != nil { + return referenceRepoRefusal(pol, fmt.Sprintf("could not confirm #%d is the hive's own item (%v)", number, err)), true + } + if pol.Login == "" || !strings.EqualFold(author, pol.Login) { + return referenceRepoRefusal(pol, fmt.Sprintf("#%d was opened by %s, not by the hive (%s); comments are allowed only on the hive's own issues and PRs", number, author, pol.Login)), true + } + return "", false + case referencePush: + if refGitInfoRefsRe.MatchString(req.URL.Path) { + return "", false // ref advertisement; the refs travel in the POST + } + def, err := p.referenceDefaultBranch(req, pol) + if err != nil { + return referenceRepoRefusal(pol, fmt.Sprintf("could not resolve its default branch to protect it (%v)", err)), true + } + refs, err := peekReceivePackRefs(req) + if err != nil { + return referenceRepoRefusal(pol, "push command list unreadable ("+err.Error()+")"), true + } + for _, ref := range refs { + if ref == "refs/heads/"+def { + return referenceRepoRefusal(pol, "pushing to its default branch ("+def+") is never allowed — push a topic branch and open a PR"), true + } + } + return "", false + } + return "", false +} + +// referenceWriteCredential says whether a request to a reference repo should +// carry the policy's write token (a write, or the push ref advertisement) — +// every other request to a reference repo goes out unauthenticated. +func referenceWriteCredential(pol ReferenceRepoPolicy, method, path, rawQuery string) bool { + if pol.Token == nil { + return false + } + d, _, _ := ReferenceRepoDecision(pol, method, path, rawQuery) + return d != referenceNotReference && d != referenceDeny +} + +// referenceAPIBase derives the REST base for the forge host a request is bound +// for: api.github.com for github.com, <host>/api/v3 for GHE. +func referenceAPIBase(req *http.Request) string { + host := canonicalHost(req.Host) + if host == "" { + host = canonicalHost(req.URL.Host) + } + switch host { + case "", "github.com", "api.github.com": + return "https://api.github.com" + } + return "https://" + host + "/api/v3" +} + +// referenceLookupFunc is the GET the own-item and default-branch checks use. +// Tests swap it; production uses an ordinary client from this process. +type referenceLookupFunc func(url, token string) ([]byte, error) + +func defaultReferenceLookup(url, token string) ([]byte, error) { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + if token != "" { + req.Header.Set("Authorization", "token "+token) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, referenceLookupLimit)) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("GitHub answered %d", resp.StatusCode) + } + return body, nil +} + +func (p *GitHubProxy) referenceLookupOrDefault() referenceLookupFunc { + if p.referenceLookup != nil { + return p.referenceLookup + } + return defaultReferenceLookup +} + +func (p *GitHubProxy) referenceToken(pol ReferenceRepoPolicy) string { + if pol.Token == nil { + return "" + } + tok, ok := pol.Token() + if !ok { + return "" + } + return tok +} + +func (p *GitHubProxy) referenceItemAuthor(req *http.Request, pol ReferenceRepoPolicy, number int) (string, error) { + body, err := p.referenceLookupOrDefault()(referenceAPIBase(req)+"/repos/"+pol.Repo+"/issues/"+strconv.Itoa(number), p.referenceToken(pol)) + if err != nil { + return "", err + } + var item struct { + User struct { + Login string `json:"login"` + } `json:"user"` + } + if err := json.Unmarshal(body, &item); err != nil { + return "", err + } + if item.User.Login == "" { + return "", fmt.Errorf("item has no author") + } + return item.User.Login, nil +} + +// referenceDefaultBranchTTL bounds how long a resolved default branch is +// trusted before it is looked up again. +const referenceDefaultBranchTTL = 10 * time.Minute + +func (p *GitHubProxy) referenceDefaultBranch(req *http.Request, pol ReferenceRepoPolicy) (string, error) { + key := strings.ToLower(pol.Repo) + p.referenceMu.RLock() + cached, ok := p.referenceDefaults[key] + p.referenceMu.RUnlock() + if ok && time.Now().Before(cached.until) { + return cached.branch, nil + } + body, err := p.referenceLookupOrDefault()(referenceAPIBase(req)+"/repos/"+pol.Repo, p.referenceToken(pol)) + if err != nil { + return "", err + } + var meta struct { + DefaultBranch string `json:"default_branch"` + } + if err := json.Unmarshal(body, &meta); err != nil { + return "", err + } + if meta.DefaultBranch == "" { + return "", fmt.Errorf("repository metadata carries no default_branch") + } + p.referenceMu.Lock() + if p.referenceDefaults == nil { + p.referenceDefaults = map[string]referenceDefaultBranch{} + } + p.referenceDefaults[key] = referenceDefaultBranch{branch: meta.DefaultBranch, until: time.Now().Add(referenceDefaultBranchTTL)} + p.referenceMu.Unlock() + return meta.DefaultBranch, nil +} + +type referenceDefaultBranch struct { + branch string + until time.Time +} + +// receivePackPeekLimit bounds the command section read from a push body. The +// command list precedes the packfile and is a handful of lines; anything +// larger is not a push shape this gate will reason about. +const receivePackPeekLimit = 64 * 1024 + +// peekReceivePackRefs reads the pkt-line command section of a git-receive-pack +// body — "<old> <new> <ref>[\0caps]" lines up to the flush — returns the refs +// it names, and puts every byte back so the request forwards unchanged. A +// content-encoded body cannot be read here and is an error (fail closed). +func peekReceivePackRefs(req *http.Request) ([]string, error) { + if req.Body == nil { + return nil, fmt.Errorf("no body") + } + if enc := req.Header.Get("Content-Encoding"); enc != "" && enc != "identity" { + return nil, fmt.Errorf("content-encoding %q", enc) + } + br := bufio.NewReader(io.LimitReader(req.Body, receivePackPeekLimit)) + var consumed bytes.Buffer + tee := io.TeeReader(br, &consumed) + var refs []string + for { + var hdr [4]byte + if _, err := io.ReadFull(tee, hdr[:]); err != nil { + return nil, fmt.Errorf("pkt-line header: %w", err) + } + n, err := hex.DecodeString(string(hdr[:])) + if err != nil { + return nil, fmt.Errorf("pkt-line length %q", hdr) + } + size := int(n[0])<<8 | int(n[1]) + if size == 0 { + break // flush: end of commands + } + if size < 4 { + return nil, fmt.Errorf("pkt-line length %d", size) + } + line := make([]byte, size-4) + if _, err := io.ReadFull(tee, line); err != nil { + return nil, fmt.Errorf("pkt-line body: %w", err) + } + if i := bytes.IndexByte(line, 0); i >= 0 { + line = line[:i] + } + fields := strings.Fields(string(line)) + if len(fields) != 3 { + return nil, fmt.Errorf("malformed command %q", line) + } + refs = append(refs, fields[2]) + } + // Whatever bufio read ahead past the flush belongs to the packfile; it is + // also in neither consumed nor req.Body any more, so splice it back too. + rest, _ := br.Peek(br.Buffered()) + req.Body = io.NopCloser(io.MultiReader(bytes.NewReader(consumed.Bytes()), bytes.NewReader(append([]byte(nil), rest...)), req.Body)) + return refs, nil +} + +// referenceWriteAllowedByMode applies the agent's ACMM mode to an opt-in +// reference write: the per-repo opt-in widens WHERE an agent may write, never +// WHAT a mode lets it do. Opening an issue or commenting needs ISSUES_ONLY; +// pushing a branch or opening a PR needs ISSUES_AND_PRS, exactly as on a +// managed repo (the PR hard-deny that routes managed-repo PRs through +// hive-open-pr does not apply — the relay cannot act on a foreign repo, and +// the reference credential is the App bot's own). +func referenceWriteAllowedByMode(mode agent.AgentMode, method, path string) bool { + switch { + case refGitReceivePackRe.MatchString(path), refGitInfoRefsRe.MatchString(path), refPullsPath.MatchString(path): + return mode >= agent.ModeIssuesAndPRs + } + return mode >= agent.ModeIssuesOnly +} diff --git a/src/crux/proxy/reference_repos_test.go b/src/crux/proxy/reference_repos_test.go new file mode 100644 index 0000000000..546ed31470 --- /dev/null +++ b/src/crux/proxy/reference_repos_test.go @@ -0,0 +1,237 @@ +package proxy + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +func newReferenceTestProxy(t *testing.T, pol ReferenceRepoPolicy, lookup referenceLookupFunc) *GitHubProxy { + t.Helper() + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.SetReferenceRepos([]ReferenceRepoPolicy{pol}) + p.referenceLookup = lookup + return p +} + +func refReq(method, path, rawQuery string, body []byte) *http.Request { + req, _ := http.NewRequest(method, "https://api.github.com"+path, bytes.NewReader(body)) + req.URL.RawQuery = rawQuery + req.Host = "api.github.com" + return req +} + +// pktLine encodes one git pkt-line. +func pktLine(s string) string { return fmt.Sprintf("%04x%s", len(s)+4, s) } + +func TestReferenceRepoDecision_Matrix(t *testing.T) { + ro := ReferenceRepoPolicy{Repo: "hivecommons/docs"} + issues := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true} + prs := ReferenceRepoPolicy{Repo: "hivecommons/docs", PRs: true} + both := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, PRs: true} + + cases := []struct { + name string + pol ReferenceRepoPolicy + method string + path string + query string + want referenceDecision + }{ + // Reads pass for every tier. + {"ro GET", ro, "GET", "/repos/hivecommons/docs/contents/README.md", "", referenceNotReference}, + {"ro fetch", ro, "POST", "/hivecommons/docs.git/git-upload-pack", "", referenceNotReference}, + {"ro info/refs upload", ro, "GET", "/hivecommons/docs.git/info/refs", "service=git-upload-pack", referenceNotReference}, + // Read-only refuses every write, including the push advertisement. + {"ro issue", ro, "POST", "/repos/hivecommons/docs/issues", "", referenceDeny}, + {"ro comment", ro, "POST", "/repos/hivecommons/docs/issues/3/comments", "", referenceDeny}, + {"ro push adv", ro, "GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", referenceDeny}, + {"ro push", ro, "POST", "/hivecommons/docs.git/git-receive-pack", "", referenceDeny}, + {"ro fork", ro, "POST", "/repos/hivecommons/docs/forks", "", referenceDeny}, + // issues tier. + {"issues open", issues, "POST", "/repos/hivecommons/docs/issues", "", referenceAllow}, + {"issues comment own-check", issues, "POST", "/repos/hivecommons/docs/issues/3/comments", "", referenceOwnItem}, + {"issues pr comment own-check", issues, "POST", "/repos/hivecommons/docs/pulls/9/comments", "", referenceOwnItem}, + {"issues edit", issues, "PATCH", "/repos/hivecommons/docs/issues/3", "", referenceDeny}, + {"issues label", issues, "POST", "/repos/hivecommons/docs/issues/3/labels", "", referenceDeny}, + {"issues assign", issues, "POST", "/repos/hivecommons/docs/issues/3/assignees", "", referenceDeny}, + {"issues open pr", issues, "POST", "/repos/hivecommons/docs/pulls", "", referenceDeny}, + {"issues push", issues, "POST", "/hivecommons/docs.git/git-receive-pack", "", referenceDeny}, + // prs tier. + {"prs open pr", prs, "POST", "/repos/hivecommons/docs/pulls", "", referenceAllow}, + {"prs push adv", prs, "GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", referencePush}, + {"prs push", prs, "POST", "/hivecommons/docs.git/git-receive-pack", "", referencePush}, + {"prs open issue", prs, "POST", "/repos/hivecommons/docs/issues", "", referenceDeny}, + {"prs merge", prs, "PUT", "/repos/hivecommons/docs/pulls/9/merge", "", referenceDeny}, + {"prs review", prs, "POST", "/repos/hivecommons/docs/pulls/9/reviews", "", referenceDeny}, + {"prs edit pr", prs, "PATCH", "/repos/hivecommons/docs/pulls/9", "", referenceDeny}, + {"prs api ref", prs, "POST", "/repos/hivecommons/docs/git/refs", "", referenceDeny}, + {"prs delete ref", prs, "DELETE", "/repos/hivecommons/docs/git/refs/heads/x", "", referenceDeny}, + {"prs contents", prs, "PUT", "/repos/hivecommons/docs/contents/x.md", "", referenceDeny}, + // both. + {"both issue", both, "POST", "/repos/hivecommons/docs/issues", "", referenceAllow}, + {"both pr", both, "POST", "/repos/hivecommons/docs/pulls", "", referenceAllow}, + {"both merge", both, "PUT", "/repos/hivecommons/docs/pulls/9/merge", "", referenceDeny}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, _, reason := ReferenceRepoDecision(tc.pol, tc.method, tc.path, tc.query) + if got != tc.want { + t.Fatalf("decision=%v want %v (reason %q)", got, tc.want, reason) + } + if got == referenceDeny && (!strings.Contains(reason, "reference_repos") || !strings.Contains(reason, tc.pol.tier())) { + t.Fatalf("refusal must name reference_repos and the tier: %q", reason) + } + }) + } +} + +func TestReferenceRepoRefusal_OwnItemCheck(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, Login: "hivecommons-hive[bot]", Token: func() (string, bool) { return "ref-tok", true }} + var gotURL, gotToken string + author := "hivecommons-hive[bot]" + p := newReferenceTestProxy(t, pol, func(url, token string) ([]byte, error) { + gotURL, gotToken = url, token + return []byte(`{"user":{"login":"` + author + `"}}`), nil + }) + + if reason, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)); refused { + t.Fatalf("comment on own issue refused: %s", reason) + } + if gotURL != "https://api.github.com/repos/hivecommons/docs/issues/12" || gotToken != "ref-tok" { + t.Fatalf("lookup url=%q token=%q", gotURL, gotToken) + } + + author = "someone-else" + reason, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)) + if !refused || !strings.Contains(reason, "someone-else") { + t.Fatalf("comment on another's issue must be refused naming the author, got refused=%v %q", refused, reason) + } + + p.referenceLookup = func(string, string) ([]byte, error) { return nil, fmt.Errorf("boom") } + if _, refused := p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil)); !refused { + t.Fatal("a failed author lookup must fail closed") + } + + // A different repo is not a reference repo at all. + if _, refused := p.referenceRepoRefusal(refReq("POST", "/repos/cncf/techdocs/issues/1/comments", "", nil)); refused { + t.Fatal("non-reference repo must not be gated here") + } +} + +func TestReferenceRepoRefusal_PushGuardsDefaultBranch(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", PRs: true, Token: func() (string, bool) { return "ref-tok", true }} + lookups := 0 + p := newReferenceTestProxy(t, pol, func(url, token string) ([]byte, error) { + lookups++ + if url != "https://api.github.com/repos/hivecommons/docs" { + t.Fatalf("unexpected lookup %s", url) + } + return []byte(`{"default_branch":"main"}`), nil + }) + zero := strings.Repeat("0", 40) + sha := strings.Repeat("a", 40) + pack := "PACK-bytes-follow" + + topic := pktLine(zero+" "+sha+" refs/heads/hive/assessment\x00report-status side-band-64k") + "0000" + pack + req := refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(topic)) + if reason, refused := p.referenceRepoRefusal(req); refused { + t.Fatalf("topic-branch push refused: %s", reason) + } + rest, _ := io.ReadAll(req.Body) + if string(rest) != topic { + t.Fatalf("push body not restored byte-for-byte:\n got %q\nwant %q", rest, topic) + } + + def := pktLine(zero+" "+sha+" refs/heads/hive/x\x00caps") + pktLine(sha+" "+sha+" refs/heads/main") + "0000" + pack + reason, refused := p.referenceRepoRefusal(refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(def))) + if !refused || !strings.Contains(reason, "default branch (main)") { + t.Fatalf("default-branch push must be refused, got refused=%v %q", refused, reason) + } + if lookups != 1 { + t.Fatalf("default branch should be cached after one lookup, got %d", lookups) + } + + gz := refReq("POST", "/hivecommons/docs.git/git-receive-pack", "", []byte(topic)) + gz.Header.Set("Content-Encoding", "gzip") + if _, refused := p.referenceRepoRefusal(gz); !refused { + t.Fatal("an encoded push body the guard cannot read must fail closed") + } + + if _, refused := p.referenceRepoRefusal(refReq("GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", nil)); refused { + t.Fatal("push ref advertisement must pass for a prs entry") + } +} + +func TestReferenceRepoAuth_StripsOnReadsInjectsOnWrites(t *testing.T) { + pol := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, PRs: true, Token: func() (string, bool) { return "ref-tok", true }} + p := newReferenceTestProxy(t, pol, nil) + p.injectGHAuth = true + p.agentTokenSource = func(string) (string, bool) { return "hive-app-token", true } + + read := refReq("GET", "/repos/hivecommons/docs/contents/README.md", "", nil) + read.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(read, "scanner") + if got := read.Header.Get("Authorization"); got != "" { + t.Fatalf("reference read must go out unauthenticated, got %q", got) + } + + fetch := refReq("POST", "/hivecommons/docs.git/git-upload-pack", "", nil) + fetch.Header.Set("Authorization", "Basic xyz") + p.rewriteGitHubAuth(fetch, "scanner") + if got := fetch.Header.Get("Authorization"); got != "" { + t.Fatalf("reference fetch must go out unauthenticated, got %q", got) + } + + write := refReq("POST", "/repos/hivecommons/docs/issues", "", nil) + write.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(write, "scanner") + if got := write.Header.Get("Authorization"); got != "token ref-tok" { + t.Fatalf("reference write must carry the reference credential, got %q", got) + } + + push := refReq("GET", "/hivecommons/docs.git/info/refs", "service=git-receive-pack", nil) + p.rewriteGitHubAuth(push, "scanner") + want := "Basic " + base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser+":ref-tok")) + if got := push.Header.Get("Authorization"); got != want || !push.Close { + t.Fatalf("push advertisement must carry basic reference credential and close, got %q close=%v", got, push.Close) + } + + // A managed repo still takes the ordinary injection path. + managed := refReq("GET", "/repos/cncf/techdocs/issues", "", nil) + p.rewriteGitHubAuth(managed, "scanner") + if got := managed.Header.Get("Authorization"); got != "token hive-app-token" { + t.Fatalf("managed repo should get the agent token, got %q", got) + } + + // Read-only entries never carry anything, even on a (refused upstream) write. + p.SetReferenceRepos([]ReferenceRepoPolicy{{Repo: "hivecommons/docs"}}) + ro := refReq("POST", "/repos/hivecommons/docs/issues", "", nil) + ro.Header.Set("Authorization", "token agent-held") + p.rewriteGitHubAuth(ro, "scanner") + if got := ro.Header.Get("Authorization"); got != "" { + t.Fatalf("read-only reference repo must never carry a credential, got %q", got) + } +} + +func TestReferenceWriteAllowedByMode(t *testing.T) { + if referenceWriteAllowedByMode(agent.ModeAdvisory, "POST", "/repos/o/r/issues") { + t.Error("ADVISORY may not open issues on a reference repo") + } + if !referenceWriteAllowedByMode(agent.ModeIssuesOnly, "POST", "/repos/o/r/issues") { + t.Error("ISSUES_ONLY may open issues") + } + if referenceWriteAllowedByMode(agent.ModeIssuesOnly, "POST", "/repos/o/r/pulls") { + t.Error("ISSUES_ONLY may not open PRs") + } + if !referenceWriteAllowedByMode(agent.ModeIssuesAndPRs, "POST", "/o/r.git/git-receive-pack") { + t.Error("ISSUES_AND_PRS may push a topic branch") + } +} diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 83820127f5..07ab917f90 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1693` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1694` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1695` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1704` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1705` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1706` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 96f6d9c44f..dc4eacc334 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1700` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1711` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index acefbb9f3c..f26adaa4ae 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1673` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1674` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1675` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1681` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1686` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1687` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1688` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1689` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1690` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1691` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1692` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1684` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1685` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1686` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1692` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1697` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1698` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1699` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1700` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1701` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1702` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1703` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index d6909a3512..8dde400125 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1763` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1774` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/github-write-surface.md b/src/docs/github-write-surface.md index bd4a8b6d1e..61ebcbd711 100644 --- a/src/docs/github-write-surface.md +++ b/src/docs/github-write-surface.md @@ -252,6 +252,21 @@ A refused request is handled like every other policy denial: Refusals are not counted as output activity. +## Reference repos + +`project.reference_repos` ([#18015](https://github.com/hivecommons/hive/issues/18015)) +names repos *outside* the project an agent may study. They are not part of +the write surface at all by default: the proxy strips the hive's App token +from every request to them and refuses every non-GET with a message naming +`reference_repos`. A per-entry `write: [issues, prs]` opt-in unlocks exactly +three direct writes on that repo — open an issue, open a PR / push a topic +branch, comment on the hive's own issue or PR — sent with a credential valid +for *that* repo (the App installed on the target org, or `write_token_file`). +Merge, default-branch push, labels, assignees, reviews and closing others' +items stay refused. The relays never act on a reference repo, so a lane under +`write_surface.enforce` cannot write to one. See the +[operator reference](operator-reference/README.md#configuration-blocks). + ## Enforcing the write surface `write_surface.enforce` lists the lanes whose **direct** GitHub writes from the diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 35267a3a32..48998e010e 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -251,6 +251,9 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `project.repo_policies[].label_driven` | Off (unset) for every repo. | Opt-in for repos whose maintainers accept and park issues with labels and their own bots (for example `needs-triage` → `triage/accepted`). On such a repo the un-park sweep posts no "What to reply" notice and never removes `needs-human`, `needs-decision` or `needs-direction` on a `/hive` comment alone; instead a trusted triager applying an approval label releases the issue and counts as human sign-off — see [Label-driven approval](#label-driven-approval). Editable from the dashboard: Settings → Repos → per-repo "Label-driven approvals (Prow-style)" (write or owner role). Details in [maintainer-commands.md](../maintainer-commands.md#label-driven-repositories). | | `project.repo_policies[].label_driven_approval_labels` | Unset: inherits `project.label_driven_approval_labels`. | Per-repo approval-label list; only read when `label_driven` is set. Editable from the dashboard sub-row of the repo's label-driven toggle. | | `project.repo_policies[].companion_repos` | Unset (no companions). | List of sibling repositories (`name` in the project org, or `org/name`) that this repo's tooling expects checked out beside it under the agent workspace root — for example a docs/website repo a spec tool registers at `<workspace>/<org>/<sibling>/…`. Contributor task prompts and resident-agent kicks carry a shallow-clone plan for each companion at `<workspace root>/<org>/<sibling>` (`$HIVE_WORKSPACE_DIR` for contributor agents, `$HOME/workspace` for resident agents), and agents are told that a tool reporting one of those paths missing means *clone it*, never park the issue. Self-references and malformed entries are dropped. See [running-at-level-6.md](../running-at-level-6.md#needless-parks). | +| `project.reference_repos` | Unset (none). | Repos **outside** the project the hive may *study* without managing them ([#18015](https://github.com/hivecommons/hive/issues/18015)) — the rare case of a hive whose deliverable is *about* another project, e.g. a CNCF TechDocs hive (authorized repo `cncf/techdocs`) assessing `hivecommons/docs`. Each entry is `owner/name` (bare names are rejected, as is any repo also in `project.repos`). Default tier is **read-only**: agents may `git clone`/fetch them unauthenticated over HTTPS and issue GitHub REST/GraphQL GETs; the proxy strips the hive's App token from every request to them and refuses every non-GET with a message naming `reference_repos`. The kick lists them under `📚 REFERENCE REPOS` so an authorized-repo item whose deliverables name one is treated as in scope (not out-of-scope/critical), and agents are told findings go in the deliverable inside the authorized repo, never as issues or PRs on the reference repo. The dashboard shows each as a minimal card with a `reference · <tier>` chip — no stats, no ACMM, no work items. Private reference repos are not readable (reads are unauthenticated). Example: <br>`project:`<br>` repos: [techdocs]`<br>` reference_repos:`<br>` - hivecommons/docs`<br>` - hivecommons/hivecommons.github.io`<br>` - {repo: hivecommons/hive, write: [issues, prs]}` | +| `project.reference_repos[].write` | Unset (read-only). | Per-entry write opt-in, any of `issues`, `prs`. The **only** writes it unlocks on that repo: open an issue; open a PR / push a topic branch; comment on an issue or PR the hive itself opened. Never: merge, push to the default branch, labels or assignees on others' items, reviews, closing others' issues — the proxy refuses those deterministically (`reference_repo` in the violations log). The write is sent with a credential GitHub honours for *that* repo, never the hive's own installation token: by default the same GitHub App must be **installed on the target org** (the hive discovers that installation and mints a token scoped to the one repo and the opted-in permissions); otherwise set `write_token_file`. Boot refuses (exit code 22) a write-mode entry with neither. Direct POSTs are used here, not the `hive-open-pr`/`hive-open-issue` relays, which only act on `project.repos`; a lane under `write_surface.enforce` therefore cannot write to a reference repo. | +| `project.reference_repos[].write_token_file` | Unset. | Path to a file holding a token (a maintainer PAT or fine-grained token) authorized for that repo; read on every use so a rotation needs no restart. Only meaningful with `write`. Its `GET /user` login is what the proxy treats as "own item" for comments. | | `project.label_driven_approval_labels` | `["triage/accepted"]` | Labels whose trusted addition approves an issue on every label-driven repo without its own list. Ignored by repos without `label_driven: true`. Dashboard: Settings → Repos → "Default approval labels". | | `project.label_driven_trust_org_members` | `true` | Whether a public member of the repo's owning org who applies an approval label is trusted (Prow's own rule). Dashboard: Settings → Repos → "trust public org members". | | `project.label_driven_trusted_bots` | `["kubestellar-prow[bot]"]` | Bot logins whose label additions count as a trusted triager (Prow command bots relaying `/triage accepted`). `github-actions[bot]` is not in the default because it is the shared identity of every workflow in the repo, including ones triggered by untrusted events; operators whose Prow emulation runs as `github-actions[bot]` must add it explicitly per deployment. Dashboard: Settings → Repos → "Trusted bots". | diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index ecf4cb63f0..8b54b34f80 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -1168,6 +1168,13 @@ type ProjectConfig struct { // Optional and additive. Absent — the state of every existing config — // means nothing is paused and the hive behaves exactly as before. PausedRepos []RepoPause `yaml:"paused_repos,omitempty"` + // ReferenceRepos are repositories agents may STUDY but do not manage + // (hivecommons/hive#18015): explicit owner/name entries, read-only by + // default, with a rare per-entry opt-in for opening issues/PRs. They are + // NOT in Repos: no work is enumerated from them, no ACMM eval runs on + // them, and writes to them are refused unless opted in. See + // reference_repos.go. + ReferenceRepos []ReferenceRepo `yaml:"reference_repos,omitempty" json:"reference_repos,omitempty"` // RepoPolicies stores optional per-repository policy overrides keyed by // repo name. project.repos remains the watched-repo identity list; this // sidecar list lets existing string-list configs keep round-tripping. diff --git a/src/pkg/config/reference_repos.go b/src/pkg/config/reference_repos.go new file mode 100644 index 0000000000..f67a976c34 --- /dev/null +++ b/src/pkg/config/reference_repos.go @@ -0,0 +1,231 @@ +package config + +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ReferenceRepo is one entry of project.reference_repos (hivecommons/hive#18015): +// a repository the hive's agents may STUDY — clone, fetch, read through the +// GitHub API — without it being an authorized repo. The motivating case is a +// documentation-assessment hive (CNCF TechDocs) whose deliverable lives in its +// own authorized repo but describes another project's repos: the agent has to +// read those repos, and must not be told they are out of scope, yet nothing +// about the assessment should ever land on them. +// +// The default tier is READ-ONLY: public clones and GETs, never carrying the +// hive's App token (which is not valid for a foreign installation anyway). +// Every write is refused by the proxy with a 403 that names reference_repos. +// +// Write is the rare opt-in for an assessor who also files findings upstream. +// It is deliberately narrow — open an issue, open a PR on a topic branch, +// comment on the hive's own items — and the credential behind it has to be one +// GitHub actually honours for that repo: the App's installation on the TARGET +// org (minted on demand, scoped to this one repo) or an operator-supplied +// write_token_file. Boot refuses a write-mode entry that has neither. +type ReferenceRepo struct { + // Repo is the explicit "owner/name" reference. Bare names are rejected: + // a reference repo is by definition outside the project's own list, so + // there is no org to qualify it against without guessing. + Repo string `yaml:"repo" json:"repo"` + // Write lists the opt-in write surfaces: "issues" and/or "prs". Empty is + // the read-only default. + Write []string `yaml:"write,omitempty" json:"write,omitempty"` + // WriteTokenFile is a path on the hive host holding a token that is + // authorized for Repo, used instead of minting from the App installation + // on the target org. Only meaningful with Write. + WriteTokenFile string `yaml:"write_token_file,omitempty" json:"write_token_file,omitempty"` +} + +const ( + ReferenceWriteIssues = "issues" + ReferenceWritePRs = "prs" +) + +// UnmarshalYAML accepts both the scalar form ("owner/name", read-only) and the +// mapping form ({repo: owner/name, write: [...]}). +func (r *ReferenceRepo) UnmarshalYAML(value *yaml.Node) error { + if value.Kind == yaml.ScalarNode { + r.Repo = strings.TrimSpace(value.Value) + r.Write = nil + r.WriteTokenFile = "" + return nil + } + type plain ReferenceRepo + var p plain + if err := value.Decode(&p); err != nil { + return err + } + *r = ReferenceRepo(p) + r.Repo = strings.TrimSpace(r.Repo) + return nil +} + +// MarshalYAML writes the scalar form back for a read-only entry so a config +// that was written as a plain list round-trips unchanged. +func (r ReferenceRepo) MarshalYAML() (any, error) { + if len(r.Write) == 0 && r.WriteTokenFile == "" { + return r.Repo, nil + } + type plain ReferenceRepo + return plain(r), nil +} + +// WritesIssues reports whether the entry opted in to opening issues (and +// commenting on its own). +func (r ReferenceRepo) WritesIssues() bool { return r.hasWrite(ReferenceWriteIssues) } + +// WritesPRs reports whether the entry opted in to pushing topic branches and +// opening PRs. +func (r ReferenceRepo) WritesPRs() bool { return r.hasWrite(ReferenceWritePRs) } + +// ReadOnly reports whether the entry has no write opt-in at all. +func (r ReferenceRepo) ReadOnly() bool { return !r.WritesIssues() && !r.WritesPRs() } + +func (r ReferenceRepo) hasWrite(kind string) bool { + for _, w := range r.Write { + if strings.EqualFold(strings.TrimSpace(w), kind) { + return true + } + } + return false +} + +// WriteLabel is the human-facing tier: "read-only", "issues", "prs" or +// "issues+prs". The dashboard chip and the kick both print it. +func (r ReferenceRepo) WriteLabel() string { + switch { + case r.WritesIssues() && r.WritesPRs(): + return "issues+prs" + case r.WritesIssues(): + return "issues" + case r.WritesPRs(): + return "prs" + } + return "read-only" +} + +// Owner returns the owner half of Repo ("" when malformed). +func (r ReferenceRepo) Owner() string { + owner, _, _ := strings.Cut(r.Repo, "/") + return owner +} + +// Name returns the repository half of Repo ("" when malformed). +func (r ReferenceRepo) Name() string { + _, name, _ := strings.Cut(r.Repo, "/") + return name +} + +// referenceRepoKey is the case-folded comparison key for a reference entry. +func referenceRepoKey(repo string) string { + return strings.ToLower(strings.TrimSpace(repo)) +} + +// ReferenceRepoFor returns the reference entry covering repo ("owner/name", +// case-insensitive), and whether one exists. +func (c *Config) ReferenceRepoFor(repo string) (ReferenceRepo, bool) { + if c == nil { + return ReferenceRepo{}, false + } + key := referenceRepoKey(repo) + if key == "" { + return ReferenceRepo{}, false + } + for _, r := range c.Project.ReferenceRepos { + if referenceRepoKey(r.Repo) == key { + return r, true + } + } + return ReferenceRepo{}, false +} + +// IsReferenceRepo reports whether repo ("owner/name") is a reference repo. +func (c *Config) IsReferenceRepo(repo string) bool { + _, ok := c.ReferenceRepoFor(repo) + return ok +} + +// ReferenceReposWithWrite returns the entries that opted in to any write. +func (c *Config) ReferenceReposWithWrite() []ReferenceRepo { + if c == nil { + return nil + } + var out []ReferenceRepo + for _, r := range c.Project.ReferenceRepos { + if !r.ReadOnly() { + out = append(out, r) + } + } + return out +} + +// validReferenceRepoSlug accepts exactly "owner/name" with no whitespace, no +// scheme and no extra path. +func validReferenceRepoSlug(repo string) bool { + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + return false + } + if strings.ContainsAny(repo, " \t\n:@") || strings.Contains(name, "/") { + return false + } + return true +} + +// validateReferenceRepos enforces the shape of project.reference_repos: every +// entry is an explicit owner/name, none duplicates another, none is also an +// authorized repo (a repo is either managed or reference, never both — the +// two tiers would otherwise contradict each other on every write), write +// kinds are known, and write_token_file is only set alongside write. +func (c *Config) validateReferenceRepos() error { + if len(c.Project.ReferenceRepos) == 0 { + return nil + } + authorized := make(map[string]bool, len(c.Project.Repos)) + for _, repo := range c.Project.Repos { + authorized[referenceRepoKey(QualifyRepo(c.Project.Org, repo))] = true + } + seen := make(map[string]bool, len(c.Project.ReferenceRepos)) + for i, r := range c.Project.ReferenceRepos { + if !validReferenceRepoSlug(r.Repo) { + return fmt.Errorf("project.reference_repos[%d]: %q must be an explicit owner/name (a reference repo lives outside this project, so there is no org to qualify a bare name against)", i, r.Repo) + } + key := referenceRepoKey(r.Repo) + if authorized[key] { + return fmt.Errorf("project.reference_repos[%d]: %q is already in project.repos — a repository is either authorized (agents work it) or reference (agents only study it), not both", i, r.Repo) + } + if seen[key] { + return fmt.Errorf("project.reference_repos[%d]: %q is listed twice", i, r.Repo) + } + seen[key] = true + for _, w := range r.Write { + switch strings.ToLower(strings.TrimSpace(w)) { + case ReferenceWriteIssues, ReferenceWritePRs: + default: + return fmt.Errorf("project.reference_repos[%d]: write %q is not one of %q, %q", i, w, ReferenceWriteIssues, ReferenceWritePRs) + } + } + if r.WriteTokenFile != "" && r.ReadOnly() { + return fmt.Errorf("project.reference_repos[%d]: write_token_file is set but write is empty — a read-only reference repo never carries a credential", i) + } + } + return nil +} + +// ReferenceRepoSlugs returns the reference repos as sorted "owner/name" +// strings, for kicks and environment. +func (c *Config) ReferenceRepoSlugs() []string { + if c == nil { + return nil + } + out := make([]string, 0, len(c.Project.ReferenceRepos)) + for _, r := range c.Project.ReferenceRepos { + out = append(out, r.Repo) + } + sort.Strings(out) + return out +} diff --git a/src/pkg/config/reference_repos_test.go b/src/pkg/config/reference_repos_test.go new file mode 100644 index 0000000000..f0973865ec --- /dev/null +++ b/src/pkg/config/reference_repos_test.go @@ -0,0 +1,101 @@ +package config + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func referenceRepoTestConfig(t *testing.T, refs string) *Config { + t.Helper() + src := ` +project: + org: cncf + repos: [techdocs] + reference_repos: +` + refs + ` +agents: + scanner: + backend: claude +` + var c Config + if err := yaml.Unmarshal([]byte(src), &c); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return &c +} + +func TestReferenceReposParseScalarAndMapping(t *testing.T) { + c := referenceRepoTestConfig(t, ` + - hivecommons/docs + - repo: hivecommons/hivecommons.github.io + write: [issues, prs] + - repo: hivecommons/hive + write: [issues] + write_token_file: /data/ref-token +`) + if err := c.validateReferenceRepos(); err != nil { + t.Fatalf("validate: %v", err) + } + if got := len(c.Project.ReferenceRepos); got != 3 { + t.Fatalf("want 3 entries, got %d", got) + } + ro, ok := c.ReferenceRepoFor("HiveCommons/Docs") + if !ok || !ro.ReadOnly() || ro.WriteLabel() != "read-only" { + t.Fatalf("docs: ok=%v entry=%+v", ok, ro) + } + rw, ok := c.ReferenceRepoFor("hivecommons/hivecommons.github.io") + if !ok || !rw.WritesIssues() || !rw.WritesPRs() || rw.WriteLabel() != "issues+prs" { + t.Fatalf("site: ok=%v entry=%+v", ok, rw) + } + tok, _ := c.ReferenceRepoFor("hivecommons/hive") + if tok.WriteTokenFile != "/data/ref-token" || tok.WriteLabel() != "issues" || tok.Owner() != "hivecommons" || tok.Name() != "hive" { + t.Fatalf("hive: %+v", tok) + } + if c.IsReferenceRepo("cncf/techdocs") { + t.Fatal("authorized repo must not read as reference") + } + if got := len(c.ReferenceReposWithWrite()); got != 2 { + t.Fatalf("want 2 write entries, got %d", got) + } + // Round-trip: the scalar entry stays a scalar. + out, err := yaml.Marshal(c.Project.ReferenceRepos) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "- hivecommons/docs\n") { + t.Fatalf("read-only entry should marshal as a scalar:\n%s", out) + } +} + +func TestReferenceReposValidationRejects(t *testing.T) { + cases := []struct { + name, refs, want string + }{ + {"bare name", " - docs\n", "explicit owner/name"}, + {"url", " - https://github.com/hivecommons/docs\n", "explicit owner/name"}, + {"overlap with project.repos", " - cncf/techdocs\n", "already in project.repos"}, + {"overlap case-insensitive", " - CNCF/TechDocs\n", "already in project.repos"}, + {"duplicate", " - hivecommons/docs\n - HiveCommons/docs\n", "listed twice"}, + {"unknown write", " - repo: hivecommons/docs\n write: [merge]\n", "not one of"}, + {"token without write", " - repo: hivecommons/docs\n write_token_file: /x\n", "write is empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := referenceRepoTestConfig(t, tc.refs) + err := c.validateReferenceRepos() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("want error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestReferenceReposValidationRunsFromValidate(t *testing.T) { + c := referenceRepoTestConfig(t, " - docs\n") + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "reference_repos") { + t.Fatalf("Validate should surface the reference_repos error, got %v", err) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 8812f3e6cf..1cc6845b18 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -99,6 +99,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := validateWorkflowBindings(c.Project.WorkflowBindings); err != nil { return err } + if err := c.validateReferenceRepos(); err != nil { + return err + } if _, err := c.Ioscan.Policy(); err != nil { return err } diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index d53f4cc24b..a8eb8e7c30 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -657,11 +657,15 @@ type StatusPayload struct { Governor FrontendGovernor `json:"governor"` Tokens FrontendTokens `json:"tokens"` Repos []FrontendRepo `json:"repos"` - Swarm SwarmStatus `json:"swarm"` - Beads FrontendBeads `json:"beads"` - Planning FrontendPlanning `json:"planning"` - Runs RunsSummary `json:"runs"` - Health map[string]any `json:"health"` + // ReferenceRepos are project.reference_repos (hivecommons/hive#18015): + // studied, not worked. Kept apart from Repos so no queue, ACMM or + // work-item consumer ever treats one as managed. + ReferenceRepos []FrontendReferenceRepo `json:"referenceRepos,omitempty"` + Swarm SwarmStatus `json:"swarm"` + Beads FrontendBeads `json:"beads"` + Planning FrontendPlanning `json:"planning"` + Runs RunsSummary `json:"runs"` + Health map[string]any `json:"health"` // DeepHealth carries the spoke's own deep health checks (HealthSummary: // ready, github_auth, agents, …) — the same checks the heartbeat reports // to the hub. The dashboard's header Health pill renders from these, NOT @@ -1211,6 +1215,13 @@ type FrontendRestartRefresh struct { ReposTotal int `json:"reposTotal"` } +// FrontendReferenceRepo is one project.reference_repos entry for the repos +// card: the slug and its tier ("read-only", "issues", "prs", "issues+prs"). +type FrontendReferenceRepo struct { + Full string `json:"full"` + Tier string `json:"tier"` +} + type FrontendRepo struct { Name string `json:"name"` Full string `json:"full"` diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 3bf0e75a3c..6c298855c7 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -482,6 +482,9 @@ project.paused_repos[].by # TODO #11854 project.paused_repos[].reason # TODO #11854 project.paused_repos[].repo # TODO #11854 project.primary_repo # TODO #11854 +project.reference_repos[].repo # file-only by design (#18015): rare study-only repos, set in the overlay +project.reference_repos[].write[] # file-only by design (#18015) +project.reference_repos[].write_token_file # file-only by design (#18015) project.repo_policies[].acmm_last_automatic.at # TODO #11854 project.repo_policies[].acmm_last_automatic.direction # TODO #11854 project.repo_policies[].acmm_last_automatic.evidence_ids[] # TODO #11854 diff --git a/src/pkg/dashboard/static-src/css/repos.css b/src/pkg/dashboard/static-src/css/repos.css index cb28002577..c8aa80659c 100644 --- a/src/pkg/dashboard/static-src/css/repos.css +++ b/src/pkg/dashboard/static-src/css/repos.css @@ -73,6 +73,13 @@ .repo-fork-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--blue) 45%, transparent); background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); cursor: help; } .repo-fork-pill.off { border-color: var(--border); background: transparent; color: var(--text-muted); } .repo-paused-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; text-transform: uppercase; border: 1px solid color-mix(in srgb, var(--yellow) 45%, transparent); background: color-mix(in srgb, var(--yellow) 15%, transparent); color: var(--yellow); cursor: help; } + /* project.reference_repos (#18015): a studied-not-worked repo. Muted + border, no stats block, one tier chip. */ + .repo-card-reference { border-style: dashed; opacity: 0.85; } + .repo-reference-provider { color: var(--muted); } + .repo-reference-link { color: inherit; text-decoration: none; } + .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } /* margin: 0 beats .config-toggle's margin-bottom (that rule is declared diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index c60dde40c0..c8f98b6833 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -346,12 +346,37 @@ <div class="repo-stat"><a href="${esc(repoUrl + cardForge.changesPath)}" target="_blank" rel="noopener" style="color:inherit;text-decoration:none"><div class="spark-row"><span class="num">${r.prs >= 0 ? r.prs : '?'}</span>${pSpark}</div><div class="label">PRs</div></a>${prBreakdown}</div> </div>${pillsHtml}${hfRepoSectionHtml(r)} </div>`; - }).join('')); + }).join('') + renderReferenceRepoCards(status.referenceRepos, hostLabel)); hfRenderEditor(); hfRenderPinned(); hfRenderCrossRepo(); } + /** + * project.reference_repos (#18015): repos outside the project the hive may + * study. Deliberately minimal — a name and a tier chip, no stats, no ACMM, + * no pause/swarm controls — so they are legible as "not managed here". + * Hidden while a pill or project filter is active (they have no work items + * to match). + */ + function renderReferenceRepoCards(refs, hostLabel) { + if (!Array.isArray(refs) || !refs.length) return ''; + if (repoPillFilterActive(repoPillFilterState()) || overviewProjectFilterState()) return ''; + const base = window._githubBaseUrl || 'https://github.com'; + return refs.map(rr => { + const full = String(rr.full || ''); + const tier = String(rr.tier || 'read-only'); + const tip = tier === 'read-only' + ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' + : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; + return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> + <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> + <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + </div>`; + }).join(''); + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 22024fe971..9ed7f34a33 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -3299,6 +3299,13 @@ .repo-fork-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--blue) 45%, transparent); background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); cursor: help; } .repo-fork-pill.off { border-color: var(--border); background: transparent; color: var(--text-muted); } .repo-paused-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; text-transform: uppercase; border: 1px solid color-mix(in srgb, var(--yellow) 45%, transparent); background: color-mix(in srgb, var(--yellow) 15%, transparent); color: var(--yellow); cursor: help; } + /* project.reference_repos (#18015): a studied-not-worked repo. Muted + border, no stats block, one tier chip. */ + .repo-card-reference { border-style: dashed; opacity: 0.85; } + .repo-reference-provider { color: var(--muted); } + .repo-reference-link { color: inherit; text-decoration: none; } + .repo-reference-pill { flex: 0 0 auto; font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; line-height: 1; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; border: 1px solid color-mix(in srgb, var(--indigo) 45%, transparent); background: color-mix(in srgb, var(--indigo) 12%, transparent); color: var(--indigo); cursor: help; } + .repo-reference-note { font-size: var(--fs-2xs); color: var(--muted); margin-top: var(--sp-2); } .repo-pause-btn { flex: 0 0 auto; margin-left: 0; font-size: 0.66rem; font-weight: 600; line-height: 1; padding: 3px 7px; border-radius: var(--r-pill); white-space: nowrap; cursor: pointer; border: 1px solid var(--border); background: transparent; color: var(--muted); opacity: 0.7; transition: opacity 0.15s, color 0.15s, border-color 0.15s; } .repo-automerge-toggle { flex: 0 0 auto; white-space: nowrap; gap: var(--sp-2); margin: 0; font-size: var(--fs-2xs); color: var(--muted); } /* margin: 0 beats .config-toggle's margin-bottom (that rule is declared @@ -36875,12 +36882,37 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> <div class="repo-stat"><a href="${esc(repoUrl + cardForge.changesPath)}" target="_blank" rel="noopener" style="color:inherit;text-decoration:none"><div class="spark-row"><span class="num">${r.prs >= 0 ? r.prs : '?'}</span>${pSpark}</div><div class="label">PRs</div></a>${prBreakdown}</div> </div>${pillsHtml}${hfRepoSectionHtml(r)} </div>`; - }).join('')); + }).join('') + renderReferenceRepoCards(status.referenceRepos, hostLabel)); hfRenderEditor(); hfRenderPinned(); hfRenderCrossRepo(); } + /** + * project.reference_repos (#18015): repos outside the project the hive may + * study. Deliberately minimal — a name and a tier chip, no stats, no ACMM, + * no pause/swarm controls — so they are legible as "not managed here". + * Hidden while a pill or project filter is active (they have no work items + * to match). + */ + function renderReferenceRepoCards(refs, hostLabel) { + if (!Array.isArray(refs) || !refs.length) return ''; + if (repoPillFilterActive(repoPillFilterState()) || overviewProjectFilterState()) return ''; + const base = window._githubBaseUrl || 'https://github.com'; + return refs.map(rr => { + const full = String(rr.full || ''); + const tier = String(rr.tier || 'read-only'); + const tip = tier === 'read-only' + ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' + : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; + return `<div class="repo-card repo-card-reference" data-repo="${esc(full)}"> + <div class="repo-provider-label repo-reference-provider" title="This repo lives on ${esc(hostLabel)} and is outside this project">${esc(hostLabel)}</div> + <div class="repo-name"><a class="repo-reference-link" href="${esc(base + '/' + full)}" target="_blank" rel="noopener" title="${esc(full)}">${esc(full)}</a><span class="repo-header-badges"><span class="repo-reference-pill" title="${esc(tip)}">📚 reference · ${esc(tier)}</span></span></div> + <div class="repo-reference-note">Outside this project — studied, not worked. No work items, no ACMM evaluation.</div> + </div>`; + }).join(''); + } + /** * Where a repo card points and what it calls its host (#14115). r.forge is * the repo's forge kind from /api/status; GitHub (and an older hive that diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index e85b6beac4..40e031e0b0 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -404,6 +404,7 @@ func BuildFrontendStatus( Governor: buildGovernor(govState, cfg, apiBudgetFromRateLimits(ghRateLimits)), Tokens: buildTokens(tokenCollector), Repos: repos, + ReferenceRepos: buildReferenceRepos(cfg), OverviewKPIIncomplete: overviewKPIIncomplete(cfg, actionable) || overviewStaleSince(repos) != "", StaleSince: earliestRFC3339(staleSinceRFC3339(actionable), overviewStaleSince(repos)), RestartRefresh: restartRefreshInfo(actionable, repos), @@ -2372,6 +2373,19 @@ func buildRepos(cfg *config.Config, actionable *github.ActionableResult, govStat return buildReposWithForge(cfg, actionable, govState, nil) } +// buildReferenceRepos lists project.reference_repos with their tier for the +// dashboard's minimal reference cards (hivecommons/hive#18015). +func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { + if cfg == nil || len(cfg.Project.ReferenceRepos) == 0 { + return nil + } + out := make([]FrontendReferenceRepo, 0, len(cfg.Project.ReferenceRepos)) + for _, rr := range cfg.Project.ReferenceRepos { + out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: rr.WriteLabel()}) + } + return out +} + func buildReposWithForge(cfg *config.Config, actionable *github.ActionableResult, govState governor.State, forge forgeOpenCounter) []FrontendRepo { repos := make([]FrontendRepo, 0, len(cfg.Project.Repos)) repoRows := make(map[string]string, len(cfg.Project.Repos)*2) diff --git a/src/pkg/github/reference_repo_token.go b/src/pkg/github/reference_repo_token.go new file mode 100644 index 0000000000..7000ddfc47 --- /dev/null +++ b/src/pkg/github/reference_repo_token.go @@ -0,0 +1,190 @@ +package github + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" + + "github.com/hivecommons/hive/pkg/config" +) + +// Reference-repo write credentials (hivecommons/hive#18015). +// +// A project.reference_repos entry that opted in to writes needs a credential +// GitHub actually honours for THAT repo. The hive's own installation token +// cannot be it: an App installation is per-account, and a reference repo is by +// definition outside the hive's org. Two sources are supported, in the order +// an operator reaches for them: +// +// - the same App installed on the target org — the hive discovers that +// installation and mints a token on it, scoped to the one listed repo and +// to exactly the permissions the opt-in needs; +// - write_token_file — an operator-supplied token (a maintainer PAT, a +// fine-grained token) read from disk on each use so a rotation is picked up +// without a restart. +// +// Boot refuses a write-mode entry that has neither (see +// VerifyReferenceRepoWriteCredential) so the failure is a config error with +// the repo named, not a 401 a day later. + +// referenceTokenRefreshMargin is how long before expiry a minted reference +// token is replaced. +const referenceTokenRefreshMargin = 5 * time.Minute + +// ErrReferenceRepoNoCredential is the sentinel for a write-mode entry with no +// usable credential source. +var ErrReferenceRepoNoCredential = errors.New("reference repo write-mode has no credential") + +// referenceRepoPermissions are the installation permissions an opt-in needs: +// metadata always, issues:write for issues, contents:write + pull_requests:write +// for prs (a PR needs a branch to be pushed first). Nothing wider — the proxy +// narrows further to the exact write paths. +func referenceRepoPermissions(rr config.ReferenceRepo) *gh.InstallationPermissions { + perms := &gh.InstallationPermissions{Metadata: gh.Ptr("read"), Contents: gh.Ptr("read")} + if rr.WritesIssues() { + perms.Issues = gh.Ptr("write") + } + if rr.WritesPRs() { + perms.Contents = gh.Ptr("write") + perms.PullRequests = gh.Ptr("write") + } + return perms +} + +// ReferenceRepoTokenSource returns a token function for the proxy: the +// write_token_file when set, else a cached installation token minted on the +// target org. auth may be replaced on key rotation, so it is taken through a +// getter rather than captured. +func ReferenceRepoTokenSource(rr config.ReferenceRepo, auth func() *AppAuth, logger *slog.Logger) func() (string, bool) { + if rr.ReadOnly() { + return nil + } + if rr.WriteTokenFile != "" { + return func() (string, bool) { + tok, err := readTokenFile(rr.WriteTokenFile) + if err != nil { + logger.Warn("reference repo write token unreadable", "repo", rr.Repo, "file", rr.WriteTokenFile, "error", err) + return "", false + } + return tok, true + } + } + var ( + mu sync.Mutex + token string + expiry time.Time + ) + return func() (string, bool) { + mu.Lock() + defer mu.Unlock() + if token != "" && time.Now().Add(referenceTokenRefreshMargin).Before(expiry) { + return token, true + } + a := auth() + if a == nil { + logger.Warn("reference repo write token: no App auth available", "repo", rr.Repo) + return "", false + } + ctx, cancel := context.WithTimeout(context.Background(), mintClientTimeout) + defer cancel() + tok, exp, err := a.MintReferenceRepoToken(ctx, rr) + if err != nil { + logger.Warn("reference repo write token mint failed", "repo", rr.Repo, "error", err) + return "", false + } + token, expiry = tok, exp + return token, true + } +} + +// MintReferenceRepoToken mints an installation token on the TARGET org's +// installation of this App, scoped to rr's single repository and to the +// permissions its write opt-in needs. +func (a *AppAuth) MintReferenceRepoToken(ctx context.Context, rr config.ReferenceRepo) (string, time.Time, error) { + if a == nil { + return "", time.Time{}, ErrNoGitHubClient + } + installationID, err := a.DiscoverInstallationID(ctx, rr.Owner()) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: %s: %v", ErrReferenceRepoNoCredential, rr.Repo, err) + } + opts := &gh.InstallationTokenOptions{ + Repositories: []string{rr.Name()}, + Permissions: referenceRepoPermissions(rr), + } + return a.mintInstallationTokenWithOptions(ctx, installationID, opts) +} + +func readTokenFile(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + tok := strings.TrimSpace(string(b)) + if tok == "" { + return "", fmt.Errorf("%s is empty", path) + } + return tok, nil +} + +// VerifyReferenceRepoWriteCredential is the boot-time check for one write-mode +// entry: the write_token_file must be readable and non-empty, or the App must +// have an installation on the target org. It returns the login the credential +// acts as — the App bot login, or the token's user — for the proxy's own-item +// check. discover and whoAmI are seams: production passes the App's installation +// discovery and a GET /user for file tokens. +func VerifyReferenceRepoWriteCredential(ctx context.Context, rr config.ReferenceRepo, botLogin string, discover func(ctx context.Context, org string) (int64, error), whoAmI func(ctx context.Context, token string) (string, error)) (string, error) { + if rr.ReadOnly() { + return "", nil + } + if rr.WriteTokenFile != "" { + tok, err := readTokenFile(rr.WriteTokenFile) + if err != nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write_token_file %s: %v", ErrReferenceRepoNoCredential, rr.Repo, rr.WriteTokenFile, err) + } + if whoAmI == nil { + return "", fmt.Errorf("%w: project.reference_repos %s: cannot resolve the write_token_file identity", ErrReferenceRepoNoCredential, rr.Repo) + } + login, err := whoAmI(ctx, tok) + if err != nil || login == "" { + return "", fmt.Errorf("%w: project.reference_repos %s: write_token_file %s is not a usable GitHub token (%v)", ErrReferenceRepoNoCredential, rr.Repo, rr.WriteTokenFile, err) + } + return login, nil + } + if discover == nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write: %v needs the GitHub App installed on %s (or write_token_file), and this hive has no App auth", ErrReferenceRepoNoCredential, rr.Repo, rr.Write, rr.Owner()) + } + if _, err := discover(ctx, rr.Owner()); err != nil { + return "", fmt.Errorf("%w: project.reference_repos %s: write: %v needs the GitHub App installed on %s (or write_token_file): %v", ErrReferenceRepoNoCredential, rr.Repo, rr.Write, rr.Owner(), err) + } + if botLogin == "" { + return "", fmt.Errorf("%w: project.reference_repos %s: App bot login unknown, cannot attribute the hive's own items", ErrReferenceRepoNoCredential, rr.Repo) + } + return botLogin, nil +} + +// TokenUserLogin resolves the login a user token acts as (GET /user). +func TokenUserLogin(ctx context.Context, token, apiURL string) (string, error) { + user, _, err := newTokenClient(token, apiURL).Users.Get(ctx, "") + if err != nil { + return "", err + } + return user.GetLogin(), nil +} + +// BotLogin returns the App bot login recorded by SetBotLogin ("" when unknown). +func (a *AppAuth) BotLogin() string { + if a == nil { + return "" + } + a.mu.Lock() + defer a.mu.Unlock() + return a.botLogin +} diff --git a/src/pkg/github/reference_repo_token_test.go b/src/pkg/github/reference_repo_token_test.go new file mode 100644 index 0000000000..0024f336aa --- /dev/null +++ b/src/pkg/github/reference_repo_token_test.go @@ -0,0 +1,120 @@ +package github + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestVerifyReferenceRepoWriteCredential(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + tokFile := filepath.Join(dir, "tok") + if err := os.WriteFile(tokFile, []byte("ghp_x\n"), 0o600); err != nil { + t.Fatal(err) + } + emptyFile := filepath.Join(dir, "empty") + if err := os.WriteFile(emptyFile, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + installed := func(_ context.Context, org string) (int64, error) { + if org == "hivecommons" { + return 42, nil + } + return 0, ErrNoInstallationForOrg + } + whoAmI := func(_ context.Context, token string) (string, error) { + if token == "ghp_x" { + return "nate", nil + } + return "", errors.New("bad credentials") + } + + cases := []struct { + name string + rr config.ReferenceRepo + botLogin string + discover func(context.Context, string) (int64, error) + wantLogin string + wantErr bool + }{ + {name: "read-only needs nothing", rr: config.ReferenceRepo{Repo: "hivecommons/docs"}, wantLogin: ""}, + {name: "app installed on target org", rr: config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"issues"}}, botLogin: "hive[bot]", discover: installed, wantLogin: "hive[bot]"}, + {name: "app not installed, no token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}}, botLogin: "hive[bot]", discover: installed, wantErr: true}, + {name: "no app auth at all", rr: config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"prs"}}, wantErr: true}, + {name: "token file wins over installation", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues", "prs"}, WriteTokenFile: tokFile}, discover: installed, wantLogin: "nate"}, + {name: "empty token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}, WriteTokenFile: emptyFile}, wantErr: true}, + {name: "missing token file", rr: config.ReferenceRepo{Repo: "other/docs", Write: []string{"issues"}, WriteTokenFile: filepath.Join(dir, "nope")}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + login, err := VerifyReferenceRepoWriteCredential(ctx, tc.rr, tc.botLogin, tc.discover, whoAmI) + if tc.wantErr { + if err == nil { + t.Fatalf("want error, got login %q", login) + } + if !errors.Is(err, ErrReferenceRepoNoCredential) { + t.Errorf("error should wrap ErrReferenceRepoNoCredential: %v", err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if login != tc.wantLogin { + t.Errorf("login = %q, want %q", login, tc.wantLogin) + } + }) + } +} + +func TestReferenceRepoTokenSource_File(t *testing.T) { + tokFile := filepath.Join(t.TempDir(), "tok") + if err := os.WriteFile(tokFile, []byte("first\n"), 0o600); err != nil { + t.Fatal(err) + } + rr := config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"issues"}, WriteTokenFile: tokFile} + src := ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger()) + if tok, ok := src(); !ok || tok != "first" { + t.Fatalf("got %q,%v want first,true", tok, ok) + } + // Rotation is picked up without a restart. + if err := os.WriteFile(tokFile, []byte("second"), 0o600); err != nil { + t.Fatal(err) + } + if tok, ok := src(); !ok || tok != "second" { + t.Fatalf("got %q,%v want second,true", tok, ok) + } + if err := os.Remove(tokFile); err != nil { + t.Fatal(err) + } + if _, ok := src(); ok { + t.Fatal("removed token file should yield ok=false") + } + if ReferenceRepoTokenSource(config.ReferenceRepo{Repo: "a/b"}, nil, testLogger()) != nil { + t.Fatal("read-only entry must have no token source") + } +} + +func TestReferenceRepoTokenSource_NoAuth(t *testing.T) { + rr := config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"prs"}} + src := ReferenceRepoTokenSource(rr, func() *AppAuth { return nil }, testLogger()) + if _, ok := src(); ok { + t.Fatal("no App auth must yield ok=false, not a credential") + } +} + +func TestReferenceRepoPermissions(t *testing.T) { + p := referenceRepoPermissions(config.ReferenceRepo{Repo: "a/b", Write: []string{"issues"}}) + if p.GetIssues() != "write" || p.GetContents() != "read" || p.GetPullRequests() != "" { + t.Errorf("issues-only perms wrong: issues=%q contents=%q prs=%q", p.GetIssues(), p.GetContents(), p.GetPullRequests()) + } + p = referenceRepoPermissions(config.ReferenceRepo{Repo: "a/b", Write: []string{"prs"}}) + if p.GetIssues() != "" || p.GetContents() != "write" || p.GetPullRequests() != "write" { + t.Errorf("prs perms wrong: issues=%q contents=%q prs=%q", p.GetIssues(), p.GetContents(), p.GetPullRequests()) + } +} diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index a767d9a60b..40681457a0 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -243,3 +243,30 @@ func TestFilterIssuesForRepos(t *testing.T) { t.Errorf("filterIssuesForRepos = %v, want the b issue", got) } } + +// project.reference_repos (hivecommons/hive#18015): the kick names each +// reference repo with its tier and tells the agent an item whose deliverables +// mention one is in scope, so the scanner stops classifying it out-of-scope. +func TestBuildReposSectionFor_ReferenceRepos(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + section := s.buildReposSectionFor("scanner") + for _, want := range []string{ + "REFERENCE REPOS", + "https://github.com/hivecommons/docs [reference · read-only]", + "https://github.com/hivecommons/hive [reference · issues+prs]", + "IN SCOPE, not a scope violation", + } { + if !strings.Contains(section, want) { + t.Errorf("%q missing:\n%s", want, section) + } + } + // A hive without the key sees no trace of it. + s.cfg.Project.ReferenceRepos = nil + if strings.Contains(s.buildReposSectionFor("scanner"), "REFERENCE REPOS") { + t.Error("reference block emitted with no reference repos configured") + } +} diff --git a/src/pkg/scheduler/kickmessage.go b/src/pkg/scheduler/kickmessage.go index bf3f40a8d7..e1ee2ed765 100644 --- a/src/pkg/scheduler/kickmessage.go +++ b/src/pkg/scheduler/kickmessage.go @@ -846,6 +846,18 @@ func (s *Scheduler) buildReposSectionFor(agentName string) string { b.WriteString(fmt.Sprintf("⏸️ PAUSED BY THE OPERATOR (watched, but OUT OF SCOPE this session): %s\n", strings.Join(pausedFull, ", "))) b.WriteString(" The hive is deliberately quiet on those repos — a release freeze, an incident, a repo declared but not yet onboarded. Writes to them are refused deterministically by the proxy and by the hive-open-pr/hive-merge relays, so retrying cannot succeed. This is an operator decision, NOT an outage and NOT scope loss: do not work them, do not route around it, and do not file an issue about it.\n") } + // project.reference_repos (hivecommons/hive#18015): repos outside the + // project an agent may STUDY — e.g. a docs-assessment hive reading the + // repos it assesses. Without this block the scanner reads any item whose + // deliverables name them as out-of-scope/critical, and agents either + // refuse the work or try to write to them. + if refs := s.cfg.Project.ReferenceRepos; len(refs) > 0 { + b.WriteString("📚 REFERENCE REPOS (outside this project — study them, do not work them):\n") + for _, rr := range refs { + b.WriteString(fmt.Sprintf(" %s/%s [reference · %s]\n", strings.TrimRight(host, "/"), rr.Repo, rr.WriteLabel())) + } + b.WriteString(" You MAY clone them (public, unauthenticated), read their files, and GET their issues/PRs/API. An authorized-repo item whose deliverables name a reference repo is IN SCOPE, not a scope violation: do NOT classify it out-of-scope, critical, or a security concern. Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo (the assessment document, the report), never as issues or PRs on the reference repo itself — unless its tier above says issues/prs, in which case you may ONLY open an issue, open a PR from your own branch, or comment on an issue/PR you opened; the proxy refuses merges, default-branch pushes, labels, assignments, reviews and closing others' items deterministically, so retrying cannot succeed.\n") + } b.WriteString("⛔ NEVER access, search, list, file issues in, or open PRs on repos not listed above.\n") b.WriteString(fmt.Sprintf("⛔ Every repo above is on %s. This hive is single-host — never touch a repo on a different GitHub host.\n", host)) // SCOPE vs PROVISIONING (#4464). This list is what `include_repos: true` From 84f39701cb1ba5044da0ca498898eb92e869c95c Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Thu, 12 Nov 2026 16:47:57 -0700 Subject: [PATCH 128/166] =?UTF-8?q?=E2=9C=A8=20Owner-facing=20GitHub=20web?= =?UTF-8?q?hook=20setup:=20Settings=20section,=20generated=20secret,=20cre?= =?UTF-8?q?ate-hook,=20owner=20guide=20(#18024)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Owner-facing GitHub webhook setup: Settings section, generated secret, create-hook, owner guide Settings → Forge App gains a GitHub webhooks section with the payload URL, events, live receiver status, Generate secret (file-backed, shown once) and Create webhook for me when the App installation has hooks write permission. Governor card gets a webhooks tile deep-linking to it. Owner guide added to docs/webhooks.md. Hosted spokes keep receiving hub relays with no action. Fixes #18017 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * Document webhook setup routes in openapi.json; move section styling to tokens Fixes TestOpenAPISpecCoversEveryRegisteredRoute, TestStyleRatchet and TestDashboardDatesUseSharedTimezoneHelpers from the first CI run. Signed-off-by: Andrew Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-webhook-owner-setup.md | 1 + dashboard/openapi.json | 258 +++++++++++++ src/docs/api-reference/agents-and-controls.md | 26 +- src/docs/api-reference/beads.md | 10 +- src/docs/api-reference/configuration.md | 29 +- src/docs/api-reference/dashboard-misc.md | 24 +- src/docs/api-reference/inception.md | 28 +- src/docs/api-reference/knowledge.md | 90 ++--- src/docs/api-reference/nous.md | 50 +-- src/docs/api-reference/packs-and-acmm.md | 36 +- src/docs/dashboard-source-layout.md | 4 +- src/docs/env-vars.md | 2 +- .../operator-reference/runtime-and-quota.md | 17 +- src/docs/webhooks.md | 77 ++++ src/pkg/dashboard/api.go | 3 + src/pkg/dashboard/api_github_webhook.go | 8 +- src/pkg/dashboard/api_github_webhook_setup.go | 339 ++++++++++++++++++ .../api_github_webhook_setup_test.go | 183 ++++++++++ .../static-src/css/config-webhooks.css | 27 ++ .../dashboard/static-src/js/config-dialog.js | 1 + .../dashboard/static-src/js/config-tabs.js | 5 +- .../static-src/js/config-webhooks.js | 217 +++++++++++ .../dashboard/static-src/js/governor-view.js | 25 ++ src/pkg/dashboard/static-src/manifest.txt | 2 + src/pkg/dashboard/static/index.html | 275 +++++++++++++- src/pkg/github/app.go | 2 + src/pkg/github/webhook_relay_key.go | 5 +- src/pkg/github/webhook_secret.go | 133 +++++++ src/pkg/github/webhook_secret_test.go | 105 ++++++ src/pkg/github/webhook_setup.go | 146 ++++++++ 30 files changed, 1965 insertions(+), 163 deletions(-) create mode 100644 changelog.d/added-webhook-owner-setup.md create mode 100644 src/pkg/dashboard/api_github_webhook_setup.go create mode 100644 src/pkg/dashboard/api_github_webhook_setup_test.go create mode 100644 src/pkg/dashboard/static-src/css/config-webhooks.css create mode 100644 src/pkg/dashboard/static-src/js/config-webhooks.js create mode 100644 src/pkg/github/webhook_secret.go create mode 100644 src/pkg/github/webhook_secret_test.go create mode 100644 src/pkg/github/webhook_setup.go diff --git a/changelog.d/added-webhook-owner-setup.md b/changelog.d/added-webhook-owner-setup.md new file mode 100644 index 0000000000..11c7755899 --- /dev/null +++ b/changelog.d/added-webhook-owner-setup.md @@ -0,0 +1 @@ +- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 651888249f..6167fff8a0 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -25525,6 +25525,264 @@ } } }, + "/api/config/github/webhooks": { + "get": { + "tags": [ + "System" + ], + "summary": "Describe GitHub webhook setup for this hive (owner)", + "description": "Owner-only. Everything the Settings → Forge App → GitHub webhooks section shows: the payload URL GitHub must call, the events to subscribe, whether a secret is configured and where it comes from (never the secret itself), whether this hive receives hub-signed relays (hosted), the live receiver health and whether the App installation's hooks permission allows creating the webhook from here. See docs/webhooks.md#setting-up-webhooks-owner-guide.", + "responses": { + "200": { + "description": "Webhook setup", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "payload_url": { + "type": "string" + }, + "payload_url_source": { + "type": "string", + "enum": [ + "config", + "hub", + "request", + "none" + ] + }, + "path": { + "type": "string" + }, + "content_type": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "optional_events": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "verification": { + "type": "string" + }, + "hosted": { + "type": "boolean" + }, + "secret_configured": { + "type": "boolean" + }, + "secret_source": { + "type": "string", + "enum": [ + "env", + "file", + "" + ] + }, + "secret_path": { + "type": "string" + }, + "can_generate_secret": { + "type": "boolean" + }, + "health": { + "type": "object", + "properties": { + "healthy": { + "type": "boolean" + }, + "last_event_at": { + "type": "string" + }, + "events_1h": { + "type": "integer" + }, + "invalidations_1h": { + "type": "integer" + }, + "targeted_refreshes_1h": { + "type": "integer" + }, + "tracker_invalidations_1h": { + "type": "integer" + }, + "last_ci_event_at": { + "type": "string" + }, + "ci_events_1h": { + "type": "integer" + }, + "ci_events_applied_1h": { + "type": "integer" + }, + "ci_refetches_1h": { + "type": "integer" + }, + "verification": { + "type": "string" + } + } + }, + "org": { + "type": "string" + }, + "repos": { + "type": "array", + "items": { + "type": "string" + } + }, + "app_org_hooks": { + "type": "boolean" + }, + "app_repo_hooks": { + "type": "boolean" + }, + "app_probe_error": { + "type": "string" + }, + "create_available": { + "type": "boolean" + }, + "docs_url": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not an owner" + }, + "503": { + "description": "Config not loaded" + } + } + } + }, + "/api/config/github/webhooks/secret": { + "post": { + "tags": [ + "System" + ], + "summary": "Generate this hive's GitHub webhook secret (owner)", + "description": "Owner-only. Writes a new random secret to the hive's secret file (mode 0600) and returns it exactly once; later GETs only report that a file secret is configured. Refused with 409 while GITHUB_WEBHOOK_SECRET is set in the environment, which always wins. Audited.", + "responses": { + "200": { + "description": "Secret, shown once", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "secret": { + "type": "string" + }, + "secret_path": { + "type": "string" + }, + "verification": { + "type": "string" + }, + "note": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not an owner" + }, + "409": { + "description": "Secret is supplied by the environment" + }, + "500": { + "description": "Could not write the secret file" + } + } + } + }, + "/api/config/github/webhooks/create": { + "post": { + "tags": [ + "System" + ], + "summary": "Create the org or repo webhook through the GitHub App (owner)", + "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Audited.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "scope": { + "type": "string", + "enum": [ + "organization", + "repository" + ] + }, + "repo": { + "type": "string", + "description": "Repository name (bare or org/name); required for scope=repository." + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Webhook created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hook_id": { + "type": "integer" + }, + "scope": { + "type": "string" + }, + "target": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "Invalid scope or repository" + }, + "403": { + "description": "Not an owner, or the App lacks the hooks permission" + }, + "409": { + "description": "No secret / no public URL / webhook already exists" + }, + "502": { + "description": "GitHub API error" + } + } + } + }, "/api/config/github": { "put": { "tags": [ diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index 7f42dc9717..01bcaaa5f4 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -23,20 +23,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:175` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:176` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:239` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:515` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:518` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:519` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:523` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:518` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:521` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:522` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:526` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:527` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:528` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:530` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:533` | -| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:534` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:320` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:321` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:322` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:323` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:530` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:531` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:533` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:536` | +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:537` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:323` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:324` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:325` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:326` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:126` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:130` | diff --git a/src/docs/api-reference/beads.md b/src/docs/api-reference/beads.md index b67e37d79d..443e2a5459 100644 --- a/src/docs/api-reference/beads.md +++ b/src/docs/api-reference/beads.md @@ -5,8 +5,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:499` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:500` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:501` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:502` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:503` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:502` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:503` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:504` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:505` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:506` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 4941f8f958..02117e6006 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -75,17 +75,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:314` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:315` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:318` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:373` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:374` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:375` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:365` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:366` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:367` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:368` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:369` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:370` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:541` | +| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:87` | +| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:88` | +| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:376` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:377` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:378` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:368` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:369` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:370` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:372` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:373` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:374` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:544` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:220` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:221` | @@ -94,8 +97,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:230` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:234` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:537` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:538` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:540` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:541` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index dc4eacc334..0b4e24ec7e 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -25,7 +25,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](../audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:63` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:510` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:513` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:66` | | `POST` | `/api/upgrade/announce` | Owner role | Announce a pod roll before it happens ([#15788](https://github.com/hivecommons/hive/issues/15788)): body `{"target": "<sha|tag>", "source": "hub"|"operator", "reason": "..."}` (`version` is an alias of `target`; all fields optional). Marks `/api/status`/`/api/version` `upgradeProgress` as `state: pulling, phase: announced` with the target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation; a SIGTERM afterwards (or with no announce at all) publishes `phase: restarting` from the same record. The hub sends the same announce through the heartbeat response (`upgrade_announce`) for hub-managed hives; an announce with no kill expires after 15 minutes. Returns `{ok, upgradeProgress}` | `pkg/dashboard/api.go:69` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:67` | @@ -40,18 +40,18 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:264` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:462` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:465` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:466` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:468` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:469` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:465` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:468` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:469` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:471` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:472` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:215` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:216` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:217` | | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:306` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {<repo>: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:307` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:548` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:551` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:118` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:123` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:137` | @@ -62,11 +62,11 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, `claimant: {kind: hive-agent|clanker|person, login, agent, hive, since, source: claim-ledger|claim-comment|assignee|kick-history|author}` naming who holds a claimed / in-progress issue or a PR under review or rework (absent when nobody is named; mirrored onto `linked_prs[]`), (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), (PRs) `lineage` on the hive's copy of an adopted PR — `{kind: adopted|superseded|rebased-by-hive, from_number, from_url, from_author, from_author_kind, to_number, to_url, at, reason}`, derived from the stage store's adoption record or the `hive/adopt-<n>` head branch, mirrored onto the issue's `linked_prs[]` entry; absent on every other row — and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; a PR in a conflict-aware merge train carries `merge_train` `{id, state, front, position, size, attempt, max_attempts, overlap_files, ahead, length, head_number, head_state, head_reason, head_since, eta_s, eta_at, eta_basis, late, stalled, stalled_s}` — `head_state` one of `needs-human` / `ci` / `soaking` / `ready` / `merging` / `fixing` / `unknown` with `head_reason` in plain words; `eta_s` / `eta_at` the estimated wait for this member's own merge attempt (own remaining soak + each PR ahead's remaining soak or one sweep + the next sweep tick; absent while the head waits on a person or CI), `eta_at` anchored to the member's first estimate at its position so a countdown survives polls, `eta_basis` its short reading ("2 ahead · head soaking 12m · next sweep in 40s"), `late` once now has overrun the first estimate twice over, `stalled` / `stalled_s` once the head has sat longer than `governor.merge_train_stall_warn_s` ([#15395](https://github.com/hivecommons/hive/issues/15395)); a PR the queue holds because another open PR declares the same `Fixes #N` carries no `merge_train` and a `merge_gate` of `{state: "duplicate-of:<winner>", duplicate_of, duplicate_issue}` plus the `needs-decision` modifier (the queue keeps one PR per issue: green first, then the earlier-opened); a PR GitHub reports conflicting on its own, or one under repair (stage/070, 071, 072, 074, 085, or a red required check), carries no `merge_train` and is not counted in any member's `size` / `length` — a repaired member the view dropped carries `left_train: {why: fixing|needs-author|returned-to-contributor|adopted-by-hive|needs-decision|checks-failing, note}` instead (mirrored onto the issue's `linked_prs[]` entry), and a train left with one member reports no `merge_train` at all; a PR short of the merge queue (below stage/080) the planner ordered rides no train either — it carries `overlap: {behind, with, files, note}` ("overlaps #15637 (sidebar.html, index.html) — will queue behind it") and no `merge_train`, and counts in no member's `size` — the state slugs (`front`, `queued-behind`) are payload values only, the SPA renders ordinal places ("Queued for merge · 1st of 2", with the train's tag when more than one train is on the page). Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=<remaining>`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:191` | | `GET` | `/api/matrix/lines` | Dashboard auth/session | Matrix Lines (deprecated name, see [Published-checkpoint regions](#published-checkpoint-regions)): serve bounded backlog of pushed lines behind dashboard.matrix with repo allowlist filtering. `max` counts only lines the repo allowlist admits, so a page is never emptied by filtered-out lines. A client paging with `max` should use the last returned line's `seq` as its next `since`, not the top-level `seq`, which is the source's latest sequence number and would skip lines still past the page. `more` is true only while another line the caller may see remains past the page; stop paging when it is false. | `pkg/dashboard/api.go:192` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:193` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:378` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:441` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:442` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:471` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:505` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:381` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:444` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:445` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:474` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:508` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:350` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:351` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:352` | diff --git a/src/docs/api-reference/inception.md b/src/docs/api-reference/inception.md index 0260a12e83..6da92d2d96 100644 --- a/src/docs/api-reference/inception.md +++ b/src/docs/api-reference/inception.md @@ -5,17 +5,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:444` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:445` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:446` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:447` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:448` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:449` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:450` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:451` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:452` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:453` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:454` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:455` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:456` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:457` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:447` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:448` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:449` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:450` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:451` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:452` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:453` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:454` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:455` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:456` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:457` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:458` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:459` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:460` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index e406cdb1a3..a20c2621ba 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -5,38 +5,38 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:380` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:381` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:390` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:382` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:383` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:384` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:385` | -| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:387` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:391` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:392` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:393` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:394` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:395` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:396` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:397` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:398` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:404` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:405` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:406` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:407` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:408` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:409` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:410` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:411` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:412` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:413` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:414` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:415` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:416` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:417` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:418` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:419` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:383` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:384` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:393` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:385` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:386` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:387` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:388` | +| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:390` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:394` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:395` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:396` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:397` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:398` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:399` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:400` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:401` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:407` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:408` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:409` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:410` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:411` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:412` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:413` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:414` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:415` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:416` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:417` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:418` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:419` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:420` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:421` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:422` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -46,19 +46,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](../settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:429` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:430` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:431` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:432` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:433` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:434` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:435` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:436` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:437` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:438` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:439` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:402` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:403` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:432` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:433` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:434` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:436` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:437` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:438` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:439` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:440` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:441` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:442` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:405` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:406` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | diff --git a/src/docs/api-reference/nous.md b/src/docs/api-reference/nous.md index 6dce3749dd..6400d1c981 100644 --- a/src/docs/api-reference/nous.md +++ b/src/docs/api-reference/nous.md @@ -5,28 +5,28 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:473` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:474` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:475` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:476` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:477` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:478` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:479` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:480` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:481` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:482` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:483` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:484` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:485` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:486` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:487` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:488` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:489` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:490` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:491` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:492` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:493` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:494` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:495` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:496` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:497` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:476` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:477` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:478` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:479` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:480` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:481` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:482` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:483` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:484` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:485` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:486` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:487` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:488` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:489` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:490` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:491` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:492` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:493` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:494` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:495` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:496` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:497` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:498` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:499` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:500` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 7dbfa576e4..b9bfed2203 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -5,29 +5,29 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:325` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:326` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:327` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:328` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:329` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:330` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](../repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:342`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:345`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage}`; `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. Rows also carry `rebase_merge: {state: approved|rebasing|ci-pending|merged|failed, actor, base, text, fail_reason, conflict_files[], armed_at}` while a *Rebase, then merge as maintainer* follow-through is live or just settled (persisted in `/data/rebase-merge-followups.json`; `fail_reason: foreign-head` when the head moved after the approval by someone other than the hive, so nothing merged), and `rebase: {attempt, max, exhausted, head, text}` on a conflicting PR the fixer is rebasing under the two-attempt loop guard (`/data/rebase-attempts.json`). | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:346`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:347`, `pkg/dashboard/api_repo_item_decision.go` | -| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_maintainer_notes.go` | -| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_maintainer_notes.go` | -| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:353`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:349`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:350`, `pkg/dashboard/api_repo_item_decision.go` | +| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:354`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:355`, `pkg/dashboard/api_maintainer_notes.go` | +| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_maintainer_notes.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `<!-- hive:delegated-decision … -->` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:557`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:362` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:364`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:359`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:360` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:361` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:560`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:364` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:365` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_hive_advice.go` | diff --git a/src/docs/dashboard-source-layout.md b/src/docs/dashboard-source-layout.md index e75eca06f9..2e907c305a 100644 --- a/src/docs/dashboard-source-layout.md +++ b/src/docs/dashboard-source-layout.md @@ -198,7 +198,7 @@ CSS (`css/`): `base` (tokens, reset, shared primitives), `sidebar`, `topbar`, `agent-detail`, `agent-cards`, `auth`, `agent-state`, `tracker` (Hive activity monitor), `tracker-native-queue`, `agent-fields`, `governor`, `advisory-lifecycle`, `evidence`, `throughput`, `repos`, `label-filter`, `overview`, `sparklines-indicators`, -`governor-matrix`, `cost`, `alerts-toasts`, `feedback-faq`, `config-dialog`, +`governor-matrix`, `cost`, `alerts-toasts`, `feedback-faq`, `config-dialog`, `config-webhooks`, `chat`, `nous`, `knowledge`, `sections-collapse`, `design-system`, `responsive`, `recipes-utilities`, `welcome`, `review-campaigns`, `variables`. @@ -224,7 +224,7 @@ JS (`js/`): `boot`, `core`, `feedback`, `sections`, `layout`, `agent-detail`, `review-queue`, `contributors`, `refresh`, `inception`, `debug-logs`, `tooltips-dialogs`, `plan-agent-controls`, `acmm`, `version`, `sse-bootstrap`, `config-state`, `settings-search`, `config-dialog`, `config-tabs`, -`config-hub`, `config-agent`, `config-governor`, `config-watchdog-security`, +`config-webhooks`, `config-hub`, `config-agent`, `config-governor`, `config-watchdog-security`, `config-features`, `config-sentinel`, `compliance`, `config-helpers`, `linear-repos-save`, `chat`, `gh-auth`, `nps`, `welcome`, `shimmy-handlers`. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 23d80031c7..e9e8264b48 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -104,7 +104,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTE_SKIP_LABELS` | No | `blocked,tracking,epic,discussion,question,needs-decision,needs-triage` | Comma-separated, case-insensitive label patterns for issues that are not contributor work and must never be offered by the relay. Patterns use `path.Match`-style `*` globs; `blocked` is always unioned into the effective set even if omitted. Same setting as `hub.contribute_skip_labels`. | | `HIVE_FEDERATION_REGISTRY_PATH` | No | `/data/federation/registry.json` | Federation registry path override. | | `HIVE_WEBHOOK_SECRET` | No | none | HMAC secret for the spoke `/webhook` channel. | -| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub only) | GitHub App webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | +| `GITHUB_WEBHOOK_SECRET` | No | `/data/saas/webhook-secret.key` when present (hub); `/data/github-webhook-secret.key` when present (spoke, written by Settings → Forge App → **Generate secret**) | GitHub App or org/repo webhook HMAC secret. The hub verifies `/api/github/webhook` with it; a self-hosted spoke verifies `/api/webhook/github` (direct or hub-relayed deliveries) with the same secret. When the variable is set it wins over the generated file and the dashboard's Generate button is disabled. See [Setting up webhooks](webhooks.md#setting-up-webhooks-owner-guide). A hosted spoke leaves it unset and verifies hub relays with its own per-hive relay key from the heartbeat; with neither, every delivery is rejected. | | `HIVE_DASHBOARD_URL` | No | none | Base URL the `hive tui` client targets (`pkg/tui/client`). A bad value surfaces as a request error on the first call, not at startup. On the hub it is also consulted (fourth) in the hub public-origin chain (see `HIVE_HUB_PUBLIC_URL`). | | `HIVE_CONVERGENCE_MODE` | No | `convergence.mode` in `hive.yaml`, else `shadow` | Process-level override of the convergence mode (`off`, `shadow`, `enforce`) so an operator can flip the mode without editing `hive.yaml`. An **unset** mode takes the default, `shadow`, which computes and records admission decisions without withholding any work. A **non-empty but unrecognised** value — a typo, or a mode this build does not know — still resolves to `off`. | | `HIVE_RELEASE_LINE_LAG_MAX` | No | `5` | Alarm threshold for the release-line drift surface ([#6960](https://github.com/hivecommons/hive/issues/6960)): how many commits the hosted edge line (`v5`) may sit behind the stable default branch (`v4`) before the dashboard status flags the line as drifting (`releaseLineLag.exceeded`). An unset, empty, non-numeric, or negative value falls back to the default — it can never silently disable the alarm. An unknown lag (tips unresolved or the compare failed) is reported as `unknown`, never a healthy zero. | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index bd85fb4553..f51851ece5 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -257,7 +257,11 @@ GraphQL query fails, or GitHub returns `mergeable: UNKNOWN` for that PR. See [GitHub webhooks: cadence, patch and reconcile](../webhooks.md) for the cadence keys, the fallback to polling when deliveries degrade, and the -accuracy guarantees of the ForgeState contract. +accuracy guarantees of the ForgeState contract. Hive owners (not cluster +operators) should start at the +[owner guide](../webhooks.md#setting-up-webhooks-owner-guide): it covers the +⚙ Settings → **Forge App** → **GitHub webhooks** section, which shows the +payload URL, generates a per-hive secret and can create the org/repo hook. A spoke accepts GitHub App webhooks on the public `POST /api/webhook/github`, either delivered directly by GitHub or relayed by the hub (the hub forwards the @@ -270,10 +274,13 @@ Deliveries for repositories the hive does not manage are ignored. #### What the operator configures -1. **Secret (self-hosted only).** Set `GITHUB_WEBHOOK_SECRET` in the spoke's - environment (a Secret mount on Kubernetes; never in `hive.yaml`) to the - App's webhook secret. A hosted spoke needs nothing here: it never holds the - App secret and verifies relays with its hub relay key instead (item 3). +1. **Secret (self-hosted only).** Either press **Generate secret** in + ⚙ Settings → Forge App → GitHub webhooks (the hive writes + `/data/github-webhook-secret.key`, mode 0600, and shows the value once), or + set `GITHUB_WEBHOOK_SECRET` in the spoke's environment (a Secret mount on + Kubernetes; never in `hive.yaml`). The environment variable wins when both + exist. A hosted spoke needs nothing here: it never holds a GitHub secret and + verifies relays with its hub relay key instead (item 3). 2. **GitHub App webhook.** Point the App's webhook URL at the hub (`POST /api/github/webhook`) for hosted spokes, or at the spoke's own `/api/webhook/github` for a stand-alone hive (a repository or organization diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 53743ab215..37865c7593 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -13,6 +13,83 @@ The priority order throughout is **accuracy → latency → API thrift**. A delivery is a free hint; every outbound GitHub call is budgeted. The rules below exist so that the hive never trades the first for the third. +## Setting up webhooks (owner guide) + +Webhooks are optional. Without them a hive polls GitHub every +`governor.eval_interval_s` (default 300 s) and finds out about a review, a red +check or a new comment at the next tick. With them the hive is told at once: +the governor wakes early, CI-failure and review kicks land within seconds +instead of minutes, the base tick stretches to `governor.eval_interval_webhook_s` +(default 900 s), and the outbound API budget drops because each delivery costs +one confirming `GET` instead of a full re-list. Open ⚙ Settings → **Forge App** +→ **GitHub webhooks** on your hive, or click the **webhooks** tile on the +Governor card, to see which of the two cases below applies to you. + +Hive never shares its own webhook secret. Every hive verifies deliveries with +a key that belongs to it alone. + +### Hosted hive (hub-managed) + +Nothing to do. GitHub sends App deliveries to the hub, the hub verifies them +and relays each one to every hive that manages that repository, signed with +the relay key it minted for your hive when it was provisioned. The Settings +section reports **Receiver: hub relay** and the events/hour count; the +Governor tile reads `N/1h`. If it stays at **no events yet** for an hour of +activity, confirm the hivecommons GitHub App is installed on the organization +or repositories your hive tracks (Settings → Forge App → Installation) — the +hub can only relay what GitHub sends it. + +### Self-hosted hive + +Your hive needs an org- or repo-level webhook in GitHub that points at it and +uses a secret only you hold. + +1. **Generate the secret.** Settings → Forge App → GitHub webhooks → + **Generate secret**. The hive writes it to `/data/github-webhook-secret.key` + (0600) and shows it once with a copy button; it is never shown again + (regenerate to rotate). If you prefer to manage it yourself, set + `GITHUB_WEBHOOK_SECRET` in the hive's environment instead — the button then + reports that the environment variable is in charge. +2. **Create the webhook.** If the section offers **Create webhook for me**, + click it: the hive's GitHub App installation holds `organization_hooks` or + `repository_hooks` write permission and the hive creates the hook with the + right URL, secret and events. Otherwise do it by hand in GitHub: + organization (or repository) → Settings → Webhooks → **Add webhook**. + - Payload URL: the value shown in the section, i.e. + `https://<your-hive>/api/webhook/github` + - Content type: `application/json` + - Secret: the value from step 1 + - SSL verification: enabled + - Events: *Let me select individual events* → `check_run`, `check_suite`, + `issue_comment`, `issues`, `pull_request`, `pull_request_review`, + `pull_request_review_comment`, `push`, `status`, `workflow_run` + (optionally `workflow_job`, and `projects_v2_item` if your work source is + GitHub Projects). +3. **Verify.** GitHub sends a `ping` immediately; the section's **Receiver** + card should read **github-signed** with a fresh *last event*, and the + Governor tile moves from **set up →** to `N/1h` after the first real event. + +The payload URL comes from `dashboard.public_url`; if the section shows no URL +or the wrong one, set that key (or the hub-reported dashboard URL) to the +address GitHub can reach. + +### Troubleshooting + +| Symptom (GitHub → Settings → Webhooks → Recent deliveries) | Cause | Fix | +| --- | --- | --- | +| `401 invalid signature` | Secret in GitHub differs from the hive's | Paste the current secret into the GitHub hook (Edit → Secret), or regenerate in Settings and update GitHub. `GITHUB_WEBHOOK_SECRET` overrides the generated file — check which one the section says is active. | +| `503 webhooks disabled: no secret configured` | Hive has neither a generated/env secret nor a hub relay key | Step 1 above. | +| `404` | Wrong path | The path is `/api/webhook/github` on the spoke (the hub's App endpoint is `/api/github/webhook` — do not point an org hook there). | +| `200` but **no events yet** | Deliveries are for repos the hive does not manage | Check `project.repos`; deliveries for other repos are acknowledged and dropped. | +| Hosted hive, no events | App not installed on the repo / relay rejected | Install the App on the org or repo; the hub relays to the dashboard URL your hive reports, which must be reachable from the hub. | +| Deliveries time out | Hive not reachable from GitHub (private network, redirects) | GitHub (and the hub relay) follow no redirects and need a public HTTPS endpoint. | + +Hub relay vs direct: a hive registered with the hub whose repos carry the +hivecommons App receives relays as well as any direct hook you add; both are +accepted, duplicates are dropped by delivery id. Only one is needed. A +self-hosted hive with a private dashboard URL cannot receive relays and must +use a direct hook. + ## Polling cadence vs webhook patch vs reconcile Three writers feed one snapshot per repository. A fourth path writes PR diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index ceebd002c0..0808453692 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -316,6 +316,9 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // Read-only inventory for the Forge App tab: every App credential this // spoke holds (active config + per-app-id PVC keys), fingerprints only. s.mux.HandleFunc("GET /api/config/github/forge-apps", s.handleConfigGitHubForgeApps) + // Owner-facing webhook setup for the same tab: payload URL, events, + // secret generation and App-side hook creation (hivecommons/hive#18017). + s.registerGitHubWebhookSetupRoutes() s.mux.HandleFunc("GET /api/agents", s.handleAgentsList) s.mux.HandleFunc("POST /api/agents", s.handleAgentCreate) diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index 539de1084a..d0c7a166ba 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -8,7 +8,6 @@ import ( "io" "log/slog" "net/http" - "os" "strconv" "strings" "time" @@ -20,7 +19,8 @@ import ( const ( // githubWebhookPath is the PUBLIC path GitHub (or the hub relay) posts App // webhooks to. Not actually open: the handler fails closed without - // GITHUB_WEBHOOK_SECRET or a hub relay key, and verifies + // a webhook secret (GITHUB_WEBHOOK_SECRET or the dashboard-generated + // secret file, github.WebhookSecret) or a hub relay key, and verifies // X-Hub-Signature-256 or the hub relay signature over the raw body. githubWebhookPath = "/api/webhook/github" @@ -61,12 +61,12 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { jsonError(w, "failed to read body", http.StatusBadRequest) return } - secret := strings.TrimSpace(os.Getenv(githubWebhookSecretEnvVar)) + secret := github.WebhookSecret() // A hosted spoke holds no App webhook secret; it verifies the hub's relay // signature with this hive's own relay key instead (#17911). relayKey := github.WebhookRelayKey() if secret == "" && relayKey == "" { - const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET to enable signed webhooks)" + const msg = "github webhook rejected: no webhook secret configured (set GITHUB_WEBHOOK_SECRET or generate one under Settings → Forge App → GitHub webhooks)" warned := false s.webhookNoSecretOnce.Do(func() { warned = true diff --git a/src/pkg/dashboard/api_github_webhook_setup.go b/src/pkg/dashboard/api_github_webhook_setup.go new file mode 100644 index 0000000000..640157b176 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_setup.go @@ -0,0 +1,339 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +// Owner-facing GitHub webhook setup (hivecommons/hive#18017): the Settings → +// Forge App → "GitHub webhooks" section. Everything an owner needs to point +// a GitHub org/repo webhook at this hive with a secret of their own, the +// live receiver status, and — when the App installation may manage hooks — +// a one-click hook creation. The secret value leaves the hive exactly once, +// in the response to the generate call; every other response reports only +// whether and from where one is configured. + +const ( + auditWebhookSecretGenerated = "github_webhook_secret_generated" + auditWebhookCreated = "github_webhook_created" + + // webhookPermissionProbeTimeout bounds the installation lookup the GET + // makes to decide whether to offer "Create webhook for me". + webhookPermissionProbeTimeout = 8 * time.Second + webhookCreateTimeout = 20 * time.Second +) + +// webhookSetupResponse is GET /api/config/github/webhooks. +type webhookSetupResponse struct { + // PayloadURL is the public receiver URL to paste into GitHub, or "" when + // this hive has no known public origin (PayloadURLSource says why). + PayloadURL string `json:"payload_url"` + PayloadURLSource string `json:"payload_url_source"` + Path string `json:"path"` + ContentType string `json:"content_type"` + // Events to tick; Optional names events that matter only with a feature. + Events []string `json:"events"` + OptionalEvents map[string]string `json:"optional_events"` + + // Verification is /api/status.webhooks.verification: github-signed, + // relay (hub-signed) or none. Hosted reports whether a hub relay key is + // held, i.e. deliveries arrive through the hub without owner action. + Verification string `json:"verification"` + Hosted bool `json:"hosted"` + // SecretConfigured / SecretSource ("env", "file", "") describe the + // GitHub-signed path; CanGenerateSecret is false while the env var owns + // the secret. + SecretConfigured bool `json:"secret_configured"` + SecretSource string `json:"secret_source"` + SecretPath string `json:"secret_path,omitempty"` + CanGenerateSecret bool `json:"can_generate_secret"` + + Health ghpkg.WebhookHealth `json:"health"` + + // Org / Repos are the targets a hook would be created for. + Org string `json:"org"` + Repos []string `json:"repos"` + // App hook-management permissions, probed live; CreateAvailable is true + // when at least one is granted and a secret is configured. + AppOrgHooks bool `json:"app_org_hooks"` + AppRepoHooks bool `json:"app_repo_hooks"` + AppProbeError string `json:"app_probe_error,omitempty"` + CreateAvailable bool `json:"create_available"` + + DocsURL string `json:"docs_url"` +} + +const ( + webhookDocsURL = "https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide" + // Where PayloadURL came from, for the card's explanation. + payloadURLSourceConfig = "dashboard.public_url" + payloadURLSourceHub = "hub dashboard URL" + payloadURLSourceRequest = "this request's host" + payloadURLSourceNone = "unknown — set dashboard.public_url" +) + +// webhookPermissionProbe is the installation lookup; tests swap it. +var webhookPermissionProbe = installationPermissionProbe + +func (s *Server) registerGitHubWebhookSetupRoutes() { + s.mux.HandleFunc("GET /api/config/github/webhooks", s.handleConfigGitHubWebhooks) + s.mux.HandleFunc("POST /api/config/github/webhooks/secret", s.handleConfigGitHubWebhooksGenerateSecret) + s.mux.HandleFunc("POST /api/config/github/webhooks/create", s.handleConfigGitHubWebhooksCreate) +} + +// webhookPayloadURL resolves this hive's public receiver URL: the configured +// dashboard.public_url, else the hub-assigned dashboard URL, else the origin +// the request arrived on (behind a proxy, X-Forwarded-Host/Proto). +func (s *Server) webhookPayloadURL(r *http.Request) (url, source string) { + var cfg *config.Config + if s.deps != nil { + cfg = s.deps.Config + } + if cfg != nil { + if u := strings.TrimRight(strings.TrimSpace(cfg.Dashboard.PublicURL), "/"); u != "" { + return u + githubWebhookPath, payloadURLSourceConfig + } + } + s.versionMu.RLock() + pushed := s.hubPushedDashboardURL + s.versionMu.RUnlock() + if cfg != nil && pushed == "" { + pushed = cfg.Hub.DashboardURL + } + if u := strings.TrimRight(strings.TrimSpace(pushed), "/"); u != "" { + return u + githubWebhookPath, payloadURLSourceHub + } + if r != nil { + host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host")) + if host == "" { + host = r.Host + } + if host != "" && !isLoopbackHost(host) { + scheme := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")) + if scheme == "" { + scheme = "https" + } + return scheme + "://" + host + githubWebhookPath, payloadURLSourceRequest + } + } + return "", payloadURLSourceNone +} + +func isLoopbackHost(host string) bool { + h := strings.ToLower(host) + if i := strings.LastIndex(h, ":"); i > 0 && !strings.Contains(h[i:], "]") { + h = h[:i] + } + return h == "localhost" || h == "127.0.0.1" || h == "[::1]" || h == "::1" +} + +func (s *Server) webhookSetupTargets() (org string, repos []string) { + if s.deps == nil || s.deps.Config == nil { + return "", nil + } + p := s.deps.Config.Project + for _, r := range p.Repos { + r = strings.TrimSpace(r) + if r == "" { + continue + } + if _, name, ok := strings.Cut(r, "/"); ok { + r = name + } + repos = append(repos, r) + } + return strings.TrimSpace(p.Org), repos +} + +func (s *Server) webhookProbeContext(timeout time.Duration) (context.Context, context.CancelFunc) { + ctx := context.Background() + if s.deps != nil && s.deps.Ctx != nil { + ctx = s.deps.Ctx + } + return context.WithTimeout(ctx, timeout) +} + +// handleConfigGitHubWebhooks serves GET /api/config/github/webhooks +// (owner-only). +func (s *Server) handleConfigGitHubWebhooks(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + payloadURL, source := s.webhookPayloadURL(r) + org, repos := s.webhookSetupTargets() + secretSource := ghpkg.WebhookSecretSource() + resp := webhookSetupResponse{ + PayloadURL: payloadURL, + PayloadURLSource: source, + Path: githubWebhookPath, + ContentType: "application/json", + Events: append([]string(nil), ghpkg.WebhookEvents...), + OptionalEvents: ghpkg.WebhookEventsOptional, + Verification: ghpkg.WebhookVerification(), + Hosted: ghpkg.WebhookRelayKey() != "", + SecretConfigured: secretSource != ghpkg.WebhookSecretSourceNone, + SecretSource: secretSource, + CanGenerateSecret: secretSource != ghpkg.WebhookSecretSourceEnv, + Health: ghpkg.WebhookHealthSnapshot(), + Org: org, + Repos: repos, + DocsURL: webhookDocsURL, + } + if secretSource == ghpkg.WebhookSecretSourceFile { + resp.SecretPath = ghpkg.WebhookSecretPath() + } + resp.Health.Verification = resp.Verification + var auth *ghpkg.AppAuth + if s.deps != nil { + auth = s.deps.GHAppAuth + } + if auth != nil && org != "" { + ctx, cancel := s.webhookProbeContext(webhookPermissionProbeTimeout) + info, err := webhookPermissionProbe(ctx, auth) + cancel() + if err != nil { + resp.AppProbeError = err.Error() + } else { + resp.AppOrgHooks, resp.AppRepoHooks = ghpkg.WebhookHookPermissions(info) + } + } + resp.CreateAvailable = resp.SecretConfigured && resp.PayloadURL != "" && (resp.AppOrgHooks || (resp.AppRepoHooks && len(repos) > 0)) + jsonResponse(w, resp) +} + +// handleConfigGitHubWebhooksGenerateSecret serves +// POST /api/config/github/webhooks/secret (owner-only). It mints a secret, +// stores it on the data volume and returns it ONCE; the receiver verifies +// GitHub-signed deliveries with it from the next request on. +func (s *Server) handleConfigGitHubWebhooksGenerateSecret(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + secret, err := ghpkg.GenerateWebhookSecret() + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, ghpkg.ErrWebhookSecretFromEnv) { + status = http.StatusConflict + } + jsonError(w, err.Error(), status) + return + } + user := strings.TrimSpace(r.Header.Get("X-Hive-User")) + s.audit.LogRecord(user, auditWebhookSecretGenerated, auditDetail("path", ghpkg.WebhookSecretPath()), "", "", 0) + s.webhookLogger().Info("github webhook secret generated from the dashboard", "user", user, "path", ghpkg.WebhookSecretPath()) + jsonResponse(w, map[string]any{ + "secret": secret, + "secret_path": ghpkg.WebhookSecretPath(), + "verification": ghpkg.WebhookVerification(), + "note": "Shown once. Paste it into the GitHub webhook's Secret field; the hive keeps only the file.", + }) +} + +type webhookCreateBody struct { + // Scope is "organization" or "repository"; Repo names the repository + // (bare name) for the repository scope. + Scope string `json:"scope"` + Repo string `json:"repo"` +} + +// handleConfigGitHubWebhooksCreate serves POST /api/config/github/webhooks/create +// (owner-only): creates the org or repo hook through the App installation. +func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var body webhookCreateBody + if r.Body != nil { + // An empty body means "organization scope". + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + jsonError(w, "invalid JSON body", http.StatusBadRequest) + return + } + } + secret := ghpkg.WebhookSecret() + if secret == "" { + jsonError(w, "no webhook secret configured — generate one first", http.StatusConflict) + return + } + payloadURL, _ := s.webhookPayloadURL(r) + if payloadURL == "" { + jsonError(w, "this hive has no public dashboard URL; set dashboard.public_url first", http.StatusConflict) + return + } + org, repos := s.webhookSetupTargets() + if org == "" { + jsonError(w, "project.org is not configured", http.StatusConflict) + return + } + var auth *ghpkg.AppAuth + if s.deps != nil { + auth = s.deps.GHAppAuth + } + if auth == nil { + jsonError(w, "GitHub App auth not configured", http.StatusConflict) + return + } + ctx, cancel := s.webhookProbeContext(webhookCreateTimeout) + defer cancel() + info, err := webhookPermissionProbe(ctx, auth) + if err != nil { + jsonError(w, "could not read the App installation's permissions: "+err.Error(), http.StatusBadGateway) + return + } + orgHooks, repoHooks := ghpkg.WebhookHookPermissions(info) + req := ghpkg.WebhookCreateRequest{Org: org, PayloadURL: payloadURL, Secret: secret, Events: ghpkg.WebhookEvents} + switch strings.ToLower(strings.TrimSpace(body.Scope)) { + case "", "organization", "org": + if !orgHooks { + jsonError(w, "the App installation has no organization_hooks: write permission", http.StatusForbidden) + return + } + case "repository", "repo": + if !repoHooks { + jsonError(w, "the App installation has no repository_hooks: write permission", http.StatusForbidden) + return + } + repo := strings.TrimSpace(body.Repo) + if _, name, ok := strings.Cut(repo, "/"); ok { + repo = name + } + if repo == "" || !containsFold(repos, repo) { + jsonError(w, "repo must be one of this hive's project.repos", http.StatusBadRequest) + return + } + req.Repo = repo + default: + jsonError(w, "scope must be organization or repository", http.StatusBadRequest) + return + } + res, err := auth.CreateWebhook(ctx, req) + if err != nil { + status := http.StatusBadGateway + if errors.Is(err, ghpkg.ErrWebhookExists) { + status = http.StatusConflict + } + jsonError(w, err.Error(), status) + return + } + user := strings.TrimSpace(r.Header.Get("X-Hive-User")) + s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL), "", res.Target, 0) + s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target) + jsonResponse(w, res) +} + +func containsFold(list []string, v string) bool { + for _, s := range list { + if strings.EqualFold(s, v) { + return true + } + } + return false +} diff --git a/src/pkg/dashboard/api_github_webhook_setup_test.go b/src/pkg/dashboard/api_github_webhook_setup_test.go new file mode 100644 index 0000000000..f657d89057 --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_setup_test.go @@ -0,0 +1,183 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +func webhookSetupGet(t *testing.T, s *Server, owner bool, host string) (*httptest.ResponseRecorder, webhookSetupResponse) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/config/github/webhooks", nil) + if host != "" { + req.Host = host + } + if owner { + markOwnerRequest(req) + } + s.mux.ServeHTTP(rec, req) + var body webhookSetupResponse + if rec.Code == http.StatusOK { + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + } + return rec, body +} + +func TestWebhookSetup_OwnerOnly(t *testing.T) { + srv := newFullServer(t) + rec, _ := webhookSetupGet(t, srv, false, "") + if rec.Code != http.StatusForbidden { + t.Fatalf("non-owner GET: %d, want 403", rec.Code) + } + rec = httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("non-owner generate: %d, want 403", rec.Code) + } +} + +// The GET describes the setup without ever carrying the secret; the generate +// call returns it once and flips the verification mode to github-signed. +func TestWebhookSetup_PayloadURLSecretLifecycle(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "") + restore := ghpkg.SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "secret.key")) + defer restore() + restoreRelay := ghpkg.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "relay.key")) + defer restoreRelay() + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org/" + + rec, body := webhookSetupGet(t, srv, true, "") + if rec.Code != http.StatusOK { + t.Fatalf("GET: %d %s", rec.Code, rec.Body.String()) + } + if body.PayloadURL != "https://hive.example.org"+githubWebhookPath { + t.Fatalf("payload_url = %q", body.PayloadURL) + } + if body.PayloadURLSource != payloadURLSourceConfig { + t.Fatalf("payload_url_source = %q", body.PayloadURLSource) + } + if body.SecretConfigured || body.SecretSource != "" || !body.CanGenerateSecret { + t.Fatalf("fresh hive secret state: configured=%v source=%q canGenerate=%v", body.SecretConfigured, body.SecretSource, body.CanGenerateSecret) + } + if body.Verification != ghpkg.WebhookVerificationNone || body.Hosted { + t.Fatalf("verification=%q hosted=%v, want none/false", body.Verification, body.Hosted) + } + if body.CreateAvailable { + t.Fatal("create must not be offered without a secret") + } + if len(body.Events) == 0 || body.Events[0] != "check_run" || body.ContentType != "application/json" { + t.Fatalf("events/content type: %v %q", body.Events, body.ContentType) + } + + rec = httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("generate: %d %s", rec.Code, rec.Body.String()) + } + var gen map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &gen) + secret, _ := gen["secret"].(string) + if len(secret) != 64 || secret != ghpkg.WebhookSecret() { + t.Fatalf("generated secret not returned/stored: %q", secret) + } + if gen["verification"] != ghpkg.WebhookVerificationGitHub { + t.Fatalf("verification after generate = %v", gen["verification"]) + } + + rec, body = webhookSetupGet(t, srv, true, "") + if !body.SecretConfigured || body.SecretSource != ghpkg.WebhookSecretSourceFile || body.Verification != ghpkg.WebhookVerificationGitHub { + t.Fatalf("after generate: configured=%v source=%q verification=%q", body.SecretConfigured, body.SecretSource, body.Verification) + } + if strings.Contains(rec.Body.String(), secret) { + t.Fatal("GET must never echo the secret") + } +} + +func TestWebhookSetup_EnvSecretBlocksGenerate(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv := newFullServer(t) + _, body := webhookSetupGet(t, srv, true, "") + if body.SecretSource != ghpkg.WebhookSecretSourceEnv || body.CanGenerateSecret { + t.Fatalf("env secret: source=%q canGenerate=%v", body.SecretSource, body.CanGenerateSecret) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/secret", nil) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusConflict { + t.Fatalf("generate with env secret: %d, want 409", rec.Code) + } +} + +func TestWebhookSetup_PayloadURLFallsBackToRequestHost(t *testing.T) { + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "" + srv.deps.Config.Hub.DashboardURL = "" + _, body := webhookSetupGet(t, srv, true, "hive.dan.example") + if body.PayloadURL != "https://hive.dan.example"+githubWebhookPath || body.PayloadURLSource != payloadURLSourceRequest { + t.Fatalf("payload_url=%q source=%q", body.PayloadURL, body.PayloadURLSource) + } + _, body = webhookSetupGet(t, srv, true, "localhost:8080") + if body.PayloadURL != "" || body.PayloadURLSource != payloadURLSourceNone { + t.Fatalf("loopback host must not be offered: %q %q", body.PayloadURL, body.PayloadURLSource) + } +} + +// Create is offered only when the App holds a hooks write permission, and +// the create endpoint refuses scopes the installation cannot manage. +func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org" + srv.deps.Config.Project.Org = "projectbluefin" + srv.deps.Config.Project.Repos = []string{"bluefin", "projectbluefin/common"} + srv.deps.GHAppAuth = &ghpkg.AppAuth{} + orig := webhookPermissionProbe + defer func() { webhookPermissionProbe = orig }() + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: map[string]string{ghpkg.PermissionRepositoryHooks: "write"}}, nil + } + + _, body := webhookSetupGet(t, srv, true, "") + if body.AppOrgHooks || !body.AppRepoHooks || !body.CreateAvailable { + t.Fatalf("org=%v repo=%v create=%v", body.AppOrgHooks, body.AppRepoHooks, body.CreateAvailable) + } + if len(body.Repos) != 2 || body.Repos[1] != "common" { + t.Fatalf("repos = %v, want bare names", body.Repos) + } + + post := func(payload string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/create", strings.NewReader(payload)) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + return rec + } + if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusForbidden { + t.Fatalf("org scope without organization_hooks: %d %s", rec.Code, rec.Body.String()) + } + if rec := post(`{"scope":"repository","repo":"not-ours"}`); rec.Code != http.StatusBadRequest { + t.Fatalf("unknown repo: %d %s", rec.Code, rec.Body.String()) + } + + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: map[string]string{}}, nil + } + _, body = webhookSetupGet(t, srv, true, "") + if body.CreateAvailable { + t.Fatal("create offered without any hooks permission") + } +} diff --git a/src/pkg/dashboard/static-src/css/config-webhooks.css b/src/pkg/dashboard/static-src/css/config-webhooks.css new file mode 100644 index 0000000000..e4ee0d2114 --- /dev/null +++ b/src/pkg/dashboard/static-src/css/config-webhooks.css @@ -0,0 +1,27 @@ + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017) */ + .wh-section-title { font-weight: 600; font-size: var(--fs-base); margin: var(--sp-5) 0 var(--sp-2); } + .wh-intro { font-size: var(--fs-sm); color: var(--muted); margin-bottom: var(--sp-3); } + .wh-muted { color: var(--muted); } + div.wh-muted { font-size: var(--fs-sm); } + .wh-small { font-size: var(--fs-sm); } + .wh-warn { color: var(--yellow); } + .wh-ok { color: var(--green); } + .wh-error { font-size: var(--fs-sm); color: var(--red); } + .wh-note { margin-top: var(--sp-2); } + .wh-card { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-4) var(--sp-5); margin-bottom: var(--sp-5); background: var(--surface); } + .wh-card-head { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; margin-bottom: var(--sp-4); } + .wh-card-title { font-weight: 600; font-size: var(--fs-md); } + .wh-step { font-size: var(--fs-sm); font-weight: 600; margin: var(--sp-2) 0 var(--sp-1); } + .wh-step-gap { margin-top: var(--sp-4); } + .wh-code { font-size: var(--fs-xs); user-select: all; } + .wh-chip { font-size: var(--fs-xs); font-weight: 600; padding: var(--sp-1) var(--sp-4); border-radius: var(--r); border: 1px solid currentColor; } + .wh-chip-muted { color: var(--muted); } + .wh-chip-ok { color: var(--green); } + .wh-chip-warn { color: var(--yellow); } + .wh-actions { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; font-size: var(--fs-sm); } + .wh-actions .hv-btn { margin-left: var(--sp-2); } + .wh-secret-once { border: 1px solid var(--green); border-radius: var(--r); padding: var(--sp-3) var(--sp-4); margin: var(--sp-2) 0; font-size: var(--fs-sm); } + .wh-secret-once-title { font-weight: 600; color: var(--green); } + .wh-secret-value { margin: var(--sp-1) 0; word-break: break-all; } + .wh-create { border-top: 1px solid var(--border); margin-top: var(--sp-4); padding-top: var(--sp-4); } + .gov-stat.wh-clickable { cursor: pointer; } diff --git a/src/pkg/dashboard/static-src/js/config-dialog.js b/src/pkg/dashboard/static-src/js/config-dialog.js index 7622fd4b20..99f72ec85d 100644 --- a/src/pkg/dashboard/static-src/js/config-dialog.js +++ b/src/pkg/dashboard/static-src/js/config-dialog.js @@ -53,6 +53,7 @@ function closeConfigDialog() { _configModalOpen = false; + if (typeof webhookSetupReset === 'function') webhookSetupReset(); // Invalidate any in-flight load so a late reply cannot repopulate a // dialog the operator has already dismissed. _configLoadSeq++; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index b8ca800a08..fd02e566d9 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -49,7 +49,7 @@ } // The Forge App tab renders a shell synchronously, then loads the // credential inventory (fingerprints only) from the spoke. - if (tabId === 'Forge App') loadForgeApps(); + if (tabId === 'Forge App') { loadForgeApps(); loadWebhookSetup(); } // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); @@ -1735,7 +1735,8 @@ when its forge matches the forge this hive's repos live on; everything else is read-only. Private keys appear as fingerprints — key material never leaves the hive. </div> - <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div>`; + <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div> + ${renderGovWebhooksSection()}`; } async function loadForgeApps() { diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js new file mode 100644 index 0000000000..986ff42f93 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -0,0 +1,217 @@ + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017). + Owner-facing setup for /api/webhook/github: the exact payload URL, + the events to tick, a per-hive secret the owner generates here and + sees once, optional App-side hook creation, and the live receiver + status. Hosted spokes get deliveries through the hub relay with no + owner action, so for them the section is a status readout. */ + + let _webhookSetup = null; + let _webhookSecretOnce = ''; + let _webhookConfirmRegenerate = false; + + function renderGovWebhooksSection() { + return ` + <div class="wh-section-title">GitHub webhooks</div> + <div class="wh-intro"> + Webhooks let GitHub tell this hive about pushes, PRs, reviews and checks as they + happen: faster kicks, fewer API calls, a slower safety-net cadence. + <a href="${webhookDocsHref()}" target="_blank" rel="noopener noreferrer">Owner guide ↗</a> + </div> + <div id="webhook-setup-host"><div class="wh-muted">Loading webhook status…</div></div>`; + } + + function webhookDocsHref() { + return (_webhookSetup && _webhookSetup.docs_url) || 'https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide'; + } + + async function loadWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + if (!host) return; + try { + const res = await fetch('/api/config/github/webhooks'); + if (res.status === 403) { + host.innerHTML = '<div class="wh-muted">Owner access required to view webhook setup.</div>'; + return; + } + if (!res.ok) throw new Error('HTTP ' + res.status); + _webhookSetup = await res.json(); + renderWebhookSetup(); + } catch (e) { + host.innerHTML = `<div class="wh-error">Failed to load webhook status: ${esc(e.message)}</div>`; + } + } + + function webhookStatusChip(d) { + const h = d.health || {}; + if (d.verification === 'none') return chip('not configured', 'muted'); + if (h.events_1h > 0 || h.healthy) return chip(`receiving · ${h.events_1h || 0}/1h`, 'ok'); + if (h.last_event_at) return chip('quiet · last ' + webhookAgo(h.last_event_at), 'warn'); + return chip('no events yet', 'warn'); + function chip(text, tone) { + return `<span class="wh-chip wh-chip-${tone}">${esc(text)}</span>`; + } + } + + function webhookAgo(iso) { + const t = Date.parse(iso); + if (!t) return '—'; + const s = Math.max(0, Math.round((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.round(s / 60) + 'm ago'; + if (s < 86400) return Math.round(s / 3600) + 'h ago'; + return Math.round(s / 86400) + 'd ago'; + } + + function webhookCopyBtn(text, label) { + return `<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCopy" data-arg-types="t" data-copy="${esc(text)}">${esc(label || 'Copy')}</button>`; + } + + async function webhookCopy(btn) { + const text = btn && btn.dataset.copy || ''; + const ok = await writeClipboardText(text); + showToast(ok ? 'Copied' : 'Could not copy — select the text and copy it by hand', ok ? 'info' : 'error'); + } + + function renderWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + const d = _webhookSetup; + if (!host || !d) return; + const h = d.health || {}; + const row = (label, valueHtml) => forgeAppRow(label, valueHtml); + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + const relayHeld = !!d.hosted; + const direct = d.secret_source !== ''; + + let html = `<div class="wh-card"> + <div class="wh-card-head"> + <span class="wh-card-title">Receiver status</span> + ${webhookStatusChip(d)} + </div> + ${row('Verification', code(d.verification || 'none') + (relayHeld && !direct ? ' <span class="wh-muted">— hub relay; the GitHub App installation delivers through the hub, nothing to set up</span>' : ''))} + ${row('Last event', h.last_event_at ? esc(formatDashboardDateTime(h.last_event_at)) + ' <span class="wh-muted">(' + esc(webhookAgo(h.last_event_at)) + ')</span>' : '<span class="wh-muted">none yet</span>')} + ${row('Deliveries (1h)', esc(String(h.events_1h || 0)) + ' <span class="wh-muted">· ' + esc(String(h.invalidations_1h || 0)) + ' cache patches · ' + esc(String(h.ci_events_1h || 0)) + ' CI events</span>')} + ${row('Cadence', code(h.mode || 'polling') + (h.mode_reason ? ' <span class="wh-warn">' + esc(h.mode_reason) + '</span>' : ''))} + ${(!h.last_event_at && d.verification !== 'none') ? '<div class="wh-warn wh-note">No events yet — on GitHub open the webhook (Settings → Webhooks → your hook → Recent deliveries) and check the response code: 401 means the secret differs, 404 means the path is wrong, 503 means this hive has no secret.</div>' : ''} + </div>`; + + html += `<div class="wh-card"> + <div class="wh-card-title wh-card-head">${relayHeld ? 'Direct webhook (optional)' : 'Set up a webhook on GitHub'}</div> + ${relayHeld ? '<div class="wh-muted wh-intro">This hive already receives deliveries the hub relays from the GitHub App installation. A direct webhook with your own secret is only needed for repositories the App is not installed on.</div>' : ''} + <div class="wh-step">1. Secret</div> + ${renderWebhookSecretBlock(d)} + <div class="wh-step wh-step-gap">2. Add the webhook in GitHub</div> + <div class="wh-muted">Organization (one hook for every repo) → <b>Settings → Webhooks → Add webhook</b>, or per repository under the repo's Settings.</div> + ${row('Payload URL', d.payload_url ? code(d.payload_url) + webhookCopyBtn(d.payload_url) + ' <span class="wh-muted">(' + esc(d.payload_url_source) + ')</span>' : '<span class="wh-warn">unknown — set <code>dashboard.public_url</code> to this hive\u2019s public origin; the path is <code>' + esc(d.path) + '</code></span>')} + ${row('Content type', code(d.content_type))} + ${row('Secret', '<span class="wh-muted">the value from step 1</span>')} + ${row('SSL verification', '<span class="wh-muted">keep enabled</span>')} + ${row('Events', '<span class="wh-muted">"Let me select individual events", then tick:</span><br>' + (d.events || []).map(e => code(e)).join(' ') + webhookCopyBtn((d.events || []).join(', '), 'Copy list'))} + ${row('Optional', Object.entries(d.optional_events || {}).map(([k, v]) => code(k) + ' <span class="wh-muted">— ' + esc(v) + '</span>').join('<br>'))} + ${renderWebhookCreateBlock(d)} + </div>`; + host.innerHTML = html; + } + + function renderWebhookSecretBlock(d) { + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + let html = ''; + if (_webhookSecretOnce) { + html += `<div class="wh-secret-once"> + <div class="wh-secret-once-title">New secret — shown once</div> + <div class="wh-secret-value">${code(_webhookSecretOnce)}${webhookCopyBtn(_webhookSecretOnce)}</div> + <div class="wh-muted">Paste it into the GitHub webhook\u2019s <b>Secret</b> field. The hive keeps only the file; closing this dialog forgets the value.</div> + </div>`; + } + if (d.secret_source === 'env') { + html += `<div class="wh-small">Configured from <code>GITHUB_WEBHOOK_SECRET</code> (a Secret mount). Use that same value in GitHub; rotate it in the deployment, not here.</div>`; + return html; + } + const state = d.secret_source === 'file' + ? `Configured · stored at <code>${esc(d.secret_path || '')}</code>${_webhookSecretOnce ? '' : ' (value not shown again — regenerate if lost)'}` + : '<span class="wh-warn">No secret yet — generate one. Without it every delivery is rejected with 503.</span>'; + const label = _webhookConfirmRegenerate ? 'Confirm: replace secret' : (d.secret_source === 'file' ? 'Regenerate secret' : 'Generate secret'); + const cls = _webhookConfirmRegenerate ? 'btn-danger' : 'btn-primary'; + html += `<div class="wh-actions wh-note"> + <span>${state}</span> + <button type="button" class="hv-btn ${cls} btn-sm" data-action="webhookGenerateSecret" data-arg-types="t">${esc(label)}</button> + ${_webhookConfirmRegenerate ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCancelRegenerate">Cancel</button><span class="wh-muted">GitHub hooks using the old secret will start failing with 401 until you update them.</span>' : ''} + </div>`; + return html; + } + + function renderWebhookCreateBlock(d) { + if (d.app_probe_error) { + return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; + } + if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + const scopes = []; + if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); + if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); + const disabled = !d.create_available; + const why = !d.secret_configured ? 'generate a secret first' : (!d.payload_url ? 'set dashboard.public_url first' : ''); + return `<div class="wh-create"> + <div class="wh-step">Or let the App create it</div> + <div class="wh-actions"> + <select id="webhook-create-scope" class="wh-small">${scopes.join('')}</select> + <button type="button" class="hv-btn btn-primary btn-sm" data-action="webhookCreateHook" data-arg-types="t" ${disabled ? 'disabled' : ''}>Create webhook for me</button> + ${why ? '<span class="wh-muted">— ' + esc(why) + '</span>' : '<span class="wh-muted">uses the App installation\u2019s hooks permission; subscribes the events above with this hive\u2019s secret</span>'} + </div> + <div id="webhook-create-result" class="wh-small wh-note"></div> + </div>`; + } + + function webhookCancelRegenerate() { + _webhookConfirmRegenerate = false; + renderWebhookSetup(); + } + + async function webhookGenerateSecret(btn) { + if (_webhookSetup && _webhookSetup.secret_source === 'file' && !_webhookConfirmRegenerate) { + _webhookConfirmRegenerate = true; + renderWebhookSetup(); + return; + } + _webhookConfirmRegenerate = false; + if (btn) btn.disabled = true; + try { + const res = await fetch('/api/config/github/webhooks/secret', { method: 'POST' }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + _webhookSecretOnce = body.secret || ''; + showToast('Webhook secret generated — copy it now, it is shown once', 'info'); + await loadWebhookSetup(); + } catch (e) { + showToast('Could not generate secret: ' + e.message, 'error'); + if (btn) btn.disabled = false; + } + } + + async function webhookCreateHook(btn) { + const sel = document.getElementById('webhook-create-scope'); + const out = document.getElementById('webhook-create-result'); + const v = (sel && sel.value) || 'organization'; + const [scope, repo] = v.split(':'); + if (btn) btn.disabled = true; + if (out) out.textContent = 'Creating…'; + try { + const res = await fetch('/api/config/github/webhooks/create', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scope, repo: repo || '' }) + }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast('Webhook created', 'info'); + } catch (e) { + if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; + } finally { + if (btn) btn.disabled = false; + } + } + + /* Forget the one-time secret whenever the Settings dialog closes so it + never lingers in a re-rendered tab. */ + function webhookSetupReset() { + _webhookSecretOnce = ''; + _webhookConfirmRegenerate = false; + } diff --git a/src/pkg/dashboard/static-src/js/governor-view.js b/src/pkg/dashboard/static-src/js/governor-view.js index 58cd25c87d..0f22bc9556 100644 --- a/src/pkg/dashboard/static-src/js/governor-view.js +++ b/src/pkg/dashboard/static-src/js/governor-view.js @@ -223,6 +223,30 @@ </div>`; } + /* Webhooks tile (hivecommons/hive#18017): whether GitHub deliveries + reach this hive (direct or hub relay) and how many in the last hour. + Owners click through to Settings → Forge App → GitHub webhooks. */ + function webhookGovStatHtml(data) { + const wh = data && data.webhooks; + if (!wh || typeof wh !== 'object') return ''; + const owner = dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + const none = !wh.verification || wh.verification === 'none'; + const n = Number(wh.events_1h || 0); + let val, cls; + if (none) { val = 'off'; cls = ' dead'; } + else if (n > 0 || wh.healthy) { val = n + '/1h'; cls = ''; } + else if (wh.last_event_at) { val = 'quiet'; cls = ' starting'; } + else { val = 'no events'; cls = ' starting'; } + const how = none + ? 'OFF — no webhook secret and no hub relay key, so every delivery is rejected and the hive polls GitHub on the base cadence.' + : `Verification: ${wh.verification}. Cadence: ${wh.mode || 'polling'}${wh.mode_reason ? ' (' + wh.mode_reason + ')' : ''}. ${n} deliveries in the last hour.`; + const tip = 'WEBHOOKS — does GitHub tell this hive about changes as they happen?\n\nWHAT: With webhooks on, pushes, PRs, reviews and checks wake the governor at once, CI failures kick the owning agent, and the safety-net cadence slows to save API calls.\n\nHOW: ' + how + + (owner ? '\n\nClick to set up or check webhooks (Settings → Forge App).' : ''); + const action = owner ? ' data-action="openConfigDialog" data-config-type="governor" data-tab="Forge App"' : ''; + const label = owner && none ? 'webhooks · set up →' : 'webhooks'; + return `<div class="gov-stat${owner ? ' wh-clickable' : ''}"${action} title="${escapeHtml(tip)}"><div class="label">${label}</div><div class="val${cls}">${escapeHtml(val)}</div></div>`; + } + function renderGovernor(gov, cadenceMatrix, data) { const el = document.getElementById('governor'); const isStarting = !gov.active && !gov.mode; @@ -296,6 +320,7 @@ <div class="gov-stat" aria-label="${escapeHtml(prEquationTitle)}"><div class="label">actionable PRs</div><div class="spark-row"><span class="val">${prCount}</span>${prsSpark}</div>${prEquationHTML ? `<div class="overview-kpi-subline">${prEquationHTML}</div>` : ''}</div> <div class="gov-stat" title="${itmTitle}"><div class="label">MTTR</div><div class="spark-row"><span class="val">${formatFixTime(itmMedian)}</span>${itmSpark}</div></div> ${fleetRunStatHtml()} + ${webhookGovStatHtml(data)} <div class="gov-stat" title="${holdTooltip}"><div class="label">on hold</div><div class="spark-row"><span class="val">${holdTotal}</span>${holdSpark}</div></div> ${(() => { // Merge hotspots tile (GET /api/hotspots): files concurrent PRs diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index d6c8e81100..fe1b1cd620 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -22,6 +22,7 @@ css/cost.css css/alerts-toasts.css css/feedback-faq.css css/config-dialog.css +css/config-webhooks.css css/workflows.css css/workflows-editor.css css/workflows-ingest.css @@ -138,6 +139,7 @@ js/config-state.js js/settings-search.js js/config-dialog.js js/config-tabs.js +js/config-webhooks.js js/config-hub.js js/config-agent.js js/config-governor.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 9ed7f34a33..065d33f36e 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5052,6 +5052,33 @@ .config-body { --config-body-pad: 14px; padding: var(--sp-0) var(--config-body-pad) var(--config-body-pad); } .config-field-row, .config-field-row4 { grid-template-columns: 1fr; } } + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017) */ + .wh-section-title { font-weight: 600; font-size: var(--fs-base); margin: var(--sp-5) 0 var(--sp-2); } + .wh-intro { font-size: var(--fs-sm); color: var(--muted); margin-bottom: var(--sp-3); } + .wh-muted { color: var(--muted); } + div.wh-muted { font-size: var(--fs-sm); } + .wh-small { font-size: var(--fs-sm); } + .wh-warn { color: var(--yellow); } + .wh-ok { color: var(--green); } + .wh-error { font-size: var(--fs-sm); color: var(--red); } + .wh-note { margin-top: var(--sp-2); } + .wh-card { border: 1px solid var(--border); border-radius: var(--r); padding: var(--sp-4) var(--sp-5); margin-bottom: var(--sp-5); background: var(--surface); } + .wh-card-head { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; margin-bottom: var(--sp-4); } + .wh-card-title { font-weight: 600; font-size: var(--fs-md); } + .wh-step { font-size: var(--fs-sm); font-weight: 600; margin: var(--sp-2) 0 var(--sp-1); } + .wh-step-gap { margin-top: var(--sp-4); } + .wh-code { font-size: var(--fs-xs); user-select: all; } + .wh-chip { font-size: var(--fs-xs); font-weight: 600; padding: var(--sp-1) var(--sp-4); border-radius: var(--r); border: 1px solid currentColor; } + .wh-chip-muted { color: var(--muted); } + .wh-chip-ok { color: var(--green); } + .wh-chip-warn { color: var(--yellow); } + .wh-actions { display: flex; align-items: center; gap: var(--sp-4); flex-wrap: wrap; font-size: var(--fs-sm); } + .wh-actions .hv-btn { margin-left: var(--sp-2); } + .wh-secret-once { border: 1px solid var(--green); border-radius: var(--r); padding: var(--sp-3) var(--sp-4); margin: var(--sp-2) 0; font-size: var(--fs-sm); } + .wh-secret-once-title { font-weight: 600; color: var(--green); } + .wh-secret-value { margin: var(--sp-1) 0; word-break: break-all; } + .wh-create { border-top: 1px solid var(--border); margin-top: var(--sp-4); padding-top: var(--sp-4); } + .gov-stat.wh-clickable { cursor: pointer; } /* Settings → Workflows (#17620 phase 3): library, detail, diagram, ingest */ .wf-nav { display: flex; gap: var(--sp-2); margin-bottom: var(--sp-5); border-bottom: var(--line-width) solid var(--border); } .wf-nav-item { background: none; border: none; border-bottom: 2px solid transparent; color: var(--muted); font: inherit; font-size: var(--fs-sm); font-weight: 700; padding: var(--sp-2) var(--sp-4); cursor: pointer; } @@ -32598,6 +32625,30 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> </div>`; } + /* Webhooks tile (hivecommons/hive#18017): whether GitHub deliveries + reach this hive (direct or hub relay) and how many in the last hour. + Owners click through to Settings → Forge App → GitHub webhooks. */ + function webhookGovStatHtml(data) { + const wh = data && data.webhooks; + if (!wh || typeof wh !== 'object') return ''; + const owner = dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); + const none = !wh.verification || wh.verification === 'none'; + const n = Number(wh.events_1h || 0); + let val, cls; + if (none) { val = 'off'; cls = ' dead'; } + else if (n > 0 || wh.healthy) { val = n + '/1h'; cls = ''; } + else if (wh.last_event_at) { val = 'quiet'; cls = ' starting'; } + else { val = 'no events'; cls = ' starting'; } + const how = none + ? 'OFF — no webhook secret and no hub relay key, so every delivery is rejected and the hive polls GitHub on the base cadence.' + : `Verification: ${wh.verification}. Cadence: ${wh.mode || 'polling'}${wh.mode_reason ? ' (' + wh.mode_reason + ')' : ''}. ${n} deliveries in the last hour.`; + const tip = 'WEBHOOKS — does GitHub tell this hive about changes as they happen?\n\nWHAT: With webhooks on, pushes, PRs, reviews and checks wake the governor at once, CI failures kick the owning agent, and the safety-net cadence slows to save API calls.\n\nHOW: ' + how + + (owner ? '\n\nClick to set up or check webhooks (Settings → Forge App).' : ''); + const action = owner ? ' data-action="openConfigDialog" data-config-type="governor" data-tab="Forge App"' : ''; + const label = owner && none ? 'webhooks · set up →' : 'webhooks'; + return `<div class="gov-stat${owner ? ' wh-clickable' : ''}"${action} title="${escapeHtml(tip)}"><div class="label">${label}</div><div class="val${cls}">${escapeHtml(val)}</div></div>`; + } + function renderGovernor(gov, cadenceMatrix, data) { const el = document.getElementById('governor'); const isStarting = !gov.active && !gov.mode; @@ -32671,6 +32722,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> <div class="gov-stat" aria-label="${escapeHtml(prEquationTitle)}"><div class="label">actionable PRs</div><div class="spark-row"><span class="val">${prCount}</span>${prsSpark}</div>${prEquationHTML ? `<div class="overview-kpi-subline">${prEquationHTML}</div>` : ''}</div> <div class="gov-stat" title="${itmTitle}"><div class="label">MTTR</div><div class="spark-row"><span class="val">${formatFixTime(itmMedian)}</span>${itmSpark}</div></div> ${fleetRunStatHtml()} + ${webhookGovStatHtml(data)} <div class="gov-stat" title="${holdTooltip}"><div class="label">on hold</div><div class="spark-row"><span class="val">${holdTotal}</span>${holdSpark}</div></div> ${(() => { // Merge hotspots tile (GET /api/hotspots): files concurrent PRs @@ -55947,6 +55999,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> function closeConfigDialog() { _configModalOpen = false; + if (typeof webhookSetupReset === 'function') webhookSetupReset(); // Invalidate any in-flight load so a late reply cannot repopulate a // dialog the operator has already dismissed. _configLoadSeq++; @@ -56533,7 +56586,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> } // The Forge App tab renders a shell synchronously, then loads the // credential inventory (fingerprints only) from the spoke. - if (tabId === 'Forge App') loadForgeApps(); + if (tabId === 'Forge App') { loadForgeApps(); loadWebhookSetup(); } // The Features tab's Linear Agent panel loads install/session state // from its own endpoint (owner-only; the render is a static shell). if (tabId === 'Features') loadLinearAgentStatus(); @@ -58219,7 +58272,8 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> when its forge matches the forge this hive's repos live on; everything else is read-only. Private keys appear as fingerprints — key material never leaves the hive. </div> - <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div>`; + <div id="forge-app-host"><div style="color:var(--muted);font-size:0.8rem">Loading forge Apps…</div></div> + ${renderGovWebhooksSection()}`; } async function loadForgeApps() { @@ -59433,6 +59487,223 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> function contribRepoMatchingDisabledEntries(repoFull, disabled) { return (disabled || []).filter(function(entry) { return contribRepoDisabledEntryMatches(repoFull, entry); }); } + /* Settings → Forge App → GitHub webhooks (hivecommons/hive#18017). + Owner-facing setup for /api/webhook/github: the exact payload URL, + the events to tick, a per-hive secret the owner generates here and + sees once, optional App-side hook creation, and the live receiver + status. Hosted spokes get deliveries through the hub relay with no + owner action, so for them the section is a status readout. */ + + let _webhookSetup = null; + let _webhookSecretOnce = ''; + let _webhookConfirmRegenerate = false; + + function renderGovWebhooksSection() { + return ` + <div class="wh-section-title">GitHub webhooks</div> + <div class="wh-intro"> + Webhooks let GitHub tell this hive about pushes, PRs, reviews and checks as they + happen: faster kicks, fewer API calls, a slower safety-net cadence. + <a href="${webhookDocsHref()}" target="_blank" rel="noopener noreferrer">Owner guide ↗</a> + </div> + <div id="webhook-setup-host"><div class="wh-muted">Loading webhook status…</div></div>`; + } + + function webhookDocsHref() { + return (_webhookSetup && _webhookSetup.docs_url) || 'https://github.com/hivecommons/hive/blob/v5/src/docs/webhooks.md#setting-up-webhooks-owner-guide'; + } + + async function loadWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + if (!host) return; + try { + const res = await fetch('/api/config/github/webhooks'); + if (res.status === 403) { + host.innerHTML = '<div class="wh-muted">Owner access required to view webhook setup.</div>'; + return; + } + if (!res.ok) throw new Error('HTTP ' + res.status); + _webhookSetup = await res.json(); + renderWebhookSetup(); + } catch (e) { + host.innerHTML = `<div class="wh-error">Failed to load webhook status: ${esc(e.message)}</div>`; + } + } + + function webhookStatusChip(d) { + const h = d.health || {}; + if (d.verification === 'none') return chip('not configured', 'muted'); + if (h.events_1h > 0 || h.healthy) return chip(`receiving · ${h.events_1h || 0}/1h`, 'ok'); + if (h.last_event_at) return chip('quiet · last ' + webhookAgo(h.last_event_at), 'warn'); + return chip('no events yet', 'warn'); + function chip(text, tone) { + return `<span class="wh-chip wh-chip-${tone}">${esc(text)}</span>`; + } + } + + function webhookAgo(iso) { + const t = Date.parse(iso); + if (!t) return '—'; + const s = Math.max(0, Math.round((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.round(s / 60) + 'm ago'; + if (s < 86400) return Math.round(s / 3600) + 'h ago'; + return Math.round(s / 86400) + 'd ago'; + } + + function webhookCopyBtn(text, label) { + return `<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCopy" data-arg-types="t" data-copy="${esc(text)}">${esc(label || 'Copy')}</button>`; + } + + async function webhookCopy(btn) { + const text = btn && btn.dataset.copy || ''; + const ok = await writeClipboardText(text); + showToast(ok ? 'Copied' : 'Could not copy — select the text and copy it by hand', ok ? 'info' : 'error'); + } + + function renderWebhookSetup() { + const host = document.getElementById('webhook-setup-host'); + const d = _webhookSetup; + if (!host || !d) return; + const h = d.health || {}; + const row = (label, valueHtml) => forgeAppRow(label, valueHtml); + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + const relayHeld = !!d.hosted; + const direct = d.secret_source !== ''; + + let html = `<div class="wh-card"> + <div class="wh-card-head"> + <span class="wh-card-title">Receiver status</span> + ${webhookStatusChip(d)} + </div> + ${row('Verification', code(d.verification || 'none') + (relayHeld && !direct ? ' <span class="wh-muted">— hub relay; the GitHub App installation delivers through the hub, nothing to set up</span>' : ''))} + ${row('Last event', h.last_event_at ? esc(formatDashboardDateTime(h.last_event_at)) + ' <span class="wh-muted">(' + esc(webhookAgo(h.last_event_at)) + ')</span>' : '<span class="wh-muted">none yet</span>')} + ${row('Deliveries (1h)', esc(String(h.events_1h || 0)) + ' <span class="wh-muted">· ' + esc(String(h.invalidations_1h || 0)) + ' cache patches · ' + esc(String(h.ci_events_1h || 0)) + ' CI events</span>')} + ${row('Cadence', code(h.mode || 'polling') + (h.mode_reason ? ' <span class="wh-warn">' + esc(h.mode_reason) + '</span>' : ''))} + ${(!h.last_event_at && d.verification !== 'none') ? '<div class="wh-warn wh-note">No events yet — on GitHub open the webhook (Settings → Webhooks → your hook → Recent deliveries) and check the response code: 401 means the secret differs, 404 means the path is wrong, 503 means this hive has no secret.</div>' : ''} + </div>`; + + html += `<div class="wh-card"> + <div class="wh-card-title wh-card-head">${relayHeld ? 'Direct webhook (optional)' : 'Set up a webhook on GitHub'}</div> + ${relayHeld ? '<div class="wh-muted wh-intro">This hive already receives deliveries the hub relays from the GitHub App installation. A direct webhook with your own secret is only needed for repositories the App is not installed on.</div>' : ''} + <div class="wh-step">1. Secret</div> + ${renderWebhookSecretBlock(d)} + <div class="wh-step wh-step-gap">2. Add the webhook in GitHub</div> + <div class="wh-muted">Organization (one hook for every repo) → <b>Settings → Webhooks → Add webhook</b>, or per repository under the repo's Settings.</div> + ${row('Payload URL', d.payload_url ? code(d.payload_url) + webhookCopyBtn(d.payload_url) + ' <span class="wh-muted">(' + esc(d.payload_url_source) + ')</span>' : '<span class="wh-warn">unknown — set <code>dashboard.public_url</code> to this hive\u2019s public origin; the path is <code>' + esc(d.path) + '</code></span>')} + ${row('Content type', code(d.content_type))} + ${row('Secret', '<span class="wh-muted">the value from step 1</span>')} + ${row('SSL verification', '<span class="wh-muted">keep enabled</span>')} + ${row('Events', '<span class="wh-muted">"Let me select individual events", then tick:</span><br>' + (d.events || []).map(e => code(e)).join(' ') + webhookCopyBtn((d.events || []).join(', '), 'Copy list'))} + ${row('Optional', Object.entries(d.optional_events || {}).map(([k, v]) => code(k) + ' <span class="wh-muted">— ' + esc(v) + '</span>').join('<br>'))} + ${renderWebhookCreateBlock(d)} + </div>`; + host.innerHTML = html; + } + + function renderWebhookSecretBlock(d) { + const code = (t) => `<code class="wh-code">${esc(t)}</code>`; + let html = ''; + if (_webhookSecretOnce) { + html += `<div class="wh-secret-once"> + <div class="wh-secret-once-title">New secret — shown once</div> + <div class="wh-secret-value">${code(_webhookSecretOnce)}${webhookCopyBtn(_webhookSecretOnce)}</div> + <div class="wh-muted">Paste it into the GitHub webhook\u2019s <b>Secret</b> field. The hive keeps only the file; closing this dialog forgets the value.</div> + </div>`; + } + if (d.secret_source === 'env') { + html += `<div class="wh-small">Configured from <code>GITHUB_WEBHOOK_SECRET</code> (a Secret mount). Use that same value in GitHub; rotate it in the deployment, not here.</div>`; + return html; + } + const state = d.secret_source === 'file' + ? `Configured · stored at <code>${esc(d.secret_path || '')}</code>${_webhookSecretOnce ? '' : ' (value not shown again — regenerate if lost)'}` + : '<span class="wh-warn">No secret yet — generate one. Without it every delivery is rejected with 503.</span>'; + const label = _webhookConfirmRegenerate ? 'Confirm: replace secret' : (d.secret_source === 'file' ? 'Regenerate secret' : 'Generate secret'); + const cls = _webhookConfirmRegenerate ? 'btn-danger' : 'btn-primary'; + html += `<div class="wh-actions wh-note"> + <span>${state}</span> + <button type="button" class="hv-btn ${cls} btn-sm" data-action="webhookGenerateSecret" data-arg-types="t">${esc(label)}</button> + ${_webhookConfirmRegenerate ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="webhookCancelRegenerate">Cancel</button><span class="wh-muted">GitHub hooks using the old secret will start failing with 401 until you update them.</span>' : ''} + </div>`; + return html; + } + + function renderWebhookCreateBlock(d) { + if (d.app_probe_error) { + return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; + } + if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + const scopes = []; + if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); + if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); + const disabled = !d.create_available; + const why = !d.secret_configured ? 'generate a secret first' : (!d.payload_url ? 'set dashboard.public_url first' : ''); + return `<div class="wh-create"> + <div class="wh-step">Or let the App create it</div> + <div class="wh-actions"> + <select id="webhook-create-scope" class="wh-small">${scopes.join('')}</select> + <button type="button" class="hv-btn btn-primary btn-sm" data-action="webhookCreateHook" data-arg-types="t" ${disabled ? 'disabled' : ''}>Create webhook for me</button> + ${why ? '<span class="wh-muted">— ' + esc(why) + '</span>' : '<span class="wh-muted">uses the App installation\u2019s hooks permission; subscribes the events above with this hive\u2019s secret</span>'} + </div> + <div id="webhook-create-result" class="wh-small wh-note"></div> + </div>`; + } + + function webhookCancelRegenerate() { + _webhookConfirmRegenerate = false; + renderWebhookSetup(); + } + + async function webhookGenerateSecret(btn) { + if (_webhookSetup && _webhookSetup.secret_source === 'file' && !_webhookConfirmRegenerate) { + _webhookConfirmRegenerate = true; + renderWebhookSetup(); + return; + } + _webhookConfirmRegenerate = false; + if (btn) btn.disabled = true; + try { + const res = await fetch('/api/config/github/webhooks/secret', { method: 'POST' }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + _webhookSecretOnce = body.secret || ''; + showToast('Webhook secret generated — copy it now, it is shown once', 'info'); + await loadWebhookSetup(); + } catch (e) { + showToast('Could not generate secret: ' + e.message, 'error'); + if (btn) btn.disabled = false; + } + } + + async function webhookCreateHook(btn) { + const sel = document.getElementById('webhook-create-scope'); + const out = document.getElementById('webhook-create-result'); + const v = (sel && sel.value) || 'organization'; + const [scope, repo] = v.split(':'); + if (btn) btn.disabled = true; + if (out) out.textContent = 'Creating…'; + try { + const res = await fetch('/api/config/github/webhooks/create', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scope, repo: repo || '' }) + }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); + if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast('Webhook created', 'info'); + } catch (e) { + if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; + } finally { + if (btn) btn.disabled = false; + } + } + + /* Forget the one-time secret whenever the Settings dialog closes so it + never lingers in a re-rendered tab. */ + function webhookSetupReset() { + _webhookSecretOnce = ''; + _webhookConfirmRegenerate = false; + } function renderGovHub(hub) { const dashUrl = hub.dashboard_url || window.location.origin || ''; const snapUrl = hub.snapshot_url || (hub.auto_snapshot && dashUrl ? dashUrl.replace(/\/$/, '') + '/snapshot' : ''); diff --git a/src/pkg/github/app.go b/src/pkg/github/app.go index f68664c810..5ac633d0cc 100644 --- a/src/pkg/github/app.go +++ b/src/pkg/github/app.go @@ -866,7 +866,9 @@ func installationPermissionMap(p *gh.InstallationPermissions) map[string]string put("contents", p.GetContents()) put("issues", p.GetIssues()) put("metadata", p.GetMetadata()) + put(PermissionOrganizationHooks, p.GetOrganizationHooks()) put("pull_requests", p.GetPullRequests()) + put(PermissionRepositoryHooks, p.GetRepositoryHooks()) put("secrets", p.GetSecrets()) put("vulnerability_alerts", p.GetVulnerabilityAlerts()) put("workflows", p.GetWorkflows()) diff --git a/src/pkg/github/webhook_relay_key.go b/src/pkg/github/webhook_relay_key.go index d850abdb66..7d7cc41278 100644 --- a/src/pkg/github/webhook_relay_key.go +++ b/src/pkg/github/webhook_relay_key.go @@ -28,7 +28,8 @@ const WebhookSecretEnvVar = "GITHUB_WEBHOOK_SECRET" // Values of /api/status.webhooks.verification: how this spoke authenticates // GitHub webhook deliveries. const ( - // WebhookVerificationGitHub: GITHUB_WEBHOOK_SECRET is set and GitHub's + // WebhookVerificationGitHub: a webhook secret is configured (WebhookSecret: + // GITHUB_WEBHOOK_SECRET or the generated secret file) and GitHub's // X-Hub-Signature-256 is verified (self-hosted, or a hub relay to a spoke // that holds the secret). WebhookVerificationGitHub = "github-signed" @@ -109,7 +110,7 @@ func WebhookRelayKeyFingerprint() string { // (WebhookVerification* above). Only the mode ever leaves this function, never // the secret or the key. func WebhookVerification() string { - if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + if WebhookSecret() != "" { return WebhookVerificationGitHub } if WebhookRelayKey() != "" { diff --git a/src/pkg/github/webhook_secret.go b/src/pkg/github/webhook_secret.go new file mode 100644 index 0000000000..49d074b16a --- /dev/null +++ b/src/pkg/github/webhook_secret.go @@ -0,0 +1,133 @@ +package github + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "sync" +) + +// GitHub webhook secret for a self-hosted spoke (hivecommons/hive#18017). +// +// A self-hosted hive verifies X-Hub-Signature-256 on /api/webhook/github +// with a secret the owner also pastes into the GitHub webhook form. It comes +// from GITHUB_WEBHOOK_SECRET when the operator mounts one, otherwise from a +// 0600 file on the data volume that the dashboard's "Generate secret" button +// writes (same pattern as the hub relay key, webhook_relay_key.go). The env +// var always wins: a Secret mount is the deployment's source of truth and the +// dashboard must not pretend it can rotate it. + +const defaultWebhookSecretPath = "/data/github-webhook-secret.key" + +// Values of WebhookSecretSource: where the active webhook secret comes from. +const ( + WebhookSecretSourceEnv = "env" + WebhookSecretSourceFile = "file" + WebhookSecretSourceNone = "" +) + +// webhookSecretBytes is the entropy of a generated secret (256 bits, hex). +const webhookSecretBytes = 32 + +// ErrWebhookSecretFromEnv is returned by GenerateWebhookSecret when +// GITHUB_WEBHOOK_SECRET is set: the file would never be consulted, so writing +// one would only mislead the owner about which secret is active. +var ErrWebhookSecretFromEnv = errors.New("webhook secret is set by " + WebhookSecretEnvVar + "; rotate it there") + +var webhookSecretState = struct { + mu sync.Mutex + path string + secret string + loaded bool +}{path: defaultWebhookSecretPath} + +func webhookSecretFileLocked() string { + st := &webhookSecretState + if !st.loaded { + st.loaded = true + if data, err := os.ReadFile(st.path); err == nil { + st.secret = strings.TrimSpace(string(data)) + } + } + return st.secret +} + +// WebhookSecret is the HMAC secret this spoke verifies GitHub-signed +// deliveries with: GITHUB_WEBHOOK_SECRET, else the generated secret file, else +// "". +func WebhookSecret() string { + if v := strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)); v != "" { + return v + } + webhookSecretState.mu.Lock() + defer webhookSecretState.mu.Unlock() + return webhookSecretFileLocked() +} + +// WebhookSecretSource reports where WebhookSecret comes from +// (WebhookSecretSource*). Never the secret itself. +func WebhookSecretSource() string { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return WebhookSecretSourceEnv + } + if WebhookSecret() != "" { + return WebhookSecretSourceFile + } + return WebhookSecretSourceNone +} + +// WebhookSecretPath is the file a generated secret is persisted to. +func WebhookSecretPath() string { + webhookSecretState.mu.Lock() + defer webhookSecretState.mu.Unlock() + return webhookSecretState.path +} + +// GenerateWebhookSecret mints a fresh random secret, persists it 0600 to the +// secret file (replacing any previous generated secret) and returns it. This +// is the only time the value leaves the hive; callers show it once. It +// refuses with ErrWebhookSecretFromEnv when the env var is set. +func GenerateWebhookSecret() (string, error) { + if strings.TrimSpace(os.Getenv(WebhookSecretEnvVar)) != "" { + return "", ErrWebhookSecretFromEnv + } + b := make([]byte, webhookSecretBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + secret := hex.EncodeToString(b) + st := &webhookSecretState + st.mu.Lock() + defer st.mu.Unlock() + if err := os.MkdirAll(filepath.Dir(st.path), 0o700); err != nil { + return "", err + } + tmp := st.path + ".tmp" + if err := os.WriteFile(tmp, []byte(secret+"\n"), 0o600); err != nil { + return "", err + } + if err := os.Rename(tmp, st.path); err != nil { + return "", err + } + st.secret = secret + st.loaded = true + return secret, nil +} + +// SetWebhookSecretPathForTest points the secret store at path, forgets any +// secret held in memory, and returns a func that restores the previous state. +func SetWebhookSecretPathForTest(path string) func() { + st := &webhookSecretState + st.mu.Lock() + prevPath, prevSecret, prevLoaded := st.path, st.secret, st.loaded + st.path, st.secret, st.loaded = path, "", false + st.mu.Unlock() + return func() { + st.mu.Lock() + st.path, st.secret, st.loaded = prevPath, prevSecret, prevLoaded + st.mu.Unlock() + } +} diff --git a/src/pkg/github/webhook_secret_test.go b/src/pkg/github/webhook_secret_test.go new file mode 100644 index 0000000000..7abdc64ad9 --- /dev/null +++ b/src/pkg/github/webhook_secret_test.go @@ -0,0 +1,105 @@ +package github + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWebhookSecret_FileFallbackAndGenerate(t *testing.T) { + t.Setenv(WebhookSecretEnvVar, "") + dir := t.TempDir() + restore := SetWebhookSecretPathForTest(filepath.Join(dir, "nested", "secret.key")) + defer restore() + + if got := WebhookSecret(); got != "" { + t.Fatalf("no file yet: WebhookSecret() = %q, want empty", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceNone { + t.Fatalf("source = %q, want none", got) + } + if got := WebhookVerification(); got == WebhookVerificationGitHub { + t.Fatalf("verification = %q before any secret", got) + } + + secret, err := GenerateWebhookSecret() + if err != nil { + t.Fatalf("GenerateWebhookSecret: %v", err) + } + if len(secret) != webhookSecretBytes*2 { + t.Fatalf("secret length %d, want %d hex chars", len(secret), webhookSecretBytes*2) + } + if got := WebhookSecret(); got != secret { + t.Fatalf("WebhookSecret() = %q, want the generated value", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceFile { + t.Fatalf("source = %q, want file", got) + } + if got := WebhookVerification(); got != WebhookVerificationGitHub { + t.Fatalf("verification = %q, want %q", got, WebhookVerificationGitHub) + } + st, err := os.Stat(WebhookSecretPath()) + if err != nil { + t.Fatalf("stat secret file: %v", err) + } + if perm := st.Mode().Perm(); perm != 0o600 { + t.Fatalf("secret file mode %o, want 0600", perm) + } + + // A fresh process reads the persisted value back. + restore2 := SetWebhookSecretPathForTest(WebhookSecretPath()) + defer restore2() + if got := WebhookSecret(); got != secret { + t.Fatalf("reloaded secret = %q, want %q", got, secret) + } + + // Regenerating replaces it. + next, err := GenerateWebhookSecret() + if err != nil || next == secret { + t.Fatalf("regenerate: err=%v same=%v", err, next == secret) + } +} + +func TestWebhookSecret_EnvWinsAndBlocksGenerate(t *testing.T) { + restore := SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "secret.key")) + defer restore() + if _, err := GenerateWebhookSecret(); err != nil { + t.Fatalf("seed file secret: %v", err) + } + t.Setenv(WebhookSecretEnvVar, "from-env") + if got := WebhookSecret(); got != "from-env" { + t.Fatalf("WebhookSecret() = %q, want env value", got) + } + if got := WebhookSecretSource(); got != WebhookSecretSourceEnv { + t.Fatalf("source = %q, want env", got) + } + if _, err := GenerateWebhookSecret(); err != ErrWebhookSecretFromEnv { + t.Fatalf("GenerateWebhookSecret with env set: err=%v, want ErrWebhookSecretFromEnv", err) + } +} + +func TestWebhookHookPermissions(t *testing.T) { + org, repo := WebhookHookPermissions(nil) + if org || repo { + t.Fatal("nil info must grant nothing") + } + info := &InstallationInfo{Permissions: map[string]string{PermissionOrganizationHooks: "write", PermissionRepositoryHooks: "read"}} + org, repo = WebhookHookPermissions(info) + if !org || repo { + t.Fatalf("org=%v repo=%v, want org write only", org, repo) + } +} + +func TestWebBaseURLForAPI(t *testing.T) { + cases := map[string]string{ + "": "https://github.com", + "https://api.github.com": "https://github.com", + "https://ghe.example.com/api/v3": "https://ghe.example.com", + "https://ghe.example.com/api/v3/": "https://ghe.example.com", + } + for in, want := range cases { + if got := webBaseURLForAPI(in); got != want { + t.Errorf("webBaseURLForAPI(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/src/pkg/github/webhook_setup.go b/src/pkg/github/webhook_setup.go new file mode 100644 index 0000000000..78aaeda11f --- /dev/null +++ b/src/pkg/github/webhook_setup.go @@ -0,0 +1,146 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + gh "github.com/google/go-github/v72/github" +) + +// Owner-facing webhook setup (hivecommons/hive#18017): the events a hive +// consumes on /api/webhook/github and the App-side hook creation the +// dashboard offers when the installation may manage hooks. + +// WebhookEvents is the list of GitHub webhook events the spoke receiver acts +// on, in the order the GitHub webhook form lists them. It is what the +// Settings card tells an owner to tick, and what CreateWebhook subscribes. +// projects_v2_item is organization-scoped and only useful with the +// github_projects work source, so it is listed separately +// (WebhookEventsOptional). +var WebhookEvents = []string{ + "check_run", + "check_suite", + "issue_comment", + "issues", + "pull_request", + "pull_request_review", + "pull_request_review_comment", + "push", + "status", + "workflow_run", +} + +// WebhookEventsOptional are events the receiver understands but that only +// matter with a specific feature enabled. +var WebhookEventsOptional = map[string]string{ + "workflow_job": "CI job counts from the job payload (needs Actions: read on the App)", + "projects_v2_item": "GitHub Projects work source (governor.work_source.type: github_projects); organization webhook only", +} + +// Installation permissions that allow the App to create hooks. +const ( + PermissionOrganizationHooks = "organization_hooks" + PermissionRepositoryHooks = "repository_hooks" +) + +// WebhookHookPermissions reports which hook-management permissions the +// installation holds at write level: org-level hooks and repo-level hooks. +func WebhookHookPermissions(info *InstallationInfo) (orgHooks, repoHooks bool) { + if info == nil { + return false, false + } + return info.Granted(PermissionOrganizationHooks, "write"), info.Granted(PermissionRepositoryHooks, "write") +} + +// ErrWebhookExists is returned when GitHub already has a hook for the same +// payload URL at that scope (GitHub answers 422 "Hook already exists"). +var ErrWebhookExists = errors.New("a webhook for this payload URL already exists") + +// WebhookCreateRequest describes the hook CreateWebhook should create. +type WebhookCreateRequest struct { + // Org is the owner login; Repo is empty for an organization hook. + Org, Repo string + // PayloadURL is the spoke's public receiver URL; Secret the HMAC secret. + PayloadURL, Secret string + Events []string +} + +// WebhookCreateResult names the hook GitHub created. +type WebhookCreateResult struct { + HookID int64 `json:"hook_id"` + Scope string `json:"scope"` // "organization" or "repository" + Target string `json:"target"` + URL string `json:"url"` // GitHub settings page for the hook +} + +// CreateWebhook creates an organization hook (req.Repo == "") or a repository +// hook through the App installation token. The token carries every granted +// permission, so the caller checks WebhookHookPermissions first and only +// offers the action when the matching write permission is held. +func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) (*WebhookCreateResult, error) { + if a == nil { + return nil, errors.New("GitHub App auth not configured") + } + if strings.TrimSpace(req.Org) == "" || strings.TrimSpace(req.PayloadURL) == "" || strings.TrimSpace(req.Secret) == "" { + return nil, errors.New("org, payload URL and secret are required") + } + token, err := a.Token(ctx) + if err != nil { + return nil, fmt.Errorf("minting installation token: %w", err) + } + client := newTokenClient(token, a.apiURL) + events := req.Events + if len(events) == 0 { + events = WebhookEvents + } + hook := &gh.Hook{ + Active: gh.Ptr(true), + Events: events, + Config: &gh.HookConfig{ + URL: gh.Ptr(req.PayloadURL), + ContentType: gh.Ptr("json"), + Secret: gh.Ptr(req.Secret), + InsecureSSL: gh.Ptr("0"), + }, + } + var ( + created *gh.Hook + resp *gh.Response + ) + res := &WebhookCreateResult{} + if req.Repo == "" { + created, resp, err = client.Organizations.CreateHook(ctx, req.Org, hook) + res.Scope, res.Target = "organization", req.Org + } else { + created, resp, err = client.Repositories.CreateHook(ctx, req.Org, req.Repo, hook) + res.Scope, res.Target = "repository", req.Org+"/"+req.Repo + } + if err != nil { + if resp != nil && resp.StatusCode == http.StatusUnprocessableEntity && strings.Contains(strings.ToLower(err.Error()), "already exists") { + return nil, ErrWebhookExists + } + return nil, err + } + res.HookID = created.GetID() + base := strings.TrimSuffix(webBaseURLForAPI(a.apiURL), "/") + if req.Repo == "" { + res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.HookID) + } else { + res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.HookID) + } + return res, nil +} + +// webBaseURLForAPI maps an API URL to the web origin a hook's settings page +// lives on: api.github.com → github.com, https://ghe.example/api/v3 → +// https://ghe.example. +func webBaseURLForAPI(apiURL string) string { + u := strings.TrimSpace(apiURL) + if u == "" || strings.Contains(u, "api.github.com") { + return "https://github.com" + } + return strings.TrimSuffix(strings.TrimSuffix(u, "/"), "/api/v3") +} From 026c6604837f66d717e97feab41a9a60a74cf386 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Thu, 12 Nov 2026 23:56:24 +0000 Subject: [PATCH 129/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.239.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.239.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 13 +++++++++++++ changelog.d/added-17987-review-perspsrc-resolver.md | 1 - ...fixed-activity-monitor-workflow-column-header.md | 1 - changelog.d/fixed-canary-kick-refusal.md | 1 - changelog.d/fixed-contributor-sweep-cancellation.md | 1 - .../fixed-spoke-installs-missing-startup-probe.md | 1 - 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-17987-review-perspsrc-resolver.md delete mode 100644 changelog.d/fixed-activity-monitor-workflow-column-header.md delete mode 100644 changelog.d/fixed-canary-kick-refusal.md delete mode 100644 changelog.d/fixed-contributor-sweep-cancellation.md delete mode 100644 changelog.d/fixed-spoke-installs-missing-startup-probe.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bf271ee566..6883be0e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-12 (v5.239.0) + +### Added + +- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). +- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). +- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. +- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". + ## 2026-11-12 (v5.238.1) ### Fixed diff --git a/changelog.d/added-17987-review-perspsrc-resolver.md b/changelog.d/added-17987-review-perspsrc-resolver.md deleted file mode 100644 index f3bbd4c8eb..0000000000 --- a/changelog.d/added-17987-review-perspsrc-resolver.md +++ /dev/null @@ -1 +0,0 @@ -- **Resolver for `review.perspectives_source`** that fetches a `kind: ReviewPerspectives` document behind the same seed-only allowlist as `definition_source`, and keeps the last-known-good set with a visible warning when a later fetch fails or a document no longer validates, instead of silently falling back to the built-in perspectives. The review relay, dispatch, head aggregation and `${REVIEW_PERSPECTIVES}` now read the resolved set via `ReviewConfig.EffectivePerspectives()`; with no `perspectives_source` configured, behaviour is unchanged. ([#17987](https://github.com/hivecommons/hive/issues/17987), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-activity-monitor-workflow-column-header.md b/changelog.d/fixed-activity-monitor-workflow-column-header.md deleted file mode 100644 index e868548c15..0000000000 --- a/changelog.d/fixed-activity-monitor-workflow-column-header.md +++ /dev/null @@ -1 +0,0 @@ -- Activity monitor: the column header now follows the workflow model the rows wear — a repository bound to a custom model gets that model's column names over its chevrons (ungrouped pages are partitioned by model with a header per run), instead of hive-default's "PR in progress / Merge queue" names (#18002). diff --git a/changelog.d/fixed-canary-kick-refusal.md b/changelog.d/fixed-canary-kick-refusal.md deleted file mode 100644 index 5c1235ffe5..0000000000 --- a/changelog.d/fixed-canary-kick-refusal.md +++ /dev/null @@ -1 +0,0 @@ -- Kicks are no longer refused wholesale as "prompt injection" because of the ioscan exfiltration canary: the preamble is now a labelled `=== HIVE HARNESS HEADER` block that states it is the operator's own configuration and not a task (Claude Sonnet 5 via the Copilot CLI refused ~65% of one hosted supervisor's kicks on the old `SECURITY CANARY:` marker), and a kick that is still refused with a canary in front is re-delivered once without it instead of burning the cadence slot (#18012). diff --git a/changelog.d/fixed-contributor-sweep-cancellation.md b/changelog.d/fixed-contributor-sweep-cancellation.md deleted file mode 100644 index be00f6d940..0000000000 --- a/changelog.d/fixed-contributor-sweep-cancellation.md +++ /dev/null @@ -1 +0,0 @@ -- Stop contributor automerge passes cleanly when a client hand-over or shutdown cancels an in-flight sweep, without filing a spurious failure report. diff --git a/changelog.d/fixed-spoke-installs-missing-startup-probe.md b/changelog.d/fixed-spoke-installs-missing-startup-probe.md deleted file mode 100644 index 3680f325a5..0000000000 --- a/changelog.d/fixed-spoke-installs-missing-startup-probe.md +++ /dev/null @@ -1 +0,0 @@ -- Spoke self-upgrade now installs the hosted template's startupProbe (10-min budget) on a hive Deployment that has none, not only raises an undersized one — pre-#9692 spokes on pull-only clusters were liveness-killed 90s into a forge-blocked boot and crash-looped (grey dot) for the length of the GHE outage; the fleet status-dot hovers now carry a colour legend and the grey dot says "no heartbeat in 5 min". From 11d1e0a0e1641628adfb01c04a2bef3e9b2d02ad Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 00:27:24 +0000 Subject: [PATCH 130/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.240.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.240.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/added-reference-repos.md | 1 - changelog.d/added-webhook-owner-setup.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-reference-repos.md delete mode 100644 changelog.d/added-webhook-owner-setup.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6883be0e10..16a3f20cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.240.0) + +### Added + +- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. +- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). + ## 2026-11-12 (v5.239.0) ### Added diff --git a/changelog.d/added-reference-repos.md b/changelog.d/added-reference-repos.md deleted file mode 100644 index 0105e3a710..0000000000 --- a/changelog.d/added-reference-repos.md +++ /dev/null @@ -1 +0,0 @@ -- **`project.reference_repos`: study-only repos outside the project** ([#18015](https://github.com/hivecommons/hive/issues/18015)). A hive whose deliverable is *about* another project (a docs assessment of `hivecommons/*` landing in `cncf/techdocs`) can list `owner/name` repos it may clone unauthenticated and GET, with the proxy stripping the hive's token and refusing every write by default; the kick tells agents such items are in scope and that findings go in the deliverable in the authorized repo, so the scanner no longer flags them out-of-scope/critical. A rare per-entry `write: [issues, prs]` opt-in allows only opening issues, opening PRs/pushing topic branches and commenting on the hive's own items, using the App's installation on the target org or `write_token_file`; boot refuses write-mode without either (exit 22). Dashboard shows a minimal `reference · <tier>` card. diff --git a/changelog.d/added-webhook-owner-setup.md b/changelog.d/added-webhook-owner-setup.md deleted file mode 100644 index 11c7755899..0000000000 --- a/changelog.d/added-webhook-owner-setup.md +++ /dev/null @@ -1 +0,0 @@ -- Owner-facing GitHub webhook setup: ⚙ Settings → Forge App gains a "GitHub webhooks" section showing the payload URL, events and live receiver status, with **Generate secret** (stored in `/data/github-webhook-secret.key`, shown once) and **Create webhook for me** when the App installation has hooks write permission; a webhooks tile on the Governor card deep-links to it; new owner guide in `docs/webhooks.md` (#18017). From bf8032ddbc72d80fd873660cef1c2fb675454ee7 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:23:20 -0500 Subject: [PATCH 131/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20show=20linked?= =?UTF-8?q?=20review=20perspectives=20read-only=20in=20Review=20Gate=20(#1?= =?UTF-8?q?8072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: show linked review perspectives read-only in Review Gate When review.perspectives_source is set, GET /api/config/review now returns the effective (linked) perspective set instead of the inline fallback, plus perspectives_linked, the source's status/provenance, any resolver warning, and the GitHub blob URL. PUT refuses perspectives/perspective_prompts edits with 409 while linked, before anything is mutated or pushed to the relay; combined_perspectives, plan_match and max_perspectives_per_pr stay editable. The Review Gate Perspectives block renders a "Linked to owner/repo/path@ref" banner with status and warning, disables the checkboxes and focus textareas, and hides "Add perspective", remove and "use built-in". Unlinked hives render exactly as before. Hive-Run: hivecommons/hive#17989 Hive-Plan: hivecommons/hive#17975 live-linked review perspectives Hive-Spec: review-perspectives-source#dashboard-read-only Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🎨 dashboard: move linked-perspectives banner styles into config-dialog.css The Review Gate's linked-source banner added three style= attributes, a raw 0.74rem font-size and a raw 8px 10px padding to the operator index.html, pushing TestStyleRatchet over its baseline. The banner, status and warning lines now use .rv-perspectives-linked* classes built from tokens (--sp-*, --fs-sm, --r, --muted, --yellow), so the counts return to baseline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🐛 github: pin clock in TestReopenedClaimPersistsAcrossLoad The fixture's merge date (2026-10-14) aged past settledClaimRetention (30 days) on 2026-11-13, so LoadClaimLedger pruned the claim on reload using the wall clock and the test began failing on every branch. Drive both the ledger and the reload from a fixed clock near the fixture. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ded-17989-dashboard-linked-perspectives.md | 1 + src/pkg/dashboard/api_config_review.go | 83 +++++++++++- src/pkg/dashboard/api_config_sections_test.go | 126 ++++++++++++++++++ .../static-src/css/config-dialog.css | 4 + .../static-src/js/config-features.js | 29 +++- src/pkg/dashboard/static/index.html | 33 ++++- src/pkg/github/prclaims_reopen_test.go | 7 +- 7 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 changelog.d/added-17989-dashboard-linked-perspectives.md diff --git a/changelog.d/added-17989-dashboard-linked-perspectives.md b/changelog.d/added-17989-dashboard-linked-perspectives.md new file mode 100644 index 0000000000..37eacda399 --- /dev/null +++ b/changelog.d/added-17989-dashboard-linked-perspectives.md @@ -0,0 +1 @@ +- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). diff --git a/src/pkg/dashboard/api_config_review.go b/src/pkg/dashboard/api_config_review.go index fb129ebb6e..3a4306da90 100644 --- a/src/pkg/dashboard/api_config_review.go +++ b/src/pkg/dashboard/api_config_review.go @@ -2,6 +2,7 @@ package dashboard import ( "net/http" + "net/url" "sort" "strings" "time" @@ -30,8 +31,32 @@ type reviewSection struct { // section can offer a pick-list for review.reviewer_agents and warn when a // fan-out would run every review on one engine (#13460). ReviewerCandidates []reviewerCandidate `json:"reviewer_candidates"` + // LinkedPerspectiveNames and LinkedPerspectivePrompts shadow the embedded + // review.perspectives / review.perspective_prompts with the EFFECTIVE set + // (ReviewConfig.EffectivePerspectives), so a hive whose perspectives are + // linked to a repo file shows what the reviewer actually runs, not the + // inline fallback the linked set overrides (hivecommons/hive#17989). + LinkedPerspectiveNames []string `json:"perspectives,omitempty"` + LinkedPerspectivePrompts map[string]string `json:"perspective_prompts,omitempty"` + // PerspectivesLinked is true while review.perspectives_source is set. The + // section is then read-only: PUT refuses perspective edits because the + // next reload would replace them with the linked set. + PerspectivesLinked bool `json:"perspectives_linked"` + // PerspectivesSourceStatus is the resolver's provenance for the linked + // set (github:owner/repo@ref, github-cache:owner/repo), or "unresolved" + // when the source is set but nothing has resolved yet, in which case the + // inline perspectives are still in effect. Empty when not linked. + PerspectivesSourceStatus string `json:"perspectives_source_status,omitempty"` + // PerspectivesSourceWarning is the last fetch or malformed-document + // failure while an older set stays in effect. + PerspectivesSourceWarning string `json:"perspectives_source_warning,omitempty"` + // PerspectivesSourceURL is the GitHub blob URL of the linked document. + PerspectivesSourceURL string `json:"perspectives_source_url,omitempty"` } +// perspectivesSourceStatusUnresolved marks a set source with no resolved set. +const perspectivesSourceStatusUnresolved = "unresolved" + // reviewerCandidate is one row of reviewSection.ReviewerCandidates. type reviewerCandidate struct { Name string `json:"name"` @@ -83,7 +108,52 @@ func reviewSectionResponse(cfg *config.Config) reviewSection { if cfg != nil { rc = cfg.Review } - return reviewSection{rc, builtin, reviewerCandidates(cfg)} + out := reviewSection{ReviewConfig: rc, BuiltinPerspectives: builtin, ReviewerCandidates: reviewerCandidates(cfg)} + out.LinkedPerspectiveNames, out.LinkedPerspectivePrompts = rc.EffectivePerspectives() + if src := rc.PerspectivesSource; src.IsSet() { + out.PerspectivesLinked = true + out.PerspectivesSourceURL = perspectivesSourceURL(*src) + out.PerspectivesSourceStatus = perspectivesSourceStatusUnresolved + if lp := rc.LinkedPerspectives; lp != nil { + if lp.Source != "" { + out.PerspectivesSourceStatus = lp.Source + } + out.PerspectivesSourceWarning = lp.Warning + } + } + return out +} + +// perspectivesSourceURL is the github.com blob URL for a linked perspectives +// document. An empty ref resolves to the repo's default branch, which +// GitHub's HEAD alias names without a second API call. Each segment is +// escaped on its own so a slashed ref (release/v5) or nested path keeps its +// separators. +func perspectivesSourceURL(src config.DefinitionSourceConfig) string { + ref := src.Ref + if ref == "" { + ref = "HEAD" + } + return "https://github.com/" + url.PathEscape(src.Owner) + "/" + url.PathEscape(src.Repo) + + "/blob/" + escapePathSegments(ref) + "/" + escapePathSegments(strings.TrimLeft(src.Path, "/")) +} + +func escapePathSegments(p string) string { + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +// perspectivesSourceLabel names a linked source as owner/repo/path[@ref] for +// operator-facing messages. +func perspectivesSourceLabel(src config.DefinitionSourceConfig) string { + label := src.Owner + "/" + src.Repo + "/" + strings.TrimLeft(src.Path, "/") + if src.Ref != "" { + label += "@" + src.Ref + } + return label } // handleReviewConfigPut updates the review-swarm merge-gate settings from the @@ -205,6 +275,17 @@ func (s *Server) handleReviewConfigPut(w http.ResponseWriter, r *http.Request) { return } + // A linked perspective set is owned by its repo file: an inline edit + // would be saved and then silently replaced by the next reload, so it is + // refused before anything is mutated. The operator-owned review knobs + // (combined_perspectives, plan_match, max_perspectives_per_pr) stay + // editable (hivecommons/hive#17989). + if src := s.deps.Config.Review.PerspectivesSource; src.IsSet() && (body.Perspectives != nil || body.PerspectivePrompts != nil) { + jsonError(w, "review perspectives are linked to "+perspectivesSourceLabel(*src)+ + "; change them in that file, or remove review.perspectives_source to edit them here", http.StatusConflict) + return + } + if body.MaxParallelReviews != nil && *body.MaxParallelReviews < 0 { jsonError(w, "max_parallel_reviews must be >= 0", http.StatusBadRequest) return diff --git a/src/pkg/dashboard/api_config_sections_test.go b/src/pkg/dashboard/api_config_sections_test.go index f73813f5fd..ea78d6c28e 100644 --- a/src/pkg/dashboard/api_config_sections_test.go +++ b/src/pkg/dashboard/api_config_sections_test.go @@ -798,3 +798,129 @@ func TestReviewConfigGet_ReviewerCandidates(t *testing.T) { } } } + +// A hive whose perspectives are linked to a repo file shows the linked set, +// its source and its provenance, never the inline fallback it overrides +// (hivecommons/hive#17989). +func TestReviewConfigGetShowsLinkedPerspectives(t *testing.T) { + s := covApiServer(t) + s.deps.Config.Review.Perspectives = []string{"correctness"} + s.deps.Config.Review.PerspectivePrompts = map[string]string{"correctness": "inline focus"} + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{ + Owner: "acme", Repo: "policy", Path: "review/perspectives.yaml", Ref: "release/v5", + } + s.deps.Config.Review.LinkedPerspectives = &config.LinkedPerspectives{ + Names: []string{"security", "api-compat"}, + Focus: map[string]string{"api-compat": "public API breakage"}, + Source: "github-cache:acme/policy", + Warning: "review perspectives document acme/policy/review/perspectives.yaml is invalid: bad", + } + + rec := doGetNoRole(s, "/api/config/review") + if rec.Code != http.StatusOK { + t.Fatalf("GET review: expected 200, got %d", rec.Code) + } + var got struct { + Perspectives []string `json:"perspectives"` + PerspectivePrompts map[string]string `json:"perspective_prompts"` + Linked bool `json:"perspectives_linked"` + Source config.DefinitionSourceConfig `json:"perspectives_source"` + Status string `json:"perspectives_source_status"` + Warning string `json:"perspectives_source_warning"` + URL string `json:"perspectives_source_url"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if !got.Linked { + t.Fatal("perspectives_linked = false with a source set") + } + if got.Source.Owner != "acme" || got.Source.Repo != "policy" || got.Source.Path != "review/perspectives.yaml" || got.Source.Ref != "release/v5" { + t.Fatalf("perspectives_source = %+v", got.Source) + } + if strings.Join(got.Perspectives, ",") != "security,api-compat" { + t.Fatalf("perspectives = %v, want the linked set", got.Perspectives) + } + if got.PerspectivePrompts["api-compat"] != "public API breakage" || got.PerspectivePrompts["correctness"] != "" { + t.Fatalf("perspective_prompts = %v, want the linked focus", got.PerspectivePrompts) + } + if got.Status != "github-cache:acme/policy" || !strings.Contains(got.Warning, "is invalid") { + t.Fatalf("status = %q warning = %q", got.Status, got.Warning) + } + if got.URL != "https://github.com/acme/policy/blob/release/v5/review/perspectives.yaml" { + t.Fatalf("perspectives_source_url = %q", got.URL) + } + + // Set but not yet resolved: still linked (and so read-only), but the + // inline set is what runs and the status says so. + s.deps.Config.Review.LinkedPerspectives = nil + got.Perspectives, got.Status, got.Warning = nil, "", "" + if err := json.Unmarshal(doGetNoRole(s, "/api/config/review").Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if !got.Linked || got.Status != perspectivesSourceStatusUnresolved || strings.Join(got.Perspectives, ",") != "correctness" { + t.Fatalf("unresolved: linked=%v status=%q perspectives=%v", got.Linked, got.Status, got.Perspectives) + } + + // Unlinked: the inline set, and no linked metadata at all. + s.deps.Config.Review.PerspectivesSource = nil + var plain map[string]any + if err := json.Unmarshal(doGetNoRole(s, "/api/config/review").Body.Bytes(), &plain); err != nil { + t.Fatalf("decode: %v", err) + } + if plain["perspectives_linked"] != false { + t.Fatalf("unlinked perspectives_linked = %v", plain["perspectives_linked"]) + } + for _, k := range []string{"perspectives_source", "perspectives_source_status", "perspectives_source_warning", "perspectives_source_url"} { + if _, ok := plain[k]; ok { + t.Fatalf("unlinked response carries %s", k) + } + } + if ps, _ := plain["perspectives"].([]any); len(ps) != 1 || ps[0] != "correctness" { + t.Fatalf("unlinked perspectives = %v", plain["perspectives"]) + } +} + +// While linked, a perspective edit is refused with 409 and nothing changes or +// is pushed to the relay; the operator-owned review knobs stay editable. +func TestReviewConfigPutRejectsPerspectiveEditWhileLinked(t *testing.T) { + s := covApiServer(t) + applied := 0 + s.deps.ReviewConfigApplied = func(config.ReviewConfig) { applied++ } + s.deps.Config.Review.Perspectives = []string{"correctness"} + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{ + Owner: "acme", Repo: "policy", Path: "review/perspectives.yaml", + } + + for _, body := range []map[string]any{ + {"perspectives": []string{"security"}}, + {"perspective_prompts": map[string]string{"security": "x"}}, + {"perspectives": []string{"security"}, "combined_perspectives": true}, + } { + rec := doPut(s, "/api/config/review", body) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "acme/policy/review/perspectives.yaml") { + t.Fatalf("%v: expected 409 naming the source, got %d: %s", body, rec.Code, rec.Body.String()) + } + } + rv := s.deps.Config.Review + if strings.Join(rv.Perspectives, ",") != "correctness" || rv.PerspectivePrompts != nil || rv.CombinedPerspectives { + t.Fatalf("refused PUT changed config: %+v", rv) + } + if applied != 0 { + t.Fatalf("ReviewConfigApplied called %d times for a refused PUT", applied) + } + + rec := doPut(s, "/api/config/review", map[string]any{"combined_perspectives": true, "max_perspectives_per_pr": 2}) + if rec.Code != http.StatusOK { + t.Fatalf("combined_perspectives while linked: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + if !s.deps.Config.Review.CombinedPerspectives || s.deps.Config.Review.MaxPerspectivesPerPR != 2 || applied != 1 { + t.Fatalf("operator-owned knobs not applied: %+v applied=%d", s.deps.Config.Review, applied) + } + + // A partially filled source is unset, so perspectives stay editable. + s.deps.Config.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "acme", Repo: "policy"} + if rec := doPut(s, "/api/config/review", map[string]any{"perspectives": []string{"security"}}); rec.Code != http.StatusOK { + t.Fatalf("partial source: expected 200, got %d: %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/static-src/css/config-dialog.css b/src/pkg/dashboard/static-src/css/config-dialog.css index 52c22bfa41..8dc476ea89 100644 --- a/src/pkg/dashboard/static-src/css/config-dialog.css +++ b/src/pkg/dashboard/static-src/css/config-dialog.css @@ -263,6 +263,10 @@ .rv-reviewer-help { margin: var(--sp-1) var(--sp-0) var(--sp-0); } .rv-reviewer-help-above { margin: var(--sp-0) var(--sp-0) var(--sp-2); } #rv-require-labels-field[hidden] { display: none; } + /* Linked review.perspectives_source banner in the Review Gate (#17989). */ + .rv-perspectives-linked { margin: var(--sp-0) var(--sp-0) var(--sp-3); padding: var(--sp-4) var(--sp-5); background: var(--surface); border: 1px solid var(--border); border-radius: var(--r); font-size: var(--fs-sm); } + .rv-perspectives-linked-status, .rv-perspectives-linked-warning { margin-top: var(--sp-2); color: var(--muted); } + .rv-perspectives-linked-warning { color: var(--yellow); } .am-trusted-authors-repos { min-height: 72px; } .am-bot-empty { color: var(--muted); margin: var(--sp-1) 0; } .am-bot-add { display: flex; gap: var(--sp-3); margin-top: var(--sp-4); align-items: center; } diff --git a/src/pkg/dashboard/static-src/js/config-features.js b/src/pkg/dashboard/static-src/js/config-features.js index 4677f288de..82a2912536 100644 --- a/src/pkg/dashboard/static-src/js/config-features.js +++ b/src/pkg/dashboard/static-src/js/config-features.js @@ -226,17 +226,31 @@ .concat(rvSelected.filter(n => !rvBuiltinNames.includes(n))) .filter((n, i, a) => a.indexOf(n) === i); const rvIsOn = (name) => rvSelected.length === 0 ? rvBuiltinNames.includes(name) || rvCustomNames.includes(name) : rvSelected.includes(name); + // Linked to a repo file (review.perspectives_source): the set shown is + // the linked one and every control is read-only, because the API + // refuses inline edits the next reload would replace (#17989). + const rvLinked = rv.perspectives_linked === true; + const rvSrc = rv.perspectives_source || {}; + const rvSrcLabel = `${rvSrc.owner || ''}/${rvSrc.repo || ''}/${rvSrc.path || ''}${rvSrc.ref ? '@' + rvSrc.ref : ''}`; + const rvSrcStatus = rv.perspectives_source_status || ''; + const rvSrcURL = /^https:\/\/github\.com\//.test(rv.perspectives_source_url || '') ? rv.perspectives_source_url : ''; + const rvLinkedBanner = !rvLinked ? '' : `<div class="rv-perspectives-linked"> + <div>Linked to ${rvSrcURL ? `<a href="${esc(rvSrcURL)}" target="_blank" rel="noopener noreferrer"><code>${esc(rvSrcLabel)}</code> ↗</a>` : `<code>${esc(rvSrcLabel)}</code>`} ${yamlHint('review.perspectives_source')} <span class="text-muted">— read-only here; change the perspectives in that file, or remove the link from hive.yaml to edit them inline.</span></div> + <div class="rv-perspectives-linked-status">${rvSrcStatus === 'unresolved' ? 'Not resolved yet (repo unreachable, not allowlisted, or invalid) — the inline perspectives below are in effect.' : `Status: <code>${esc(rvSrcStatus)}</code>`}</div> + ${rv.perspectives_source_warning ? `<div class="rv-perspectives-linked-warning">⚠ ${esc(rv.perspectives_source_warning)}</div>` : ''} + </div>`; + const rvLockAttr = rvLinked ? ' disabled' : ''; const rvPerspectiveRow = (name, defaultFocus) => { const custom = rvPrompts[name] || ''; const isCustom = !rvBuiltinNames.includes(name); return `<div class="rv-perspective" data-perspective="${esc(name)}" style="margin-top:var(--sp-4);padding:8px 10px;border:1px solid var(--border);border-radius:var(--r)"> - <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:pointer;font-weight:600;font-size:0.8rem"> - <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''} data-change-action="gh91"> + <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:${rvLinked ? 'default' : 'pointer'};font-weight:600;font-size:0.8rem"> + <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''}${rvLockAttr} data-change-action="gh91"> <code>${esc(name)}</code>${isCustom ? ' <span style="font-size:var(--fs-xs);color:var(--muted);font-weight:400">custom</span>' : ''} - ${isCustom ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} + ${isCustom && !rvLinked ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} </label> - <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}" data-change-action="gh92">${esc(custom)}</textarea> - ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} + <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}"${rvLockAttr} data-change-action="gh92">${esc(custom)}</textarea> + ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom && !rvLinked ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} </div>`; }; const rvRec = rv.recommendations || {}; @@ -486,13 +500,14 @@ </div> <div id="rv-perspectives" style="margin-top:10px"> <p style="font-size:0.72rem;color:var(--muted);margin:var(--sp-0) var(--sp-0) var(--sp-2)">Which questions the reviewer asks of every PR, and what each one looks for. Edit the text to tell this hive's reviewer what matters in <em>your</em> repos — e.g. name the conventions doc for <code>style</code>. Blank means the built-in. Unchecking a perspective stops it running and stops its verdicts being accepted.</p> + ${rvLinkedBanner} ${rvBuiltin.map(b => rvPerspectiveRow(b.name, b.focus)).join('')} ${rvCustomNames.map(n => rvPerspectiveRow(n, '')).join('')} - <div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> + ${rvLinked ? '' : `<div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> <input type="text" id="rv-new-perspective" placeholder="new perspective, e.g. api-compatibility" style="flex:1;font-size:0.76rem" pattern="[a-z][a-z0-9]*(-[a-z0-9]+)*"> <button type="button" class="hv-btn btn-secondary btn-sm btn-small" data-action="gh95">Add perspective</button> </div> - <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p> + <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p>`} </div> <div class="config-subgroup">Reviewers</div> <div class="config-field" id="rv-reviewer-agents"> diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 065d33f36e..e869b6c771 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -4728,6 +4728,10 @@ .rv-reviewer-help { margin: var(--sp-1) var(--sp-0) var(--sp-0); } .rv-reviewer-help-above { margin: var(--sp-0) var(--sp-0) var(--sp-2); } #rv-require-labels-field[hidden] { display: none; } + /* Linked review.perspectives_source banner in the Review Gate (#17989). */ + .rv-perspectives-linked { margin: var(--sp-0) var(--sp-0) var(--sp-3); padding: var(--sp-4) var(--sp-5); background: var(--surface); border: 1px solid var(--border); border-radius: var(--r); font-size: var(--fs-sm); } + .rv-perspectives-linked-status, .rv-perspectives-linked-warning { margin-top: var(--sp-2); color: var(--muted); } + .rv-perspectives-linked-warning { color: var(--yellow); } .am-trusted-authors-repos { min-height: 72px; } .am-bot-empty { color: var(--muted); margin: var(--sp-1) 0; } .am-bot-add { display: flex; gap: var(--sp-3); margin-top: var(--sp-4); align-items: center; } @@ -68580,17 +68584,31 @@ <h3 class="features-section-title">${esc(s.title)}</h3> .concat(rvSelected.filter(n => !rvBuiltinNames.includes(n))) .filter((n, i, a) => a.indexOf(n) === i); const rvIsOn = (name) => rvSelected.length === 0 ? rvBuiltinNames.includes(name) || rvCustomNames.includes(name) : rvSelected.includes(name); + // Linked to a repo file (review.perspectives_source): the set shown is + // the linked one and every control is read-only, because the API + // refuses inline edits the next reload would replace (#17989). + const rvLinked = rv.perspectives_linked === true; + const rvSrc = rv.perspectives_source || {}; + const rvSrcLabel = `${rvSrc.owner || ''}/${rvSrc.repo || ''}/${rvSrc.path || ''}${rvSrc.ref ? '@' + rvSrc.ref : ''}`; + const rvSrcStatus = rv.perspectives_source_status || ''; + const rvSrcURL = /^https:\/\/github\.com\//.test(rv.perspectives_source_url || '') ? rv.perspectives_source_url : ''; + const rvLinkedBanner = !rvLinked ? '' : `<div class="rv-perspectives-linked"> + <div>Linked to ${rvSrcURL ? `<a href="${esc(rvSrcURL)}" target="_blank" rel="noopener noreferrer"><code>${esc(rvSrcLabel)}</code> ↗</a>` : `<code>${esc(rvSrcLabel)}</code>`} ${yamlHint('review.perspectives_source')} <span class="text-muted">— read-only here; change the perspectives in that file, or remove the link from hive.yaml to edit them inline.</span></div> + <div class="rv-perspectives-linked-status">${rvSrcStatus === 'unresolved' ? 'Not resolved yet (repo unreachable, not allowlisted, or invalid) — the inline perspectives below are in effect.' : `Status: <code>${esc(rvSrcStatus)}</code>`}</div> + ${rv.perspectives_source_warning ? `<div class="rv-perspectives-linked-warning">⚠ ${esc(rv.perspectives_source_warning)}</div>` : ''} + </div>`; + const rvLockAttr = rvLinked ? ' disabled' : ''; const rvPerspectiveRow = (name, defaultFocus) => { const custom = rvPrompts[name] || ''; const isCustom = !rvBuiltinNames.includes(name); return `<div class="rv-perspective" data-perspective="${esc(name)}" style="margin-top:var(--sp-4);padding:8px 10px;border:1px solid var(--border);border-radius:var(--r)"> - <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:pointer;font-weight:600;font-size:0.8rem"> - <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''} data-change-action="gh91"> + <label style="display:flex;align-items:center;gap:var(--sp-4);cursor:${rvLinked ? 'default' : 'pointer'};font-weight:600;font-size:0.8rem"> + <input type="checkbox" class="rv-perspective-on" ${rvIsOn(name) ? 'checked' : ''}${rvLockAttr} data-change-action="gh91"> <code>${esc(name)}</code>${isCustom ? ' <span style="font-size:var(--fs-xs);color:var(--muted);font-weight:400">custom</span>' : ''} - ${isCustom ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} + ${isCustom && !rvLinked ? `<button type="button" class="hv-btn btn-secondary btn-sm btn-small btn-primary" data-action="gh93" data-arg0="${esc(name)}" data-arg-types="s">remove</button>` : ''} </label> - <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}" data-change-action="gh92">${esc(custom)}</textarea> - ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} + <textarea class="rv-perspective-focus" rows="2" style="width:100%;margin-top:var(--sp-3);font-size:0.76rem" placeholder="${esc(defaultFocus || 'what should this perspective look for?')}"${rvLockAttr} data-change-action="gh92">${esc(custom)}</textarea> + ${defaultFocus ? `<p style="font-size:var(--fs-xs);color:var(--muted);margin:3px 0 0">Built-in: ${esc(defaultFocus)}${custom && !rvLinked ? ' — <a href="#" data-action="gh94" data-prevent="1" data-arg0="' + esc(name) + '" data-arg-types="s">use built-in</a>' : ''}</p>` : ''} </div>`; }; const rvRec = rv.recommendations || {}; @@ -68840,13 +68858,14 @@ <h3 class="features-section-title">${esc(s.title)}</h3> </div> <div id="rv-perspectives" style="margin-top:10px"> <p style="font-size:0.72rem;color:var(--muted);margin:var(--sp-0) var(--sp-0) var(--sp-2)">Which questions the reviewer asks of every PR, and what each one looks for. Edit the text to tell this hive's reviewer what matters in <em>your</em> repos — e.g. name the conventions doc for <code>style</code>. Blank means the built-in. Unchecking a perspective stops it running and stops its verdicts being accepted.</p> + ${rvLinkedBanner} ${rvBuiltin.map(b => rvPerspectiveRow(b.name, b.focus)).join('')} ${rvCustomNames.map(n => rvPerspectiveRow(n, '')).join('')} - <div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> + ${rvLinked ? '' : `<div style="display:flex;gap:var(--sp-3);margin-top:var(--sp-4);align-items:center"> <input type="text" id="rv-new-perspective" placeholder="new perspective, e.g. api-compatibility" style="flex:1;font-size:0.76rem" pattern="[a-z][a-z0-9]*(-[a-z0-9]+)*"> <button type="button" class="hv-btn btn-secondary btn-sm btn-small" data-action="gh95">Add perspective</button> </div> - <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p> + <p style="font-size:var(--fs-xs);color:var(--muted);margin:var(--sp-2) var(--sp-0) var(--sp-0)">A new perspective needs focus text before it can be saved. Names: lowercase letters, digits, single dashes.</p>`} </div> <div class="config-subgroup">Reviewers</div> <div class="config-field" id="rv-reviewer-agents"> diff --git a/src/pkg/github/prclaims_reopen_test.go b/src/pkg/github/prclaims_reopen_test.go index d95255b93e..651cfc234c 100644 --- a/src/pkg/github/prclaims_reopen_test.go +++ b/src/pkg/github/prclaims_reopen_test.go @@ -233,8 +233,13 @@ func TestReconcileReopenedIssuesVoidsClaimFromTimeline(t *testing.T) { } func TestReopenedClaimPersistsAcrossLoad(t *testing.T) { + // Pin the clock near the fixture dates: with the wall clock, the merged + // claim ages past settledClaimRetention 30 days after reopenTestMerged and + // the reload prunes it, which is retention working, not a reopen bug. + now := func() time.Time { return reopenTestReopened.Add(time.Hour) } path := filepath.Join(t.TempDir(), "ledger.json") ledger := NewClaimLedger(path, testLogger()) + ledger.SetClock(now) ledger.Reconcile([]IssueClaim{reopenTestClaim()}, true) if !ledger.MarkIssueReopened("r", 7, reopenTestReopened) { t.Fatal("setup: reopen not recorded") @@ -242,7 +247,7 @@ func TestReopenedClaimPersistsAcrossLoad(t *testing.T) { if err := ledger.Save(); err != nil { t.Fatal(err) } - reloaded, err := LoadClaimLedger(path, testLogger()) + reloaded, err := loadClaimLedgerAt(path, testLogger(), now) if err != nil { t.Fatal(err) } From b163989c2ddce566b8b381f66dc8c13f35e32fd6 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:33:23 -0500 Subject: [PATCH 132/166] =?UTF-8?q?=F0=9F=93=96=20docs:=20project.referenc?= =?UTF-8?q?e=5Frepos=20in=20agent-configuration=20and=20hive.yaml.example?= =?UTF-8?q?=20(#18073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 docs: project.reference_repos in agent-configuration and hive.yaml.example The operator reference and github-write-surface already document reference_repos (merged in #18023). Add the agent-facing explanation — config shape, read-only default, the exact opt-in write surface and what it still refuses, the credential rule, kick and dashboard rendering — to agent-configuration.md, and a commented example to hive.yaml.example. Hive-Run: hivecommons/hive#18022 Hive-Plan: #18015 child 5 of 5 (docs) Hive-Spec: project.reference_repos#docs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * changelog: fragment for #18073 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-18073.md | 1 + src/docs/agent-configuration.md | 26 ++++++++++++++++++++++++++ src/hive.yaml.example | 16 ++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 changelog.d/changed-18073.md diff --git a/changelog.d/changed-18073.md b/changelog.d/changed-18073.md new file mode 100644 index 0000000000..82b08a446e --- /dev/null +++ b/changelog.d/changed-18073.md @@ -0,0 +1 @@ +- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 49a8f6bba0..fbff124ed5 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1157,6 +1157,32 @@ project: agent_issue_decision_policy: resolve # resolve (default) or escalate ``` +### Reference repos: what agents may study (`project.reference_repos`) + +Some hives produce a deliverable *about* another project — a CNCF TechDocs hive whose authorized repo is `cncf/techdocs` assesses `hivecommons/docs`, for example. Without configuration, the scanner reads such an item as out-of-scope or critical, and the proxy has no way to say "read this repo but do not work it". `project.reference_repos` names those repos ([#18015](https://github.com/hivecommons/hive/issues/18015)). A reference repo is never an authorized repo: it gets no queue, no ACMM evaluation and no work items, and it cannot also be in `project.repos`. + +Each entry is either a string (`owner/name`, read-only) or a mapping `{repo, write, write_token_file}`. Bare names are rejected because a reference repo lives outside the project org. + +```yaml +project: + repos: [techdocs] + reference_repos: + - hivecommons/docs # read-only (default) + - hivecommons/hivecommons.github.io + - repo: hivecommons/hive + write: [issues, prs] # opt-in tier + # write_token_file: /data/secrets/hive-ref-pat # only if the App is not installed on that org +``` + +| Tier | Agents may | Proxy refuses | +|---|---|---| +| `read-only` (string form, or no `write`) | `git clone`/fetch over HTTPS, GitHub REST/GraphQL GETs. Every request goes out without auth because the hive's App token is removed, so a private reference repo is not readable. | Every non-GET, with a 403 that names `reference_repos`. | +| `issues`, `prs` or `issues+prs` (`write:`) | The reads above, plus only: open an issue (`issues`), open a PR or push a topic branch (`prs`), and comment on an issue or PR the hive itself opened. | Merges, pushes to the default branch, labels, assignees, reviews, and closing other people's items. Each refusal is logged as `reference_repo` in the violations log, so retrying cannot succeed. | + +Writes on the opt-in tier use a credential valid for *that* repo, never the hive's own installation token. By default the same GitHub App must be installed on the target org: the hive discovers that installation and mints a token limited to the one repo and the opted-in permissions. Otherwise, set `write_token_file`. If a write-mode entry has neither, boot fails with exit code 22. These are direct POSTs. The `hive-open-issue`/`hive-open-pr` relays only act on `project.repos`, so a lane under `write_surface.enforce` cannot write to a reference repo at all ([GitHub write surface](github-write-surface.md#reference-repos)). + +What agents see: every kick carries a `📚 REFERENCE REPOS` block listing each repo with its tier (`[reference · read-only]`, `[reference · issues+prs]`, …). It tells agents that an authorized-repo item whose deliverables name a reference repo is **in scope**. Findings belong in the deliverable inside the authorized repo, unless the tier allows an issue or PR. On the dashboard, each reference repo is a minimal card with a `reference · <tier>` chip and no stats, ACMM level or work items. Reference repos come in a separate payload field, so no queue or ACMM consumer counts them. Field-by-field defaults are in the [operator reference](operator-reference/README.md#notable-fields). + ## Label policy: which issues agents may work There is exactly **one** label-policy surface for the hive's own agents — the **Governor Configuration → Labels** tab — and it has two polarities: diff --git a/src/hive.yaml.example b/src/hive.yaml.example index cbd09db9a6..2dcfdf7cf7 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -39,6 +39,22 @@ project: # - repo: repo-two # by: your-github-login # reason: release freeze until Friday + # Optional: repos OUTSIDE the project that agents may study but not work + # (src/docs/agent-configuration.md#reference-repos-what-agents-may-study-projectreference_repos). + # Always owner/name; never also in `repos`. No queue, ACMM or work items. + # String form = read-only: clone/fetch and API GETs, sent unauthenticated + # (the hive token is stripped, so private repos are unreadable); every + # non-GET is refused with a 403 naming reference_repos. `write` opts in to + # ONLY: open an issue, open a PR / push a topic branch, comment on the hive's + # own issue/PR — never merge, default-branch push, labels, assignees, reviews + # or closing others' items. Writes use the App installed on that org (minted + # for the one repo) or write_token_file; boot exits 22 with neither. Kicks + # list them as 📚 REFERENCE REPOS; the dashboard shows a `reference · <tier>` chip. + # reference_repos: + # - hivecommons/docs # read-only + # - repo: hivecommons/hive + # write: [issues, prs] # issues, prs, or both + # write_token_file: /data/secrets/ref-pat # optional; only with write # Optional opt-in "clanker-requested" policy, under project.issue_filter.reporter_trust. # Off by default. While on, a PR carrying the label below is treated as held. From 79d5c016c086e1cabf81588ad3d4c073e55b412a Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 10:53:07 -0500 Subject: [PATCH 133/166] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20webhooks=20?= =?UTF-8?q?card=20names=20the=20App=20hook=20permission;=20pin=20card=20co?= =?UTF-8?q?py=20in=20tests=20(#18077)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Forge App → GitHub webhooks (#18017, shipped in #18024) hid the "Create webhook for me" option without explanation when the App installation lacks organization_hooks / repository_hooks write. Render a one-line hint naming the permission to grant instead. Tests: node-executed render tests for the relay, none and env-secret verification modes (relay copy vs set-up steps, generate-secret button, create button vs permission hint), and a Settings parity test that every endpoint the card fetches is in the page, routed with that method and owner-only. Hive-Run: hivecommons/hive#18029 Hive-Plan: #18017 child 4 of 5 (dashboard webhooks card) Hive-Spec: github-webhooks-owner-setup#dashboard-card Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ged-18029-webhooks-card-permission-hint.md | 1 + .../dashboard/config_webhooks_card_js_test.go | 89 +++++++++++++++++++ .../dashboard/github_settings_parity_test.go | 23 +++++ .../static-src/js/config-webhooks.js | 4 +- src/pkg/dashboard/static/index.html | 4 +- 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 changelog.d/changed-18029-webhooks-card-permission-hint.md create mode 100644 src/pkg/dashboard/config_webhooks_card_js_test.go diff --git a/changelog.d/changed-18029-webhooks-card-permission-hint.md b/changelog.d/changed-18029-webhooks-card-permission-hint.md new file mode 100644 index 0000000000..8b89d7cf4a --- /dev/null +++ b/changelog.d/changed-18029-webhooks-card-permission-hint.md @@ -0,0 +1 @@ +- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. diff --git a/src/pkg/dashboard/config_webhooks_card_js_test.go b/src/pkg/dashboard/config_webhooks_card_js_test.go new file mode 100644 index 0000000000..64d9b5b5c2 --- /dev/null +++ b/src/pkg/dashboard/config_webhooks_card_js_test.go @@ -0,0 +1,89 @@ +package dashboard + +import ( + "os/exec" + "strings" + "testing" +) + +// Settings → Forge App → GitHub webhooks (hivecommons/hive#18029): the card +// is rendered from GET /api/config/github/webhooks alone. These tests lift the +// render functions out of index.html and execute them under node against the +// three verification modes the endpoint reports, so the copy an owner reads +// for each mode is pinned, not only pattern-matched in the source. + +func webhooksCardHarness(t *testing.T, html string) string { + t.Helper() + return strings.Join([]string{ + "const assert = require('assert');", + "let _webhookSetup = null; let _webhookSecretOnce = ''; let _webhookConfirmRegenerate = false;", + "const host = { innerHTML: '' };", + "global.document = { getElementById(id) { return id === 'webhook-setup-host' ? host : null; } };", + jsFunc(t, html, "esc"), + jsFunc(t, html, "forgeAppRow"), + jsFunc(t, html, "webhookStatusChip"), + jsFunc(t, html, "webhookAgo"), + jsFunc(t, html, "webhookCopyBtn"), + jsFunc(t, html, "renderWebhookSecretBlock"), + jsFunc(t, html, "renderWebhookCreateBlock"), + jsFunc(t, html, "renderWebhookSetup"), + "function render(d) { _webhookSetup = d; renderWebhookSetup(); return host.innerHTML; }", + `const base = { payload_url: 'https://spoke.example/api/webhook/github', payload_url_source: 'config', + path: '/api/webhook/github', content_type: 'application/json', events: ['check_run', 'pull_request', 'push'], + optional_events: {}, org: 'o', repos: ['r'], app_org_hooks: false, app_repo_hooks: false };`, + }, "\n") +} + +func runWebhooksCardScript(t *testing.T, script string) { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not on PATH — the webhooks card render was NOT executed by this run") + } + if out, err := runNodeStdin(node, script); err != nil { + t.Fatalf("node failed: %v\n%s", err, out) + } +} + +func TestWebhooksCardRendersRelayAndSetupCopy(t *testing.T) { + html := indexHTML(t) + runWebhooksCardScript(t, webhooksCardHarness(t, html)+` +// relay: a hosted spoke receives hub-relayed deliveries; nothing to set up, live status shown. +const relay = render(Object.assign({}, base, { verification: 'relay (hub-signed)', hosted: true, + secret_source: '', secret_configured: false, + health: { events_1h: 4, last_event_at: new Date(Date.now() - 120000).toISOString(), mode: 'webhook' } })); +assert.ok(relay.includes('hub relay; the GitHub App installation delivers through the hub, nothing to set up'), relay); +assert.ok(relay.includes('Direct webhook (optional)'), relay); +assert.ok(relay.includes('receiving · 4/1h'), relay); +assert.ok(!relay.includes('Set up a webhook on GitHub'), relay); +assert.ok(!relay.includes('No events yet'), relay); + +// none: the set-up steps — generate a secret, payload URL, content type, events to tick. +const none = render(Object.assign({}, base, { verification: 'none', hosted: false, + secret_source: '', secret_configured: false, health: {} })); +assert.ok(none.includes('Set up a webhook on GitHub'), none); +assert.ok(none.includes('not configured'), none); +assert.ok(none.includes('No secret yet — generate one'), none); +assert.ok(none.includes('data-action="webhookGenerateSecret"') && none.includes('>Generate secret<'), none); +assert.ok(none.includes('https://spoke.example/api/webhook/github'), none); +assert.ok(none.includes('application/json'), none); +for (const e of base.events) assert.ok(none.includes('<code class="wh-code">' + e + '</code>'), e); +assert.ok(!none.includes('hub relay'), none); +// Without the App's hook permission there is no create button, only a hint naming the permission. +assert.ok(!none.includes('Create webhook for me'), none); +assert.ok(none.includes('<code>organization_hooks</code>') && none.includes('<code>repository_hooks</code>'), none); + +// direct, env secret: no generate button, the env var is named; no events yet → test-delivery hint. +const env = render(Object.assign({}, base, { verification: 'github-signed', hosted: false, + secret_source: 'env', secret_configured: true, health: {} })); +assert.ok(env.includes('GITHUB_WEBHOOK_SECRET'), env); +assert.ok(!env.includes('webhookGenerateSecret'), env); +assert.ok(env.includes('No events yet'), env); + +// With the App's hook permission the create button replaces the hint. +const app = render(Object.assign({}, base, { verification: 'github-signed', hosted: false, + secret_source: 'file', secret_configured: true, app_org_hooks: true, create_available: true, health: {} })); +assert.ok(app.includes('Create webhook for me'), app); +assert.ok(!app.includes('<code>organization_hooks</code>'), app); +`) +} diff --git a/src/pkg/dashboard/github_settings_parity_test.go b/src/pkg/dashboard/github_settings_parity_test.go index ac31266870..e7f8b21b41 100644 --- a/src/pkg/dashboard/github_settings_parity_test.go +++ b/src/pkg/dashboard/github_settings_parity_test.go @@ -3,6 +3,7 @@ package dashboard import ( "encoding/json" "net/http" + "net/http/httptest" "path/filepath" "strings" "testing" @@ -259,3 +260,25 @@ func TestParityBatch6SettingsRenderEveryKey(t *testing.T) { t.Error("saveForgeGitHubSettings does not save through PUT /api/config/github") } } + +// Settings → Forge App → GitHub webhooks (hivecommons/hive#18029): every +// endpoint the card calls is in the page and routed with the method the card +// uses, and each is owner-only (403, never 404/405, for a non-owner). +func TestConfigGitHubWebhooksCardEndpointsParity(t *testing.T) { + html := indexHTML(t) + srv := newFullServer(t) + for _, ep := range []struct{ method, path string }{ + {http.MethodGet, "/api/config/github/webhooks"}, + {http.MethodPost, "/api/config/github/webhooks/secret"}, + {http.MethodPost, "/api/config/github/webhooks/create"}, + } { + if !strings.Contains(html, "fetch('"+ep.path+"'") { + t.Errorf("Settings UI does not call %s", ep.path) + } + rec := httptest.NewRecorder() + srv.mux.ServeHTTP(rec, httptest.NewRequest(ep.method, ep.path, nil)) + if rec.Code != http.StatusForbidden { + t.Errorf("non-owner %s %s: %d, want 403", ep.method, ep.path, rec.Code) + } + } +} diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js index 986ff42f93..257a1ba44a 100644 --- a/src/pkg/dashboard/static-src/js/config-webhooks.js +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -143,7 +143,9 @@ if (d.app_probe_error) { return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; } - if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + if (!d.app_org_hooks && !d.app_repo_hooks) { + return `<div class="wh-muted wh-note">To have the App create the hook for you, grant it <b>Organization → Webhooks</b> (<code>organization_hooks</code>) or <b>Repository → Webhooks</b> (<code>repository_hooks</code>) at <b>Read and write</b>, then reopen this tab.</div>`; + } const scopes = []; if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index e869b6c771..342530dd61 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59636,7 +59636,9 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> if (d.app_probe_error) { return `<div class="wh-muted wh-note">Could not read the App installation\u2019s permissions (${esc(d.app_probe_error)}); create the hook by hand as above.</div>`; } - if (!d.app_org_hooks && !d.app_repo_hooks) return ''; + if (!d.app_org_hooks && !d.app_repo_hooks) { + return `<div class="wh-muted wh-note">To have the App create the hook for you, grant it <b>Organization → Webhooks</b> (<code>organization_hooks</code>) or <b>Repository → Webhooks</b> (<code>repository_hooks</code>) at <b>Read and write</b>, then reopen this tab.</div>`; + } const scopes = []; if (d.app_org_hooks) scopes.push(`<option value="organization">organization ${esc(d.org)}</option>`); if (d.app_repo_hooks) (d.repos || []).forEach(r => scopes.push(`<option value="repository:${esc(r)}">repository ${esc(d.org)}/${esc(r)}</option>`)); From cdc97909d5609ef640be59aa98b43e72f5d94ab9 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:59:09 +0000 Subject: [PATCH 134/166] [scanner] fix: de-flake TestMarketplaceRefreshRoutesAndStatus and TestBackfillCoveredByLedger_HealsRecordBelow040 (#18079) Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: hivecommons-hive[bot] <hivecommons-hive@hive.kubestellar.io> --- src/pkg/github/stage_ledger_backfill_test.go | 16 ++++++++++++---- src/pkg/hub/marketplace_test.go | 7 ++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pkg/github/stage_ledger_backfill_test.go b/src/pkg/github/stage_ledger_backfill_test.go index 70d182408a..d93aa802cd 100644 --- a/src/pkg/github/stage_ledger_backfill_test.go +++ b/src/pkg/github/stage_ledger_backfill_test.go @@ -177,14 +177,22 @@ func TestBackfillCoveredByLedger_HealsRecordBelow040(t *testing.T) { c.SetIssueStageLabels(func() IssueStagePolicy { return IssueStagePolicy{Enabled: true} }) store := stagelabels.Shared() issue := c.IssueStageRef("o/r", 7) - store.Observe(issue.Key(), stagelabels.Actionable, "actionable") covered := c.IssueStageRef("o/r", 9) - store.Observe(covered.Key(), stagelabels.CoveredByPR, "open PR covers it") - t.Cleanup(func() { + unseen := c.IssueStageRef("o/r", 11) + // The store is process-wide and other tests in this package drive stage + // facts for o/r#9 / o/r#11 without forgetting them; under -shuffle a + // leaked sub-040 record for the "never seen" #11 made the backfill count + // it too (checked=2). Start from a clean slate for every key we assert on. + forget := func() { store.Forget(issue.Key()) store.Forget(covered.Key()) + store.Forget(unseen.Key()) store.Forget(c.PRStageRef("o/r", 8).Key()) - }) + } + forget() + t.Cleanup(forget) + store.Observe(issue.Key(), stagelabels.Actionable, "actionable") + store.Observe(covered.Key(), stagelabels.CoveredByPR, "open PR covers it") ledger := backfillLedger(t, IssueClaim{Repo: "r", Issue: 7, PRNumber: 8, PRRepo: "r", Closing: true}, IssueClaim{Repo: "r", Issue: 9, PRNumber: 8, PRRepo: "r", Closing: true}, diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index b364fea6dd..531ef4dc70 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -598,6 +598,11 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { } rf.mu.Lock() rf.running = false + // The single-repo refresh above already left an "on-demand:" result in + // rf.last, and rf.running stays false until the background goroutine + // starts — so waiting on those alone can pass before the full run even + // begins. Wait for the full run to replace this result instead. + previous := rf.last rf.mu.Unlock() t.Setenv(hubMarketplaceExtraReposEnv, "konflux-ci/konflux-ci") if rec = mktDo(s, admin("/api/marketplace/refresh")); rec.Code != 202 { @@ -606,7 +611,7 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() defer rf.mu.Unlock() - return !rf.running && rf.last != nil && strings.HasPrefix(rf.last.Trigger, "on-demand:") + return !rf.running && rf.last != nil && rf.last != previous && strings.HasPrefix(rf.last.Trigger, "on-demand:") }, "background refresh did not finish") rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) From 3c17d4d38321cd38a16c5f0956bf16cf0ad8d052 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 16:03:20 +0000 Subject: [PATCH 135/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.241.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.241.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 6 ++++++ changelog.d/added-17989-dashboard-linked-perspectives.md | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/added-17989-dashboard-linked-perspectives.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a3f20cf5..03461444e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.241.0) + +### Added + +- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). + ## 2026-11-13 (v5.240.0) ### Added diff --git a/changelog.d/added-17989-dashboard-linked-perspectives.md b/changelog.d/added-17989-dashboard-linked-perspectives.md deleted file mode 100644 index 37eacda399..0000000000 --- a/changelog.d/added-17989-dashboard-linked-perspectives.md +++ /dev/null @@ -1 +0,0 @@ -- **Review Gate shows linked review perspectives read-only.** When `review.perspectives_source` links the perspective set to a repo file, Settings → Features → Review Gate → Perspectives now shows the linked set (not the inline fallback), a "Linked to owner/repo/path@ref" line pointing at the file on GitHub, and the resolver's status and any fetch or malformed-document warning. The checkboxes and focus text are read-only and "Add perspective" is hidden, and `PUT /api/config/review` answers 409 for `perspectives` / `perspective_prompts` edits that the next reload would silently replace; `combined_perspectives`, `plan_match` and `max_perspectives_per_pr` stay editable. Unlinked hives see no change ([#17989](https://github.com/hivecommons/hive/issues/17989)). From 6ccd83ad734c9b3c82dc22b733f1403c6ccce726 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:13:47 -0500 Subject: [PATCH 136/166] =?UTF-8?q?=F0=9F=90=9B=20benchmarks:=20emit=20can?= =?UTF-8?q?onical=20model=20id=20and=20effort=20alongside=20the=20reported?= =?UTF-8?q?=20id=20(#18037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 benchmarks: emit canonical model id and effort alongside the reported id Different CLIs report one model under different spellings (claude-sonnet-5-5 vs claude-sonnet-5.5, openai-codex/gpt-6.1-sol vs gpt-6.1-sol, gemini-3.8-flash-low), so the leaderboard showed one model as several rows. modelfamily.Resolve folds a raw id into canonical id, vendor and reasoning effort; the benchmarks exporter publishes model.canonical / model.effort next to the raw model.name and derives model.provider from the model family instead of the CLI prefix. Fixes #18036 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🧪 modelfamily: cover canonical-id edge cases for the coverage gate Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --- .../fixed-benchmarks-canonical-model-id.md | 1 + src/docs/benchmarks.md | 2 +- src/pkg/evidence/benchmarks/convert.go | 29 ++-- src/pkg/evidence/benchmarks/convert_test.go | 17 ++ src/pkg/evidence/benchmarks/record.go | 12 +- .../benchmarks/testdata/result.schema.json | 6 +- src/pkg/modelfamily/modelfamily.go | 159 +++++++++++++++++- src/pkg/modelfamily/modelfamily_test.go | 49 ++++++ 8 files changed, 243 insertions(+), 32 deletions(-) create mode 100644 changelog.d/fixed-benchmarks-canonical-model-id.md diff --git a/changelog.d/fixed-benchmarks-canonical-model-id.md b/changelog.d/fixed-benchmarks-canonical-model-id.md new file mode 100644 index 0000000000..044a0042fc --- /dev/null +++ b/changelog.d/fixed-benchmarks-canonical-model-id.md @@ -0,0 +1 @@ +- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index ef2ef3c072..5641dae5fe 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -22,7 +22,7 @@ One record per PR per role, conforming to | `source.hive_id` | **Salted hash** of the hive id: `sha256("hivecommons-benchmarks/v1/" + hive_id)[:16]`. The hive id itself never leaves the spoke. | | `source.hive_version`, `source.cluster` | Build version and `HIVE_CLUSTER_ID`. | | `agent.name`, `agent.role` (`author` / `reviewer`), `agent.harness` | Bundle `author.agent`, `author.backend`; reviewer records come from the dominant `(backend, model)` among the swarm verdicts. | -| `model.provider`, `model.name` | Bundle `author.model`; the provider is inferred from the model name prefix. | +| `model.provider`, `model.name`, `model.canonical`, `model.effort` | Bundle `author.model`. `name` is the id exactly as the harness reported it; `canonical` folds CLI spellings to one id (`claude-sonnet-5-5`, `anthropic/claude-sonnet-5.5` → `claude-sonnet-5.5`; `openai-codex/gpt-6.1-sol` → `gpt-6.1-sol`; `auto` for a runtime auto-select) and `effort` holds a stripped reasoning-effort suffix (`gemini-3.8-flash-low` → `low`). `provider` is the vendor of the canonical family, never a CLI or gateway name. Rules: `pkg/modelfamily.Resolve`. | | `sandbox_runtime` | Stamped on new bundles from the agent's effective crux sandbox (`podman`, `kubernetes-job`, …) or `none` when agents run in-process. Old bundles fall back to the current config. | | `task.kind` | Stamped from the agent's lane (`docs`, `security-fix`, `ci-repair`, `release`, `dependency-update`, `triage`, `refactor`), else `unknown`. Reviewer records are `pr-review`. | | `task.acmm_level` | `policy.acmm_level` at review time. | diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 6f52f14ab5..f3f1fcf001 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -7,6 +7,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/evidence" + "github.com/hivecommons/hive/pkg/modelfamily" ) // Options parameterise one export. @@ -65,12 +66,6 @@ var kindHints = []struct{ hint, kind string }{ {"dep", "dependency-update"}, {"renovate", "dependency-update"}, {"triage", "triage"}, {"refactor", "refactor"}, } -var providerPrefixes = []struct{ prefix, provider string }{ - {"claude", "anthropic"}, {"gpt", "openai"}, {"o1", "openai"}, {"o3", "openai"}, {"o4", "openai"}, - {"gemini", "google"}, {"granite", "ibm"}, {"llama", "meta"}, {"mistral", "mistral"}, - {"qwen", "alibaba"}, {"deepseek", "deepseek"}, {"grok", "xai"}, {"kimi", "moonshot"}, -} - // Convert folds bundles per PR and emits an author record for each PR a hive // agent opened and a reviewer record for each PR the swarm scored. Output is // sorted by id so exports are byte-stable. @@ -133,7 +128,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { name = "agent" } rec.Agent = Agent{Name: name, Role: RoleAuthor, Harness: harnessOf(a.Backend)} - rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} + rec.Model = modelOf(a.Model) rec.SandboxRuntime = runtimeOf(opts, a, name) rec.Task.Kind = kindOf(a.TaskKind, name) rec.Cost = costOf(opts.CostLookup, authorCostRefs(latest)) @@ -143,7 +138,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec := base rec.ID = recordID(hash, latest.Repo, latest.Number, RoleReviewer) rec.Agent = Agent{Name: reviewSwarmAgent, Role: RoleReviewer, Harness: harnessOf(backend)} - rec.Model = Model{Provider: providerOf(model), Name: model} + rec.Model = modelOf(model) rec.SandboxRuntime = SandboxRuntimeNone rec.Task.Kind = kindPRReview rec.Cost = costOf(opts.CostLookup, prCostRefs(latest.Repo, latest.Number, costKindPR)) @@ -367,17 +362,15 @@ func harnessOf(backend string) string { return harnessOther } -func providerOf(model string) string { - m := strings.ToLower(strings.TrimSpace(model)) - if p, _, ok := strings.Cut(m, "/"); ok && p != "" { - return p - } - for _, pp := range providerPrefixes { - if strings.HasPrefix(m, pp.prefix) { - return pp.provider - } +// modelOf keeps the reported id and adds its canonical reading so one model +// aggregates under one row whichever CLI reported it. +func modelOf(reported string) Model { + r := modelfamily.Resolve(reported, "") + provider := r.Vendor + if provider == modelfamily.Unknown { + provider = providerUnknown } - return providerUnknown + return Model{Provider: provider, Name: strings.TrimSpace(reported), Canonical: r.Canonical, Effort: r.Effort} } // kindOf prefers the task kind stamped on the bundle and otherwise guesses diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index 6bef3b93b8..b9c91c8e27 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -331,3 +331,20 @@ func TestConvertEdgeCases(t *testing.T) { t.Errorf("qualityOf with nothing to report = %+v, want nil", got) } } + +func TestConvertCanonicalisesReportedModel(t *testing.T) { + bundles := fixtureBundles() + bundles[1].Author.Model = "openai-codex/gpt-5.6-sol:high" + for i := range bundles[1].Verdicts { + bundles[1].Verdicts[i].Model = "claude-sonnet-5-5" + } + recs := Convert(bundles, testOptions()) + a := byRole(recs, "hivecommons/hive", RoleAuthor) + if a == nil || a.Model.Name != "openai-codex/gpt-5.6-sol:high" || a.Model.Canonical != "gpt-5.6-sol" || a.Model.Provider != "openai" || a.Model.Effort != "high" { + t.Errorf("author model not canonicalised: %+v", a) + } + r := byRole(recs, "hivecommons/hive", RoleReviewer) + if r == nil || r.Model.Name != "claude-sonnet-5-5" || r.Model.Canonical != "claude-sonnet-5.5" || r.Model.Provider != "anthropic" || r.Model.Effort != "" { + t.Errorf("reviewer model not canonicalised: %+v", r) + } +} diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go index 875297febd..9e985f1f3a 100644 --- a/src/pkg/evidence/benchmarks/record.go +++ b/src/pkg/evidence/benchmarks/record.go @@ -68,10 +68,16 @@ type Agent struct { Harness string `json:"harness"` } -// Model is the model the agent ran on. +// Model is the model the agent ran on. Name is the id exactly as the +// harness reported it; Canonical folds the spellings different CLIs use for +// one model ("claude-sonnet-5-5", "anthropic/claude-sonnet-5.5" → +// "claude-sonnet-5.5", modelfamily.Resolve) so the leaderboard shows one +// row per model, and Provider is the vendor derived the same way. type Model struct { - Provider string `json:"provider"` - Name string `json:"name"` + Provider string `json:"provider"` + Name string `json:"name"` + Canonical string `json:"canonical,omitempty"` + Effort string `json:"effort,omitempty"` } // Task describes the unit of work. diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json index 87f9edd6bb..a946e89d0d 100644 --- a/src/pkg/evidence/benchmarks/testdata/result.schema.json +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -142,12 +142,16 @@ "type": "string", "description": "Exact model id as reported by the harness (evidence author.model)." }, + "canonical": { + "type": "string", + "description": "Canonical model id the reported name folds to (pkg/modelfamily.Resolve): lowercase, CLI/gateway prefix and reasoning-effort suffix stripped, version separators as dots, e.g. claude-sonnet-5.5; 'auto' for a runtime auto-select." + }, "version": { "type": "string" }, "effort": { "type": "string", - "description": "Reasoning effort / thinking budget label if the harness exposes one." + "description": "Reasoning effort / thinking budget label if the harness exposes one (low, medium, high, xhigh, max); split off the model id by the exporter." } } }, diff --git a/src/pkg/modelfamily/modelfamily.go b/src/pkg/modelfamily/modelfamily.go index 7cf5b683a0..4a27eb75f0 100644 --- a/src/pkg/modelfamily/modelfamily.go +++ b/src/pkg/modelfamily/modelfamily.go @@ -43,29 +43,170 @@ func Normalize(value string) string { // ("openai-codex/gpt-6.1-sol" → "gpt-6.1-sol", "anthropic/claude-opus-5" → // "claude-opus-5"), a ":variant" suffix ("gpt-5.6-sol:high" → "gpt-5.6-sol") // and a dash between two digits ("claude-opus-5-5" → "claude-opus-5.5"). -// Unknown stays Unknown. +// Unknown stays Unknown. Reasoning-effort suffixes are kept; Resolve strips +// them. func Canonical(model string) string { m := Normalize(model) if m == Unknown { return Unknown } + m = canonicalID(m) + if m == "" { + return Unknown + } + return m +} + +// Auto is the canonical id and vendor of a runtime's auto-selected model: +// the harness picked, the hive was not told which. +const Auto = "auto" + +// Resolved is the canonical reading of a raw model id. +type Resolved struct { + // Canonical is the id spellings fold to ("claude-sonnet-5.5"); Auto for + // a runtime auto-select; Unknown when nothing was reported. + Canonical string + // Vendor is the model's vendor family ("anthropic", "openai", ...), + // inferred from the model name when the reported prefix is a CLI or + // gateway ("openai-codex/", "aws/"); Auto for Auto, Unknown otherwise. + Vendor string + // Effort is the reasoning-effort label stripped from the id + // ("gemini-3.8-flash-low" → "low", "gpt-5.6-sol:high" → "high"); empty + // when none was reported. + Effort string +} + +// effortLabels are the reasoning-effort suffixes harnesses glue onto ids. +var effortLabels = map[string]bool{"low": true, "medium": true, "high": true, "xhigh": true, "max": true} + +// gateways are prefixes that name the CLI or routing layer, not the vendor, +// so they never become Resolved.Vendor. +var gateways = map[string]bool{ + "openai-codex": true, "github-copilot": true, "copilot": true, "openrouter": true, "aws": true, "bedrock": true, + "vertex": true, "azure": true, "rits": true, "pi-claude-code-provider": true, "kilocode": true, "lab-worker": true, + "stealth": true, +} + +// vendorFamilies maps a canonical id's family (text before the first dash) +// to its vendor. +var vendorFamilies = map[string]string{ + "claude": "anthropic", "gpt": "openai", "o1": "openai", "o3": "openai", "o4": "openai", "codex": "openai", + "gemini": "google", "gemma": "google", "granite": "ibm", "llama": "meta", "mistral": "mistral", "qwen": "alibaba", + "deepseek": "deepseek", "grok": "xai", "kimi": "moonshot", "glm": "z-ai", "minimax": "minimax", "mimo": "xiaomi", + "muse": "meta", +} + +// canonicalOverrides are ids that do not follow the generic rules. +var canonicalOverrides = map[string]string{ + "sonnet": "claude-sonnet", "opus": "claude-opus", "haiku": "claude-haiku", "default": Auto, +} + +// Resolve reads a raw model id into its canonical id, vendor and effort. +// reportedProvider is the vendor the harness claimed, if any; it is used +// only when the model name itself does not identify a vendor and it is not +// a CLI or gateway name. +func Resolve(model, reportedProvider string) Resolved { + raw := strings.TrimSpace(strings.ToLower(model)) + raw = strings.Trim(raw, "`'\"“”‘’") + if raw == Auto || raw == "default" { + return Resolved{Canonical: Auto, Vendor: Auto} + } + m := Normalize(raw) + if m == Unknown { + return Resolved{Canonical: Unknown, Vendor: Unknown} + } + prefix := "" + if i := strings.Index(m, "/"); i >= 0 { + prefix = m[:i] + } + var effort string + if i := strings.Index(m, ":"); i >= 0 { + if effortLabels[m[i+1:]] { + effort = m[i+1:] + } + } + id := canonicalID(m) + if parts := strings.Split(id, "-"); len(parts) > 1 && effortLabels[parts[len(parts)-1]] { + if effort == "" { + effort = parts[len(parts)-1] + } + id = strings.Join(parts[:len(parts)-1], "-") + } + if o, ok := canonicalOverrides[id]; ok { + id = o + } + if id == "" { + return Resolved{Canonical: Unknown, Vendor: Unknown, Effort: effort} + } + if id == Auto { + return Resolved{Canonical: Auto, Vendor: Auto, Effort: effort} + } + vendor := Unknown + family := id + if i := strings.Index(id, "-"); i >= 0 { + family = id[:i] + } + if v, ok := vendorFamilies[family]; ok { + vendor = v + } else if prefix != "" && !gateways[prefix] { + vendor = prefix + } else if p := strings.TrimSpace(strings.ToLower(reportedProvider)); p != "" && p != Unknown && !gateways[p] { + vendor = p + } + return Resolved{Canonical: id, Vendor: vendor, Effort: effort} +} + +// canonicalID applies the spelling folds to an already-normalized id: drop +// path prefixes and ":variant", spaces and underscores to dashes, join +// consecutive all-digit segments with dots ("sonnet-5-5" → "sonnet-5.5" but +// "ornith-1.5-35b" stays), drop an 8-digit date stamp ("-20250929"). +func canonicalID(m string) string { if i := strings.LastIndex(m, "/"); i >= 0 { m = m[i+1:] } if i := strings.Index(m, ":"); i >= 0 { m = m[:i] } - b := []byte(m) - for i := 1; i+1 < len(b); i++ { - if b[i] == '-' && isDigit(b[i-1]) && isDigit(b[i+1]) { - b[i] = '.' + m = strings.NewReplacer(" ", "-", "_", "-").Replace(m) + for strings.Contains(m, "--") { + m = strings.ReplaceAll(m, "--", "-") + } + parts := strings.Split(strings.Trim(m, "-"), "-") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p == "" { + continue + } + if len(p) == 8 && allDigits(p) && len(out) > 0 { + continue } + if n := len(out); n > 0 && allDigits(p) && allDigits(lastSegment(out[n-1])) { + out[n-1] += "." + p + continue + } + out = append(out, p) } - m = strings.TrimSpace(string(b)) - if m == "" { - return Unknown + return strings.Join(out, "-") +} + +// lastSegment is the text after the final dot, so "5.5" reads as digits. +func lastSegment(s string) string { + if i := strings.LastIndex(s, "."); i >= 0 { + return s[i+1:] } - return m + return s +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if !isDigit(s[i]) { + return false + } + } + return true } func isDigit(c byte) bool { return c >= '0' && c <= '9' } diff --git a/src/pkg/modelfamily/modelfamily_test.go b/src/pkg/modelfamily/modelfamily_test.go index 39061e481c..a052b41fa4 100644 --- a/src/pkg/modelfamily/modelfamily_test.go +++ b/src/pkg/modelfamily/modelfamily_test.go @@ -38,6 +38,8 @@ func TestCanonicalFoldsSpellings(t *testing.T) { "lemonade/qwen-3": "qwen-3", "": Unknown, "auto": Unknown, + "-": Unknown, + "gemini-3.8-flash-low": "gemini-3.8-flash-low", } for in, want := range cases { if got := Canonical(in); got != want { @@ -57,3 +59,50 @@ func TestSameFamily(t *testing.T) { t.Fatalf("unknown author: got same=%v known=%v", same, known) } } + +func TestResolveFoldsRuntimeSpellings(t *testing.T) { + cases := []struct { + in, provider string + want Resolved + }{ + {"claude-sonnet-5-5", "anthropic", Resolved{Canonical: "claude-sonnet-5.5", Vendor: "anthropic"}}, + {"claude-sonnet-5.5", "anthropic", Resolved{Canonical: "claude-sonnet-5.5", Vendor: "anthropic"}}, + {"openai-codex/gpt-6.1-sol", "openai-codex", Resolved{Canonical: "gpt-6.1-sol", Vendor: "openai"}}, + {"gpt-6.1-sol", "openai", Resolved{Canonical: "gpt-6.1-sol", Vendor: "openai"}}, + {"auto", "unknown", Resolved{Canonical: Auto, Vendor: Auto}}, + {"default", "", Resolved{Canonical: Auto, Vendor: Auto}}, + {"claude-opus-4-6", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + {"claude-opus-4.6", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + {"gemini-3.8-flash-low", "google", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google", Effort: "low"}}, + {"anthropic/claude-sonnet-5:max", "anthropic", Resolved{Canonical: "claude-sonnet-5", Vendor: "anthropic", Effort: "max"}}, + {"openai-codex/gpt-5.6-sol:high", "openai-codex", Resolved{Canonical: "gpt-5.6-sol", Vendor: "openai", Effort: "high"}}, + {"aws/claude-haiku-4-5", "aws", Resolved{Canonical: "claude-haiku-4.5", Vendor: "anthropic"}}, + {"github-copilot/gemini-3.8-flash", "github-copilot", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google"}}, + {"rits/zai-org/glm-5-3", "rits", Resolved{Canonical: "glm-5.3", Vendor: "z-ai"}}, + {"lemonade/ornith-1.5-35b-a3b-gguf-q6_k", "lemonade", Resolved{Canonical: "ornith-1.5-35b-a3b-gguf-q6-k", Vendor: "lemonade"}}, + {"pi-claude-code-provider/sonnet", "pi-claude-code-provider", Resolved{Canonical: "claude-sonnet", Vendor: "anthropic"}}, + {"GPT-5.6 Terra", "openai", Resolved{Canonical: "gpt-5.6-terra", Vendor: "openai"}}, + {"claude-sonnet-4-5-20250929", "", Resolved{Canonical: "claude-sonnet-4.5", Vendor: "anthropic"}}, + {"kimi-k3", "moonshot", Resolved{Canonical: "kimi-k3", Vendor: "moonshot"}}, + {"unknown", "unknown", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"", "", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"ornith-1.5", " Lemonade ", Resolved{Canonical: "ornith-1.5", Vendor: "lemonade"}}, + {"ornith-1.5", "aws", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"ornith-1.5", "unknown", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"ornith-1.5", "", Resolved{Canonical: "ornith-1.5", Vendor: Unknown}}, + {"-", "", Resolved{Canonical: Unknown, Vendor: Unknown}}, + {"-:high", "", Resolved{Canonical: Unknown, Vendor: Unknown, Effort: "high"}}, + {"default:high", "", Resolved{Canonical: Auto, Vendor: Auto, Effort: "high"}}, + {"claude--opus-high", "", Resolved{Canonical: "claude-opus", Vendor: "anthropic", Effort: "high"}}, + {"opus", "", Resolved{Canonical: "claude-opus", Vendor: "anthropic"}}, + {"haiku:low", "", Resolved{Canonical: "claude-haiku", Vendor: "anthropic", Effort: "low"}}, + {"gpt-5.6-sol:fast", "openai", Resolved{Canonical: "gpt-5.6-sol", Vendor: "openai"}}, + {"gemini-3.8-flash-low:high", "google", Resolved{Canonical: "gemini-3.8-flash", Vendor: "google", Effort: "high"}}, + {"`'Claude-Opus-4-6'`", "", Resolved{Canonical: "claude-opus-4.6", Vendor: "anthropic"}}, + } + for _, c := range cases { + if got := Resolve(c.in, c.provider); got != c.want { + t.Errorf("Resolve(%q,%q)=%+v want %+v", c.in, c.provider, got, c.want) + } + } +} From 527bf15b477f8e16a42ee63b175e9e4e1db64e33 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:13:52 -0500 Subject: [PATCH 137/166] =?UTF-8?q?=E2=9C=A8=20activity=20monitor:=20stage?= =?UTF-8?q?-meaning=20icons=20in=20inactive=20workflow=20chevrons,=20optio?= =?UTF-8?q?nal=20column=20icon=20override=20(#18038)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ activity monitor: stage-meaning icons in inactive workflow chevrons, optional column icon override Every column across hive-default, the curated marketplace bundles, the ingest skeleton and the editor palette resolves to a monochrome inline-SVG glyph by its normalised name (unknown → a neutral dot). Done / todo model chevrons draw the glyph alone at 60%; the active one leads its word with it. spec.columns[].icon overrides the glyph with an allowlisted id or a single emoji, validated in workflowmodel.Validate and in the editor, so no markup reaches a chevron. The editor's column inspector previews the icon and offers a pick-list plus an emoji box. Fixes #18033 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 activity monitor: fix CI for chevron stage icons - lift stageTrackerWorkflowColumnIconHtml into the workflow-columns node fixture so stageTrackerWorkflowStripItems resolves it - ValidateColumnIcon trims only blanks, so a trailing newline is rejected - TestValidateRejectsBadColumnIcon uses the techdocs fixture (hive-default has no columns) - chevron / editor emoji glyphs size with --fs tokens instead of raw px so the style ratchet stays at its baseline - drop the '#' from the tracker.css comment that sits inside the repo-chip tokens-only guard window Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-chevron-stage-icons.md | 1 + src/pkg/dashboard/api_workflows_test.go | 24 ++ .../stage_tracker_stage_icons_ui_test.go | 123 ++++++++++ src/pkg/dashboard/stage_tracker_workflows.go | 4 +- .../stage_tracker_workflows_ui_test.go | 2 +- src/pkg/dashboard/static-src/css/tracker.css | 8 + .../static-src/css/workflows-editor.css | 6 + .../static-src/js/config-workflows-editor.js | 32 +++ .../static-src/js/tracker-stage-icons.js | 160 +++++++++++++ .../static-src/js/tracker-workflows.js | 16 +- src/pkg/dashboard/static-src/manifest.txt | 1 + src/pkg/dashboard/static/index.html | 222 +++++++++++++++++- src/pkg/workflowmodel/icons.go | 97 ++++++++ src/pkg/workflowmodel/icons_test.go | 60 +++++ src/pkg/workflowmodel/lifecycle.go | 10 +- src/pkg/workflowmodel/model.go | 5 + 16 files changed, 762 insertions(+), 9 deletions(-) create mode 100644 changelog.d/added-chevron-stage-icons.md create mode 100644 src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go create mode 100644 src/pkg/dashboard/static-src/js/tracker-stage-icons.js create mode 100644 src/pkg/workflowmodel/icons.go create mode 100644 src/pkg/workflowmodel/icons_test.go diff --git a/changelog.d/added-chevron-stage-icons.md b/changelog.d/added-chevron-stage-icons.md new file mode 100644 index 0000000000..341298d6e9 --- /dev/null +++ b/changelog.d/added-chevron-stage-icons.md @@ -0,0 +1 @@ +- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). diff --git a/src/pkg/dashboard/api_workflows_test.go b/src/pkg/dashboard/api_workflows_test.go index 7a7a7d9c59..1b106149eb 100644 --- a/src/pkg/dashboard/api_workflows_test.go +++ b/src/pkg/dashboard/api_workflows_test.go @@ -378,3 +378,27 @@ func TestWorkflowEnabledPut(t *testing.T) { t.Fatalf("no config: %d", rec.Code) } } + +// A column's `icon` rides the tracker block as the SPA reads it (trimmed), +// and a model whose column names a non-allowlisted icon is refused at the +// API, so a chevron can only ever draw an allowlisted glyph +// (hivecommons/hive#18033). +func TestStageTrackerWorkflowsColumnIcon(t *testing.T) { + m := techdocsModel(t) + m.Spec.Columns[0].Icon = " bolt " + m.Spec.Columns[1].Icon = "📝" + p, _, err := workflowmodel.Compile(m) + if err != nil { + t.Fatal(err) + } + blk := stageTrackerWorkflowBlock(p) + if blk.Columns[0].Icon != "bolt" || blk.Columns[1].Icon != "📝" || blk.Columns[2].Icon != "" { + t.Fatalf("icons not carried: %+v", blk.Columns) + } + s := workflowTestServer(t, true) + bad := techdocsModel(t) + bad.Spec.Columns[0].Icon = "<svg onload=alert(1)>" + if rec := wfReq(s, http.MethodPost, "/api/workflows", bad, true); rec.Code == http.StatusCreated { + t.Fatalf("model with markup icon accepted: %s", rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go b/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go new file mode 100644 index 0000000000..be262eb2a6 --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_stage_icons_ui_test.go @@ -0,0 +1,123 @@ +package dashboard + +import ( + "os" + "os/exec" + "regexp" + "sort" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// The SPA's icon set and the server's allowlist are one list +// (hivecommons/hive#18033): an id the server accepts must draw, and the +// SPA must not draw an id the server would refuse. +func TestStageIconSetMatchesAllowlist(t *testing.T) { + html := indexHTML(t) + block := jsConstBlock(t, html, "STAGE_ICON_SET") + keyRe := regexp.MustCompile(`(?m)^\s*'?([a-z][a-z-]*)'?: '`) + var js []string + for _, m := range keyRe.FindAllStringSubmatch(block, -1) { + js = append(js, m[1]) + } + want := append([]string{}, workflowmodel.ColumnIconIDs...) + sort.Strings(js) + sort.Strings(want) + if strings.Join(js, ",") != strings.Join(want, ",") { + t.Errorf("STAGE_ICON_SET ids %v != workflowmodel.ColumnIconIDs %v", js, want) + } + if !strings.Contains(html, `const STAGE_ICON_DEFAULT = '`+workflowmodel.ColumnIconDefault+`';`) { + t.Errorf("STAGE_ICON_DEFAULT is not %q", workflowmodel.ColumnIconDefault) + } + for _, want := range []string{ + `js/tracker-stage-icons.js`, + `function stageIconColumnHtml(col, cls, label)`, + `function wfEdColumnIconField(c, base, dis)`, + `function wfEdSetColumnIcon(path, value)`, + `.ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; }`, + `.ham-emoji-ico {`, + } { + if !strings.Contains(html, want) && !strings.Contains(manifestText(t), want) { + t.Errorf("index.html missing stage-icon wiring %q", want) + } + } +} + +func manifestText(t *testing.T) string { + t.Helper() + b, err := os.ReadFile("static-src/manifest.txt") + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// Under node: names normalise and resolve (unknown → the neutral dot), an +// override must be an allowlisted id or one emoji (markup and words are +// refused and never rendered), and the model strip draws the icon alone +// in done / todo chevrons and ahead of the word in the active one. +func TestStageIconsNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +const STAGE_TRACKER_TERMINAL = 110; +const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; +function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } +` + jsConstBlock(t, html, "STAGE_ICON_SET") + "\n" + jsConstBlock(t, html, "STAGE_ICON_BY_NAME") + "\nconst STAGE_ICON_DEFAULT = 'dot';\n" + + jsFunc(t, html, "stageIconNormalizeName") + jsFunc(t, html, "stageIconIsId") + jsFunc(t, html, "stageIconIsEmoji") + + jsFunc(t, html, "stageIconIsValid") + jsFunc(t, html, "stageIconIdForName") + jsFunc(t, html, "stageIconFor") + + jsFunc(t, html, "stageIconHtml") + jsFunc(t, html, "stageIconColumnHtml") + + jsFunc(t, html, "stageTrackerWorkflowColumnIconHtml") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + ` +// normalisation and the canonical map +assert.strictEqual(stageIconNormalizeName(' Reviewer Check '), 'reviewer check'); +assert.strictEqual(stageIconNormalizeName('CI / Review'), 'ci review'); +assert.strictEqual(stageIconNormalizeName('Deployed · staging'), 'deployed staging'); +const expect = { 'Open': 'inbox', 'Actionable': 'bolt', 'Claimed': 'hand', 'PR in progress': 'pull-request', 'CI / Review': 'flask', 'Review': 'eye', 'Merge queue': 'merge', 'Merged': 'check', + 'Requested': 'inbox', 'Accepted': 'checkbox', 'Drafting': 'pencil', 'Reviewer check': 'eye', 'Stakeholder review': 'people', 'Approval': 'thumbs-up', 'Done': 'check', + 'Intake': 'inbox', 'Triage': 'tag', 'Analysis': 'search', 'Writing': 'pencil', 'Publish': 'rocket', 'No such stage': 'dot' }; +for (const k of Object.keys(expect)) assert.strictEqual(stageIconIdForName(k), expect[k], k); +for (const id of Object.values(STAGE_ICON_BY_NAME)) assert.ok(stageIconIsId(id), 'map names unknown id ' + id); +// the allowlist +for (const ok of ['', 'bolt', ' merge ', '✅', '🚀', '👍🏽', '👩‍💻', '🇺🇸']) assert.ok(stageIconIsValid(ok), 'valid ' + ok); +for (const bad of ['Bolt', 'sparkle', '<svg>', '<img src=x onerror=alert(1)>', 'a', '✅✅✅', 'ok ✅', 'javascript:']) assert.ok(!stageIconIsValid(bad), 'invalid ' + bad); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: '<b>x</b>' }), 'pencil', 'invalid override falls back to the label'); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: 'rocket' }), 'rocket'); +assert.strictEqual(stageIconFor({ label: 'Drafting', icon: '🚀' }), '🚀'); +assert.strictEqual(stageIconHtml('<svg onload=alert(1)>', 'c'), '', 'markup never renders'); +assert.strictEqual(stageIconHtml('sparkle', 'c'), ''); +const svg = stageIconHtml('bolt', 'ham-ico ham-stage-ico'); +assert.ok(svg.startsWith('<svg class="ham-ico ham-stage-ico"') && svg.includes('stroke="currentColor"') && svg.includes('aria-hidden="true"')); +assert.ok(stageIconHtml('bolt', 'c', 'Actionable').includes('role="img" aria-label="Actionable"')); +const emo = stageIconHtml('🚀', 'ham-ico'); +assert.ok(emo.includes('ham-emoji-ico') && emo.includes('🚀') && !emo.includes('<svg')); +// the strip: icon alone when inactive, icon + word when active +const wf = { name: 't', version: '1', columns: [ + { id: 'a', label: 'Requested', states: [], numbers: [10] }, + { id: 'b', label: 'Drafting', states: [], numbers: [20], icon: '🚀' }, + { id: 'c', label: 'Review', states: [], numbers: [30], icon: '<script>' }, + { id: 'd', label: 'Done', terminal: true, states: [], numbers: [] }, +]}; +const lis = stageTrackerWorkflowStripItems({ stage: 20 }, wf, []).split('</li>').filter(Boolean); +assert.strictEqual(lis.length, 4); +assert.ok(lis[0].includes(' done') && lis[0].includes('<svg') && lis[0].includes('ham-stage-ico') && !lis[0].includes('ham-seg-t'), 'done chevron = icon alone: ' + lis[0]); +assert.ok(lis[1].includes(' active') && lis[1].includes('ham-emoji-ico') && lis[1].includes('<span class="ham-seg-t">Drafting</span>'), 'active = icon + word: ' + lis[1]); +assert.ok(lis[2].includes(' todo') && lis[2].includes(STAGE_ICON_SET.eye) && !lis[2].includes('<script>'), 'invalid override falls back and never renders: ' + lis[2]); +assert.ok(lis[3].includes(' todo') && lis[3].includes('cap') && lis[3].includes(STAGE_ICON_SET.check)); +assert.ok(lis[0].includes('title="Requested"'), 'label stays the hover'); +// header keeps the label and gains the icon +const head = stageTrackerWorkflowHeaderHtml(wf); +assert.ok(head.includes('ham-col-ico') && head.includes('Requested') && head.includes('Done')); +console.log('ok'); +` + cmd := exec.Command(node, "-e", script) + out, err := cmd.CombinedOutput() + if err != nil || !strings.Contains(string(out), "ok") { + t.Fatalf("node: %v\n%s", err, out) + } +} diff --git a/src/pkg/dashboard/stage_tracker_workflows.go b/src/pkg/dashboard/stage_tracker_workflows.go index 639739a670..9b3b3fe83f 100644 --- a/src/pkg/dashboard/stage_tracker_workflows.go +++ b/src/pkg/dashboard/stage_tracker_workflows.go @@ -3,6 +3,7 @@ package dashboard import ( "sort" "strconv" + "strings" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/workflowmodel" @@ -45,6 +46,7 @@ type StageTrackerWorkflowColumn struct { ID string `json:"id"` Label string `json:"label"` Hint string `json:"hint,omitempty"` + Icon string `json:"icon,omitempty"` Terminal bool `json:"terminal,omitempty"` States []string `json:"states"` Numbers []int `json:"numbers"` @@ -57,7 +59,7 @@ type workflowProfileLookup func(name, version string) (*workflowmodel.Profile, e func stageTrackerWorkflowBlock(p *workflowmodel.Profile) StageTrackerWorkflow { out := StageTrackerWorkflow{Name: p.Name, Version: p.Version, Hash: p.Hash} for _, c := range p.Columns { - col := StageTrackerWorkflowColumn{ID: c.ID, Label: c.Label, Hint: c.Hint, Terminal: c.Terminal, States: append([]string{}, c.States...), Numbers: []int{}} + col := StageTrackerWorkflowColumn{ID: c.ID, Label: c.Label, Hint: c.Hint, Icon: strings.TrimSpace(c.Icon), Terminal: c.Terminal, States: append([]string{}, c.States...), Numbers: []int{}} for _, id := range c.States { if st := p.States[id]; st != nil && st.Lifecycle.Number > 0 { col.Numbers = append(col.Numbers, st.Lifecycle.Number) diff --git a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go index 3b9437a5f1..401ec67714 100644 --- a/src/pkg/dashboard/stage_tracker_workflows_ui_test.go +++ b/src/pkg/dashboard/stage_tracker_workflows_ui_test.go @@ -59,7 +59,7 @@ const STAGE_TRACKER_TERMINAL = 110; function esc(s) { return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); } ` + jsFunc(t, html, "stageTrackerWorkflowOf") + jsFunc(t, html, "stageTrackerWorkflowColumnIndex") + jsFunc(t, html, "stageTrackerWorkflowBadgeHtml") + jsFunc(t, html, "stageTrackerWorkflowHeaderHtml") + - jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + + jsFunc(t, html, "stageTrackerWorkflowColumnIconHtml") + jsFunc(t, html, "stageTrackerWorkflowStripItems") + jsFunc(t, html, "stageTrackerWorkflowKey") + jsFunc(t, html, "stageTrackerColumnsHeaderHtml") + jsFunc(t, html, "stageTrackerWorkflowRuns") + jsFunc(t, html, "stageTrackerWorkflowRunHeaderRows") + jsFunc(t, html, "stageTrackerGroupColumnsHtml") + ` const STAGE_TRACKER_WORKFLOW_UNCOVERED_WORD = 'Not covered'; diff --git a/src/pkg/dashboard/static-src/css/tracker.css b/src/pkg/dashboard/static-src/css/tracker.css index 53a8eadfc8..e9411e4da6 100644 --- a/src/pkg/dashboard/static-src/css/tracker.css +++ b/src/pkg/dashboard/static-src/css/tracker.css @@ -272,6 +272,14 @@ .ham-wf-badge { flex: 0 0 auto; font-family: var(--font-mono); font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; color: var(--indigo); background: color-mix(in srgb, var(--indigo) 12%, transparent); border: 1px solid color-mix(in srgb, var(--indigo) 40%, transparent); } .ham-wf-cols { position: static; } .ham-wf-cols .ham-wf-col { color: var(--indigo); } + /* Model chevrons wear the column's stage icon (hivecommons/hive issue 18033): + alone and dimmed in a done / todo step, leading the word in the lit + one. An emoji override is sized to the SVG glyph so the row keeps + its height. */ + .ham-seg.ham-wf-seg .ham-stage-ico { flex: 0 0 auto; } + .ham-seg.ham-wf-seg.active .ham-stage-ico { margin-right: var(--sp-1); } + .ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; } + .ham-emoji-ico { display: inline-block; width: 12px; height: 12px; font-size: var(--fs-xs); line-height: 12px; text-align: center; font-style: normal; } .ham-seg.ham-wf-uncovered { background: color-mix(in srgb, var(--amber) 18%, transparent); color: var(--amber); outline: 1px dashed var(--amber); } .ham-group-dist { flex: 0 0 auto; margin-left: auto; margin-right: calc(4em + var(--sp-3)); display: inline-flex; align-items: flex-end; gap: var(--sp-1); height: 0.9em; } .ham-gd { display: inline-block; width: 0.45em; height: 15%; border-radius: var(--sp-1); background: color-mix(in srgb, var(--text-faint) 35%, transparent); } diff --git a/src/pkg/dashboard/static-src/css/workflows-editor.css b/src/pkg/dashboard/static-src/css/workflows-editor.css index 7d8276cd3d..3189f213fc 100644 --- a/src/pkg/dashboard/static-src/css/workflows-editor.css +++ b/src/pkg/dashboard/static-src/css/workflows-editor.css @@ -108,6 +108,12 @@ .wf-ed-field input[readonly] { color: var(--muted); } .wf-ed-inline { display: flex; gap: var(--sp-2); align-items: center; } .wf-ed-inline input { flex: 1 1 auto; min-width: 0; } + /* Column icon row (hivecommons/hive#18033): preview glyph, id pick-list, emoji box. */ + .wf-ed-icon-row .wf-ed-icon-preview { display: inline-flex; align-items: center; justify-content: center; width: 24px; height: 24px; flex: 0 0 auto; border-radius: var(--r-sm, 4px); background: var(--panel-strong); color: var(--text); } + .wf-ed-icon-row .wf-ed-icon-preview .ham-ico { width: 14px; height: 14px; } + .wf-ed-icon-row .wf-ed-icon-preview .ham-emoji-ico { width: 14px; height: 14px; font-size: var(--fs-sm); line-height: 14px; } + .wf-ed-icon-row select { flex: 1 1 auto; min-width: 0; } + .wf-ed-icon-row .wf-ed-icon-emoji { flex: 0 0 7em; } .wf-ed-checks { display: flex; gap: var(--sp-4); align-items: center; flex-wrap: wrap; } .wf-ed-warn { color: var(--status-warn); } .wf-ed-translist { margin: var(--sp-0); padding-left: var(--sp-5); font-size: var(--fs-xs); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-editor.js b/src/pkg/dashboard/static-src/js/config-workflows-editor.js index 6649151761..4dfd898aab 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-editor.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-editor.js @@ -877,6 +877,24 @@ options.map(o => { const v = Array.isArray(o) ? o[0] : o, l = Array.isArray(o) ? o[1] : (o || '\u2014'); return `<option value="${esc(v)}" ${v === value ? 'selected' : ''}>${esc(l)}</option>`; }).join('') + '</select>'; } + // The column's chevron icon (hivecommons/hive#18033): a preview of what + // the Activity Monitor draws, a pick-list of the allowlisted ids with + // the label's default first, and a box for a single emoji. Both write + // spec.columns[id].icon through wfEdSetColumnIcon, which refuses + // anything outside the allowlist. + function wfEdColumnIconField(c, base, dis) { + if (typeof stageIconFor !== 'function') return ''; + const cur = String(c.icon == null ? '' : c.icon).trim(); + const def = stageIconIdForName(c.label || c.id); + const isEmoji = !!cur && !stageIconIsId(cur); + const ids = Object.keys(STAGE_ICON_SET); + const options = [['', 'default \u00b7 ' + def]].concat(ids.map(id => [id, id])); + const preview = `<span class="wf-ed-icon-preview" title="${esc('as drawn in the Activity Monitor')}">${stageIconHtml(stageIconFor(c), 'ham-ico ham-stage-ico')}</span>`; + const select = `<select data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>${options.map(o => `<option value="${esc(o[0])}" ${!isEmoji && o[0] === cur ? 'selected' : ''}>${esc(o[1])}</option>`).join('')}${isEmoji ? `<option value="${esc(cur)}" selected>${esc(cur)} (emoji)</option>` : ''}</select>`; + const emoji = `<input type="text" class="wf-ed-icon-emoji" maxlength="16" placeholder="or an emoji" value="${esc(isEmoji ? cur : '')}" data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>`; + return wfEdField('Icon', `<div class="wf-ed-inline wf-ed-icon-row">${preview}${select}${emoji}</div>`, 'drawn in this column\u2019s chevrons; blank = by label'); + } + function renderWfEdInspector(st) { const m = st.model, spec = m.spec, sel = st.selection; const can = wfEdCanEdit(); @@ -927,6 +945,7 @@ wfEdField('Id', `<div class="wf-ed-inline"><input type="text" value="${esc(c.id)}" readonly><button type="button" class="hv-btn btn-secondary btn-sm" data-action="wfEdRenameColumnAction" data-arg0="${esc(c.id)}" ${dis}>Rename\u2026</button></div>`) + wfEdField('Label', wfEdText(base + '.label', c.label, dis)) + wfEdField('Hint', wfEdText(base + '.hint', c.hint, dis), 'shown as the lane\u2019s tooltip in the Activity Monitor') + + wfEdColumnIconField(c, base, dis) + `<div class="config-field wf-ed-field"><label><input type="checkbox" ${c.terminal ? 'checked' : ''} data-change-action="wfEdFieldSetBool" data-arg0="${esc(base + '.terminal')}" data-arg-types="s,c" ${dis}> Terminal column</label><span class="text-muted fs-xs">states here need no exit</span></div>` + wfEdField('Width', `<input type="range" min="${WF_ED.minColW}" max="${WF_ED.maxColW}" step="${WF_ED.grid}" value="${st.view.laneWidths[c.id] || WF_ED.colW}" data-input-action="wfEdLaneWidth" data-arg0="${esc(c.id)}" data-arg-types="s,v">`) + `<p class="text-muted fs-sm">${(c.states || []).length} state${(c.states || []).length === 1 ? '' : 's'}</p>` + @@ -1498,6 +1517,19 @@ wfEdCommit(wfEdUpdate(_wfEd.model, path, isNaN(n) || n <= 0 ? '' : n)); } function wfEdFieldSetBool(path, checked) { wfEdCommit(wfEdUpdate(_wfEd.model, path, checked ? true : '')); } + // An icon is '' (by label), an allowlisted id or one emoji; anything + // else is refused with a toast and the inspector redraws the last good + // value, so no markup ever reaches a chevron (hivecommons/hive#18033). + function wfEdSetColumnIcon(path, value) { + if (!_wfEd.model) return; + const v = String(value == null ? '' : value).trim(); + if (typeof stageIconIsValid === 'function' && !stageIconIsValid(v)) { + if (typeof showToast === 'function') showToast('Icon must be one of the built-in icon ids or a single emoji.', 'error'); + wfEdRefresh(['inspector']); + return; + } + wfEdCommit(wfEdUpdate(_wfEd.model, path, v), { parts: ['canvas', 'inspector', 'drawer'] }); + } function wfEdSetList(path, value) { wfEdCommit(wfEdUpdate(_wfEd.model, path, String(value || '').split(/[,\s]+/).map(s => s.trim()).filter(Boolean))); } function wfEdSetStateColumn(id, columnId) { wfEdCommit(wfEdMoveState(_wfEd.model, id, columnId === WF_ED_UNASSIGNED ? '' : columnId)); } function wfEdSetInitial(id) { wfEdCommit(wfEdUpdate(_wfEd.model, 'spec.initial', id)); } diff --git a/src/pkg/dashboard/static-src/js/tracker-stage-icons.js b/src/pkg/dashboard/static-src/js/tracker-stage-icons.js new file mode 100644 index 0000000000..df7464dfbb --- /dev/null +++ b/src/pkg/dashboard/static-src/js/tracker-stage-icons.js @@ -0,0 +1,160 @@ +/* ---- Stage icons for workflow-model chevrons (hivecommons/hive#18033) --- + A chevron that is not the row's active stage used to carry only its + clipped column label. Now every column has a glyph that says what the + stage means: picked by the column's name (STAGE_ICON_BY_NAME, keyed + by the normalised label), or by the model's own `icon` on the column + — an allowlisted id below or a single emoji, never markup. The set is + 16-unit monochrome strokes in currentColor, so it follows the theme + like STAGE_TRACKER_ICONS does. Pure helpers: the node harness + (stage_tracker_stage_icons_ui_test.go) lifts them by name, and a Go + test keeps the id list equal to workflowmodel.ColumnIconIDs. */ + const STAGE_ICON_SET = { + inbox: '<path d="M2 9.5h3.5l1 2h3l1-2H14"/><path d="M2 9.5 4 3.5h8l2 6v4H2z"/>', + circle: '<circle cx="8" cy="8" r="5"/>', + bolt: '<path d="M9 1 3 9h4l-1 6 6-8H8z"/>', + hand: '<path d="M4.5 8.5V4a1 1 0 0 1 2 0v3.5M6.5 7V3a1 1 0 0 1 2 0v4M8.5 7V3.5a1 1 0 0 1 2 0V8M10.5 8V5a1 1 0 0 1 2 0v5.5c0 2.5-1.8 4-4.3 4-2 0-3-.9-3.9-2.3L2.6 9.9a1 1 0 0 1 1.6-1.2L4.5 9"/>', + person: '<circle cx="8" cy="5" r="3"/><path d="M2.5 14.5a5.5 5.5 0 0 1 11 0"/>', + pencil: '<path d="M11.5 2.5 13.5 4.5 5.5 12.5 2.5 13.5 3.5 10.5z"/><path d="M9.5 4.5l2 2"/>', + 'pull-request': '<circle cx="4" cy="3" r="1.5"/><circle cx="4" cy="13" r="1.5"/><circle cx="12" cy="13" r="1.5"/><path d="M4 4.5v7M12 11.5V7a2 2 0 0 0-2-2H8"/>', + flask: '<path d="M5.5 1.5h5M6.5 1.5v4.6L2.6 12.9a1 1 0 0 0 .9 1.6h9a1 1 0 0 0 .9-1.6L9.5 6.1V1.5"/><path d="M4.4 10.5h7.2"/>', + gear: '<circle cx="8" cy="8" r="2.2"/><path d="M8 1.5v2M8 12.5v2M1.5 8h2M12.5 8h2M3.4 3.4l1.4 1.4M11.2 11.2l1.4 1.4M3.4 12.6l1.4-1.4M11.2 4.8l1.4-1.4"/>', + eye: '<path d="M1 8s2.5-5 7-5 7 5 7 5-2.5 5-7 5-7-5-7-5z"/><circle cx="8" cy="8" r="2"/>', + people: '<circle cx="5.5" cy="5" r="2.5"/><path d="M1 14a4.5 4.5 0 0 1 9 0"/><path d="M10.5 3a2.5 2.5 0 0 1 0 4.5M12 10a4 4 0 0 1 3 4"/>', + 'thumbs-up': '<path d="M5 7.5 8 1.5c1.5 0 2 1 2 2L9.5 6H13a1.5 1.5 0 0 1 1.5 1.7l-1 5A1.5 1.5 0 0 1 12 14H5z"/><path d="M5 7.5H2v6.5h3"/>', + 'shield-check': '<path d="M8 1.5 13.5 3.5v4c0 3.5-2.5 5.9-5.5 7-3-1.1-5.5-3.5-5.5-7v-4z"/><path d="M5.5 8l1.8 1.8L10.8 6"/>', + merge: '<path stroke-width="2" d="M6 14.5V3M3 6l3-3 3 3M13 14.5c0-3.8-3.4-4.6-7-6.5"/>', + check: '<path d="M3 8l3 3 7-7"/>', + checkbox: '<rect x="2.5" y="2.5" width="11" height="11" rx="1.5"/><path d="M5 8l2 2 4-4"/>', + list: '<path d="M3 5h10M3 8h10M3 11h10"/>', + tag: '<path d="M2.5 2.5h5.5l5.5 5.5-5.5 5.5-5.5-5.5z"/><circle cx="5.5" cy="5.5" r="1"/>', + flag: '<path d="M3.5 14.5V2.5h8.5l-2 3 2 3H3.5"/>', + rocket: '<path d="M8 1.5c2.5 1.5 3.5 4.5 3 8l-3 3-3-3c-.5-3.5.5-6.5 3-8z"/><circle cx="8" cy="6.5" r="1.2"/><path d="M5 9.5 3 11.5l.5 2 2-1M11 9.5l2 2-.5 2-2-1"/>', + book: '<path d="M2.5 2.5h4a1.5 1.5 0 0 1 1.5 1.5v9.5a1 1 0 0 0-1-1h-4.5zM13.5 2.5h-4A1.5 1.5 0 0 0 8 4v9.5a1 1 0 0 1 1-1h4.5z"/>', + search: '<circle cx="7" cy="7" r="4.5"/><path d="M10.5 10.5 14 14"/>', + clock: '<circle cx="8" cy="8" r="6"/><path d="M8 4.5V8l2.5 1.5"/>', + pause: '<path d="M5.5 3v10M10.5 3v10"/>', + archive: '<rect x="2" y="2.5" width="12" height="3"/><path d="M3 5.5v8h10v-8M6.5 8.5h3"/>', + dot: '<circle cx="8" cy="8" r="1.75" fill="currentColor"/>', + }; + const STAGE_ICON_DEFAULT = 'dot'; + + // Pure: the key a column name is matched by — lower-case, trimmed, + // inner whitespace and the " / " and " · " separators collapsed to one + // space, so "Reviewer check", "reviewer check" and "CI / Review" each + // land on one entry. + function stageIconNormalizeName(name) { + return String(name == null ? '' : name).toLowerCase().replace(/[\s·/]+/g, ' ').trim(); + } + + // Canonical stage name → icon id. Every column label across hive-default, + // the curated marketplace bundles (cncf-techdocs-assessment), the ingest + // skeleton (Intake, Triage) and the editor palette has an entry; the + // workflow editor's column inspector shows the one its label resolves to. + const STAGE_ICON_BY_NAME = { + // hive-default + 'open': 'inbox', + 'actionable': 'bolt', + 'claimed': 'hand', + 'pr in progress': 'pull-request', + 'ci review': 'flask', + 'ci': 'flask', + 'review': 'eye', + 'merge queue': 'merge', + 'merged': 'check', + // cncf-techdocs-assessment + 'requested': 'inbox', + 'accepted': 'checkbox', + 'drafting': 'pencil', + 'reviewer check': 'eye', + 'stakeholder review': 'people', + 'approval': 'thumbs-up', + 'done': 'check', + // ingest skeleton and editor palette + 'intake': 'inbox', + 'triage': 'tag', + 'analysis': 'search', + 'writing': 'pencil', + 'publish': 'rocket', + 'published': 'rocket', + 'needs decision': 'person', + 'work in progress': 'pencil', + 'in progress': 'pencil', + 'in review': 'eye', + 'ci failing': 'flask', + 'released': 'rocket', + 'deployed': 'rocket', + 'deployed staging': 'rocket', + 'deployed production': 'rocket', + 'closed': 'archive', + 'verified': 'shield-check', + 'verification': 'shield-check', + 'approved': 'thumbs-up', + 'blocked': 'pause', + 'hold': 'pause', + 'on hold': 'pause', + 'waiting': 'clock', + 'backlog': 'list', + 'todo': 'list', + 'to do': 'list', + 'docs': 'book', + 'documentation': 'book', + }; + + // Pure: is v an allowlisted icon id? + function stageIconIsId(v) { return Object.prototype.hasOwnProperty.call(STAGE_ICON_SET, String(v || '')); } + + // Pure: is v a single emoji — one or two pictographs (a ZWJ sequence or + // a flag) plus modifiers, no letters, digits, spaces or markup? The + // mirror of workflowmodel.IsColumnIconEmoji. + function stageIconIsEmoji(v) { + const s = String(v == null ? '' : v); + if (!s || Array.from(s).length > 8) return false; + let pict = 0; + for (const ch of s) { + const cp = ch.codePointAt(0); + if (cp === 0x200D || cp === 0xFE0F || cp === 0xFE0E || cp === 0x20E3 || (cp >= 0x1F3FB && cp <= 0x1F3FF)) continue; + if (cp < 0x80 || /[\p{L}\p{N}\s\p{Cc}]/u.test(ch)) return false; + if ((cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2B00 && cp <= 0x2BFF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2190 && cp <= 0x21FF) || (cp >= 0x25A0 && cp <= 0x25FF) || (cp >= 0x2900 && cp <= 0x297F) || (cp >= 0x1F1E6 && cp <= 0x1F1FF) || [0xA9, 0xAE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299].indexOf(cp) >= 0) { pict++; continue; } + return false; + } + return pict >= 1 && pict <= 2; + } + + // Pure: '' (default), an icon id or a single emoji validate; anything + // else is refused — the editor shows why and keeps the last good value. + function stageIconIsValid(v) { + const s = String(v == null ? '' : v).trim(); + return s === '' || stageIconIsId(s) || stageIconIsEmoji(s); + } + + // Pure: the icon id a column name resolves to; STAGE_ICON_DEFAULT when + // the name is unknown. + function stageIconIdForName(name) { + return STAGE_ICON_BY_NAME[stageIconNormalizeName(name)] || STAGE_ICON_DEFAULT; + } + + // Pure: what a column draws — its valid `icon` override (id or emoji), + // else the id its label resolves to. An invalid override is ignored, + // never rendered. + function stageIconFor(col) { + const c = col || {}; + const over = String(c.icon == null ? '' : c.icon).trim(); + if (over && (stageIconIsId(over) || stageIconIsEmoji(over))) return over; + return stageIconIdForName(c.label || c.id); + } + + // The icon as markup: an allowlisted id becomes its inline SVG, an + // emoji a text span (escaped), anything else nothing. label is for + // assistive tech only; the chevron's title carries the hover text. + function stageIconHtml(icon, cls, label) { + const v = String(icon == null ? '' : icon).trim(); + const aria = label ? ` role="img" aria-label="${esc(label)}"` : ' aria-hidden="true"'; + if (stageIconIsId(v)) return `<svg class="${cls}" viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"${aria}>${STAGE_ICON_SET[v]}</svg>`; + if (stageIconIsEmoji(v)) return `<span class="${cls} ham-emoji-ico"${aria}>${esc(v)}</span>`; + return ''; + } + + // The glyph a column's chevron wears, by its override or its name. + function stageIconColumnHtml(col, cls, label) { + return stageIconHtml(stageIconFor(col), cls, label); + } diff --git a/src/pkg/dashboard/static-src/js/tracker-workflows.js b/src/pkg/dashboard/static-src/js/tracker-workflows.js index fa24eacf3d..5004eac845 100644 --- a/src/pkg/dashboard/static-src/js/tracker-workflows.js +++ b/src/pkg/dashboard/static-src/js/tracker-workflows.js @@ -67,7 +67,7 @@ // The model's column header, drawn under a bound repository's group // header in place of the shared hive-default header. function stageTrackerWorkflowHeaderHtml(wf) { - const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${esc(c.label)}</span></li>`).join(''); + const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-col-ico ham-stage-ico')}${esc(c.label)}</span></li>`).join(''); return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } @@ -127,10 +127,20 @@ return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; } + // The glyph a model column wears (hivecommons/hive#18033): its `icon` + // override when valid, else the icon its label resolves to. Guarded so + // harnesses that lift the strip alone still render. + function stageTrackerWorkflowColumnIconHtml(c, cls, label) { + return typeof stageIconColumnHtml === 'function' ? stageIconColumnHtml(c, cls, label) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber // with the "Not covered" word so the gap is visible, not silent. + // Done and todo chevrons wear the column's icon alone (the label is + // the header's and the hover's; in a chevron it only clipped); the + // active one leads its word with the icon (hivecommons/hive#18033). function stageTrackerWorkflowStripItems(item, wf, stages) { const cur = Number(item.stage) || 0; const terminal = cur >= (typeof STAGE_TRACKER_TERMINAL === 'number' ? STAGE_TRACKER_TERMINAL : 110); @@ -148,6 +158,8 @@ ? `stage/${String(cur).padStart(3, '0')} is not a state of ${wf.name}@${wf.version}; the engine leaves this item where it is.` : (c.hint || c.label) + (state === 'active' && cur ? ` — stage/${String(cur).padStart(3, '0')}` : ''); const cls = ['ham-seg', 'ham-wf-seg', state, c.terminal ? 'cap' : '', uncovered && state === 'active' ? 'ham-wf-uncovered' : ''].filter(Boolean).join(' '); - return `<li class="${cls}" title="${esc(tip)}"><span class="ham-seg-t">${esc(word)}</span></li>`; + const ico = stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-stage-ico'); + const inner = state === 'active' ? `${ico}<span class="ham-seg-t">${esc(word)}</span>` : (ico || `<span class="ham-seg-t">${esc(word)}</span>`); + return `<li class="${cls}" title="${esc(tip)}">${inner}</li>`; }).join(''); } diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index fe1b1cd620..310a469a9b 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -81,6 +81,7 @@ js/tracker-words-gate.js js/tracker-words.js js/tracker-row-anchor.js js/tracker-row.js +js/tracker-stage-icons.js js/tracker-row-strip.js js/tracker-workflows.js js/tracker-row-acted.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 342530dd61..a59be35a8e 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -1615,6 +1615,14 @@ .ham-wf-badge { flex: 0 0 auto; font-family: var(--font-mono); font-size: var(--fs-2xs); font-weight: 700; letter-spacing: 0.02em; padding: var(--sp-1) var(--sp-3); border-radius: var(--r-pill); white-space: nowrap; color: var(--indigo); background: color-mix(in srgb, var(--indigo) 12%, transparent); border: 1px solid color-mix(in srgb, var(--indigo) 40%, transparent); } .ham-wf-cols { position: static; } .ham-wf-cols .ham-wf-col { color: var(--indigo); } + /* Model chevrons wear the column's stage icon (hivecommons/hive issue 18033): + alone and dimmed in a done / todo step, leading the word in the lit + one. An emoji override is sized to the SVG glyph so the row keeps + its height. */ + .ham-seg.ham-wf-seg .ham-stage-ico { flex: 0 0 auto; } + .ham-seg.ham-wf-seg.active .ham-stage-ico { margin-right: var(--sp-1); } + .ham-seg.ham-wf-seg.done .ham-stage-ico { opacity: 0.6; } + .ham-emoji-ico { display: inline-block; width: 12px; height: 12px; font-size: var(--fs-xs); line-height: 12px; text-align: center; font-style: normal; } .ham-seg.ham-wf-uncovered { background: color-mix(in srgb, var(--amber) 18%, transparent); color: var(--amber); outline: 1px dashed var(--amber); } .ham-group-dist { flex: 0 0 auto; margin-left: auto; margin-right: calc(4em + var(--sp-3)); display: inline-flex; align-items: flex-end; gap: var(--sp-1); height: 0.9em; } .ham-gd { display: inline-block; width: 0.45em; height: 15%; border-radius: var(--sp-1); background: color-mix(in srgb, var(--text-faint) 35%, transparent); } @@ -5298,6 +5306,12 @@ .wf-ed-field input[readonly] { color: var(--muted); } .wf-ed-inline { display: flex; gap: var(--sp-2); align-items: center; } .wf-ed-inline input { flex: 1 1 auto; min-width: 0; } + /* Column icon row (hivecommons/hive#18033): preview glyph, id pick-list, emoji box. */ + .wf-ed-icon-row .wf-ed-icon-preview { display: inline-flex; align-items: center; justify-content: center; width: 24px; height: 24px; flex: 0 0 auto; border-radius: var(--r-sm, 4px); background: var(--panel-strong); color: var(--text); } + .wf-ed-icon-row .wf-ed-icon-preview .ham-ico { width: 14px; height: 14px; } + .wf-ed-icon-row .wf-ed-icon-preview .ham-emoji-ico { width: 14px; height: 14px; font-size: var(--fs-sm); line-height: 14px; } + .wf-ed-icon-row select { flex: 1 1 auto; min-width: 0; } + .wf-ed-icon-row .wf-ed-icon-emoji { flex: 0 0 7em; } .wf-ed-checks { display: flex; gap: var(--sp-4); align-items: center; flex-wrap: wrap; } .wf-ed-warn { color: var(--status-warn); } .wf-ed-translist { margin: var(--sp-0); padding-left: var(--sp-5); font-size: var(--fs-xs); } @@ -18803,6 +18817,166 @@ <h3 class="faq-group-title">Where to get help</h3> if (hours < 48) return hours + 'h'; return Math.floor(hours / 24) + 'd'; } +/* ---- Stage icons for workflow-model chevrons (hivecommons/hive#18033) --- + A chevron that is not the row's active stage used to carry only its + clipped column label. Now every column has a glyph that says what the + stage means: picked by the column's name (STAGE_ICON_BY_NAME, keyed + by the normalised label), or by the model's own `icon` on the column + — an allowlisted id below or a single emoji, never markup. The set is + 16-unit monochrome strokes in currentColor, so it follows the theme + like STAGE_TRACKER_ICONS does. Pure helpers: the node harness + (stage_tracker_stage_icons_ui_test.go) lifts them by name, and a Go + test keeps the id list equal to workflowmodel.ColumnIconIDs. */ + const STAGE_ICON_SET = { + inbox: '<path d="M2 9.5h3.5l1 2h3l1-2H14"/><path d="M2 9.5 4 3.5h8l2 6v4H2z"/>', + circle: '<circle cx="8" cy="8" r="5"/>', + bolt: '<path d="M9 1 3 9h4l-1 6 6-8H8z"/>', + hand: '<path d="M4.5 8.5V4a1 1 0 0 1 2 0v3.5M6.5 7V3a1 1 0 0 1 2 0v4M8.5 7V3.5a1 1 0 0 1 2 0V8M10.5 8V5a1 1 0 0 1 2 0v5.5c0 2.5-1.8 4-4.3 4-2 0-3-.9-3.9-2.3L2.6 9.9a1 1 0 0 1 1.6-1.2L4.5 9"/>', + person: '<circle cx="8" cy="5" r="3"/><path d="M2.5 14.5a5.5 5.5 0 0 1 11 0"/>', + pencil: '<path d="M11.5 2.5 13.5 4.5 5.5 12.5 2.5 13.5 3.5 10.5z"/><path d="M9.5 4.5l2 2"/>', + 'pull-request': '<circle cx="4" cy="3" r="1.5"/><circle cx="4" cy="13" r="1.5"/><circle cx="12" cy="13" r="1.5"/><path d="M4 4.5v7M12 11.5V7a2 2 0 0 0-2-2H8"/>', + flask: '<path d="M5.5 1.5h5M6.5 1.5v4.6L2.6 12.9a1 1 0 0 0 .9 1.6h9a1 1 0 0 0 .9-1.6L9.5 6.1V1.5"/><path d="M4.4 10.5h7.2"/>', + gear: '<circle cx="8" cy="8" r="2.2"/><path d="M8 1.5v2M8 12.5v2M1.5 8h2M12.5 8h2M3.4 3.4l1.4 1.4M11.2 11.2l1.4 1.4M3.4 12.6l1.4-1.4M11.2 4.8l1.4-1.4"/>', + eye: '<path d="M1 8s2.5-5 7-5 7 5 7 5-2.5 5-7 5-7-5-7-5z"/><circle cx="8" cy="8" r="2"/>', + people: '<circle cx="5.5" cy="5" r="2.5"/><path d="M1 14a4.5 4.5 0 0 1 9 0"/><path d="M10.5 3a2.5 2.5 0 0 1 0 4.5M12 10a4 4 0 0 1 3 4"/>', + 'thumbs-up': '<path d="M5 7.5 8 1.5c1.5 0 2 1 2 2L9.5 6H13a1.5 1.5 0 0 1 1.5 1.7l-1 5A1.5 1.5 0 0 1 12 14H5z"/><path d="M5 7.5H2v6.5h3"/>', + 'shield-check': '<path d="M8 1.5 13.5 3.5v4c0 3.5-2.5 5.9-5.5 7-3-1.1-5.5-3.5-5.5-7v-4z"/><path d="M5.5 8l1.8 1.8L10.8 6"/>', + merge: '<path stroke-width="2" d="M6 14.5V3M3 6l3-3 3 3M13 14.5c0-3.8-3.4-4.6-7-6.5"/>', + check: '<path d="M3 8l3 3 7-7"/>', + checkbox: '<rect x="2.5" y="2.5" width="11" height="11" rx="1.5"/><path d="M5 8l2 2 4-4"/>', + list: '<path d="M3 5h10M3 8h10M3 11h10"/>', + tag: '<path d="M2.5 2.5h5.5l5.5 5.5-5.5 5.5-5.5-5.5z"/><circle cx="5.5" cy="5.5" r="1"/>', + flag: '<path d="M3.5 14.5V2.5h8.5l-2 3 2 3H3.5"/>', + rocket: '<path d="M8 1.5c2.5 1.5 3.5 4.5 3 8l-3 3-3-3c-.5-3.5.5-6.5 3-8z"/><circle cx="8" cy="6.5" r="1.2"/><path d="M5 9.5 3 11.5l.5 2 2-1M11 9.5l2 2-.5 2-2-1"/>', + book: '<path d="M2.5 2.5h4a1.5 1.5 0 0 1 1.5 1.5v9.5a1 1 0 0 0-1-1h-4.5zM13.5 2.5h-4A1.5 1.5 0 0 0 8 4v9.5a1 1 0 0 1 1-1h4.5z"/>', + search: '<circle cx="7" cy="7" r="4.5"/><path d="M10.5 10.5 14 14"/>', + clock: '<circle cx="8" cy="8" r="6"/><path d="M8 4.5V8l2.5 1.5"/>', + pause: '<path d="M5.5 3v10M10.5 3v10"/>', + archive: '<rect x="2" y="2.5" width="12" height="3"/><path d="M3 5.5v8h10v-8M6.5 8.5h3"/>', + dot: '<circle cx="8" cy="8" r="1.75" fill="currentColor"/>', + }; + const STAGE_ICON_DEFAULT = 'dot'; + + // Pure: the key a column name is matched by — lower-case, trimmed, + // inner whitespace and the " / " and " · " separators collapsed to one + // space, so "Reviewer check", "reviewer check" and "CI / Review" each + // land on one entry. + function stageIconNormalizeName(name) { + return String(name == null ? '' : name).toLowerCase().replace(/[\s·/]+/g, ' ').trim(); + } + + // Canonical stage name → icon id. Every column label across hive-default, + // the curated marketplace bundles (cncf-techdocs-assessment), the ingest + // skeleton (Intake, Triage) and the editor palette has an entry; the + // workflow editor's column inspector shows the one its label resolves to. + const STAGE_ICON_BY_NAME = { + // hive-default + 'open': 'inbox', + 'actionable': 'bolt', + 'claimed': 'hand', + 'pr in progress': 'pull-request', + 'ci review': 'flask', + 'ci': 'flask', + 'review': 'eye', + 'merge queue': 'merge', + 'merged': 'check', + // cncf-techdocs-assessment + 'requested': 'inbox', + 'accepted': 'checkbox', + 'drafting': 'pencil', + 'reviewer check': 'eye', + 'stakeholder review': 'people', + 'approval': 'thumbs-up', + 'done': 'check', + // ingest skeleton and editor palette + 'intake': 'inbox', + 'triage': 'tag', + 'analysis': 'search', + 'writing': 'pencil', + 'publish': 'rocket', + 'published': 'rocket', + 'needs decision': 'person', + 'work in progress': 'pencil', + 'in progress': 'pencil', + 'in review': 'eye', + 'ci failing': 'flask', + 'released': 'rocket', + 'deployed': 'rocket', + 'deployed staging': 'rocket', + 'deployed production': 'rocket', + 'closed': 'archive', + 'verified': 'shield-check', + 'verification': 'shield-check', + 'approved': 'thumbs-up', + 'blocked': 'pause', + 'hold': 'pause', + 'on hold': 'pause', + 'waiting': 'clock', + 'backlog': 'list', + 'todo': 'list', + 'to do': 'list', + 'docs': 'book', + 'documentation': 'book', + }; + + // Pure: is v an allowlisted icon id? + function stageIconIsId(v) { return Object.prototype.hasOwnProperty.call(STAGE_ICON_SET, String(v || '')); } + + // Pure: is v a single emoji — one or two pictographs (a ZWJ sequence or + // a flag) plus modifiers, no letters, digits, spaces or markup? The + // mirror of workflowmodel.IsColumnIconEmoji. + function stageIconIsEmoji(v) { + const s = String(v == null ? '' : v); + if (!s || Array.from(s).length > 8) return false; + let pict = 0; + for (const ch of s) { + const cp = ch.codePointAt(0); + if (cp === 0x200D || cp === 0xFE0F || cp === 0xFE0E || cp === 0x20E3 || (cp >= 0x1F3FB && cp <= 0x1F3FF)) continue; + if (cp < 0x80 || /[\p{L}\p{N}\s\p{Cc}]/u.test(ch)) return false; + if ((cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2B00 && cp <= 0x2BFF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2190 && cp <= 0x21FF) || (cp >= 0x25A0 && cp <= 0x25FF) || (cp >= 0x2900 && cp <= 0x297F) || (cp >= 0x1F1E6 && cp <= 0x1F1FF) || [0xA9, 0xAE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299].indexOf(cp) >= 0) { pict++; continue; } + return false; + } + return pict >= 1 && pict <= 2; + } + + // Pure: '' (default), an icon id or a single emoji validate; anything + // else is refused — the editor shows why and keeps the last good value. + function stageIconIsValid(v) { + const s = String(v == null ? '' : v).trim(); + return s === '' || stageIconIsId(s) || stageIconIsEmoji(s); + } + + // Pure: the icon id a column name resolves to; STAGE_ICON_DEFAULT when + // the name is unknown. + function stageIconIdForName(name) { + return STAGE_ICON_BY_NAME[stageIconNormalizeName(name)] || STAGE_ICON_DEFAULT; + } + + // Pure: what a column draws — its valid `icon` override (id or emoji), + // else the id its label resolves to. An invalid override is ignored, + // never rendered. + function stageIconFor(col) { + const c = col || {}; + const over = String(c.icon == null ? '' : c.icon).trim(); + if (over && (stageIconIsId(over) || stageIconIsEmoji(over))) return over; + return stageIconIdForName(c.label || c.id); + } + + // The icon as markup: an allowlisted id becomes its inline SVG, an + // emoji a text span (escaped), anything else nothing. label is for + // assistive tech only; the chevron's title carries the hover text. + function stageIconHtml(icon, cls, label) { + const v = String(icon == null ? '' : icon).trim(); + const aria = label ? ` role="img" aria-label="${esc(label)}"` : ' aria-hidden="true"'; + if (stageIconIsId(v)) return `<svg class="${cls}" viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"${aria}>${STAGE_ICON_SET[v]}</svg>`; + if (stageIconIsEmoji(v)) return `<span class="${cls} ham-emoji-ico"${aria}>${esc(v)}</span>`; + return ''; + } + + // The glyph a column's chevron wears, by its override or its name. + function stageIconColumnHtml(col, cls, label) { + return stageIconHtml(stageIconFor(col), cls, label); + } // Column header: the stage names once, aligned with the chevrons below. // It is sticky under the Merged bar (tracker-cues CSS), so the names @@ -19966,7 +20140,7 @@ <h3 class="faq-group-title">Where to get help</h3> // The model's column header, drawn under a bound repository's group // header in place of the shared hive-default header. function stageTrackerWorkflowHeaderHtml(wf) { - const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${esc(c.label)}</span></li>`).join(''); + const cols = wf.columns.map(c => `<li class="ham-seg ham-wf-col${c.terminal ? ' cap' : ''}" title="${esc(c.hint || c.label)}"><span class="ham-seg-t">${stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-col-ico ham-stage-ico')}${esc(c.label)}</span></li>`).join(''); return `<div class="ham-cols ham-wf-cols" aria-hidden="true" data-ham-workflow="${esc(wf.name + '@' + wf.version)}"><div class="ham-strip-row"><ol class="ham-strip">${cols}</ol><span class="ham-since">since</span></div></div>`; } @@ -20026,10 +20200,20 @@ <h3 class="faq-group-title">Where to get help</h3> return ctx && ctx.leadWorkflow ? stageTrackerHeaderHtml(stages) : ''; } + // The glyph a model column wears (hivecommons/hive#18033): its `icon` + // override when valid, else the icon its label resolves to. Guarded so + // harnesses that lift the strip alone still render. + function stageTrackerWorkflowColumnIconHtml(c, cls, label) { + return typeof stageIconColumnHtml === 'function' ? stageIconColumnHtml(c, cls, label) : ''; + } + // One row's strip under its model: done before the anchored column, // active at it (the stage's own word from `stages`, or the column // label), todo after. An uncovered stage lights the first column amber // with the "Not covered" word so the gap is visible, not silent. + // Done and todo chevrons wear the column's icon alone (the label is + // the header's and the hover's; in a chevron it only clipped); the + // active one leads its word with the icon (hivecommons/hive#18033). function stageTrackerWorkflowStripItems(item, wf, stages) { const cur = Number(item.stage) || 0; const terminal = cur >= (typeof STAGE_TRACKER_TERMINAL === 'number' ? STAGE_TRACKER_TERMINAL : 110); @@ -20047,7 +20231,9 @@ <h3 class="faq-group-title">Where to get help</h3> ? `stage/${String(cur).padStart(3, '0')} is not a state of ${wf.name}@${wf.version}; the engine leaves this item where it is.` : (c.hint || c.label) + (state === 'active' && cur ? ` — stage/${String(cur).padStart(3, '0')}` : ''); const cls = ['ham-seg', 'ham-wf-seg', state, c.terminal ? 'cap' : '', uncovered && state === 'active' ? 'ham-wf-uncovered' : ''].filter(Boolean).join(' '); - return `<li class="${cls}" title="${esc(tip)}"><span class="ham-seg-t">${esc(word)}</span></li>`; + const ico = stageTrackerWorkflowColumnIconHtml(c, 'ham-ico ham-stage-ico'); + const inner = state === 'active' ? `${ico}<span class="ham-seg-t">${esc(word)}</span>` : (ico || `<span class="ham-seg-t">${esc(word)}</span>`); + return `<li class="${cls}" title="${esc(tip)}">${inner}</li>`; }).join(''); } const STAGE_TRACKER_ACTED_STATES = ['waiting', 'addressed', 'stuck', 'pending', 'failed', 'executing', 'blocked']; @@ -65606,6 +65792,24 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> options.map(o => { const v = Array.isArray(o) ? o[0] : o, l = Array.isArray(o) ? o[1] : (o || '\u2014'); return `<option value="${esc(v)}" ${v === value ? 'selected' : ''}>${esc(l)}</option>`; }).join('') + '</select>'; } + // The column's chevron icon (hivecommons/hive#18033): a preview of what + // the Activity Monitor draws, a pick-list of the allowlisted ids with + // the label's default first, and a box for a single emoji. Both write + // spec.columns[id].icon through wfEdSetColumnIcon, which refuses + // anything outside the allowlist. + function wfEdColumnIconField(c, base, dis) { + if (typeof stageIconFor !== 'function') return ''; + const cur = String(c.icon == null ? '' : c.icon).trim(); + const def = stageIconIdForName(c.label || c.id); + const isEmoji = !!cur && !stageIconIsId(cur); + const ids = Object.keys(STAGE_ICON_SET); + const options = [['', 'default \u00b7 ' + def]].concat(ids.map(id => [id, id])); + const preview = `<span class="wf-ed-icon-preview" title="${esc('as drawn in the Activity Monitor')}">${stageIconHtml(stageIconFor(c), 'ham-ico ham-stage-ico')}</span>`; + const select = `<select data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>${options.map(o => `<option value="${esc(o[0])}" ${!isEmoji && o[0] === cur ? 'selected' : ''}>${esc(o[1])}</option>`).join('')}${isEmoji ? `<option value="${esc(cur)}" selected>${esc(cur)} (emoji)</option>` : ''}</select>`; + const emoji = `<input type="text" class="wf-ed-icon-emoji" maxlength="16" placeholder="or an emoji" value="${esc(isEmoji ? cur : '')}" data-change-action="wfEdSetColumnIcon" data-arg0="${esc(base + '.icon')}" data-arg-types="s,v" ${dis}>`; + return wfEdField('Icon', `<div class="wf-ed-inline wf-ed-icon-row">${preview}${select}${emoji}</div>`, 'drawn in this column\u2019s chevrons; blank = by label'); + } + function renderWfEdInspector(st) { const m = st.model, spec = m.spec, sel = st.selection; const can = wfEdCanEdit(); @@ -65656,6 +65860,7 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> wfEdField('Id', `<div class="wf-ed-inline"><input type="text" value="${esc(c.id)}" readonly><button type="button" class="hv-btn btn-secondary btn-sm" data-action="wfEdRenameColumnAction" data-arg0="${esc(c.id)}" ${dis}>Rename\u2026</button></div>`) + wfEdField('Label', wfEdText(base + '.label', c.label, dis)) + wfEdField('Hint', wfEdText(base + '.hint', c.hint, dis), 'shown as the lane\u2019s tooltip in the Activity Monitor') + + wfEdColumnIconField(c, base, dis) + `<div class="config-field wf-ed-field"><label><input type="checkbox" ${c.terminal ? 'checked' : ''} data-change-action="wfEdFieldSetBool" data-arg0="${esc(base + '.terminal')}" data-arg-types="s,c" ${dis}> Terminal column</label><span class="text-muted fs-xs">states here need no exit</span></div>` + wfEdField('Width', `<input type="range" min="${WF_ED.minColW}" max="${WF_ED.maxColW}" step="${WF_ED.grid}" value="${st.view.laneWidths[c.id] || WF_ED.colW}" data-input-action="wfEdLaneWidth" data-arg0="${esc(c.id)}" data-arg-types="s,v">`) + `<p class="text-muted fs-sm">${(c.states || []).length} state${(c.states || []).length === 1 ? '' : 's'}</p>` + @@ -66227,6 +66432,19 @@ <h4 style="margin:18px 0 8px">Agent Watchdog</h4> wfEdCommit(wfEdUpdate(_wfEd.model, path, isNaN(n) || n <= 0 ? '' : n)); } function wfEdFieldSetBool(path, checked) { wfEdCommit(wfEdUpdate(_wfEd.model, path, checked ? true : '')); } + // An icon is '' (by label), an allowlisted id or one emoji; anything + // else is refused with a toast and the inspector redraws the last good + // value, so no markup ever reaches a chevron (hivecommons/hive#18033). + function wfEdSetColumnIcon(path, value) { + if (!_wfEd.model) return; + const v = String(value == null ? '' : value).trim(); + if (typeof stageIconIsValid === 'function' && !stageIconIsValid(v)) { + if (typeof showToast === 'function') showToast('Icon must be one of the built-in icon ids or a single emoji.', 'error'); + wfEdRefresh(['inspector']); + return; + } + wfEdCommit(wfEdUpdate(_wfEd.model, path, v), { parts: ['canvas', 'inspector', 'drawer'] }); + } function wfEdSetList(path, value) { wfEdCommit(wfEdUpdate(_wfEd.model, path, String(value || '').split(/[,\s]+/).map(s => s.trim()).filter(Boolean))); } function wfEdSetStateColumn(id, columnId) { wfEdCommit(wfEdMoveState(_wfEd.model, id, columnId === WF_ED_UNASSIGNED ? '' : columnId)); } function wfEdSetInitial(id) { wfEdCommit(wfEdUpdate(_wfEd.model, 'spec.initial', id)); } diff --git a/src/pkg/workflowmodel/icons.go b/src/pkg/workflowmodel/icons.go new file mode 100644 index 0000000000..20cae6bcd4 --- /dev/null +++ b/src/pkg/workflowmodel/icons.go @@ -0,0 +1,97 @@ +package workflowmodel + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +// ColumnIconIDs is the allowlist of icon ids a Column.Icon may name. The +// SPA draws each id from its own inline-SVG set (STAGE_ICON_SET in +// static-src/js/tracker-stage-icons.js; a dashboard test keeps the two +// lists equal), so a model can never inject markup into a chevron +// (hivecommons/hive#18033). Order is the editor's pick-list order. +var ColumnIconIDs = []string{ + "inbox", "circle", "bolt", "hand", "person", "pencil", "pull-request", + "flask", "gear", "eye", "people", "thumbs-up", "shield-check", "merge", + "check", "checkbox", "list", "tag", "flag", "rocket", "book", "search", + "clock", "pause", "archive", "dot", +} + +// ColumnIconDefault is the icon an unknown column name falls back to. +const ColumnIconDefault = "dot" + +// columnIconMaxEmojiRunes bounds an emoji override: one pictograph plus +// its variation selector, skin tone or ZWJ-joined companions. +const columnIconMaxEmojiRunes = 8 + +var columnIconIDSet = func() map[string]bool { + m := make(map[string]bool, len(ColumnIconIDs)) + for _, id := range ColumnIconIDs { + m[id] = true + } + return m +}() + +// IsColumnIconID reports whether id is an allowlisted icon id. +func IsColumnIconID(id string) bool { return columnIconIDSet[id] } + +// IsColumnIconEmoji reports whether s is a single emoji (one pictograph +// with optional modifiers and joiners): no letters, digits, spaces, +// punctuation or markup, so a user-typed override can only ever render as +// a glyph. +func IsColumnIconEmoji(s string) bool { + if s == "" || !utf8.ValidString(s) || utf8.RuneCountInString(s) > columnIconMaxEmojiRunes { + return false + } + pictographs := 0 + for _, r := range s { + switch { + case columnIconJoiner(r): + case columnIconPictograph(r): + pictographs++ + default: + return false + } + } + return pictographs >= 1 && pictographs <= 2 +} + +// columnIconJoiner: ZWJ, variation selectors, keycap and skin tones — the +// code points that modify a pictograph without being one. +func columnIconJoiner(r rune) bool { + return r == 0x200D || r == 0xFE0F || r == 0xFE0E || r == 0x20E3 || (r >= 0x1F3FB && r <= 0x1F3FF) +} + +// columnIconPictograph covers the emoji blocks (Supplemental Symbols and +// Pictographs, Misc Symbols, Dingbats, Arrows, Geometric Shapes, …) and +// the regional indicators that pair into flags. +func columnIconPictograph(r rune) bool { + if unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsSpace(r) || unicode.IsControl(r) || r < 0x80 { + return false + } + ranges := [][2]rune{{0x1F000, 0x1FAFF}, {0x2600, 0x27BF}, {0x2B00, 0x2BFF}, {0x2300, 0x23FF}, {0x2190, 0x21FF}, {0x25A0, 0x25FF}, {0x2900, 0x297F}, {0x1F1E6, 0x1F1FF}} + for _, rg := range ranges { + if r >= rg[0] && r <= rg[1] { + return true + } + } + switch r { + case 0x00A9, 0x00AE, 0x2122, 0x2139, 0x3030, 0x303D, 0x3297, 0x3299: + return true + } + return false +} + +// ValidateColumnIcon accepts the empty value (default by name), an +// allowlisted icon id, or a single emoji; anything else is an error naming +// what was rejected. Only surrounding blanks are forgiven: a newline or +// other control character is never part of an icon. +func ValidateColumnIcon(icon string) error { + v := strings.Trim(icon, " \t") + if v == "" || IsColumnIconID(v) || IsColumnIconEmoji(v) { + return nil + } + return fmt.Errorf("icon %q is neither an icon id (%s) nor a single emoji", icon, strings.Join(ColumnIconIDs, ", ")) +} diff --git a/src/pkg/workflowmodel/icons_test.go b/src/pkg/workflowmodel/icons_test.go new file mode 100644 index 0000000000..60d8f8ff66 --- /dev/null +++ b/src/pkg/workflowmodel/icons_test.go @@ -0,0 +1,60 @@ +package workflowmodel + +import ( + "errors" + "strings" + "testing" +) + +// A column's icon is the empty default, an allowlisted id or a single +// emoji; letters, markup and multi-glyph strings are refused so the SPA's +// chevron renderer never sees anything it would have to escape +// (hivecommons/hive#18033). +func TestValidateColumnIcon(t *testing.T) { + for _, ok := range []string{"", "bolt", "merge", " eye ", "✅", "🚀", "👍🏽", "👩‍💻", "🇺🇸", "⏸️"} { + if err := ValidateColumnIcon(ok); err != nil { + t.Errorf("ValidateColumnIcon(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{"sparkle", "Bolt", "<svg>", "<img src=x onerror=alert(1)>", "a", "✅✅✅", "ok ✅", "🚀 launch", "javascript:", "✅\n"} { + if err := ValidateColumnIcon(bad); err == nil { + t.Errorf("ValidateColumnIcon(%q) = nil, want error", bad) + } + } +} + +func TestValidateRejectsBadColumnIcon(t *testing.T) { + m := techdocs(t) + m.Spec.Columns[0].Icon = "<b>x</b>" + m.Spec.Columns[1].Icon = "bolt" + err := m.Validate() + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v", err) + } + if !strings.Contains(err.Error(), `column "`+m.Spec.Columns[0].ID+`": icon "<b>x</b>"`) { + t.Errorf("error does not name the column and icon: %v", err) + } + if strings.Contains(err.Error(), `"bolt"`) { + t.Errorf("allowlisted id reported: %v", err) + } + m.Spec.Columns[0].Icon = "📥" + if err := m.Validate(); err != nil { + t.Errorf("emoji icon rejected: %v", err) + } +} + +func TestColumnIconIDsAreUnique(t *testing.T) { + seen := map[string]bool{} + for _, id := range ColumnIconIDs { + if seen[id] { + t.Errorf("duplicate icon id %q", id) + } + seen[id] = true + if strings.TrimSpace(id) != id || strings.ToLower(id) != id { + t.Errorf("icon id %q is not a trimmed lower-case token", id) + } + } + if !IsColumnIconID(ColumnIconDefault) { + t.Errorf("default %q is not in the allowlist", ColumnIconDefault) + } +} diff --git a/src/pkg/workflowmodel/lifecycle.go b/src/pkg/workflowmodel/lifecycle.go index 6606df5364..ef3c1db93d 100644 --- a/src/pkg/workflowmodel/lifecycle.go +++ b/src/pkg/workflowmodel/lifecycle.go @@ -64,9 +64,13 @@ type Citation struct { // Column is one Activity Monitor column; several states may share it. type Column struct { - ID string `json:"id"` - Label string `json:"label"` - Hint string `json:"hint,omitempty"` + ID string `json:"id"` + Label string `json:"label"` + Hint string `json:"hint,omitempty"` + // Icon overrides the glyph the Activity Monitor draws in this column's + // chevrons: an allowlisted id from ColumnIconIDs or a single emoji; + // empty picks the icon by the column's label (hivecommons/hive#18033). + Icon string `json:"icon,omitempty"` States []string `json:"states"` Terminal bool `json:"terminal,omitempty"` } diff --git a/src/pkg/workflowmodel/model.go b/src/pkg/workflowmodel/model.go index 7ec6c90386..4d30b96249 100644 --- a/src/pkg/workflowmodel/model.go +++ b/src/pkg/workflowmodel/model.go @@ -341,6 +341,11 @@ func (m *Model) Validate() error { if m.Spec.Scope.Repository == "" { fail("spec.scope.repository is empty") } + for _, c := range m.Spec.Columns { + if err := ValidateColumnIcon(c.Icon); err != nil { + fail("column %q: %v", c.ID, err) + } + } ids := map[string]bool{} var walk func(states []State, parallelParent bool) walk = func(states []State, parallelParent bool) { From ad8f8e935e55cb777ecd1e5c167892dd36319afa Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:16:50 +0000 Subject: [PATCH 138/166] [scanner] fix: dedupe expected agent role claim rejection log (#18076) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- changelog.d/fixed-role-claim-reject-log.md | 1 + src/pkg/dashboard/contribute_ws.go | 27 ++++++++++++++++--- .../contribute_ws_role_reject_log_test.go | 25 +++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-role-claim-reject-log.md create mode 100644 src/pkg/dashboard/contribute_ws_role_reject_log_test.go diff --git a/changelog.d/fixed-role-claim-reject-log.md b/changelog.d/fixed-role-claim-reject-log.md new file mode 100644 index 0000000000..e5ca0de6d8 --- /dev/null +++ b/changelog.d/fixed-role-claim-reject-log.md @@ -0,0 +1 @@ +- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index f9c4e86eed..4023c44b25 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -1089,6 +1089,9 @@ type ContributeWSHub struct { // stays bounded. Guarded by rateMu. assignmentTimes map[string][]time.Time rateMu sync.Mutex + // roleRejectLogged dedupes the "agent role claim rejected" log per + // (user, role) so a client retrying a forbidden role does not flood it. + roleRejectLogged sync.Map // map[string]time.Time // contributorFailureStreaks tracks, per identity, CONSECUTIVE hub-measured // sub-minute task failures (kubestellar/hive#6450, contribute_failure_streak.go). // Once a streak reaches contributorFailureStreakThreshold, selectTask pauses @@ -2843,9 +2846,7 @@ func (s *wsSession) handleAuthResponse(msg WSMessage) (stop bool) { if requestedRole != "" { if ok, reason := h.roleClaimAllowed(probeContributor, requestedRole); !ok { _ = sendJSON(s.conn, WSMessage{Type: "auth_failed", Reason: reason, Role: requestedRole}) - h.logger.Warn("[contribute-ws] agent role claim rejected", - "username", profile.GitHubUsername, "tier", profile.TrustTier, - "role", requestedRole, "reason", reason) + h.logRoleClaimRejected(profile.GitHubUsername, profile.TrustTier, requestedRole, reason) h.auditRelayAs(profile.GitHubUsername, "", agentaudit.AuditRelayRejected, nil, "reason", "role_claim_rejected") closeWithReason(s.conn, websocket.ClosePolicyViolation, "agent role claim rejected") return true @@ -5196,3 +5197,23 @@ func contributorUsername(c *ContributorConnection) string { } return c.profile.GitHubUsername } + +// roleRejectLogWindow is how long a repeated identical role-claim rejection is +// suppressed from the log. +const roleRejectLogWindow = 10 * time.Minute + +// logRoleClaimRejected logs an expected trust-tier role rejection at Info (it is +// a policy decision, not a hub error) and at most once per (user, role) per +// roleRejectLogWindow. The authorization decision is unchanged. +func (h *ContributeWSHub) logRoleClaimRejected(username, tier, role, reason string) { + key := username + "\x00" + role + now := time.Now() + if last, ok := h.roleRejectLogged.Load(key); ok { + if t, _ := last.(time.Time); now.Sub(t) < roleRejectLogWindow { + return + } + } + h.roleRejectLogged.Store(key, now) + h.logger.Info("[contribute-ws] agent role claim rejected", + "username", username, "tier", tier, "role", role, "reason", reason) +} diff --git a/src/pkg/dashboard/contribute_ws_role_reject_log_test.go b/src/pkg/dashboard/contribute_ws_role_reject_log_test.go new file mode 100644 index 0000000000..facb67851b --- /dev/null +++ b/src/pkg/dashboard/contribute_ws_role_reject_log_test.go @@ -0,0 +1,25 @@ +package dashboard + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +// A client retrying a forbidden role must log once per (user, role), at Info. +func TestLogRoleClaimRejectedDedupes(t *testing.T) { + var buf bytes.Buffer + h := &ContributeWSHub{logger: slog.New(slog.NewTextHandler(&buf, nil))} + for i := 0; i < 50; i++ { + h.logRoleClaimRejected("alice", "advisor", "scanner", "nope") + } + h.logRoleClaimRejected("alice", "advisor", "quality", "nope") + out := buf.String() + if n := strings.Count(out, "agent role claim rejected"); n != 2 { + t.Fatalf("got %d log lines, want 2:\n%s", n, out) + } + if strings.Contains(out, "level=WARN") { + t.Fatalf("expected rejection must not log at WARN:\n%s", out) + } +} From 856055f230bcbdfc2de03fc684f430d2f4c36b0e Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:17:39 -0500 Subject: [PATCH 139/166] =?UTF-8?q?=F0=9F=8C=B1=20test(dashboard):=20recei?= =?UTF-8?q?ver=20accepts=20a=20delivery=20signed=20with=20the=20file-backe?= =?UTF-8?q?d=20webhook=20secret=20(#18082)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #18024 added github.WebhookSecret() (env, else /data/github-webhook-secret.key) and switched the receiver to it, with unit tests in pkg/github. The receiver-level case the issue asked for was missing: with only the secret file present, a correctly signed delivery is accepted (no restart), a wrong signature is rejected, and the env var still wins when both are set. Hive-Run: hivecommons/hive#18025 Hive-Plan: hivecommons/hive#18017 child 1 of 5 Hive-Spec: hivecommons/hive#18025#tests Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../api_github_webhook_secret_file_test.go | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/pkg/dashboard/api_github_webhook_secret_file_test.go diff --git a/src/pkg/dashboard/api_github_webhook_secret_file_test.go b/src/pkg/dashboard/api_github_webhook_secret_file_test.go new file mode 100644 index 0000000000..ebc7cbb7bd --- /dev/null +++ b/src/pkg/dashboard/api_github_webhook_secret_file_test.go @@ -0,0 +1,59 @@ +package dashboard + +import ( + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/github" +) + +// TestGitHubWebhookReceiverAcceptsFileSecret: with no GITHUB_WEBHOOK_SECRET, +// the receiver verifies deliveries against the secret file the owner wrote +// (/data/github-webhook-secret.key in production), with no restart; the env +// var still wins when both are present (hivecommons/hive#18025). +func TestGitHubWebhookReceiverAcceptsFileSecret(t *testing.T) { + const fileSecret = "from-the-file" + t.Setenv(githubWebhookSecretEnvVar, "") + path := filepath.Join(t.TempDir(), "github-webhook-secret.key") + t.Cleanup(github.SetWebhookSecretPathForTest(path)) + t.Cleanup(github.SetWebhookRelayKeyPathForTest(filepath.Join(t.TempDir(), "relay.key"))) + s := newGitHubWebhookTestServer(t) + body := []byte(`{"number":5,"repository":{"full_name":"someone/else"}}`) + + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no env, no file: status = %d, want 503", rec.Code) + } + + if err := os.WriteFile(path, []byte(fileSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + // The receiver read (and cached) "no secret" above; a fresh store load + // stands in for the dashboard writing the file through the package. + t.Cleanup(github.SetWebhookSecretPathForTest(path)) + + if got := github.WebhookSecretSource(); got != github.WebhookSecretSourceFile { + t.Fatalf("source = %q, want file", got) + } + if got := github.WebhookVerification(); got != github.WebhookVerificationGitHub { + t.Fatalf("verification = %q, want %q", got, github.WebhookVerificationGitHub) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusOK { + t.Fatalf("file secret, correctly signed: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("wrong", body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("file secret, wrong signature: status = %d, want 401", rec.Code) + } + + t.Setenv(githubWebhookSecretEnvVar, "from-the-env") + if got := github.WebhookSecretSource(); got != github.WebhookSecretSourceEnv { + t.Fatalf("source with env set = %q, want env", got) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest(fileSecret, body), body); rec.Code != http.StatusUnauthorized { + t.Fatalf("env set, signed with file secret: status = %d, want 401", rec.Code) + } + if rec := postGitHubWebhook(s, "pull_request", signGitHubWebhookForTest("from-the-env", body), body); rec.Code != http.StatusOK { + t.Fatalf("env set, signed with env secret: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } +} From 11b4355affc736578c94081f1d63baa77c630b52 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:31:14 -0500 Subject: [PATCH 140/166] =?UTF-8?q?=F0=9F=8C=B1=20kicks:=20register=20?= =?UTF-8?q?=F0=9F=93=9A=20REFERENCE=20REPOS=20as=20layout=20section=2010a?= =?UTF-8?q?=20and=20golden=20its=20wording=20(#18074)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 kicks: register 📚 REFERENCE REPOS as layout section 10a and golden its wording The reference-repos kick block and its in-scope rule shipped in #18023; this registers it in the kick layout map next to section 10 and pins the wording the scanner's scope classification depends on: - kick_layout.go: document 10a (inside AUTHORIZED REPOS, before the ⛔ scope lines, so "repos not listed above" covers reference repos). - golden for the block with a read-only and an issues+prs entry, plus per-tier labels (read-only, issues, prs, issues+prs). - the cncf/techdocs#395 shape: an item citing only reference repos is listed above the scope line and told it is in scope. - assembled-kick order: reference block after AUTHORIZED REPOS and ahead of the dynamic tail. Closes #18020 Hive-Run: hivecommons/hive#18020 Hive-Plan: hivecommons/hive#18015 child 3 of 5 (kick reference block + in-scope classification) Hive-Spec: reference-repos#kick-wording Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * changelog: fragment for #18074 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/changed-18074.md | 1 + src/pkg/scheduler/agent_repo_scope_test.go | 100 +++++++++++++++++++++ src/pkg/scheduler/kick_layout.go | 3 + src/pkg/scheduler/kick_layout_test.go | 32 +++++++ 4 files changed, 136 insertions(+) create mode 100644 changelog.d/changed-18074.md diff --git a/changelog.d/changed-18074.md b/changelog.d/changed-18074.md new file mode 100644 index 0000000000..beeb0a4829 --- /dev/null +++ b/changelog.d/changed-18074.md @@ -0,0 +1 @@ +- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) diff --git a/src/pkg/scheduler/agent_repo_scope_test.go b/src/pkg/scheduler/agent_repo_scope_test.go index 40681457a0..4d066fedf2 100644 --- a/src/pkg/scheduler/agent_repo_scope_test.go +++ b/src/pkg/scheduler/agent_repo_scope_test.go @@ -270,3 +270,103 @@ func TestBuildReposSectionFor_ReferenceRepos(t *testing.T) { t.Error("reference block emitted with no reference repos configured") } } + +// referenceReposGolden is the exact 📚 REFERENCE REPOS block for one +// read-only entry and one issues+prs entry (hivecommons/hive#18020). It is a +// golden on purpose: this wording is the only thing that tells the scanner an +// item citing a reference repo is in scope, so a change to it must be a +// deliberate edit here, not a side effect. +const referenceReposGolden = "📚 REFERENCE REPOS (outside this project — study them, do not work them):\n" + + " https://github.com/hivecommons/docs [reference · read-only]\n" + + " https://github.com/hivecommons/hive [reference · issues+prs]\n" + + " You MAY clone them (public, unauthenticated), read their files, and GET their issues/PRs/API. " + + "An authorized-repo item whose deliverables name a reference repo is IN SCOPE, not a scope violation: " + + "do NOT classify it out-of-scope, critical, or a security concern. " + + "Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo (the assessment document, the report), " + + "never as issues or PRs on the reference repo itself — unless its tier above says issues/prs, in which case you may ONLY open an issue, " + + "open a PR from your own branch, or comment on an issue/PR you opened; the proxy refuses merges, default-branch pushes, labels, assignments, " + + "reviews and closing others' items deterministically, so retrying cannot succeed.\n" + +func TestBuildReposSectionFor_ReferenceReposGolden(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + section := s.buildReposSectionFor("scanner") + start := strings.Index(section, "📚 REFERENCE REPOS") + end := strings.Index(section, "⛔ NEVER access") + if start < 0 || end < 0 || end < start { + t.Fatalf("reference block must precede the ⛔ scope line (start=%d end=%d):\n%s", start, end, section) + } + if got := section[start:end]; got != referenceReposGolden { + t.Errorf("reference block drifted from the golden:\n--- got\n%s\n--- want\n%s", got, referenceReposGolden) + } + // It belongs to the scope list, after the authorized repos: "repos not + // listed above" must include the reference repos. + if auth := strings.Index(section, "AUTHORIZED REPOS"); auth < 0 || auth > start { + t.Errorf("reference block must follow AUTHORIZED REPOS:\n%s", section) + } +} + +// Each tier renders on its own: a read-only-only hive never advertises a +// write tier, and an issues-only or prs-only entry prints exactly that tier. +func TestBuildReposSectionFor_ReferenceRepoTiers(t *testing.T) { + for _, tc := range []struct { + write []string + label string + }{ + {nil, "read-only"}, + {[]string{"issues"}, "issues"}, + {[]string{"prs"}, "prs"}, + {[]string{"prs", "issues"}, "issues+prs"}, + } { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{{Repo: "cncf/landscape", Write: tc.write}} + section := s.buildReposSectionFor("scanner") + want := " https://github.com/cncf/landscape [reference · " + tc.label + "]\n" + if !strings.Contains(section, want) { + t.Errorf("write=%v: %q missing:\n%s", tc.write, want, section) + } + if !strings.Contains(section, "IN SCOPE, not a scope violation") { + t.Errorf("write=%v: in-scope rule missing:\n%s", tc.write, section) + } + } +} + +// The cncf/techdocs#395 shape: the hive's only authorized repo is the docs +// repo, and the item's deliverables cite reference repos only. The kick the +// scanner reads lists those repos inside the scope block and tells it the +// item is in scope — the scope classification is the scanner's, driven by +// this text, so this is the classifier contract. +func TestBuildReposSectionFor_ItemCitingOnlyReferenceReposIsInScope(t *testing.T) { + s := scopeScheduler(t, map[string]config.AgentConfig{"scanner": {Backend: "claude"}}) + s.cfg.Project.Org = "cncf" + s.cfg.Project.Repos = []string{"techdocs"} + s.cfg.Project.PrimaryRepo = "techdocs" + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hivecommons.github.io"}, + {Repo: "hivecommons/hive"}, + } + section := s.buildReposSectionFor("scanner") + never := strings.Index(section, "⛔ NEVER access") + if never < 0 { + t.Fatalf("scope line missing:\n%s", section) + } + for _, cited := range []string{"hivecommons/docs", "hivecommons/hivecommons.github.io", "hivecommons/hive"} { + at := strings.Index(section, "https://github.com/"+cited+" [reference · read-only]") + if at < 0 || at > never { + t.Errorf("%s must be listed above the ⛔ scope line so it is not 'a repo not listed above':\n%s", cited, section) + } + } + for _, want := range []string{ + "https://github.com/cncf/techdocs\n", + "do NOT classify it out-of-scope, critical, or a security concern", + "Findings about a reference repo belong in the deliverable inside the AUTHORIZED repo", + } { + if !strings.Contains(section, want) { + t.Errorf("%q missing:\n%s", want, section) + } + } +} diff --git a/src/pkg/scheduler/kick_layout.go b/src/pkg/scheduler/kick_layout.go index a755a06a7d..8be79d0629 100644 --- a/src/pkg/scheduler/kick_layout.go +++ b/src/pkg/scheduler/kick_layout.go @@ -25,6 +25,9 @@ import ( // 8. <resolved policy body> static text, then ${ISSUE_LIST}/${PR_LIST}/… where the template puts them // 9. REPO-SCOPED CADENCE TARGET semi-static (repo-scoped kicks only) // 10. AUTHORIZED REPOS semi-static (include_repos) +// 10a. 📚 REFERENCE REPOS semi-static (project.reference_repos; rendered inside 10, after +// the authorized/paused lists and before its ⛔ scope lines, so +// "repos not listed above" covers them — #18015/#18020) // 11. hold-gated PR coordination dynamic (held PR snapshot) // 12. red-PR fix-before-new dynamic (ci-failing.json) // 13. review-thread fix-before-new dynamic (review-threads.json) diff --git a/src/pkg/scheduler/kick_layout_test.go b/src/pkg/scheduler/kick_layout_test.go index 971ef98432..db7e0955e1 100644 --- a/src/pkg/scheduler/kick_layout_test.go +++ b/src/pkg/scheduler/kick_layout_test.go @@ -268,3 +268,35 @@ func TestKickLayoutManualKickSharesPrefix(t *testing.T) { t.Errorf("manual kick must not grow the repos block:\n%s", manual) } } + +// Section 10a: project.reference_repos render inside the AUTHORIZED REPOS +// section of the assembled kick, ahead of the dynamic tail +// (hivecommons/hive#18020). +func TestKickLayoutReferenceReposFollowAuthorizedRepos(t *testing.T) { + s := kickLayoutScheduler(t) + s.cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{"issues", "prs"}}, + } + msg := kickLayoutMessage(t, s, makeIssue("test-org/console", 1, "first issue", "scanner", 5, nil, false)) + order := []string{ + "AUTHORIZED REPOS", + "📚 REFERENCE REPOS", + "[reference · read-only]", + "[reference · issues+prs]", + "⛔ NEVER access", + "## Open hold-gated PR coordination", + concreteKickClosingInstruction, + } + last := -1 + for _, want := range order { + at := strings.Index(msg, want) + if at < 0 { + t.Fatalf("kick missing %q:\n%s", want, msg) + } + if at < last { + t.Errorf("%q at %d is out of documented order (previous at %d):\n%s", want, at, last, msg) + } + last = at + } +} From 57782546155b3af9488c3a876619c53cfff14b61 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:33:27 -0500 Subject: [PATCH 141/166] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20gh=202.?= =?UTF-8?q?103.3=20->=202.104.0=20(#18063)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌱 images: bump gh 2.103.3 -> 2.104.0 Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.104.0 SHA256_AMD64=98cb101ad87dbbd3f6b2f2bfcbd43197ccd07643140121d8a9cfaa91eadbc24e SHA256_ARM64=cb34cefc6f4e77b90731374a2b737516bb8f3033c6ca46aa949e5344f68069e3 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> * changelog: fragment for #18063 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> --------- Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- changelog.d/changed-18063.md | 1 + src/Dockerfile | 6 +++--- src/deploy/ci-runners/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/changed-18063.md diff --git a/changelog.d/changed-18063.md b/changelog.d/changed-18063.md new file mode 100644 index 0000000000..8aa025ec82 --- /dev/null +++ b/changelog.d/changed-18063.md @@ -0,0 +1 @@ +- images: bump gh 2.103.3 -> 2.104.0 (#18063) diff --git a/src/Dockerfile b/src/Dockerfile index 70476b97bc..274c7caf41 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -245,9 +245,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ chmod +x /usr/local/bin/ttyd # --- Layer 2b: GitHub CLI (off-PATH — only snapshot publisher uses it) --- -ARG GH_VERSION=2.103.3 -ARG GH_SHA256_AMD64=4c806978ffce1cec70853302a56d70ac6da4f19bae7adca4e1a8ab00633dd91d -ARG GH_SHA256_ARM64=ee32df2323c8847f5fac6cb6b4b69ca75828e9329a8f22f677021f7b3306ce63 +ARG GH_VERSION=2.104.0 +ARG GH_SHA256_AMD64=98cb101ad87dbbd3f6b2f2bfcbd43197ccd07643140121d8a9cfaa91eadbc24e +ARG GH_SHA256_ARM64=cb34cefc6f4e77b90731374a2b737516bb8f3033c6ca46aa949e5344f68069e3 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ case "$ARCH" in \ amd64) GH_ARCH=amd64; GH_SHA256="$GH_SHA256_AMD64" ;; \ diff --git a/src/deploy/ci-runners/Dockerfile b/src/deploy/ci-runners/Dockerfile index 73c205c0a8..e9628381ab 100644 --- a/src/deploy/ci-runners/Dockerfile +++ b/src/deploy/ci-runners/Dockerfile @@ -48,7 +48,7 @@ ARG RUNNER_BASE_IMAGE=summerwind/actions-runner:v2.337.0-ubuntu-24.04 FROM ${RUNNER_BASE_IMAGE} -ARG GITHUB_CLI_VERSION=2.103.3 +ARG GITHUB_CLI_VERSION=2.104.0 ARG GITHUB_CLI_KEYRING_SHA256=6084d5d7bd8e288441e0e94fc6275570895da18e6751f70f057485dc2d1a811b # The base image runs as uid 1001 (`runner`), which does not own /etc/apt. From 3b7e61819d5266f763d71301babe804bc93c8248 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 16:41:56 +0000 Subject: [PATCH 142/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.241.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.241.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 7 +++++++ changelog.d/changed-18029-webhooks-card-permission-hint.md | 1 - changelog.d/changed-18073.md | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/changed-18029-webhooks-card-permission-hint.md delete mode 100644 changelog.d/changed-18073.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 03461444e8..5290f196c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.241.1) + +### Changed + +- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. +- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) + ## 2026-11-13 (v5.241.0) ### Added diff --git a/changelog.d/changed-18029-webhooks-card-permission-hint.md b/changelog.d/changed-18029-webhooks-card-permission-hint.md deleted file mode 100644 index 8b89d7cf4a..0000000000 --- a/changelog.d/changed-18029-webhooks-card-permission-hint.md +++ /dev/null @@ -1 +0,0 @@ -- **Settings → Forge App → GitHub webhooks names the App permission to grant** (#18029): when the GitHub App installation cannot manage hooks, the card now says which permission to grant (Organization → Webhooks `organization_hooks`, or Repository → Webhooks `repository_hooks`, Read and write) so **Create webhook for me** appears, instead of silently omitting the option. Node-executed tests pin the card's copy for relay, none and env-secret modes, and a parity test checks every endpoint the card calls is routed and owner-only. diff --git a/changelog.d/changed-18073.md b/changelog.d/changed-18073.md deleted file mode 100644 index 82b08a446e..0000000000 --- a/changelog.d/changed-18073.md +++ /dev/null @@ -1 +0,0 @@ -- docs: project.reference_repos in agent-configuration and hive.yaml.example (#18073) From 1b1c8210f5fdd6162e6f5be06428c1a945d003a8 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:53:29 +0000 Subject: [PATCH 143/166] =?UTF-8?q?=E2=9C=A8=20review:=20resolve=20perspec?= =?UTF-8?q?tives=5Fsource=20at=20startup=20and=20reload=20and=20reinstall?= =?UTF-8?q?=20the=20relay=20perspective=20set=20(#18035)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...anged-review-perspectives-source-reload.md | 1 + src/cmd/hive/boot.go | 2 + src/cmd/hive/main.go | 18 +++- src/cmd/hive/main_sweeps.go | 10 ++ .../hive/review_perspectives_source_test.go | 101 ++++++++++++++++++ src/pkg/review/review_heads_decision_test.go | 37 +++++++ 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 changelog.d/changed-review-perspectives-source-reload.md create mode 100644 src/cmd/hive/review_perspectives_source_test.go diff --git a/changelog.d/changed-review-perspectives-source-reload.md b/changelog.d/changed-review-perspectives-source-reload.md new file mode 100644 index 0000000000..c517a8ac1f --- /dev/null +++ b/changelog.d/changed-review-perspectives-source-reload.md @@ -0,0 +1 @@ +- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 91d87cde38..99ed7f91c1 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -31,6 +31,7 @@ import ( "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/priorart" "github.com/hivecommons/hive/pkg/promptsrc" @@ -190,6 +191,7 @@ type boot struct { promptFetcher promptsrc.Fetcher defFetcher defsrc.Fetcher definitionResolver *defsrc.Resolver + perspectivesResolver *perspsrc.Resolver pendingTokenSeed []dashboard.TokenSparklineEntry pendingFactSeed []dashboard.FactHistoryEntry pendingCostSeed []dashboard.CostHistoryEntry diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index e58fe9c42a..68a979c6e3 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -61,6 +61,7 @@ import ( "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/maintainernotes" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/policies" "github.com/hivecommons/hive/pkg/promptsrc" @@ -1311,7 +1312,9 @@ func (b *boot) wireBootClosures() { if b.ghClient == nil { return } - installReviewRelaySettings(b.ghClient, b.cfg, b.logger) + // LinkedPerspectives is runtime-only, so re-resolve it before + // the relay reinstall or a dashboard PUT would drop the link. + applyReviewPerspectivesSource(context.Background(), b.cfg, b.perspectivesResolver, b.ghClient, b.logger) // review_bots.min_priority and fix_human_prs are edited // through the same PUT; reinstall so the thread reconciler // applies them without a restart (#10481). @@ -1473,6 +1476,18 @@ func (b *boot) bootGitHubWith(deps bootGitHubDeps) { // The same function every rebuild path uses (#9614). Only the tiers whose // dependencies exist yet apply here; bootAdvisory, bootAgents and // bootDashboard apply the rest as those dependencies come up. + // Resolve review.perspectives_source before the first relay install so + // the relay never boots on a set the link has since replaced. + var persFetcher defsrc.Fetcher + if b.ghClient != nil { + persFetcher = b.ghClient + } + b.perspectivesResolver = perspsrc.NewResolver( + persFetcher, + func(slug string) bool { return b.cfg.GitHubPromptAllowed(slug) }, + b.logger, + ) + perspsrc.ApplyToConfig(b.ctx, b.cfg, b.perspectivesResolver, b.logger) b.configureGitHubClient(b.ghClient) // Long-lived consumers read the client through this provider, never a // captured b.ghClient, so every rebuild reaches them (#9621). @@ -3909,6 +3924,7 @@ func (b *boot) bootWatchersWith(deps bootWatchersDeps) { // fields; a fetch failure keeps each agent's baked definition. Runs before // initAgentConfigDrivenSystems so downstream systems see the merged config. defsrc.ApplyToConfig(context.Background(), b.cfg, b.definitionResolver, b.logger) + applyReviewPerspectivesSource(context.Background(), b.cfg, b.perspectivesResolver, b.ghClient, b.logger) if err := b.cfg.ExpandAgentReplicas(); err != nil { b.logger.Warn("failed to expand agent replicas after config reload", "error", err) } diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 7487b7007d..eebae1bf67 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -26,6 +26,7 @@ import ( "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hooks" "github.com/hivecommons/hive/pkg/notify" + "github.com/hivecommons/hive/pkg/perspsrc" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/rotation" @@ -1988,6 +1989,15 @@ func reviewAuthorBase(cfg *config.Config, agent string) string { return agent } +// applyReviewPerspectivesSource re-resolves review.perspectives_source and +// reinstalls the relay settings, so the relay validates verdicts against the +// set the link now names rather than the one it booted with. With no source +// set it fetches nothing and the inline set stands. +func applyReviewPerspectivesSource(ctx context.Context, cfg *config.Config, resolver *perspsrc.Resolver, client *github.Client, logger *slog.Logger) { + perspsrc.ApplyToConfig(ctx, cfg, resolver, logger) + installReviewRelaySettings(client, cfg, logger) +} + // installReviewRelaySettings gives a (possibly rebuilt) GitHub client the // review-relay knobs that live in cfg.Review: which repos may be revised in // place, which perspectives a verdict may name, and whether comments carry a diff --git a/src/cmd/hive/review_perspectives_source_test.go b/src/cmd/hive/review_perspectives_source_test.go new file mode 100644 index 0000000000..d9ae106839 --- /dev/null +++ b/src/cmd/hive/review_perspectives_source_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/perspsrc" + "github.com/hivecommons/hive/pkg/review" +) + +type fakePerspectivesFetcher struct { + doc string + err error + calls int +} + +func (f *fakePerspectivesFetcher) GetFileContentRef(_ context.Context, _, _, _, _ string) (string, error) { + f.calls++ + return f.doc, f.err +} + +const ( + perspectivesDocA = "kind: ReviewPerspectives\nperspectives:\n - name: correctness\n - name: spyre-numerics\n focus: Check numerics on the Spyre path.\n" + perspectivesDocB = "kind: ReviewPerspectives\nperspectives:\n - name: correctness\n" +) + +func perspectivesSourceFixture(f *fakePerspectivesFetcher) (*config.Config, *perspsrc.Resolver, *github.Client, *slog.Logger) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{} + cfg.Review.PerspectivesSource = &config.DefinitionSourceConfig{Owner: "acme", Repo: "review", Path: "perspectives.yaml"} + resolver := perspsrc.NewResolver(f, func(string) bool { return true }, logger) + client := github.NewClientForTest("http://127.0.0.1:0", "acme", nil, logger) + return cfg, resolver, client, logger +} + +func relayAccepts(cfg *config.Config, logger *slog.Logger, name string) bool { + return reviewPerspectiveSet(cfg, logger).Known(review.Perspective(name)) +} + +func TestApplyReviewPerspectivesSourceReinstallsRelaySet(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("relay set should accept spyre-numerics from doc A") + } + + f.doc = perspectivesDocB + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("relay set still accepts spyre-numerics after doc B removed it") + } + if !relayAccepts(cfg, logger, "correctness") { + t.Fatal("relay set lost correctness") + } +} + +func TestApplyReviewPerspectivesSourceMalformedKeepsRelaySet(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + + f.doc = "kind: Nope\n" + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("malformed document dropped the last-known-good set") + } + if cfg.Review.LinkedPerspectives == nil || cfg.Review.LinkedPerspectives.Warning == "" { + t.Fatalf("want a warning on the kept set, got %+v", cfg.Review.LinkedPerspectives) + } + + f.doc, f.err = "", errors.New("boom") + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if !relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("fetch failure dropped the last-known-good set") + } +} + +func TestApplyReviewPerspectivesSourceUnsetMakesNoFetch(t *testing.T) { + f := &fakePerspectivesFetcher{doc: perspectivesDocA} + cfg, resolver, client, logger := perspectivesSourceFixture(f) + cfg.Review.PerspectivesSource = nil + cfg.Review.Perspectives = []string{"correctness", "security"} + + applyReviewPerspectivesSource(context.Background(), cfg, resolver, client, logger) + if f.calls != 0 { + t.Fatalf("unset perspectives_source fetched %d times", f.calls) + } + if cfg.Review.LinkedPerspectives != nil { + t.Fatalf("LinkedPerspectives = %+v, want nil", cfg.Review.LinkedPerspectives) + } + if !relayAccepts(cfg, logger, "security") || relayAccepts(cfg, logger, "spyre-numerics") { + t.Fatal("boot relay set changed") + } +} diff --git a/src/pkg/review/review_heads_decision_test.go b/src/pkg/review/review_heads_decision_test.go index 5f5dd86801..46112870dc 100644 --- a/src/pkg/review/review_heads_decision_test.go +++ b/src/pkg/review/review_heads_decision_test.go @@ -362,6 +362,43 @@ func TestRemovedPerspectiveDoesNotBlockHeads(t *testing.T) { } } +// A set that changes through a perspectives_source link is built by +// NewPerspectiveSet from the resolved document; the removed perspective must +// then neither block waiting heads nor count as coverage (#14905, #17988). +func TestLinkedPerspectiveRemovalSettlesWaitingHeads(t *testing.T) { + docB, err := NewPerspectiveSet([]string{"correctness", "security"}, nil) + if err != nil { + t.Fatal(err) + } + opts := AggregateOptions{Perspectives: docB, ExpectedHeads: []string{"cr-sol", "cr-sonnet"}} + linked := Perspective("spyre-numerics") + reports := []PerspectiveReport{ + headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), + headRep(PerspectiveCorrectness, "cr-sonnet", VerdictApprove), + headRep(PerspectiveSecurity, "cr-sol", VerdictApprove), + headRep(PerspectiveSecurity, "cr-sonnet", VerdictApprove), + headRep(linked, "cr-sol", VerdictChangesRequested, nilDeref), + } + agg := AggregateReports(reports, opts) + if !agg.AwaitingDecision || agg.DecisionSkipped { + t.Fatalf("aggregate = %+v, want awaiting decision", agg) + } + for _, p := range decisionPerspectives(agg, opts.Perspectives) { + if p == linked { + t.Fatalf("decision includes removed linked perspective: %+v", agg) + } + } + + partial := AggregateReports([]PerspectiveReport{ + headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), + headRep(PerspectiveCorrectness, "cr-sonnet", VerdictApprove), + headRep(linked, "cr-sol", VerdictApprove), + }, opts) + if partial.Verdict != VerdictIncomplete || partial.DecisionSkipped || partial.AwaitingDecision { + t.Fatalf("aggregate = %+v, want incomplete configured coverage", partial) + } +} + func TestRemovedPerspectiveDoesNotCountAsCoverage(t *testing.T) { agg := AggregateReports([]PerspectiveReport{ headRep(PerspectiveCorrectness, "cr-sol", VerdictApprove), From 717597f71e99f6a73c1df93bde30b18118eb0bc2 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 11:53:43 -0500 Subject: [PATCH 144/166] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20claude?= =?UTF-8?q?=202.1.322=20->=202.1.323=20(#18065)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=2.1.323 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 274c7caf41..e802b98fed 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -342,7 +342,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_V echo "WARN: OpenAI Codex CLI install failed — skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.322 +ARG CLAUDE_CODE_VERSION=2.1.323 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 0b4f73bf04..bc5e3349ad 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -239,7 +239,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.322 +ARG CLAUDE_CODE_VERSION=2.1.323 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform From 0ae4ff788aff7534e54b1895b478cdbc65f13f79 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:53:56 -0500 Subject: [PATCH 145/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20reference=5Fti?= =?UTF-8?q?er=20on=20/api/status=20reference=20repos=20(#18021)=20(#18075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ dashboard: reference_tier on /api/status reference repos (#18021) project.reference_repos entries in /api/status referenceRepos[] gain an additive reference_tier field (read-only / issues / prs / issues+prs); the existing tier field is kept for older clients and the repos card now prefers reference_tier. A test pins that /api/status reports the tier for both a read-only and an issues+prs entry, that neither leaks into repos[] or the contribute enable/disable list, and that no ACMM, work-item or pause field is present on a reference entry. Docs describe the field; citations re-pointed after the server.go shift. Hive-Run: hivecommons/hive#18021 Hive-Plan: hivecommons/hive#18015 child 4 of 5 (dashboard) Hive-Spec: project.reference_repos#dashboard Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * ✅ dashboard: make the #18021 reference-leak assertions able to fail The test checked buildRepos and contributeAvailableRepos output with IsReferenceRepo, but newFullServer only manages testorg/testrepo and config validation rejects a repo listed in both project.repos and project.reference_repos, so the checks could never trip. Pin the exact managed set instead; a builder that starts reading reference_repos now fails the test (verified by injecting a reference row). Addresses the P3 review follow-up on hivecommons/hive#18075. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- changelog.d/added-18021-reference-tier.md | 1 + src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 ++--- src/docs/api-reference/snapshots-and-style.md | 2 +- .../reference_repos_status_18021_test.go | 81 +++++++++++++++++++ src/pkg/dashboard/server.go | 4 + .../dashboard/static-src/js/repos-render.js | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/dashboard/status_builder.go | 3 +- 10 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 changelog.d/added-18021-reference-tier.md create mode 100644 src/pkg/dashboard/reference_repos_status_18021_test.go diff --git a/changelog.d/added-18021-reference-tier.md b/changelog.d/added-18021-reference-tier.md new file mode 100644 index 0000000000..358b39b129 --- /dev/null +++ b/changelog.d/added-18021-reference-tier.md @@ -0,0 +1 @@ +- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 07ab917f90..a332a30544 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1704` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1705` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1706` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1708` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1709` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1710` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 0b4e24ec7e..5d10bcbcba 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1711` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1715` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index f26adaa4ae..7d08eb0603 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1684` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1685` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1686` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1692` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1697` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1698` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1699` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1700` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1701` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1702` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1703` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1688` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1689` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1690` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1696` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `referenceRepos[]` `{full, tier, reference_tier}` lists `project.reference_repos` apart from `repos[]` — `reference_tier` is `read-only`, `issues`, `prs` or `issues+prs` (`tier` is the same value, kept for older clients) and an entry carries no ACMM, work-item or pause fields, since a reference repo is studied, not worked (#18021); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1701` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1702` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1703` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1704` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1705` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1706` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1707` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 8dde400125..714349077f 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1774` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1778` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/pkg/dashboard/reference_repos_status_18021_test.go b/src/pkg/dashboard/reference_repos_status_18021_test.go new file mode 100644 index 0000000000..ca6dde9572 --- /dev/null +++ b/src/pkg/dashboard/reference_repos_status_18021_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "slices" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +// hivecommons/hive#18021: /api/status lists project.reference_repos apart +// from the managed repos, each with a reference_tier, and with none of the +// ACMM, work-item or pause fields a managed repo row carries. +func TestStatusReferenceReposCarryTierAndNoManagedFields(t *testing.T) { + srv := newFullServer(t) + cfg := srv.deps.Config + cfg.Project.ReferenceRepos = []config.ReferenceRepo{ + {Repo: "hivecommons/docs"}, + {Repo: "hivecommons/hive", Write: []string{config.ReferenceWriteIssues, config.ReferenceWritePRs}}, + } + + // Config validation rejects a repo in both project.repos and + // project.reference_repos, so a reference repo can only leak if the + // builders start reading reference_repos. Pin the exact managed set + // (newFullServer's testorg/testrepo) so that regression fails here. + wantManaged := []string{"testorg/testrepo"} + + repos := buildRepos(cfg, nil, governor.State{}) + var managed []string + for _, r := range repos { + managed = append(managed, r.Full) + } + if !slices.Equal(managed, wantManaged) { + t.Fatalf("managed repos list = %v, want exactly %v (reference repos must not appear)", managed, wantManaged) + } + srv.UpdateStatus(&StatusPayload{Repos: repos, ReferenceRepos: buildReferenceRepos(cfg)}) + + if got := srv.contributeAvailableRepos(); !slices.Equal(got, wantManaged) { + t.Fatalf("contribute enable/disable list = %v, want exactly %v (reference repos must not be offered)", got, wantManaged) + } + + rec := doGet(srv, "/api/status") + if rec.Code != http.StatusOK { + t.Fatalf("/api/status = %d %s", rec.Code, rec.Body.String()) + } + var body struct { + ReferenceRepos []map[string]any `json:"referenceRepos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode /api/status: %v", err) + } + if len(body.ReferenceRepos) != 2 { + t.Fatalf("referenceRepos = %v, want both entries", body.ReferenceRepos) + } + + want := map[string]string{"hivecommons/docs": "read-only", "hivecommons/hive": "issues+prs"} + managedOnly := []string{ + "issues", "prs", "mode", "paused", "acmmLevel", "acmmLevelConfigured", + "acmmPackAgents", "acmmAdvice", "sla", "countsIncomplete", "countsStale", + } + for _, rr := range body.ReferenceRepos { + full, _ := rr["full"].(string) + tier, ok := want[full] + if !ok { + t.Fatalf("unexpected reference entry %v", rr) + } + if got := rr["reference_tier"]; got != tier { + t.Errorf("%s reference_tier = %v, want %q", full, got, tier) + } + if got := rr["tier"]; got != tier { + t.Errorf("%s tier = %v, want %q (kept for existing consumers)", full, got, tier) + } + for _, k := range managedOnly { + if _, present := rr[k]; present { + t.Errorf("%s carries managed-repo field %q: %v", full, k, rr) + } + } + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index a8eb8e7c30..a650064ffc 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1217,9 +1217,13 @@ type FrontendRestartRefresh struct { // FrontendReferenceRepo is one project.reference_repos entry for the repos // card: the slug and its tier ("read-only", "issues", "prs", "issues+prs"). +// It deliberately carries no ACMM, work-item or pause fields (#18021). type FrontendReferenceRepo struct { Full string `json:"full"` Tier string `json:"tier"` + // ReferenceTier repeats Tier under the name API consumers key on to tell + // a reference entry from a managed repo row (hivecommons/hive#18021). + ReferenceTier string `json:"reference_tier"` } type FrontendRepo struct { diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index c8f98b6833..b6f16b1201 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -365,7 +365,7 @@ const base = window._githubBaseUrl || 'https://github.com'; return refs.map(rr => { const full = String(rr.full || ''); - const tier = String(rr.tier || 'read-only'); + const tier = String(rr.reference_tier || rr.tier || 'read-only'); const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index a59be35a8e..2bc465b916 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -37143,7 +37143,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const base = window._githubBaseUrl || 'https://github.com'; return refs.map(rr => { const full = String(rr.full || ''); - const tier = String(rr.tier || 'read-only'); + const tier = String(rr.reference_tier || rr.tier || 'read-only'); const tip = tier === 'read-only' ? 'Reference repo: agents may clone and read it; every write is refused by the proxy.' : `Reference repo: agents may clone and read it, and may open ${tier.replace('+', ' and ')} (never merge, push to the default branch, label, assign, review, or close others' items).`; diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 40e031e0b0..18be7c3f88 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -2381,7 +2381,8 @@ func buildReferenceRepos(cfg *config.Config) []FrontendReferenceRepo { } out := make([]FrontendReferenceRepo, 0, len(cfg.Project.ReferenceRepos)) for _, rr := range cfg.Project.ReferenceRepos { - out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: rr.WriteLabel()}) + tier := rr.WriteLabel() + out = append(out, FrontendReferenceRepo{Full: rr.Repo, Tier: tier, ReferenceTier: tier}) } return out } From 2b63c9bdbb4e9e48121053351b9cfacea5ee28d9 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:54:30 -0500 Subject: [PATCH 146/166] =?UTF-8?q?=E2=9C=A8=20dashboard:=20idempotent=20w?= =?UTF-8?q?ebhook=20create=20through=20the=20App,=20412=20on=20missing=20h?= =?UTF-8?q?ook=20permission=20(#18086)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/config/github/webhooks/create now lists the target's hooks first and returns an existing hook with the same config.url (created: false) instead of creating a duplicate; a 422 "already exists" race resolves to the winner the same way. A missing organization_hooks / repository_hooks write permission answers 412 naming the permission. The response carries id, events, payload_url and created next to the existing fields; only an actual creation is audited. The Settings card says "Already set up" when the hook was found. Tests use an httptest GitHub stub: 409 without a secret, 412 without the permission, create on first call and return existing on second, repo scope hits the repo endpoint. Hive-Run: hivecommons/hive#18028 Hive-Plan: hivecommons/hive#18017 child 3 of 5 (webhook create endpoint) Hive-Spec: hivecommons/hive#18028#webhooks-create Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...changed-18028-webhook-create-idempotent.md | 1 + dashboard/openapi.json | 30 ++- src/docs/api-reference/configuration.md | 6 +- src/docs/webhooks.md | 4 +- src/pkg/dashboard/api_github_webhook_setup.go | 19 +- .../api_github_webhook_setup_test.go | 181 +++++++++++++++++- .../static-src/js/config-webhooks.js | 5 +- src/pkg/dashboard/static/index.html | 5 +- src/pkg/github/webhook_setup.go | 107 +++++++++-- src/pkg/github/webhook_setup_test.go | 153 +++++++++++++++ 10 files changed, 478 insertions(+), 33 deletions(-) create mode 100644 changelog.d/changed-18028-webhook-create-idempotent.md create mode 100644 src/pkg/github/webhook_setup_test.go diff --git a/changelog.d/changed-18028-webhook-create-idempotent.md b/changelog.d/changed-18028-webhook-create-idempotent.md new file mode 100644 index 0000000000..01d2c011f1 --- /dev/null +++ b/changelog.d/changed-18028-webhook-create-idempotent.md @@ -0,0 +1 @@ +- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 6167fff8a0..9d1e34573a 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -25719,7 +25719,7 @@ "System" ], "summary": "Create the org or repo webhook through the GitHub App (owner)", - "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Audited.", + "description": "Owner-only. Uses the App installation's organization_hooks or repository_hooks write permission to create a webhook on GitHub pointing at this hive's payload URL with the configured secret and event list. Requires a configured secret, a public dashboard URL and the matching permission; the repository must be one of project.repos. Idempotent: when a hook with the same payload URL already exists on the target it is returned with created=false instead of creating a duplicate. Creation is audited.", "requestBody": { "required": true, "content": { @@ -25745,15 +25745,32 @@ }, "responses": { "200": { - "description": "Webhook created", + "description": "Webhook created, or the existing hook for this payload URL", "content": { "application/json": { "schema": { "type": "object", "properties": { - "hook_id": { + "id": { "type": "integer" }, + "hook_id": { + "type": "integer", + "description": "Same as id" + }, + "created": { + "type": "boolean", + "description": "false when an existing hook for this payload URL was returned" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "payload_url": { + "type": "string" + }, "scope": { "type": "string" }, @@ -25772,10 +25789,13 @@ "description": "Invalid scope or repository" }, "403": { - "description": "Not an owner, or the App lacks the hooks permission" + "description": "Not an owner" }, "409": { - "description": "No secret / no public URL / webhook already exists" + "description": "No secret / no public URL / no project.org / no App auth" + }, + "412": { + "description": "The App installation lacks organization_hooks or repository_hooks write for the scope" }, "502": { "description": "GitHub API error" diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 02117e6006..c11dea097d 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -75,9 +75,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:314` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:315` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:318` | -| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:87` | -| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:88` | -| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `GET` | `/api/config/github/webhooks` | Owner only | Config GitHub Webhooks (payload URL, events, secret source, receiver health, App hook permissions) | `pkg/dashboard/api_github_webhook_setup.go:88` | +| `POST` | `/api/config/github/webhooks/secret` | Owner only | Config GitHub Webhooks Generate Secret (returned once) | `pkg/dashboard/api_github_webhook_setup.go:89` | +| `POST` | `/api/config/github/webhooks/create` | Owner only | Config GitHub Webhooks Create Hook (through the App installation) | `pkg/dashboard/api_github_webhook_setup.go:90` | | `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:376` | | `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:377` | | `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:378` | diff --git a/src/docs/webhooks.md b/src/docs/webhooks.md index 37865c7593..ab1d9ade5e 100644 --- a/src/docs/webhooks.md +++ b/src/docs/webhooks.md @@ -53,7 +53,9 @@ uses a secret only you hold. 2. **Create the webhook.** If the section offers **Create webhook for me**, click it: the hive's GitHub App installation holds `organization_hooks` or `repository_hooks` write permission and the hive creates the hook with the - right URL, secret and events. Otherwise do it by hand in GitHub: + right URL, secret and events. Clicking it again is safe: when a hook for + this hive's payload URL already exists it is reported, not duplicated. + Otherwise do it by hand in GitHub: organization (or repository) → Settings → Webhooks → **Add webhook**. - Payload URL: the value shown in the section, i.e. `https://<your-hive>/api/webhook/github` diff --git a/src/pkg/dashboard/api_github_webhook_setup.go b/src/pkg/dashboard/api_github_webhook_setup.go index 640157b176..4c38e68487 100644 --- a/src/pkg/dashboard/api_github_webhook_setup.go +++ b/src/pkg/dashboard/api_github_webhook_setup.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "strconv" "strings" "time" @@ -246,6 +247,9 @@ type webhookCreateBody struct { // handleConfigGitHubWebhooksCreate serves POST /api/config/github/webhooks/create // (owner-only): creates the org or repo hook through the App installation. +// 409 without a webhook secret, 412 when the installation lacks the hooks +// write permission for the scope; when a hook for this hive's payload URL +// already exists it is returned with created=false instead of duplicated. func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http.Request) { if !requireOwnerRole(w, r) { return @@ -293,12 +297,12 @@ func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http switch strings.ToLower(strings.TrimSpace(body.Scope)) { case "", "organization", "org": if !orgHooks { - jsonError(w, "the App installation has no organization_hooks: write permission", http.StatusForbidden) + jsonError(w, "the App installation lacks organization_hooks: write permission — grant it on the App and accept the installation update", http.StatusPreconditionFailed) return } case "repository", "repo": if !repoHooks { - jsonError(w, "the App installation has no repository_hooks: write permission", http.StatusForbidden) + jsonError(w, "the App installation lacks repository_hooks: write permission — grant it on the App and accept the installation update", http.StatusPreconditionFailed) return } repo := strings.TrimSpace(body.Repo) @@ -324,8 +328,15 @@ func (s *Server) handleConfigGitHubWebhooksCreate(w http.ResponseWriter, r *http return } user := strings.TrimSpace(r.Header.Get("X-Hive-User")) - s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL), "", res.Target, 0) - s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target) + if !res.Created { + // Idempotent: a hook for this payload URL already exists on the + // target; nothing was mutated, so there is nothing to audit. + s.webhookLogger().Info("github webhook already present; returning it", "user", user, "scope", res.Scope, "target", res.Target, "hook_id", res.ID) + jsonResponse(w, res) + return + } + s.audit.LogRecord(user, auditWebhookCreated, auditDetail("scope", res.Scope, "target", res.Target, "payload_url", payloadURL, "hook_id", strconv.FormatInt(res.ID, 10)), "", res.Target, 0) + s.webhookLogger().Info("github webhook created through the App", "user", user, "scope", res.Scope, "target", res.Target, "hook_id", res.ID) jsonResponse(w, res) } diff --git a/src/pkg/dashboard/api_github_webhook_setup_test.go b/src/pkg/dashboard/api_github_webhook_setup_test.go index f657d89057..856c2b7346 100644 --- a/src/pkg/dashboard/api_github_webhook_setup_test.go +++ b/src/pkg/dashboard/api_github_webhook_setup_test.go @@ -2,12 +2,22 @@ package dashboard import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" "encoding/json" + "encoding/pem" + "fmt" + "io" + "log/slog" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" + "sync" "testing" + "time" ghpkg "github.com/hivecommons/hive/pkg/github" ) @@ -166,7 +176,7 @@ func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { srv.mux.ServeHTTP(rec, req) return rec } - if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusForbidden { + if rec := post(`{"scope":"organization"}`); rec.Code != http.StatusPreconditionFailed { t.Fatalf("org scope without organization_hooks: %d %s", rec.Code, rec.Body.String()) } if rec := post(`{"scope":"repository","repo":"not-ours"}`); rec.Code != http.StatusBadRequest { @@ -181,3 +191,172 @@ func TestWebhookSetup_CreateGatedOnAppPermission(t *testing.T) { t.Fatal("create offered without any hooks permission") } } + +// webhookGitHubStub serves the installation-token mint plus the org/repo +// hooks list+create endpoints, remembering created hooks per path. +type webhookGitHubStub struct { + mu sync.Mutex + hooks map[string][]map[string]any + posts map[string]int +} + +func (g *webhookGitHubStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.mu.Lock() + defer g.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + path := strings.TrimSuffix(r.URL.Path, "/") + switch { + case strings.HasSuffix(path, "/access_tokens"): + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, `{"token":"ghs_webhook_stub","expires_at":%q}`, time.Now().Add(time.Hour).UTC().Format(time.RFC3339)) + case strings.HasSuffix(path, "/hooks") && r.Method == http.MethodGet: + list := g.hooks[path] + if list == nil { + list = []map[string]any{} + } + _ = json.NewEncoder(w).Encode(list) + case strings.HasSuffix(path, "/hooks") && r.Method == http.MethodPost: + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + g.posts[path]++ + hook := map[string]any{"id": 4200 + len(g.hooks[path]) + g.posts[path], "events": body["events"], "config": body["config"], "active": true} + g.hooks[path] = append(g.hooks[path], hook) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(hook) + default: + http.NotFound(w, r) + } +} + +func (g *webhookGitHubStub) postCount(path string) int { + g.mu.Lock() + defer g.mu.Unlock() + return g.posts[path] +} + +func (g *webhookGitHubStub) totalPosts() int { + g.mu.Lock() + defer g.mu.Unlock() + n := 0 + for _, c := range g.posts { + n += c + } + return n +} + +func newWebhookCreateServer(t *testing.T, perms map[string]string) (*Server, *webhookGitHubStub) { + t.Helper() + stub := &webhookGitHubStub{hooks: map[string][]map[string]any{}, posts: map[string]int{}} + api := httptest.NewServer(stub) + t.Cleanup(api.Close) + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate key: %v", err) + } + keyFile := filepath.Join(t.TempDir(), "app-key.pem") + if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + auth, err := ghpkg.NewAppAuthWithCache(1, 2, keyFile, filepath.Join(t.TempDir(), "token.cache"), + slog.New(slog.NewTextHandler(io.Discard, nil)), api.URL) + if err != nil { + t.Fatalf("NewAppAuthWithCache: %v", err) + } + + srv := newFullServer(t) + srv.deps.Config.Dashboard.PublicURL = "https://hive.example.org" + srv.deps.Config.Project.Org = "projectbluefin" + srv.deps.Config.Project.Repos = []string{"bluefin", "projectbluefin/common"} + srv.deps.GHAppAuth = auth + orig := webhookPermissionProbe + t.Cleanup(func() { webhookPermissionProbe = orig }) + webhookPermissionProbe = func(context.Context, *ghpkg.AppAuth) (*ghpkg.InstallationInfo, error) { + return &ghpkg.InstallationInfo{Permissions: perms}, nil + } + return srv, stub +} + +func postWebhookCreate(srv *Server, payload string) (*httptest.ResponseRecorder, ghpkg.WebhookCreateResult) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config/github/webhooks/create", strings.NewReader(payload)) + markOwnerRequest(req) + srv.mux.ServeHTTP(rec, req) + var res ghpkg.WebhookCreateResult + if rec.Code == http.StatusOK { + _ = json.Unmarshal(rec.Body.Bytes(), &res) + } + return rec, res +} + +func TestWebhookCreate_NoSecretIs409(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "") + restore := ghpkg.SetWebhookSecretPathForTest(filepath.Join(t.TempDir(), "absent.key")) + defer restore() + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "write"}) + if rec, _ := postWebhookCreate(srv, `{"scope":"org"}`); rec.Code != http.StatusConflict { + t.Fatalf("no secret: %d %s, want 409", rec.Code, rec.Body.String()) + } + if stub.totalPosts() != 0 { + t.Fatal("GitHub hook created without a secret") + } +} + +func TestWebhookCreate_MissingPermissionIs412(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "read"}) + rec, _ := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusPreconditionFailed || !strings.Contains(rec.Body.String(), "organization_hooks") { + t.Fatalf("org without write: %d %s, want 412 naming organization_hooks", rec.Code, rec.Body.String()) + } + rec, _ = postWebhookCreate(srv, `{"scope":"repo","repo":"projectbluefin/bluefin"}`) + if rec.Code != http.StatusPreconditionFailed || !strings.Contains(rec.Body.String(), "repository_hooks") { + t.Fatalf("repo without write: %d %s, want 412 naming repository_hooks", rec.Code, rec.Body.String()) + } + if stub.totalPosts() != 0 { + t.Fatal("GitHub hook created without the permission") + } +} + +// First call creates the org hook (audited); the second returns it instead +// of creating a duplicate. +func TestWebhookCreate_OrgCreatesThenReturnsExisting(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionOrganizationHooks: "write"}) + + rec, first := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusOK || !first.Created || first.ID == 0 || len(first.Events) != len(ghpkg.WebhookEvents) { + t.Fatalf("first create: %d %s", rec.Code, rec.Body.String()) + } + if first.PayloadURL != "https://hive.example.org"+githubWebhookPath { + t.Fatalf("payload url = %q", first.PayloadURL) + } + rec, second := postWebhookCreate(srv, `{"scope":"org"}`) + if rec.Code != http.StatusOK || second.Created || second.ID != first.ID { + t.Fatalf("second create: %d %s, want existing hook %d", rec.Code, rec.Body.String(), first.ID) + } + if n := stub.postCount("/orgs/projectbluefin/hooks"); n != 1 { + t.Fatalf("org POSTs = %d, want 1", n) + } + var audited int + for _, e := range srv.audit.Recent(50) { + if e.Action == auditWebhookCreated { + audited++ + } + } + if audited != 1 { + t.Fatalf("audit entries = %d, want 1 (only the real creation)", audited) + } +} + +func TestWebhookCreate_RepoScopeHitsRepoEndpoint(t *testing.T) { + t.Setenv(ghpkg.WebhookSecretEnvVar, "mounted") + srv, stub := newWebhookCreateServer(t, map[string]string{ghpkg.PermissionRepositoryHooks: "write"}) + rec, res := postWebhookCreate(srv, `{"scope":"repo","repo":"projectbluefin/common"}`) + if rec.Code != http.StatusOK || !res.Created || res.Target != "projectbluefin/common" { + t.Fatalf("repo create: %d %s", rec.Code, rec.Body.String()) + } + if stub.postCount("/repos/projectbluefin/common/hooks") != 1 || stub.postCount("/orgs/projectbluefin/hooks") != 0 { + t.Fatal("want exactly one POST, to the repo endpoint") + } +} diff --git a/src/pkg/dashboard/static-src/js/config-webhooks.js b/src/pkg/dashboard/static-src/js/config-webhooks.js index 257a1ba44a..18cc6682ec 100644 --- a/src/pkg/dashboard/static-src/js/config-webhooks.js +++ b/src/pkg/dashboard/static-src/js/config-webhooks.js @@ -202,8 +202,9 @@ }); const body = await res.json().catch(() => ({})); if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); - if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; - showToast('Webhook created', 'info'); + const verb = body.created === false ? 'Already set up:' : 'Created'; + if (out) out.innerHTML = `<span class="wh-ok">${verb} ${esc(body.scope)} hook #${esc(String(body.id || body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast(body.created === false ? 'Webhook already exists' : 'Webhook created', 'info'); } catch (e) { if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; } finally { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 2bc465b916..01999d74af 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -59881,8 +59881,9 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> }); const body = await res.json().catch(() => ({})); if (!res.ok) throw new Error(body.error || ('HTTP ' + res.status)); - if (out) out.innerHTML = `<span class="wh-ok">Created ${esc(body.scope)} hook #${esc(String(body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; - showToast('Webhook created', 'info'); + const verb = body.created === false ? 'Already set up:' : 'Created'; + if (out) out.innerHTML = `<span class="wh-ok">${verb} ${esc(body.scope)} hook #${esc(String(body.id || body.hook_id))} on ${esc(body.target)}.</span> <a href="${esc(body.url)}" target="_blank" rel="noopener noreferrer">View on GitHub ↗</a> — push a commit or comment on a PR and watch “Deliveries (1h)” move.`; + showToast(body.created === false ? 'Webhook already exists' : 'Webhook created', 'info'); } catch (e) { if (out) out.innerHTML = `<span class="wh-error">${esc(e.message)}</span>`; } finally { diff --git a/src/pkg/github/webhook_setup.go b/src/pkg/github/webhook_setup.go index 78aaeda11f..0e95da85a6 100644 --- a/src/pkg/github/webhook_setup.go +++ b/src/pkg/github/webhook_setup.go @@ -68,18 +68,32 @@ type WebhookCreateRequest struct { Events []string } -// WebhookCreateResult names the hook GitHub created. +// WebhookCreateResult names the hook CreateWebhook created or found. type WebhookCreateResult struct { - HookID int64 `json:"hook_id"` - Scope string `json:"scope"` // "organization" or "repository" - Target string `json:"target"` - URL string `json:"url"` // GitHub settings page for the hook + // ID is the GitHub hook id; HookID repeats it for the Settings card. + ID int64 `json:"id"` + HookID int64 `json:"hook_id"` + Scope string `json:"scope"` // "organization" or "repository" + Target string `json:"target"` + URL string `json:"url"` // GitHub settings page for the hook + Events []string `json:"events"` + // PayloadURL is the hook's config.url (this hive's receiver). + PayloadURL string `json:"payload_url"` + // Created is false when a hook with the same payload URL already existed + // on the target and was returned instead of creating a duplicate. + Created bool `json:"created"` } +// webhookListPages bounds the hook listing; GitHub caps hooks per target at +// 20 for repositories and organizations, so one page is normally enough. +const webhookListPages = 5 + // CreateWebhook creates an organization hook (req.Repo == "") or a repository -// hook through the App installation token. The token carries every granted -// permission, so the caller checks WebhookHookPermissions first and only -// offers the action when the matching write permission is held. +// hook through the App installation token. It is idempotent: the target's +// hooks are listed first and a hook whose config.url equals req.PayloadURL is +// returned (Created=false) instead of creating a duplicate. The token carries +// every granted permission, so the caller checks WebhookHookPermissions first +// and only offers the action when the matching write permission is held. func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) (*WebhookCreateResult, error) { if a == nil { return nil, errors.New("GitHub App auth not configured") @@ -96,6 +110,15 @@ func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) ( if len(events) == 0 { events = WebhookEvents } + + existing, err := findWebhookByURL(ctx, client, req.Org, req.Repo, req.PayloadURL) + if err != nil { + return nil, fmt.Errorf("listing existing hooks: %w", err) + } + if existing != nil { + return a.webhookResult(req, existing, false), nil + } + hook := &gh.Hook{ Active: gh.Ptr(true), Events: events, @@ -110,28 +133,82 @@ func (a *AppAuth) CreateWebhook(ctx context.Context, req WebhookCreateRequest) ( created *gh.Hook resp *gh.Response ) - res := &WebhookCreateResult{} if req.Repo == "" { created, resp, err = client.Organizations.CreateHook(ctx, req.Org, hook) - res.Scope, res.Target = "organization", req.Org } else { created, resp, err = client.Repositories.CreateHook(ctx, req.Org, req.Repo, hook) - res.Scope, res.Target = "repository", req.Org+"/"+req.Repo } if err != nil { if resp != nil && resp.StatusCode == http.StatusUnprocessableEntity && strings.Contains(strings.ToLower(err.Error()), "already exists") { + // Lost a race with a concurrent create: return the winner. + if found, ferr := findWebhookByURL(ctx, client, req.Org, req.Repo, req.PayloadURL); ferr == nil && found != nil { + return a.webhookResult(req, found, false), nil + } return nil, ErrWebhookExists } return nil, err } - res.HookID = created.GetID() + if len(created.Events) == 0 { + created.Events = events + } + return a.webhookResult(req, created, true), nil +} + +// findWebhookByURL returns the hook on the org (repo == "") or repository +// whose config.url equals payloadURL, or nil when there is none. +func findWebhookByURL(ctx context.Context, client *gh.Client, org, repo, payloadURL string) (*gh.Hook, error) { + want := strings.TrimRight(strings.TrimSpace(payloadURL), "/") + opts := &gh.ListOptions{PerPage: 100} + for page := 0; page < webhookListPages; page++ { + var ( + hooks []*gh.Hook + resp *gh.Response + err error + ) + if repo == "" { + hooks, resp, err = client.Organizations.ListHooks(ctx, org, opts) + } else { + hooks, resp, err = client.Repositories.ListHooks(ctx, org, repo, opts) + } + if err != nil { + return nil, err + } + for _, h := range hooks { + if h.GetConfig() == nil { + continue + } + if strings.TrimRight(strings.TrimSpace(h.GetConfig().GetURL()), "/") == want { + return h, nil + } + } + if resp == nil || resp.NextPage == 0 { + return nil, nil + } + opts.Page = resp.NextPage + } + return nil, nil +} + +func (a *AppAuth) webhookResult(req WebhookCreateRequest, hook *gh.Hook, created bool) *WebhookCreateResult { + res := &WebhookCreateResult{ + ID: hook.GetID(), + HookID: hook.GetID(), + Events: append([]string(nil), hook.Events...), + PayloadURL: req.PayloadURL, + Created: created, + } + if hook.GetConfig() != nil && hook.GetConfig().GetURL() != "" { + res.PayloadURL = hook.GetConfig().GetURL() + } base := strings.TrimSuffix(webBaseURLForAPI(a.apiURL), "/") if req.Repo == "" { - res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.HookID) + res.Scope, res.Target = "organization", req.Org + res.URL = fmt.Sprintf("%s/organizations/%s/settings/hooks/%d", base, req.Org, res.ID) } else { - res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.HookID) + res.Scope, res.Target = "repository", req.Org+"/"+req.Repo + res.URL = fmt.Sprintf("%s/%s/%s/settings/hooks/%d", base, req.Org, req.Repo, res.ID) } - return res, nil + return res } // webBaseURLForAPI maps an API URL to the web origin a hook's settings page diff --git a/src/pkg/github/webhook_setup_test.go b/src/pkg/github/webhook_setup_test.go new file mode 100644 index 0000000000..2a4f86b737 --- /dev/null +++ b/src/pkg/github/webhook_setup_test.go @@ -0,0 +1,153 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// hookStub is a minimal GitHub hooks API: GET/POST on /orgs/{org}/hooks and +// /repos/{owner}/{repo}/hooks, keeping created hooks per path. +type hookStub struct { + mu sync.Mutex + hooks map[string][]map[string]any + posts map[string]int + nextID int64 + lastReq map[string]any +} + +func newHookStub() *hookStub { + return &hookStub{hooks: map[string][]map[string]any{}, posts: map[string]int{}, nextID: 100} +} + +func (s *hookStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + path := strings.TrimSuffix(r.URL.Path, "/") + if !strings.HasSuffix(path, "/hooks") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + switch r.Method { + case http.MethodGet: + list := s.hooks[path] + if list == nil { + list = []map[string]any{} + } + _ = json.NewEncoder(w).Encode(list) + case http.MethodPost: + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + s.lastReq = body + s.posts[path]++ + s.nextID++ + hook := map[string]any{ + "id": s.nextID, + "active": body["active"], + "events": body["events"], + "config": body["config"], + } + s.hooks[path] = append(s.hooks[path], hook) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(hook) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (s *hookStub) postCount(path string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.posts[path] +} + +func (s *hookStub) lastBody() map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + return s.lastReq +} + +func hookStubAuth(t *testing.T, stub http.Handler) *AppAuth { + t.Helper() + srv := httptest.NewServer(stub) + t.Cleanup(srv.Close) + return &AppAuth{ + cachedToken: "ghs_hook_stub", + tokenExpiry: time.Now().Add(2 * time.Hour), + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + installationID: 1, + apiURL: srv.URL + "/", + } +} + +func TestCreateWebhook_OrgCreatesThenReturnsExisting(t *testing.T) { + stub := newHookStub() + auth := hookStubAuth(t, stub) + req := WebhookCreateRequest{Org: "acme", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "s3cret"} + + res, err := auth.CreateWebhook(context.Background(), req) + if err != nil { + t.Fatalf("first create: %v", err) + } + if !res.Created || res.ID == 0 || res.ID != res.HookID || res.Scope != "organization" || res.Target != "acme" { + t.Fatalf("first result = %+v", res) + } + if len(res.Events) != len(WebhookEvents) { + t.Fatalf("events = %v, want default WebhookEvents", res.Events) + } + body := stub.lastBody() + cfg, _ := body["config"].(map[string]any) + if cfg["url"] != req.PayloadURL || cfg["content_type"] != "json" || cfg["secret"] != "s3cret" || cfg["insecure_ssl"] != "0" || body["active"] != true { + t.Fatalf("create body = %v", body) + } + + again, err := auth.CreateWebhook(context.Background(), req) + if err != nil { + t.Fatalf("second create: %v", err) + } + if again.Created || again.ID != res.ID { + t.Fatalf("second result = %+v, want existing hook %d", again, res.ID) + } + if n := stub.postCount("/orgs/acme/hooks"); n != 1 { + t.Fatalf("POST count = %d, want 1 (no duplicate)", n) + } +} + +func TestCreateWebhook_RepoScopeHitsRepoEndpoint(t *testing.T) { + stub := newHookStub() + auth := hookStubAuth(t, stub) + res, err := auth.CreateWebhook(context.Background(), WebhookCreateRequest{ + Org: "acme", Repo: "widgets", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "s3cret", + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if stub.postCount("/repos/acme/widgets/hooks") != 1 || stub.postCount("/orgs/acme/hooks") != 0 { + t.Fatal("want exactly one POST, to the repo endpoint") + } + if !res.Created || res.Scope != "repository" || res.Target != "acme/widgets" || !strings.HasSuffix(res.URL, "/acme/widgets/settings/hooks/101") { + t.Fatalf("result = %+v", res) + } +} + +// A hook for another URL on the same target does not count as existing. +func TestCreateWebhook_OtherURLDoesNotMatch(t *testing.T) { + stub := newHookStub() + stub.hooks["/orgs/acme/hooks"] = []map[string]any{{"id": 7, "config": map[string]any{"url": "https://ci.example.org/hook"}}} + auth := hookStubAuth(t, stub) + res, err := auth.CreateWebhook(context.Background(), WebhookCreateRequest{Org: "acme", PayloadURL: "https://hive.example.org/api/webhook/github", Secret: "x"}) + if err != nil { + t.Fatalf("create: %v", err) + } + if !res.Created || res.ID == 7 { + t.Fatalf("result = %+v, want a new hook", res) + } +} From ef2148718fff350a3b7bd050e89592665f740b0b Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 11:54:51 -0500 Subject: [PATCH 147/166] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20stop=20a=20?= =?UTF-8?q?Hold=20click=20down=20the=20Projects=20card=20from=20releasing?= =?UTF-8?q?=20a=20neighbour=20(#18085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hold toggle moves its row into the held group and the status render that follows re-sorts the groups, so the button under the pointer changes a moment after every click. Working down a card clicking Hold, the next click landed on another item's Release and that item's hold came straight off: on cncf/techdocs #151 was released in the same second #283 was held, #153 three seconds after #54, #221 two seconds after #139 — every label removal was the hive bot acting on a dashboard Release. renderRepos now notes when the card's rows move (order or membership), and a Release that arrives within 1.5s of a move is not sent; a toast tells the operator to click Release again. Hold is never delayed. The in-flight guard moves from the clicked button (which the optimistic repaint replaces) to the item, so one item never has two toggles racing. Fixes #18054 Hive-Run: hivecommons/hive#18054 Hive-Plan: issue-18054 Hive-Spec: issue-18054#hold-sticks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: nate-double-u <4453979+nate-double-u@users.noreply.github.com> --- changelog.d/fixed-18054-hold-release-guard.md | 1 + .../repo_card_held_pills_7896_test.go | 2 + .../repo_hold_shift_guard_18054_test.go | 164 ++++++++++++++++++ .../dashboard/static-src/js/repos-render.js | 1 + src/pkg/dashboard/static-src/js/repos.js | 40 +++++ src/pkg/dashboard/static/index.html | 41 +++++ 6 files changed, 249 insertions(+) create mode 100644 changelog.d/fixed-18054-hold-release-guard.md create mode 100644 src/pkg/dashboard/repo_hold_shift_guard_18054_test.go diff --git a/changelog.d/fixed-18054-hold-release-guard.md b/changelog.d/fixed-18054-hold-release-guard.md new file mode 100644 index 0000000000..6e43c24352 --- /dev/null +++ b/changelog.d/fixed-18054-hold-release-guard.md @@ -0,0 +1 @@ +- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) diff --git a/src/pkg/dashboard/repo_card_held_pills_7896_test.go b/src/pkg/dashboard/repo_card_held_pills_7896_test.go index 9d7899debc..b51013ca28 100644 --- a/src/pkg/dashboard/repo_card_held_pills_7896_test.go +++ b/src/pkg/dashboard/repo_card_held_pills_7896_test.go @@ -271,6 +271,7 @@ func TestRepoHoldToggleOptimisticRollback(t *testing.T) { jsFunc(t, html, "repoParts") + "\n" + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + jsFunc(t, html, "toggleRepoItemHold") + "\n" + holdToggleRollbackAssertions path := filepath.Join(t.TempDir(), "hold-toggle.js") @@ -305,6 +306,7 @@ func TestRepoHoldToggleSuccessRaisesStatusFloor(t *testing.T) { jsFunc(t, html, "repoParts") + "\n" + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + jsFunc(t, html, "toggleRepoItemHold") + "\n" + holdToggleSuccessAssertions path := filepath.Join(t.TempDir(), "hold-toggle-success.js") diff --git a/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go b/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go new file mode 100644 index 0000000000..665ecb135d --- /dev/null +++ b/src/pkg/dashboard/repo_hold_shift_guard_18054_test.go @@ -0,0 +1,164 @@ +package dashboard + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// hivecommons/hive#18054: on the Projects card, Hold on cncf/techdocs #151, +// #153 and #221 "did not stick". GitHub's timeline showed why: each hold +// label came off again a moment after a Hold click on a DIFFERENT row (#151 +// released in the same second #283 was held, #153 three seconds after #54, +// #221 two seconds after #139). A hold toggle moves its row into the held +// group and the next status render re-sorts the groups, so the operator +// working down the list hit another item's Release button. These tests pin +// the guard: a Release that lands while the rows are moving is not sent, a +// Hold still is, and one item never has two toggles in flight. + +// repoHoldShiftGuardJS is the guard's declarations as shipped: everything +// from REPO_HOLD_SHIFT_GUARD_MS through repoHoldReleaseTooSoon. +func repoHoldShiftGuardJS(t *testing.T, html string) string { + t.Helper() + start := strings.Index(html, "const REPO_HOLD_SHIFT_GUARD_MS") + if start < 0 { + t.Fatal("index.html does not define REPO_HOLD_SHIFT_GUARD_MS") + } + last := jsFunc(t, html, "repoHoldReleaseTooSoon") + end := strings.Index(html[start:], last) + if end < 0 { + t.Fatal("repoHoldReleaseTooSoon is not declared after REPO_HOLD_SHIFT_GUARD_MS") + } + return html[start : start+end+len(last)] +} + +func runRepoHoldShiftGuardScript(t *testing.T, name, harness, assertions string) { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not on PATH — the hold shift guard was NOT executed by this run") + } + html := indexHTML(t) + start := strings.Index(html, "const HOLD_LABEL_SPELLINGS = [") + if start < 0 { + t.Fatal("index.html does not define HOLD_LABEL_SPELLINGS") + } + spellings := html[start : start+strings.Index(html[start:], "\n")] + script := "let clock = 1000000; Date.now = () => clock;\n" + + "const window = { _lastStatus: { hiveId: 'h1', repos: [{ full: 'o/r', workBreakdown: { issues: { actionable: 2, hold: 0 } }, actionableIssues: [{ number: 151, labels: [] }, { number: 283, labels: [] }], heldIssues: [], openPrs: [], heldPrs: [] }] } };\n" + + "let _statusSeqFloor = 0;\n" + + "const posts = []; const toasts = [];\n" + + "function showToast(m, k){ toasts.push(k + ':' + m); }\n" + + "function render(){}\n" + + // The real repaint path: renderRepos notes the layout on every render. + "function repaintReposForWidth(){ noteRepoHoldLayout(window._lastStatus.repos, Date.now()); }\n" + + harness + "\n" + + spellings + "\n" + + jsFunc(t, html, "canonicalHiveHoldLabel") + "\n" + + jsFunc(t, html, "holdLabels") + "\n" + + jsFunc(t, html, "repoParts") + "\n" + + jsFunc(t, html, "updateRepoBreakdownForHold") + "\n" + + jsFunc(t, html, "moveRepoItemHold") + "\n" + + repoHoldShiftGuardJS(t, html) + "\n" + + jsFunc(t, html, "toggleRepoItemHold") + "\n" + + assertions + path := filepath.Join(t.TempDir(), name+".js") + if err := os.WriteFile(path, []byte(script), 0o600); err != nil { + t.Fatal(err) + } + if out, err := exec.Command(node, path).CombinedOutput(); err != nil { + t.Fatalf("%s failed:\n%s", name, strings.TrimSpace(string(out))) + } +} + +const repoHoldPostingFetch = ` +function fetch(url, opts) { + if (opts) posts.push({ url, body: JSON.parse(opts.body) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }); +} +` + +func TestRepoHoldReleaseIgnoredWhileRowsMove(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-shift-guard", repoHoldPostingFetch, ` +(async () => { + const fails = []; + const released = n => posts.some(p => p.url === '/api/repos/o/r/items/' + n + '/hold' && p.body.held === false); + noteRepoHoldLayout(window._lastStatus.repos, clock); // first paint + // Hold #151: its row moves into the held group. + await toggleRepoItemHold('o/r', 151, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + // 300 ms later the click aimed at the next Hold lands on #151's Release. + clock += 300; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (released(151)) fails.push('a Release that landed while the rows moved was sent'); + if (!toasts.some(t => t.includes('o/r#151') && t.includes('Release again'))) fails.push('no toast named the unsent Release: ' + JSON.stringify(toasts)); + if (!window._lastStatus.repos[0].heldIssues.some(i => i.number === 151)) fails.push('#151 left the held group'); + // A Hold in the same window is still sent: clicking Hold down a list works. + await toggleRepoItemHold('o/r', 283, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + if (!posts.some(p => p.url === '/api/repos/o/r/items/283/hold' && p.body.held === true)) fails.push('a Hold was delayed by the guard'); + // Once the list has been still for the guard window, Release goes through. + clock += REPO_HOLD_SHIFT_GUARD_MS; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (!released(151)) fails.push('a deliberate Release on a still list was blocked'); + if (fails.length) { console.log(fails.join('\n')); process.exit(1); } + console.log('ok'); +})().catch(e => { console.log(e && e.stack || e); process.exit(1); }); +`) +} + +func TestRepoHoldToggleOneRequestPerItemInFlight(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-in-flight", ` +let finishFirst = null; +function fetch(url, opts) { + if (opts) { + posts.push({ url, body: JSON.parse(opts.body) }); + if (!finishFirst) return new Promise(r => { finishFirst = () => r({ ok: true, json: () => Promise.resolve({}) }); }); + } + return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }); +} +`, ` +(async () => { + const fails = []; + noteRepoHoldLayout(window._lastStatus.repos, clock); + // The optimistic repaint replaces the clicked button, so its busy mark is + // gone; a second click on the item's new Release must not race the Hold. + const first = toggleRepoItemHold('o/r', 151, 'issue', true, { dataset: {}, textContent: '⏸ Hold' }); + clock += 10 * REPO_HOLD_SHIFT_GUARD_MS; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (posts.length !== 1) fails.push('second toggle sent while the first was in flight: ' + JSON.stringify(posts)); + finishFirst(); + await first; + await toggleRepoItemHold('o/r', 151, 'issue', false, { dataset: {}, textContent: '▶ Release' }); + if (posts.length !== 2 || posts[1].body.held !== false) fails.push('Release after the Hold settled was not sent: ' + JSON.stringify(posts)); + if (fails.length) { console.log(fails.join('\n')); process.exit(1); } + console.log('ok'); +})().catch(e => { console.log(e && e.stack || e); process.exit(1); }); +`) +} + +func TestRepoHoldLayoutSignatureMarksMoves(t *testing.T) { + runRepoHoldShiftGuardScript(t, "hold-layout-signature", repoHoldPostingFetch, ` +const fails = []; +const repos = [{ full: 'o/r', actionableIssues: [{ number: 1 }, { number: 2 }], heldIssues: [{ number: 3 }, { number: 4 }], openPrs: [], heldPrs: [] }]; +noteRepoHoldLayout(repos, 100); +if (repoHoldReleaseTooSoon(150)) fails.push('the first paint counted as a move'); +noteRepoHoldLayout(repos, 200); +if (repoHoldReleaseTooSoon(250)) fails.push('an unchanged repaint counted as a move'); +// A status render that only re-sorts the held group still moves the rows. +repos[0].heldIssues = [{ number: 4 }, { number: 3 }]; +noteRepoHoldLayout(repos, 300); +if (!repoHoldReleaseTooSoon(300 + REPO_HOLD_SHIFT_GUARD_MS - 1)) fails.push('a changed held group was not a move'); +if (repoHoldReleaseTooSoon(300 + REPO_HOLD_SHIFT_GUARD_MS)) fails.push('the guard outlived its window'); +if (fails.length) { console.log(fails.join('\n')); process.exit(1); } +console.log('ok'); +`) +} + +// renderRepos is what notes each layout; without the call the guard never +// sees the status render that re-sorts the card. +func TestRenderReposNotesHoldLayout(t *testing.T) { + if src := jsFunc(t, indexHTML(t), "renderRepos"); !strings.Contains(src, "noteRepoHoldLayout(repos, Date.now())") { + t.Fatal("renderRepos does not call noteRepoHoldLayout; a status re-sort would not arm the Release guard") + } +} diff --git a/src/pkg/dashboard/static-src/js/repos-render.js b/src/pkg/dashboard/static-src/js/repos-render.js index b6f16b1201..d9ad371d0e 100644 --- a/src/pkg/dashboard/static-src/js/repos-render.js +++ b/src/pkg/dashboard/static-src/js/repos-render.js @@ -2,6 +2,7 @@ refreshSectionCardShell('repos-section'); if (_repoResizeDrag || _repoOrderDrag) return; repos = repoOrderApply(repos || []); + noteRepoHoldLayout(repos, Date.now()); const el = document.getElementById('repos'); const showPlan = planningAllowed(); const showDesign = showPlan && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); diff --git a/src/pkg/dashboard/static-src/js/repos.js b/src/pkg/dashboard/static-src/js/repos.js index 5224b72543..4926bb31c9 100644 --- a/src/pkg/dashboard/static-src/js/repos.js +++ b/src/pkg/dashboard/static-src/js/repos.js @@ -657,11 +657,49 @@ repaintReposForWidth(); return { repo, item, fromKey, toKey }; } + // A hold toggle moves its row between the held and actionable groups, and + // the status render that follows re-sorts them, so the button under the + // pointer changes a moment after every click. Clicking Hold down a card + // then landed the next click on another item's Release, and that item's + // hold came straight back off (hivecommons/hive#18054: #151 released in + // the same second #283 was held). A Release that arrives within + // REPO_HOLD_SHIFT_GUARD_MS of the rows moving is not sent; the operator + // clicks it again once the list is still. Hold is never delayed: under + // the pointer it can only park an item, and parking is what that click + // was doing anyway. + const REPO_HOLD_SHIFT_GUARD_MS = 1500; + let _repoHoldLayoutSig = null; + let _repoHoldShiftAt = 0; + const _repoHoldInFlight = {}; + // Pure: the order and membership of every card's hold-toggle rows. + function repoHoldLayoutSignature(repos) { + const nums = list => (list || []).map(x => Number(x && x.number) || 0).join(','); + return (repos || []).map(r => (r.full || r.name || '') + ':' + nums(r.actionableIssues) + '|' + nums(r.heldIssues) + '|' + nums(r.openPrs) + '|' + nums(r.heldPrs)).join(';'); + } + // Called on every repos render: remembers when the rows last moved. The + // first render is not a move; nothing was under the pointer before it. + function noteRepoHoldLayout(repos, now) { + const sig = repoHoldLayoutSignature(repos); + if (_repoHoldLayoutSig !== null && sig !== _repoHoldLayoutSig) _repoHoldShiftAt = now; + _repoHoldLayoutSig = sig; + } + function repoHoldReleaseTooSoon(now) { + return _repoHoldShiftAt > 0 && now - _repoHoldShiftAt < REPO_HOLD_SHIFT_GUARD_MS; + } async function toggleRepoItemHold(repoFull, number, type, held, btn) { if (btn && btn.dataset.busy === '1') return; const p = repoParts(repoFull); if (!p.owner || !p.repo || !number) return; const wantHeld = !!held; + // The busy mark lives on a button the optimistic repaint replaces, so + // the item itself carries the in-flight guard. + const flightKey = String(repoFull).toLowerCase() + '/' + type + '#' + number; + if (_repoHoldInFlight[flightKey]) return; + if (!wantHeld && repoHoldReleaseTooSoon(Date.now())) { + showToast('The list moved just as you clicked, so ' + repoFull + '#' + number + ' is still held. Click Release again to release it.', 'info'); + return; + } + _repoHoldInFlight[flightKey] = true; const prevText = btn ? btn.textContent : ''; const moved = moveRepoItemHold(repoFull, number, type, wantHeld, canonicalHiveHoldLabel()); if (btn) { btn.dataset.busy = '1'; btn.textContent = '…'; } @@ -680,6 +718,8 @@ if (moved) moveRepoItemHold(repoFull, number, type, !wantHeld, canonicalHiveHoldLabel()); showToast('Hold toggle failed: ' + (e && e.message ? e.message : 'network error'), 'error'); if (btn) { btn.dataset.busy = ''; btn.textContent = prevText || (wantHeld ? '⏸ Hold' : '▶ Release'); } + } finally { + delete _repoHoldInFlight[flightKey]; } } diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 01999d74af..01b9fd6063 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -33934,11 +33934,49 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> repaintReposForWidth(); return { repo, item, fromKey, toKey }; } + // A hold toggle moves its row between the held and actionable groups, and + // the status render that follows re-sorts them, so the button under the + // pointer changes a moment after every click. Clicking Hold down a card + // then landed the next click on another item's Release, and that item's + // hold came straight back off (hivecommons/hive#18054: #151 released in + // the same second #283 was held). A Release that arrives within + // REPO_HOLD_SHIFT_GUARD_MS of the rows moving is not sent; the operator + // clicks it again once the list is still. Hold is never delayed: under + // the pointer it can only park an item, and parking is what that click + // was doing anyway. + const REPO_HOLD_SHIFT_GUARD_MS = 1500; + let _repoHoldLayoutSig = null; + let _repoHoldShiftAt = 0; + const _repoHoldInFlight = {}; + // Pure: the order and membership of every card's hold-toggle rows. + function repoHoldLayoutSignature(repos) { + const nums = list => (list || []).map(x => Number(x && x.number) || 0).join(','); + return (repos || []).map(r => (r.full || r.name || '') + ':' + nums(r.actionableIssues) + '|' + nums(r.heldIssues) + '|' + nums(r.openPrs) + '|' + nums(r.heldPrs)).join(';'); + } + // Called on every repos render: remembers when the rows last moved. The + // first render is not a move; nothing was under the pointer before it. + function noteRepoHoldLayout(repos, now) { + const sig = repoHoldLayoutSignature(repos); + if (_repoHoldLayoutSig !== null && sig !== _repoHoldLayoutSig) _repoHoldShiftAt = now; + _repoHoldLayoutSig = sig; + } + function repoHoldReleaseTooSoon(now) { + return _repoHoldShiftAt > 0 && now - _repoHoldShiftAt < REPO_HOLD_SHIFT_GUARD_MS; + } async function toggleRepoItemHold(repoFull, number, type, held, btn) { if (btn && btn.dataset.busy === '1') return; const p = repoParts(repoFull); if (!p.owner || !p.repo || !number) return; const wantHeld = !!held; + // The busy mark lives on a button the optimistic repaint replaces, so + // the item itself carries the in-flight guard. + const flightKey = String(repoFull).toLowerCase() + '/' + type + '#' + number; + if (_repoHoldInFlight[flightKey]) return; + if (!wantHeld && repoHoldReleaseTooSoon(Date.now())) { + showToast('The list moved just as you clicked, so ' + repoFull + '#' + number + ' is still held. Click Release again to release it.', 'info'); + return; + } + _repoHoldInFlight[flightKey] = true; const prevText = btn ? btn.textContent : ''; const moved = moveRepoItemHold(repoFull, number, type, wantHeld, canonicalHiveHoldLabel()); if (btn) { btn.dataset.busy = '1'; btn.textContent = '…'; } @@ -33957,6 +33995,8 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> if (moved) moveRepoItemHold(repoFull, number, type, !wantHeld, canonicalHiveHoldLabel()); showToast('Hold toggle failed: ' + (e && e.message ? e.message : 'network error'), 'error'); if (btn) { btn.dataset.busy = ''; btn.textContent = prevText || (wantHeld ? '⏸ Hold' : '▶ Release'); } + } finally { + delete _repoHoldInFlight[flightKey]; } } @@ -36780,6 +36820,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> refreshSectionCardShell('repos-section'); if (_repoResizeDrag || _repoOrderDrag) return; repos = repoOrderApply(repos || []); + noteRepoHoldLayout(repos, Date.now()); const el = document.getElementById('repos'); const showPlan = planningAllowed(); const showDesign = showPlan && dashboardRoleAtLeast(window._hiveRole || 'read', 'owner'); From 16564d7de983fa37717e11671a3da89edbee682d Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 12:04:38 -0500 Subject: [PATCH 148/166] =?UTF-8?q?=E2=9C=A8=20hub:=20credential=20fence?= =?UTF-8?q?=20for=20self-service=20(byo-)=20hive=20heartbeats=20(#18081)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ hub: credential fence for self-service (byo-) hive heartbeats Step 1 of the #18060 RFC. Hive IDs in the reserved byo- namespace are listed as directory metadata only, so their heartbeat response must not carry hub-held credentials or management actions. Several delivery paths resolve what to send from spoke-reported identity (appKeySyncForHeartbeat falls back to the payload cluster_id for a hive with no SaaS record), so a per-site check is fragile. One guard in signHeartbeatResponse, the single point every heartbeat response passes through, zeroes each field that heartbeatResponseFieldFence classifies as fenced, and fails closed for any unclassified field. TestHeartbeatResponseFieldFenceInventory fails when a HeartbeatResponse field is added without a classification, following the TestNotificationWriteSitesInventory pattern. Hive-Run: hivecommons/hive#18060 Hive-Plan: RFC #18060 suggested order of work, step 1 (fence and inventory test) Hive-Spec: hivecommons/hive#18060#proposed-solution-4-credential-fence Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🚨 hub: use reflect.Pointer in self-service fence test (govet inline) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hivecommons-hive[bot] <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...rity-18060-selfservice-credential-fence.md | 1 + src/docs/hub-deployment.md | 4 + src/pkg/hub/heartbeat_selfservice_fence.go | 121 ++++++++++ .../hub/heartbeat_selfservice_fence_test.go | 221 ++++++++++++++++++ src/pkg/hub/heartbeat_signing.go | 10 + 5 files changed, 357 insertions(+) create mode 100644 changelog.d/security-18060-selfservice-credential-fence.md create mode 100644 src/pkg/hub/heartbeat_selfservice_fence.go create mode 100644 src/pkg/hub/heartbeat_selfservice_fence_test.go diff --git a/changelog.d/security-18060-selfservice-credential-fence.md b/changelog.d/security-18060-selfservice-credential-fence.md new file mode 100644 index 0000000000..961ff47b53 --- /dev/null +++ b/changelog.d/security-18060-selfservice-credential-fence.md @@ -0,0 +1 @@ +- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index fd842d5b06..fb132e303f 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -88,6 +88,10 @@ env: Without `HIVE_HUB_SECRET`, a protected hub returns `401 unauthorized`, the spoke logs `hub heartbeat rejected status=401`, and the hub shows the hive offline. This matches the gotcha in `manual-provisioning.md`. +### Reserved `byo-` hive IDs (credential fence) + +Hive IDs that start with `byo-` are reserved for hives that their owners run and register themselves ([#18060](https://github.com/hivecommons/hive/issues/18060)). A hub lists such a hive as directory metadata only, so its heartbeat response never carries hub-held credentials or management actions, whatever key the spoke authenticated with. The hub withholds these fields: the GitHub App config (including cluster, secondary and webhook relay keys), the pending gateway, the claimed project config, the cluster ID, the authorized-user list, visibility, upgrade and branch-switch instructions, the upgrade announcement and policy, spoke restarts, agent-restart resets, fleet-breaker instructions, and self-experiment flags. Display-only data such as admin banners still flows. When the fence withholds a field, the hub logs `heartbeat: credential fence withheld fields from self-service hive` with the field names, never the values. Do not give an operator-provisioned hive a `byo-` ID. + ## Hosted spoke provisioning model The hub's SaaS provisioner creates or records: diff --git a/src/pkg/hub/heartbeat_selfservice_fence.go b/src/pkg/hub/heartbeat_selfservice_fence.go new file mode 100644 index 0000000000..191ce9d28c --- /dev/null +++ b/src/pkg/hub/heartbeat_selfservice_fence.go @@ -0,0 +1,121 @@ +package hub + +import ( + "reflect" + "strings" +) + +// Credential fence for self-service (self-hosted, owner-registered) hives +// (#18060). +// +// A self-service hive runs on its owner's infrastructure and is listed on the +// hub as directory metadata only (#12571). The hub holds no authority over its +// policy, credentials or upgrades, and every identity field such a spoke puts +// in its heartbeat payload (cluster, namespace, App installation) is reported +// by the spoke itself and cannot be trusted. Several delivery paths in the +// heartbeat response builder resolve what to send from exactly those +// payload-reported fields: appKeySyncForHeartbeat, for example, falls back to +// the spoke-reported cluster_id for a hive with no SaaS record and would hand +// that cluster's App private key to any spoke that claims it. +// +// So instead of a check at each delivery site, ONE guard runs on every +// heartbeat response before it is signed and written +// (signHeartbeatResponse is the single point every response passes through). +// The guard is driven by heartbeatResponseFieldFence, which classifies every +// HeartbeatResponse field. TestHeartbeatResponseFieldFenceInventory fails when +// a field is added to HeartbeatResponse without a classification here, so a +// new credential or management field cannot reach a self-service hive by +// default. + +// SelfServiceHiveIDPrefix is the reserved hive ID namespace for hives that +// owners register themselves (for example "byo-<github-login>-<name>"). Every +// hive ID in this namespace is fenced, whatever key it authenticated with. +const SelfServiceHiveIDPrefix = "byo-" + +// IsSelfServiceHiveID reports whether hiveID is in the reserved self-service +// namespace. +func IsSelfServiceHiveID(hiveID string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(hiveID)), SelfServiceHiveIDPrefix) +} + +// heartbeatFieldClass is how the credential fence treats one +// HeartbeatResponse field for a self-service hive. +type heartbeatFieldClass int + +const ( + // heartbeatFieldAllowed is protocol, signature or display-only data that + // carries no hub-held secret and no instruction to the spoke. + heartbeatFieldAllowed heartbeatFieldClass = iota + 1 + // heartbeatFieldFenced is a hub-held credential, a management action, hub + // policy over the hive, or data resolved from spoke-reported identity. It + // is zeroed before the response leaves the hub. + heartbeatFieldFenced +) + +// heartbeatResponseFieldFence classifies every HeartbeatResponse field by its +// Go field name. Adding a field to HeartbeatResponse requires adding it here. +var heartbeatResponseFieldFence = map[string]heartbeatFieldClass{ + // Protocol and descriptive data. + "OK": heartbeatFieldAllowed, + "HubGitHash": heartbeatFieldAllowed, + "LatestSHA": heartbeatFieldAllowed, + "LatestTag": heartbeatFieldAllowed, + "HubBanner": heartbeatFieldAllowed, // admin message, display only + "ActivityETag": heartbeatFieldAllowed, + "ActivityEventsAck": heartbeatFieldAllowed, + // Response signature binding. + "SigHiveID": heartbeatFieldAllowed, + "SigSeq": heartbeatFieldAllowed, + "SigSignedAt": heartbeatFieldAllowed, + "SigVersion": heartbeatFieldAllowed, + + // Hub-held credentials. + "GitHubAppConfig": heartbeatFieldFenced, // App, cluster and secondary keys, webhook relay key + "PendingGateway": heartbeatFieldFenced, // funded gateway API key + // Upgrade and lifecycle management. + "UpgradeTo": heartbeatFieldFenced, + "UpgradeInterrupt": heartbeatFieldFenced, + "SwitchToTag": heartbeatFieldFenced, + "RestartSpoke": heartbeatFieldFenced, + "UpgradeAnnounce": heartbeatFieldFenced, + "UpgradePolicy": heartbeatFieldFenced, // the owner, not the hub, sets upgrade posture + // Operational actions. + "ResetAgentRestarts": heartbeatFieldFenced, + "Breaker": heartbeatFieldFenced, + "SelfExperimentFlag": heartbeatFieldFenced, + // Hub policy over the hive's config and access. + "IsPublic": heartbeatFieldFenced, + "AuthorizedUsers": heartbeatFieldFenced, + "AuthorizedUserNames": heartbeatFieldFenced, + "ProjectConfig": heartbeatFieldFenced, // claimed project config + // Resolved from spoke-reported or hub-provisioned identity. + "ClusterID": heartbeatFieldFenced, +} + +// fenceSelfServiceHeartbeatResponse zeroes every fenced field of resp when +// hiveID is a self-service hive, and returns the names of fields that held a +// value (never the values). Any field missing from heartbeatResponseFieldFence +// is treated as fenced, so an unclassified field fails closed even in a build +// that skipped the inventory test. For any other hive it does nothing. +func fenceSelfServiceHeartbeatResponse(resp *HeartbeatResponse, hiveID string) []string { + if resp == nil || !IsSelfServiceHiveID(hiveID) { + return nil + } + v := reflect.ValueOf(resp).Elem() + t := v.Type() + var withheld []string + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if heartbeatResponseFieldFence[f.Name] == heartbeatFieldAllowed { + continue + } + fv := v.Field(i) + if !fv.CanSet() || fv.IsZero() { + // Unexported fields are never marshalled to the wire. + continue + } + fv.Set(reflect.Zero(f.Type)) + withheld = append(withheld, f.Name) + } + return withheld +} diff --git a/src/pkg/hub/heartbeat_selfservice_fence_test.go b/src/pkg/hub/heartbeat_selfservice_fence_test.go new file mode 100644 index 0000000000..83d7aa4be2 --- /dev/null +++ b/src/pkg/hub/heartbeat_selfservice_fence_test.go @@ -0,0 +1,221 @@ +package hub + +import ( + "reflect" + "sort" + "testing" +) + +// TestHeartbeatResponseFieldFenceInventory fails when a HeartbeatResponse +// field has no credential-fence classification, or when the fence names a +// field that no longer exists. A new field must be classified as allowed or +// fenced before it can ship (#18060). +func TestHeartbeatResponseFieldFenceInventory(t *testing.T) { + typ := reflect.TypeOf(HeartbeatResponse{}) + seen := map[string]bool{} + var missing []string + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + seen[name] = true + switch heartbeatResponseFieldFence[name] { + case heartbeatFieldAllowed, heartbeatFieldFenced: + default: + missing = append(missing, name) + } + } + if len(missing) > 0 { + sort.Strings(missing) + t.Errorf("HeartbeatResponse fields with no credential-fence classification: %v\n"+ + "add each to heartbeatResponseFieldFence in heartbeat_selfservice_fence.go as "+ + "heartbeatFieldAllowed (protocol/display only) or heartbeatFieldFenced "+ + "(credential, management action, hub policy, or spoke-reported identity)", missing) + } + var stale []string + for name := range heartbeatResponseFieldFence { + if !seen[name] { + stale = append(stale, name) + } + } + if len(stale) > 0 { + sort.Strings(stale) + t.Errorf("heartbeatResponseFieldFence names fields HeartbeatResponse no longer has: %v", stale) + } +} + +// The fields the RFC names explicitly must stay fenced, so a later edit to the +// table cannot quietly reclassify a credential as allowed. +func TestHeartbeatResponseFieldFenceCoversCredentialsAndManagement(t *testing.T) { + for _, name := range []string{ + "GitHubAppConfig", "PendingGateway", "ProjectConfig", "ClusterID", + "UpgradeTo", "UpgradeInterrupt", "SwitchToTag", "UpgradeAnnounce", + "RestartSpoke", "AuthorizedUsers", "AuthorizedUserNames", + } { + if heartbeatResponseFieldFence[name] != heartbeatFieldFenced { + t.Errorf("%s must be heartbeatFieldFenced", name) + } + } +} + +func TestIsSelfServiceHiveID(t *testing.T) { + cases := map[string]bool{ + "byo-alice-lab": true, + "BYO-alice-lab": true, + " byo-x": true, + "byo": false, + "hosted-acme": false, + "saas-acme": false, + "kelly-headwaters": false, + "my-byo-hive": false, + "": false, + "byoverlay-cluster": false, + } + for id, want := range cases { + if got := IsSelfServiceHiveID(id); got != want { + t.Errorf("IsSelfServiceHiveID(%q) = %v, want %v", id, got, want) + } + } +} + +// filledHeartbeatResponse sets every exported field of a HeartbeatResponse to +// a non-zero value, so the fence test covers fields added after it was written. +func filledHeartbeatResponse(t *testing.T) *HeartbeatResponse { + t.Helper() + resp := &HeartbeatResponse{} + v := reflect.ValueOf(resp).Elem() + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + if !fv.CanSet() { + continue + } + fillNonZero(t, v.Type().Field(i).Name, fv) + if fv.IsZero() { + t.Fatalf("could not give HeartbeatResponse.%s a non-zero value", v.Type().Field(i).Name) + } + } + return resp +} + +func fillNonZero(t *testing.T, name string, fv reflect.Value) { + t.Helper() + switch fv.Kind() { + case reflect.Bool: + fv.SetBool(true) + case reflect.String: + fv.SetString("x") + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + fv.SetInt(1) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + fv.SetUint(1) + case reflect.Slice: + fv.Set(reflect.MakeSlice(fv.Type(), 1, 1)) + case reflect.Map: + fv.Set(reflect.MakeMap(fv.Type())) + fv.SetMapIndex(reflect.Zero(fv.Type().Key()), reflect.Zero(fv.Type().Elem())) + case reflect.Pointer: + fv.Set(reflect.New(fv.Type().Elem())) + case reflect.Struct: + for i := 0; i < fv.NumField(); i++ { + if fv.Field(i).CanSet() { + fillNonZero(t, name, fv.Field(i)) + return + } + } + default: + t.Fatalf("fillNonZero: unhandled kind %s for HeartbeatResponse.%s", fv.Kind(), name) + } +} + +func TestFenceSelfServiceHeartbeatResponseZeroesFencedFields(t *testing.T) { + resp := filledHeartbeatResponse(t) + withheld := fenceSelfServiceHeartbeatResponse(resp, "byo-alice-lab") + + withheldSet := map[string]bool{} + for _, name := range withheld { + withheldSet[name] = true + } + v := reflect.ValueOf(resp).Elem() + for i := 0; i < v.NumField(); i++ { + name := v.Type().Field(i).Name + switch heartbeatResponseFieldFence[name] { + case heartbeatFieldAllowed: + if v.Field(i).IsZero() { + t.Errorf("allowed field %s was cleared", name) + } + if withheldSet[name] { + t.Errorf("allowed field %s reported as withheld", name) + } + default: + if !v.Field(i).IsZero() { + t.Errorf("fenced field %s still set for a self-service hive", name) + } + if !withheldSet[name] { + t.Errorf("fenced field %s cleared but not reported as withheld", name) + } + } + } +} + +func TestFenceSelfServiceHeartbeatResponseLeavesOtherHivesAlone(t *testing.T) { + for _, id := range []string{"hosted-acme", "kelly-headwaters", ""} { + resp := filledHeartbeatResponse(t) + want := *resp + if withheld := fenceSelfServiceHeartbeatResponse(resp, id); withheld != nil { + t.Errorf("hive %q: withheld %v, want nothing", id, withheld) + } + if !reflect.DeepEqual(*resp, want) { + t.Errorf("hive %q: response changed by the fence", id) + } + } + if fenceSelfServiceHeartbeatResponse(nil, "byo-x") != nil { + t.Error("nil response must be a no-op") + } +} + +// End to end through the heartbeat handler: hub-held credentials queued for +// a self-service hive ID never reach the wire, while the same queue for an +// operator-listed hive still delivers, and display-only data still flows. +func TestHeartbeatHandlerFencesSelfServiceHive(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + + queue := func(s *HubServer, id string) { + s.pendingGitHubAppConfigs[id] = &HeartbeatGitHubAppConfig{AppID: 42, InstallationID: 7, PrivateKey: "-----BEGIN KEY-----"} + s.pendingGateways[id] = &HeartbeatGatewayConfig{Name: "openrouter", Kind: "openrouter", Endpoint: "https://example.invalid", Key: "sk-secret"} + s.hubBanners[id] = &HubBannerEntry{ID: "b1", Message: "maintenance tonight", Color: "info"} + } + + t.Run("self-service hive gets no credentials", func(t *testing.T) { + s := newHeartbeatHub() + const id = "byo-alice-lab" + queue(s, id) + resp := decodeHeartbeatResponse(t, s, `{"hive_id":"`+id+`","org":"alice","cluster_id":"hive-oke"}`) + if resp.GitHubAppConfig != nil { + t.Errorf("GitHubAppConfig delivered to a self-service hive: %+v", resp.GitHubAppConfig) + } + if resp.PendingGateway != nil { + t.Errorf("PendingGateway delivered to a self-service hive: %+v", resp.PendingGateway) + } + if resp.UpgradePolicy != nil { + t.Errorf("UpgradePolicy delivered to a self-service hive: %+v", resp.UpgradePolicy) + } + if !resp.OK || resp.HubGitHash == "" { + t.Errorf("protocol fields lost: ok=%v hub_git_hash=%q", resp.OK, resp.HubGitHash) + } + if resp.HubBanner == nil || resp.HubBanner.Message != "maintenance tonight" { + t.Errorf("display-only banner not delivered: %+v", resp.HubBanner) + } + }) + + t.Run("operator-listed hive still gets them", func(t *testing.T) { + s := newHeartbeatHub() + const id = "alice-lab" + queue(s, id) + resp := decodeHeartbeatResponse(t, s, `{"hive_id":"`+id+`","org":"alice","cluster_id":"hive-oke"}`) + if resp.GitHubAppConfig == nil || resp.GitHubAppConfig.PrivateKey == "" { + t.Errorf("GitHubAppConfig not delivered to an operator-listed hive: %+v", resp.GitHubAppConfig) + } + if resp.PendingGateway == nil { + t.Error("PendingGateway not delivered to an operator-listed hive") + } + }) +} diff --git a/src/pkg/hub/heartbeat_signing.go b/src/pkg/hub/heartbeat_signing.go index 8329642c27..94691f7362 100644 --- a/src/pkg/hub/heartbeat_signing.go +++ b/src/pkg/hub/heartbeat_signing.go @@ -3,6 +3,7 @@ package hub import ( "encoding/json" "net/http" + "strings" "sync" "time" @@ -54,7 +55,16 @@ func (h *heartbeatSeqState) next(hiveID string, now time.Time) int64 { // and it marshals resp once here to sign the exact bytes; the caller marshals // again to write. The two marshals are byte-identical (same struct, same // process), so the signature covers what the spoke receives. +// +// It also applies the self-service credential fence (#18060) first, and +// before the keyless early return, because every heartbeat response passes +// through here on its way to the wire: a self-service hive never receives a +// fenced field, signed or not. func (s *HubServer) signHeartbeatResponse(w http.ResponseWriter, resp *HeartbeatResponse, hiveID string) { + if withheld := fenceSelfServiceHeartbeatResponse(resp, hiveID); len(withheld) > 0 && s.logger != nil { + s.logger.Warn("heartbeat: credential fence withheld fields from self-service hive", + "hive_id", hiveID, "fields", strings.Join(withheld, ",")) + } seed := s.ssoSigningSeed() if seed == "" || hiveID == "" { // No key or no identity: leave the response unsigned (legacy behaviour). From 475a3604d8f4c8d8b52bc7ff2c373fb940ecbb5f Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 12:12:36 -0500 Subject: [PATCH 149/166] =?UTF-8?q?=F0=9F=90=9B=20knowledge:=20persist=20d?= =?UTF-8?q?ashboard-imported=20documents=20so=20upgrades=20keep=20them=20(?= =?UTF-8?q?#18088)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git sources connected from the dashboard are written to knowledge.git_sources and re-connected at boot, but an imported document (PDF, HTML, DOCX, text) had no record outside its /data/knowledge/documents/<slug>/metadata.json. When that on-disk record did not survive an upgrade, the document vanished from the knowledge sources list while the GitHub sources stayed. Record dashboard imports in knowledge.documents (deduped by slug), drop the entry on delete, and switch the boot loop to EnsureDocument, which keeps a document already reloaded from disk and re-imports a configured one that is missing. This also stops the boot loop from logging an "already imported" warning for every configured document on restart. Fixes #18058 Hive-Run: hivecommons/hive#18058 Hive-Plan: issue-18058-document-source-persistence Hive-Spec: hive#18058-upgrade-keeps-knowledge-sources Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: nate-double-u <4453979+nate-double-u@users.noreply.github.com> --- ...-18058-document-sources-survive-upgrade.md | 1 + src/cmd/hive/main.go | 10 ++- src/pkg/dashboard/api_knowledge.go | 53 ++++++++++++ .../api_knowledge_documents_persist_test.go | 82 +++++++++++++++++++ src/pkg/knowledge/api.go | 20 +++++ src/pkg/knowledge/api_ensure_document_test.go | 64 +++++++++++++++ 6 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-18058-document-sources-survive-upgrade.md create mode 100644 src/pkg/dashboard/api_knowledge_documents_persist_test.go create mode 100644 src/pkg/knowledge/api_ensure_document_test.go diff --git a/changelog.d/fixed-18058-document-sources-survive-upgrade.md b/changelog.d/fixed-18058-document-sources-survive-upgrade.md new file mode 100644 index 0000000000..473e1afa0a --- /dev/null +++ b/changelog.d/fixed-18058-document-sources-survive-upgrade.md @@ -0,0 +1 @@ +- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 68a979c6e3..4f5841fb67 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3125,12 +3125,20 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { FilePath: doc.FilePath, Layer: knowledge.LayerType(doc.Layer), } - meta, err := b.knowledgeAPI.ImportDocument(b.ctx, docConfig) + // EnsureDocument keeps a document already reloaded from disk and + // re-imports one whose on-disk state is gone, so a configured + // document survives an upgrade the way git sources do (#18058). + meta, imported, err := b.knowledgeAPI.EnsureDocument(b.ctx, docConfig) if err != nil { b.logger.Warn("failed to import document source", "name", doc.Name, "error", err, ) + } else if !imported { + b.logger.Debug("document source already loaded", + "name", doc.Name, + "facts", meta.FactCount, + ) } else { b.logger.Info("document source imported", "name", doc.Name, diff --git a/src/pkg/dashboard/api_knowledge.go b/src/pkg/dashboard/api_knowledge.go index f0643d563d..b3414363cf 100644 --- a/src/pkg/dashboard/api_knowledge.go +++ b/src/pkg/dashboard/api_knowledge.go @@ -1283,10 +1283,62 @@ func (s *Server) handleDocumentsImport(w http.ResponseWriter, r *http.Request) { jsonError(w, err.Error(), http.StatusInternalServerError) return } + s.persistDocumentSource(req) s.auditFromRequest(r, "document_import", auditDetail("name", req.Name), "") jsonResponse(w, meta) } +// persistDocumentSource records a dashboard-imported document in +// knowledge.documents, as handleGitSourcesConnect does for git sources. The +// document's metadata.json on the data volume was its only record, so an +// upgrade that lost it dropped the document while config-backed git sources +// survived (#18058); boot re-imports configured documents that are missing. +// Context7 imports are not recorded: knowledge.documents has no context7_id +// and Context7 has its own auto_import list. +func (s *Server) persistDocumentSource(req knowledge.DocSourceConfig) { + if s.deps.Config == nil || req.Context7ID != "" { + return + } + slug := knowledge.DocumentSlug(req.Name) + for _, d := range s.deps.Config.Knowledge.Documents { + if knowledge.DocumentSlug(d.Name) == slug { + return + } + } + s.deps.Config.Knowledge.Documents = append(s.deps.Config.Knowledge.Documents, config.DocSourceConfigYAML{ + Name: req.Name, + URL: req.URL, + FilePath: req.FilePath, + Layer: string(req.Layer), + }) + if err := s.saveConfig(); err != nil { + s.logger.Error("failed to persist config after document import", "error", err) + } +} + +// unpersistDocumentSource drops a deleted document from knowledge.documents +// so boot does not re-import it. +func (s *Server) unpersistDocumentSource(slug string) { + if s.deps.Config == nil { + return + } + docs := s.deps.Config.Knowledge.Documents + filtered := make([]config.DocSourceConfigYAML, 0, len(docs)) + for _, d := range docs { + if knowledge.DocumentSlug(d.Name) == slug { + continue + } + filtered = append(filtered, d) + } + if len(filtered) == len(docs) { + return + } + s.deps.Config.Knowledge.Documents = filtered + if err := s.saveConfig(); err != nil { + s.logger.Error("failed to persist config after document delete", "error", err) + } +} + func (s *Server) handleDocumentGet(w http.ResponseWriter, r *http.Request) { if !s.ensureKnowledge() { jsonError(w, "knowledge not configured", http.StatusServiceUnavailable) @@ -1313,6 +1365,7 @@ func (s *Server) handleDocumentDelete(w http.ResponseWriter, r *http.Request) { jsonError(w, err.Error(), http.StatusNotFound) return } + s.unpersistDocumentSource(slug) s.auditFromRequest(r, "document_delete", auditDetail("slug", slug), "") okResponse(w, map[string]string{"status": "deleted", "slug": slug}) } diff --git a/src/pkg/dashboard/api_knowledge_documents_persist_test.go b/src/pkg/dashboard/api_knowledge_documents_persist_test.go new file mode 100644 index 0000000000..247335c2f2 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_documents_persist_test.go @@ -0,0 +1,82 @@ +package dashboard + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge" +) + +// TestDocumentsImportPersistsAndDeleteUnpersists pins #18058: a document +// imported from the dashboard is recorded in knowledge.documents (as git +// sources are in knowledge.git_sources) so boot can restore it after an +// upgrade, a repeat import does not duplicate the entry, and deleting the +// document drops it from config so boot does not resurrect it. +func TestDocumentsImportPersistsAndDeleteUnpersists(t *testing.T) { + base := t.TempDir() + knowledge.SetBaseDirForTest(t, base) + source := filepath.Join(base, "uploads", "guide.txt") + if err := os.MkdirAll(filepath.Dir(source), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("Guide\n\nAgents must cite the tech docs guide."), 0o644); err != nil { + t.Fatal(err) + } + + s, deps := apiServer(t) + deps.Config.SourcePath = filepath.Join(t.TempDir(), "hive.yaml") + deps.Knowledge = knowledge.NewKnowledgeAPI(nil, knowledge.KnowledgeConfig{Enabled: true, Engine: "file"}, deps.Logger) + deps.Config.Knowledge.Documents = []config.DocSourceConfigYAML{{Name: "other-doc", URL: "https://example.com/other.pdf", Layer: "project"}} + + rec := doPost(s, "/api/knowledge/documents", map[string]any{"name": "Tech Docs Guide", "file_path": source}) + if rec.Code != http.StatusOK { + t.Fatalf("import: expected 200, got %d (body %q)", rec.Code, rec.Body.String()) + } + want := []config.DocSourceConfigYAML{ + {Name: "other-doc", URL: "https://example.com/other.pdf", Layer: "project"}, + {Name: "Tech Docs Guide", FilePath: source, Layer: "project"}, + } + assertDocs := func(stage string, want []config.DocSourceConfigYAML) { + t.Helper() + got := deps.Config.Knowledge.Documents + if len(got) != len(want) { + t.Fatalf("%s: knowledge.documents = %+v, want %+v", stage, got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%s: knowledge.documents = %+v, want %+v", stage, got, want) + } + } + } + assertDocs("after import", want) + persisted, err := os.ReadFile(deps.Config.SourcePath) + if err != nil { + t.Fatalf("read persisted config: %v", err) + } + if !strings.Contains(string(persisted), "Tech Docs Guide") { + t.Fatalf("persisted config is missing the imported document:\n%s", persisted) + } + + // A config entry for the same slug (e.g. a re-import after delete-from-disk) + // must not be duplicated; the knowledge API rejects the second import, so + // exercise the dedupe directly. + s.persistDocumentSource(knowledge.DocSourceConfig{Name: "tech docs guide", FilePath: source, Layer: knowledge.LayerProject}) + assertDocs("after duplicate persist", want) + + rec = doDelete(s, "/api/knowledge/documents/"+knowledge.DocumentSlug("Tech Docs Guide"), nil) + if rec.Code != http.StatusOK { + t.Fatalf("delete: expected 200, got %d (body %q)", rec.Code, rec.Body.String()) + } + assertDocs("after delete", want[:1]) + persisted, err = os.ReadFile(deps.Config.SourcePath) + if err != nil { + t.Fatalf("read persisted config: %v", err) + } + if text := string(persisted); strings.Contains(text, "Tech Docs Guide") || !strings.Contains(text, "other-doc") { + t.Fatalf("persisted config did not drop only the deleted document:\n%s", text) + } +} diff --git a/src/pkg/knowledge/api.go b/src/pkg/knowledge/api.go index c83db1ea61..d26886445e 100644 --- a/src/pkg/knowledge/api.go +++ b/src/pkg/knowledge/api.go @@ -1105,6 +1105,26 @@ func (k *KnowledgeAPI) ImportDocument(ctx context.Context, config DocSourceConfi return meta, nil } +// DocumentSlug returns the slug a document named name is stored under, the +// same key GetDocument and DeleteDocument take. +func DocumentSlug(name string) string { return slugify(name) } + +// EnsureDocument imports config unless a document with the same slug is +// already loaded (typically reloaded from its on-disk metadata at startup). +// imported reports whether a fresh import ran. It is the boot path for +// knowledge.documents: a configured document whose on-disk state was lost is +// re-imported instead of silently disappearing (#18058). +func (k *KnowledgeAPI) EnsureDocument(ctx context.Context, config DocSourceConfig) (meta *DocMetadata, imported bool, err error) { + if existing, getErr := k.GetDocument(DocumentSlug(config.Name)); getErr == nil { + return existing, false, nil + } + meta, err = k.ImportDocument(ctx, config) + if err != nil { + return nil, false, err + } + return meta, true, nil +} + // ListDocuments returns metadata for all imported documents. func (k *KnowledgeAPI) ListDocuments() []DocMetadata { k.mu.RLock() diff --git a/src/pkg/knowledge/api_ensure_document_test.go b/src/pkg/knowledge/api_ensure_document_test.go new file mode 100644 index 0000000000..1f276f68cb --- /dev/null +++ b/src/pkg/knowledge/api_ensure_document_test.go @@ -0,0 +1,64 @@ +package knowledge + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// TestEnsureDocumentReimportsAfterLostDiskState pins #18058: a configured +// document whose on-disk metadata is gone after an upgrade must be imported +// again at boot rather than silently dropped, while one that was reloaded +// from disk must not be imported a second time. +func TestEnsureDocumentReimportsAfterLostDiskState(t *testing.T) { + base := withKnowledgeBaseDir(t) + source := filepath.Join(base, "uploads", "guide.txt") + if err := os.MkdirAll(filepath.Dir(source), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("Guide\n\nAgents must cite the tech docs guide."), 0o644); err != nil { + t.Fatal(err) + } + cfg := DocSourceConfig{Name: "Tech Docs Guide", FilePath: source, Layer: LayerProject} + + first := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + meta, imported, err := first.EnsureDocument(context.Background(), cfg) + if err != nil || !imported { + t.Fatalf("first EnsureDocument = %+v, imported=%v, err=%v; want a fresh import", meta, imported, err) + } + slug := DocumentSlug(cfg.Name) + if meta.Slug != slug { + t.Fatalf("slug = %q, want %q", meta.Slug, slug) + } + + // Restart with the disk state intact: the reloaded document is kept as-is. + restarted := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + got, imported, err := restarted.EnsureDocument(context.Background(), cfg) + if err != nil { + t.Fatalf("EnsureDocument after restart: %v", err) + } + if imported { + t.Fatal("EnsureDocument re-imported a document that was reloaded from disk") + } + if got.Slug != slug || got.FactCount != meta.FactCount { + t.Fatalf("EnsureDocument after restart = %+v, want reloaded %+v", got, meta) + } + + // Upgrade lost the document's on-disk record: the configured source must + // come back instead of vanishing from the knowledge sources list. + if err := os.RemoveAll(filepath.Join(base, docStorageDir)); err != nil { + t.Fatal(err) + } + upgraded := NewKnowledgeAPI(nil, KnowledgeConfig{Enabled: true}, importDocTestLogger()) + if docs := upgraded.ListDocuments(); len(docs) != 0 { + t.Fatalf("precondition: expected no documents reloaded, got %+v", docs) + } + got, imported, err = upgraded.EnsureDocument(context.Background(), cfg) + if err != nil || !imported { + t.Fatalf("EnsureDocument after lost disk state = %+v, imported=%v, err=%v; want a re-import", got, imported, err) + } + if docs := upgraded.ListDocuments(); len(docs) != 1 || docs[0].Slug != slug { + t.Fatalf("ListDocuments after re-import = %+v, want %q", docs, slug) + } +} From 51921a7bac8660a5b94056f389f6abfb9af1b7ec Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 17:15:23 +0000 Subject: [PATCH 150/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.242.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.242.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 16 ++++++++++++++++ changelog.d/added-chevron-stage-icons.md | 1 - changelog.d/changed-18063.md | 1 - changelog.d/changed-18074.md | 1 - .../fixed-benchmarks-canonical-model-id.md | 1 - changelog.d/fixed-role-claim-reject-log.md | 1 - 6 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/added-chevron-stage-icons.md delete mode 100644 changelog.d/changed-18063.md delete mode 100644 changelog.d/changed-18074.md delete mode 100644 changelog.d/fixed-benchmarks-canonical-model-id.md delete mode 100644 changelog.d/fixed-role-claim-reject-log.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5290f196c0..79c94c0d44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.242.0) + +### Added + +- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). + +### Changed + +- images: bump gh 2.103.3 -> 2.104.0 (#18063) +- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) + +### Fixed + +- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. +- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). + ## 2026-11-13 (v5.241.1) ### Changed diff --git a/changelog.d/added-chevron-stage-icons.md b/changelog.d/added-chevron-stage-icons.md deleted file mode 100644 index 341298d6e9..0000000000 --- a/changelog.d/added-chevron-stage-icons.md +++ /dev/null @@ -1 +0,0 @@ -- Activity monitor: a workflow-model chevron that is not the row's active stage now wears a glyph for what the stage means (inbox, bolt, hand, pencil, pull-request, flask, eye, people, thumbs-up, merge, check, …) picked by the column's name, dimmed to 60%, instead of its clipped label; the active chevron leads its word with the same glyph. Workflow columns take an optional `icon` (an allowlisted icon id or a single emoji, validated server- and client-side; never markup), and the workflow editor's column inspector previews the default and lets the operator replace it (#18033). diff --git a/changelog.d/changed-18063.md b/changelog.d/changed-18063.md deleted file mode 100644 index 8aa025ec82..0000000000 --- a/changelog.d/changed-18063.md +++ /dev/null @@ -1 +0,0 @@ -- images: bump gh 2.103.3 -> 2.104.0 (#18063) diff --git a/changelog.d/changed-18074.md b/changelog.d/changed-18074.md deleted file mode 100644 index beeb0a4829..0000000000 --- a/changelog.d/changed-18074.md +++ /dev/null @@ -1 +0,0 @@ -- kicks: register REFERENCE REPOS as layout section 10a and golden its wording (#18074) diff --git a/changelog.d/fixed-benchmarks-canonical-model-id.md b/changelog.d/fixed-benchmarks-canonical-model-id.md deleted file mode 100644 index 044a0042fc..0000000000 --- a/changelog.d/fixed-benchmarks-canonical-model-id.md +++ /dev/null @@ -1 +0,0 @@ -- Benchmarks export now adds `model.canonical` and `model.effort` next to the reported id and derives `model.provider` from the model family, so `claude-sonnet-5-5` (Copilot CLI), `claude-sonnet-5.5` (Claude Code) and `openai-codex/gpt-6.1-sol` no longer appear as separate models on hivecommons.dev/benchmarks. diff --git a/changelog.d/fixed-role-claim-reject-log.md b/changelog.d/fixed-role-claim-reject-log.md deleted file mode 100644 index e5ca0de6d8..0000000000 --- a/changelog.d/fixed-role-claim-reject-log.md +++ /dev/null @@ -1 +0,0 @@ -- Repeated trust-tier agent role claim rejections over contribute-ws are now logged once per user and role at INFO instead of flooding WARN, so they no longer surface as unclassified self-reported errors (#18071). From 487b3be8f236d46e8294880a6826e2b7dcb5c92a Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:42:44 -0500 Subject: [PATCH 151/166] fix(dashboard): surface Check result for knowledge connectors (#18096) The Knowledge Integrations Check button showed a 'checking' toast and then nothing. kbPlaudAuthLoad now returns the state it stored and kbPlaudRefreshAuth reports it: an error toast with the server's error string verbatim (e.g. the 404 'no knowledge connectors are configured'), or a success/info toast with the sign-in state. The card still re-renders. Fixes #18092 Signed-off-by: Andy Anderson <andy@clubanderson.com> --- .../fixed-18092-knowledge-check-toast.md | 1 + .../knowledge_plaud_check_feedback.test.js | 96 +++++++++++++++++++ src/pkg/dashboard/static-src/js/knowledge.js | 14 ++- src/pkg/dashboard/static/index.html | 14 ++- 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18092-knowledge-check-toast.md create mode 100644 src/pkg/dashboard/knowledge_plaud_check_feedback.test.js diff --git a/changelog.d/fixed-18092-knowledge-check-toast.md b/changelog.d/fixed-18092-knowledge-check-toast.md new file mode 100644 index 0000000000..1500f367c0 --- /dev/null +++ b/changelog.d/fixed-18092-knowledge-check-toast.md @@ -0,0 +1 @@ +- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). diff --git a/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js b/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js new file mode 100644 index 0000000000..26aa1006a3 --- /dev/null +++ b/src/pkg/dashboard/knowledge_plaud_check_feedback.test.js @@ -0,0 +1,96 @@ +// Knowledge Integrations "Check" (hivecommons/hive#18092): the per-connector +// Check button must always report an outcome. A failed status call surfaces +// the server's `error` string verbatim in an error toast; a successful one +// surfaces the sign-in state in a success/info toast and re-renders the card. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +// Like the other dashboard tests' fnSrc, but keeps a leading `async` so the +// extracted handlers can still await. +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; + +const NOT_CONFIGURED = 'no knowledge connectors are configured'; +const HTTP_NOT_FOUND = 404; +const HTTP_OK = 200; + +const setup = ({ status, body, cardPresent = true }) => { + const toasts = []; + const calls = { urls: [], renders: 0 }; + const fetch = async url => { + calls.urls.push(url); + return { ok: status === HTTP_OK, status, json: async () => body }; + }; + const document = { querySelector: () => (cardPresent ? {} : null) }; + const CSS = { escape: s => s }; + const h = new Function('fetch', 'document', 'CSS', 'showToast', 'kbRenderIntegrationsModal', + 'const _kbPlaudAuth = {};\n' + fnSrc('kbPlaudAuthLoad') + '\n' + fnSrc('kbPlaudAuthSummary') + '\n' + fnSrc('kbPlaudRefreshAuth') + + '\nreturn { kbPlaudRefreshAuth, _kbPlaudAuth };')(fetch, document, CSS, (m, k) => toasts.push([m, k]), () => { calls.renders++; }); + return { h, toasts, calls }; +}; + +test('a 404 from the auth endpoint shows an error toast with the server error verbatim', async () => { + const { h, toasts, calls } = setup({ status: HTTP_NOT_FOUND, body: { error: NOT_CONFIGURED } }); + await h.kbPlaudRefreshAuth('plaud-voice'); + assert.deepStrictEqual(calls.urls, ['/api/knowledge/connectors/plaud-voice/auth?refresh=1']); + assert.strictEqual(toasts[0][1], 'info'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'error'); + assert.ok(last[0].includes(NOT_CONFIGURED), 'toast must carry the server error: ' + last[0]); + assert.strictEqual(calls.renders, 1, 'card re-rendered with the error state'); + assert.strictEqual(h._kbPlaudAuth['plaud-voice'].error, NOT_CONFIGURED); +}); + +test('a non-JSON failure falls back to the HTTP status in the error toast', async () => { + const { h, toasts } = setup({ status: HTTP_NOT_FOUND, body: {} }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'error'); + assert.ok(last[0].includes('HTTP ' + HTTP_NOT_FOUND), last[0]); +}); + +test('a connected state shows a success toast naming the account and refreshes the card', async () => { + const { h, toasts, calls } = setup({ status: HTTP_OK, body: { connected: true, account: 'andy' } }); + await h.kbPlaudRefreshAuth('plaud-voice'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'success'); + assert.ok(last[0].includes('plaud-voice') && last[0].includes('connected as andy'), last[0]); + assert.strictEqual(calls.renders, 1); +}); + +test('a not-connected state is reported as info, with the server message', async () => { + const { h, toasts } = setup({ status: HTTP_OK, body: { connected: false, message: 'token expired' } }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'info'); + assert.ok(last[0].includes('not connected') && last[0].includes('token expired'), last[0]); +}); + +test('a pending state tells the user where the authorization link is', async () => { + const { h, toasts } = setup({ status: HTTP_OK, body: { pending: true, auth_url: 'https://example.invalid/auth' } }); + await h.kbPlaudRefreshAuth('p'); + const last = toasts[toasts.length - 1]; + assert.strictEqual(last[1], 'info'); + assert.ok(last[0].includes('waiting for authorization'), last[0]); +}); + +test('the toast is still shown when the card is no longer in the DOM', async () => { + const { h, toasts, calls } = setup({ status: HTTP_NOT_FOUND, body: { error: NOT_CONFIGURED }, cardPresent: false }); + await h.kbPlaudRefreshAuth('p'); + assert.strictEqual(calls.renders, 0); + assert.strictEqual(toasts[toasts.length - 1][1], 'error'); +}); diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index ac6380b949..4cd1dfa12c 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2596,10 +2596,22 @@ _kbPlaudAuth[name] = { error: e.message }; } if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + return _kbPlaudAuth[name]; } + // One-line summary of a sign-in state for the Check toast (hivecommons/hive#18092). + function kbPlaudAuthSummary(a) { + if (a.connected) return 'connected' + (a.account ? ' as ' + a.account : ''); + if (a.pending) return 'waiting for authorization' + (a.auth_url ? ' — open the link on the card' : ''); + return 'not connected' + (a.message ? ' — ' + a.message : ''); + } + // Check: re-query the sign-in state and always report the outcome, so a + // 404 "no knowledge connectors are configured" reads as an error toast + // with the server's message verbatim, not as silence (hivecommons/hive#18092). async function kbPlaudRefreshAuth(name) { showToast('Checking Plaud sign-in…', 'info'); - await kbPlaudAuthLoad(name, true); + const a = (await kbPlaudAuthLoad(name, true)) || {}; + if (a.error) showToast('Check failed: ' + a.error, 'error'); + else showToast('Plaud ' + name + ': ' + kbPlaudAuthSummary(a), a.connected ? 'success' : 'info'); } async function kbPlaudConnect(name) { try { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 01b9fd6063..a78345f735 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -46526,10 +46526,22 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> _kbPlaudAuth[name] = { error: e.message }; } if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + return _kbPlaudAuth[name]; } + // One-line summary of a sign-in state for the Check toast (hivecommons/hive#18092). + function kbPlaudAuthSummary(a) { + if (a.connected) return 'connected' + (a.account ? ' as ' + a.account : ''); + if (a.pending) return 'waiting for authorization' + (a.auth_url ? ' — open the link on the card' : ''); + return 'not connected' + (a.message ? ' — ' + a.message : ''); + } + // Check: re-query the sign-in state and always report the outcome, so a + // 404 "no knowledge connectors are configured" reads as an error toast + // with the server's message verbatim, not as silence (hivecommons/hive#18092). async function kbPlaudRefreshAuth(name) { showToast('Checking Plaud sign-in…', 'info'); - await kbPlaudAuthLoad(name, true); + const a = (await kbPlaudAuthLoad(name, true)) || {}; + if (a.error) showToast('Check failed: ' + a.error, 'error'); + else showToast('Plaud ' + name + ': ' + kbPlaudAuthSummary(a), a.connected ? 'success' : 'info'); } async function kbPlaudConnect(name) { try { From 9c3d8874885baa7f1209c4224897e81336e15cde Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:45:06 -0500 Subject: [PATCH 152/166] =?UTF-8?q?=E2=9C=A8=20Link=20the=20fleet=20benchm?= =?UTF-8?q?arks=20from=20the=20hub=20nav,=20My=20Hives=20and=20the=20spoke?= =?UTF-8?q?=20FAQ=20(#18039)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Link the fleet benchmarks from the hub nav, My Hives and the spoke FAQ Fixes #18034 Signed-off-by: Andrew Anderson <andy@clubanderson.com> * 🐛 benchmarks links: use tokens instead of inline styles Signed-off-by: Andy Anderson <andy@clubanderson.com> * 🐛 dashboard: Twitch link test tolerates text after the FAQ link The FAQ intro now continues with a benchmarks sentence, so the test no longer pins the closing </p> right after the Twitch link. Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andrew Anderson <andy@clubanderson.com> Signed-off-by: Andy Anderson <andy@clubanderson.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/added-benchmarks-links.md | 1 + src/pkg/dashboard/static-src/html/faq.html | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/dashboard/twitch_link_test.go | 2 +- src/pkg/hub/assets/dashboard.html | 4 ++++ src/pkg/hub/static/index.html | 1 + src/pkg/hub/static/my-hives.html | 1 + 7 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-benchmarks-links.md diff --git a/changelog.d/added-benchmarks-links.md b/changelog.d/added-benchmarks-links.md new file mode 100644 index 0000000000..26cf95b2bb --- /dev/null +++ b/changelog.d/added-benchmarks-links.md @@ -0,0 +1 @@ +- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). diff --git a/src/pkg/dashboard/static-src/html/faq.html b/src/pkg/dashboard/static-src/html/faq.html index 6075f22dea..41ace5bf37 100644 --- a/src/pkg/dashboard/static-src/html/faq.html +++ b/src/pkg/dashboard/static-src/html/faq.html @@ -6,7 +6,7 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" <p class="faq-intro">Short answers to the questions people ask once a hive is running. Everything below describes what this hive actually does today — where behaviour is surprising, it is called out rather than smoothed over. Each answer links to the source doc, so this panel stays a map, not a second source of truth.</p> - <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>.</p> + <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>. Wondering which model to run? <a class="text-accent fw-semibold" href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Compare models on real fleet PR outcomes</a> (merge rate, CI first-try, estimated cost).</p> <div class="faq-filter-row"> <label for="faq-filter" class="faq-filter-label">Filter</label> diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index a78345f735..af856dff8b 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -9198,7 +9198,7 @@ <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" <p class="faq-intro">Short answers to the questions people ask once a hive is running. Everything below describes what this hive actually does today — where behaviour is surprising, it is called out rather than smoothed over. Each answer links to the source doc, so this panel stays a map, not a second source of truth.</p> - <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>.</p> + <p class="faq-intro">Need help from humans? <a class="text-accent fw-semibold" href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a>; the Hive Commons community can help you pick the right next step. Want to see a hive at work? <a class="text-accent fw-semibold" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener">Watch the hive live on Twitch</a>. Wondering which model to run? <a class="text-accent fw-semibold" href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Compare models on real fleet PR outcomes</a> (merge rate, CI first-try, estimated cost).</p> <div class="faq-filter-row"> <label for="faq-filter" class="faq-filter-label">Filter</label> diff --git a/src/pkg/dashboard/twitch_link_test.go b/src/pkg/dashboard/twitch_link_test.go index 0ad4f8aa70..d5d462a828 100644 --- a/src/pkg/dashboard/twitch_link_test.go +++ b/src/pkg/dashboard/twitch_link_test.go @@ -12,7 +12,7 @@ func TestTwitchLiveStreamLinkInDashboardShell(t *testing.T) { for _, want := range []string{ `<a class="oc-nav-item" href="https://www.twitch.tv/hivecommons" target="_blank" rel="noopener"`, `<span class="oc-nav-text">Watch live on Twitch</span>`, - `Watch the hive live on Twitch</a>.</p>`, + `Watch the hive live on Twitch</a>.`, } { if !strings.Contains(html, want) { t.Fatalf("index.html missing %q", want) diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index 2e910c1e5e..c1a2da62b5 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -107,6 +107,8 @@ } h1 { letter-spacing: 0; font-size: clamp(2rem, 4vw, 3rem); line-height: .98; margin-bottom: .8rem; } .subtitle { color: var(--muted); font-size: 1.02rem; line-height: 1.7; margin-bottom: var(--sp-9); } + .benchmarks-pointer { margin-top: var(--sp-3); padding: var(--sp-3) var(--sp-5); border: 1px solid var(--line); border-radius: var(--r-sm); font-size: var(--fs-sm); color: var(--muted); } + .benchmarks-pointer a { color: var(--amber); font-weight: 700; } /* ── Table ── */ /* Status filter chips above the My Hives table. --chip-color is set inline @@ -764,6 +766,7 @@ <a href="/fleet" id="nav-fleet" style="display:none" title="Fleet health — agents the governor expects on but that can't work">Fleet</a> <a href="/api/docs" target="_blank">API</a> <a href="https://kubestellar.io/docs/hive/overview/introduction" target="_blank" rel="noopener">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <span id="nav-user" class="nav-user"></span> </nav> <div class="header-right"> @@ -779,6 +782,7 @@ <h1>My Hives</h1> <p class="subtitle">Hive instances you own or have access to</p> <p style="margin-top:var(--sp-4);padding:8px 14px;border:1px solid var(--line);border-radius:8px;background:color-mix(in srgb,var(--hub-dashboard-accent) 8%,transparent);font-size:0.85rem">👋 New to Hive? Read the <a href="https://docs.kubestellar.io/docs/hive/getting-started" target="_blank" rel="noopener" style="color:var(--amber);font-weight:700">Getting Started Guide</a> before diving in.</p> + <p id="benchmarks-pointer" class="benchmarks-pointer">📊 See how models perform across the fleet — merge rate, CI first-try and estimated cost per PR, from real hive output. <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener">Open the benchmarks ↗</a></p> <p id="latest-image-sha" style="font-size:0.7rem;color:var(--muted);margin-top:var(--sp-2)"></p> <!-- Image-pulls bar chart: per-release container-image PULLS of the public spoke image (ghcr.io/hivecommons/hive), bucketed by the diff --git a/src/pkg/hub/static/index.html b/src/pkg/hub/static/index.html index 09c9211c9e..b423cf2a99 100644 --- a/src/pkg/hub/static/index.html +++ b/src/pkg/hub/static/index.html @@ -623,6 +623,7 @@ <a href="#how-it-works">How it works</a> <a href="/learn">Learn</a> <a href="https://docs.hivecommons.dev" target="_blank" rel="noopener noreferrer">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener noreferrer" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <a href="/reading">Reading</a> <a href="/get-started">Get Started</a> <a href="https://hivecommons.dev/discord" target="_blank" rel="noopener">Join our Discord</a> diff --git a/src/pkg/hub/static/my-hives.html b/src/pkg/hub/static/my-hives.html index 5f66da02e0..23b2345128 100644 --- a/src/pkg/hub/static/my-hives.html +++ b/src/pkg/hub/static/my-hives.html @@ -242,6 +242,7 @@ <a href="/fleet" id="nav-fleet" class="active" aria-current="page" title="Fleet health — agents the governor expects on but that can't work">Fleet</a> <a href="/api/docs">API</a> <a href="https://docs.hivecommons.dev" target="_blank" rel="noopener">Docs</a> + <a href="https://hivecommons.dev/benchmarks/" target="_blank" rel="noopener" title="Real PR outcomes per model and agent across the fleet">Benchmarks</a> <span id="nav-user" class="nav-user"></span> </nav> <div class="header-right"> From fd3a496548a0f4098fe6009093c87fab79dd0533 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 17:48:05 +0000 Subject: [PATCH 153/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.243.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.243.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 20 +++++++++++++++++++ changelog.d/added-18021-reference-tier.md | 1 - ...changed-18028-webhook-create-idempotent.md | 1 - ...anged-review-perspectives-source-reload.md | 1 - changelog.d/fixed-18054-hold-release-guard.md | 1 - ...-18058-document-sources-survive-upgrade.md | 1 - ...rity-18060-selfservice-credential-fence.md | 1 - 7 files changed, 20 insertions(+), 6 deletions(-) delete mode 100644 changelog.d/added-18021-reference-tier.md delete mode 100644 changelog.d/changed-18028-webhook-create-idempotent.md delete mode 100644 changelog.d/changed-review-perspectives-source-reload.md delete mode 100644 changelog.d/fixed-18054-hold-release-guard.md delete mode 100644 changelog.d/fixed-18058-document-sources-survive-upgrade.md delete mode 100644 changelog.d/security-18060-selfservice-credential-fence.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c94c0d44..72f6020238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,26 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.243.0) + +### Added + +- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). + +### Changed + +- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) +- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) + +### Fixed + +- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) +- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. + +### Security + +- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). + ## 2026-11-13 (v5.242.0) ### Added diff --git a/changelog.d/added-18021-reference-tier.md b/changelog.d/added-18021-reference-tier.md deleted file mode 100644 index 358b39b129..0000000000 --- a/changelog.d/added-18021-reference-tier.md +++ /dev/null @@ -1 +0,0 @@ -- `/api/status` `referenceRepos[]` entries now carry an additive `reference_tier` field (`read-only`, `issues`, `prs` or `issues+prs`) so API consumers can tell a `project.reference_repos` entry from a managed repo; the entries stay out of `repos[]` and carry no ACMM, work-item or pause fields, and the dashboard's `📚 reference · <tier>` chip reads it ([#18021](https://github.com/hivecommons/hive/issues/18021)). diff --git a/changelog.d/changed-18028-webhook-create-idempotent.md b/changelog.d/changed-18028-webhook-create-idempotent.md deleted file mode 100644 index 01d2c011f1..0000000000 --- a/changelog.d/changed-18028-webhook-create-idempotent.md +++ /dev/null @@ -1 +0,0 @@ -- Dashboard: "Create webhook for me" is idempotent — it returns the existing hook for this hive's payload URL instead of creating a duplicate, and answers 412 naming the missing `organization_hooks`/`repository_hooks` write permission (#18028) diff --git a/changelog.d/changed-review-perspectives-source-reload.md b/changelog.d/changed-review-perspectives-source-reload.md deleted file mode 100644 index c517a8ac1f..0000000000 --- a/changelog.d/changed-review-perspectives-source-reload.md +++ /dev/null @@ -1 +0,0 @@ -- **`review.perspectives_source` is now resolved at startup and on every config reload, and the review relay perspective set is reinstalled each time**, so a verdict naming a perspective the link removed is refused and one it added is accepted without a restart. A dashboard edit of the review config no longer drops the linked set, and a malformed or unreachable document keeps the last-known-good set with a warning. ([#17988](https://github.com/hivecommons/hive/issues/17988), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-18054-hold-release-guard.md b/changelog.d/fixed-18054-hold-release-guard.md deleted file mode 100644 index 6e43c24352..0000000000 --- a/changelog.d/fixed-18054-hold-release-guard.md +++ /dev/null @@ -1 +0,0 @@ -- dashboard: a Projects-card Release that lands while the rows are re-sorting after a Hold is no longer sent, so holding items down a list no longer releases a neighbouring item's hold (#18054) diff --git a/changelog.d/fixed-18058-document-sources-survive-upgrade.md b/changelog.d/fixed-18058-document-sources-survive-upgrade.md deleted file mode 100644 index 473e1afa0a..0000000000 --- a/changelog.d/fixed-18058-document-sources-survive-upgrade.md +++ /dev/null @@ -1 +0,0 @@ -- Documents imported from the dashboard (PDF, HTML, DOCX, text) are now saved to `knowledge.documents` in `hive.yaml` like git sources, and boot re-imports any configured document whose on-disk record is missing, so an upgrade no longer drops a PDF knowledge source while keeping GitHub ones. diff --git a/changelog.d/security-18060-selfservice-credential-fence.md b/changelog.d/security-18060-selfservice-credential-fence.md deleted file mode 100644 index 961ff47b53..0000000000 --- a/changelog.d/security-18060-selfservice-credential-fence.md +++ /dev/null @@ -1 +0,0 @@ -- The hub now withholds hub-held credentials and management actions (GitHub App config and keys, pending gateway, project config, cluster ID, access list, upgrade, restart and breaker instructions) from the heartbeat response of any hive whose ID is in the reserved `byo-` self-service namespace, through one guard driven by a classification of every response field ([#18060](https://github.com/hivecommons/hive/issues/18060)). From bb010a649fec9406c566bba44669e89b8aedb93f Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 12:59:45 -0500 Subject: [PATCH 154/166] =?UTF-8?q?=F0=9F=90=9B=20knowledge:=20'not=20load?= =?UTF-8?q?ed'=20badge=20and=20disabled=20actions=20when=20the=20connector?= =?UTF-8?q?=20runtime=20is=20not=20running=20(#18097)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dashboard): show 'not loaded' badge when the connector runtime is not running When the status response carries runtime:false (hive booted with no knowledge.connectors, so the syncer is nil), connector cards showed a 'pending' badge, which reads as 'awaiting OAuth sign-in'. Cards now show a distinct 'not loaded' badge with a one-line restart hint and docs link, disable Connect / Check / Sync with the same hint instead of letting them 404, and skip the Plaud auth probe that would also 404. 'pending' remains reserved for a loaded connector awaiting sign-in or first sync. Fixes #18093 Signed-off-by: Andy Anderson <andy@clubanderson.com> * fix(dashboard): only an explicit runtime:false marks a connector not loaded A failed or absent /status response was coerced to runtime:false, which would have rendered every card as 'not loaded' with a restart instruction after a transient 5xx. Store st.runtime !== false instead, drop the unused kc-not-loaded class, and cover the failed-status, missing-key and explicit false cases in the test. Signed-off-by: Andy Anderson <andy@clubanderson.com> * test(dashboard): match the Plaud card button call after the not-loaded change kbPlaudAuthButtons now takes a disabledAttr argument, so the index.html snippet asserted by TestKnowledgeIntegrationsModalPlaudCard is updated to the new call shape; the auth-line snippet gains its else prefix. Signed-off-by: Andy Anderson <andy@clubanderson.com> --------- Signed-off-by: Andy Anderson <andy@clubanderson.com> --- ...93-knowledge-connector-not-loaded-badge.md | 1 + .../knowledge_connector_not_loaded.test.js | 132 ++++++++++++++++++ .../knowledge_integrations_modal_test.go | 4 +- .../dashboard/static-src/css/knowledge.css | 1 + src/pkg/dashboard/static-src/js/knowledge.js | 37 +++-- src/pkg/dashboard/static/index.html | 38 +++-- 6 files changed, 193 insertions(+), 20 deletions(-) create mode 100644 changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md create mode 100644 src/pkg/dashboard/knowledge_connector_not_loaded.test.js diff --git a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md new file mode 100644 index 0000000000..319a50e047 --- /dev/null +++ b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md @@ -0,0 +1 @@ +- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). diff --git a/src/pkg/dashboard/knowledge_connector_not_loaded.test.js b/src/pkg/dashboard/knowledge_connector_not_loaded.test.js new file mode 100644 index 0000000000..b04bd8cc41 --- /dev/null +++ b/src/pkg/dashboard/knowledge_connector_not_loaded.test.js @@ -0,0 +1,132 @@ +// Knowledge Integrations connector card when the backend syncer is not running +// (hivecommons/hive#18093): the status response's `runtime: false` turns the +// `pending` badge into a distinct `not loaded` badge with a one-line hint and +// a docs link, and disables Connect / Check / Sync with the same hint. A loaded +// connector awaiting sign-in keeps `pending`; a disabled entry keeps `disabled`. +// Run: node --test src/pkg/dashboard/*.test.js +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); +// Keeps a leading `async` so extracted loaders can still await. +const ASYNC_PREFIX = 'async '; +const fnSrc = name => { + let start = html.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + if (html.slice(start - ASYNC_PREFIX.length, start) === ASYNC_PREFIX) start -= ASYNC_PREFIX.length; + let depth = 0; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) return html.slice(start, i + 1); + } + throw new Error('unbalanced ' + name); +}; +// The two string constants next to the helpers, lifted from the source so the +// test does not duplicate the badge text or hint. +const constSrc = name => { + const m = html.match(new RegExp('const ' + name + " = '([^']*)';")); + assert.ok(m, name + ' not found'); + return m[1]; +}; + +const NOT_LOADED = constSrc('KC_STATUS_NOT_LOADED'); +const HINT = constSrc('KC_NOT_LOADED_HINT'); +const OWNER = true; + +const render = ({ runtime, c, row, type = 'plaud', owner = OWNER }) => { + const calls = { authLines: 0 }; + const escapeHtml = s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); + const h = new Function('_kcState', '_kbPlaudAuth', 'escapeHtml', 'kcAgo', 'kbPlaudAuthLine', 'kbPlaudPasteHtml', 'kbWisprFlowSourceLine', + fnSrc('kbConnectorNotLoaded') + '\n' + fnSrc('kbConnectorNotLoadedHtml') + '\n' + fnSrc('kbPlaudAuthButtons') + '\n' + fnSrc('kbConnectorRow') + + "\nconst KC_STATUS_NOT_LOADED = '" + NOT_LOADED + "'; const KC_NOT_LOADED_HINT = '" + HINT + "';" + + '\nreturn kbConnectorRow;')( + { runtime, rows: row ? [row] : [], publish: null }, {}, escapeHtml, () => '—', + () => { calls.authLines++; return '<div data-kb-plaud-auth></div>'; }, () => '', () => ''); + return { html: h(type, c, owner), calls }; +}; + +const badge = out => { const m = out.match(/<span class="kc-pill" data-status="([^"]*)">([^<]*)<\/span>/); assert.ok(m, 'badge missing'); assert.strictEqual(m[1], m[2]); return m[1]; }; +const buttons = out => [...out.matchAll(/<button [^>]*data-action="([^"]+)"[^>]*>/g)].map(m => ({ action: m[1], disabled: / disabled\b/.test(m[0]), title: (m[0].match(/title="([^"]*)"/) || [])[1] })); + +test('runtime:false shows a not-loaded badge, the hint and a docs link instead of pending', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out, calls } = render({ runtime: false, c, row: { name: 'plaud-voice', status: 'pending' } }); + assert.strictEqual(badge(out), NOT_LOADED); + assert.ok(!out.includes('>pending<'), 'pending badge must not appear'); + assert.ok(out.includes('data-kb-not-loaded'), 'hint line rendered'); + assert.ok(out.includes(HINT), 'hint text rendered'); + assert.ok(out.includes('href="/docs/knowledge-connectors.md"'), 'docs link rendered'); + assert.strictEqual(calls.authLines, 0, 'the Plaud auth probe (which would 404) is not rendered'); +}); + +test('runtime:false disables Connect, Check and Sync and explains why in the title', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out } = render({ runtime: false, c, row: null }); + const b = buttons(out); + const byAction = Object.fromEntries(b.map(x => [x.action, x])); + for (const action of ['kbPlaudConnect', 'kbPlaudRefreshAuth', 'kbIntegrationSync']) { + assert.ok(byAction[action], action + ' rendered'); + assert.ok(byAction[action].disabled, action + ' disabled'); + assert.strictEqual(byAction[action].title, HINT, action + ' carries the hint'); + } + assert.ok(!byAction.kbIntegrationEdit.disabled, 'Edit stays enabled'); + assert.ok(!byAction.kbIntegrationToggle.disabled, 'Enable/Disable stays enabled'); +}); + +test('runtime:true keeps pending for a loaded connector awaiting sign-in, with live buttons', () => { + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + const { html: out, calls } = render({ runtime: true, c, row: { name: 'plaud-voice', status: 'pending' } }); + assert.strictEqual(badge(out), 'pending'); + assert.ok(!out.includes('data-kb-not-loaded')); + assert.strictEqual(calls.authLines, 1); + const b = buttons(out); + assert.ok(b.some(x => x.action === 'kbPlaudRefreshAuth' && !x.disabled), 'Check enabled'); + assert.ok(b.some(x => x.action === 'kbIntegrationSync' && !x.disabled), 'Sync enabled'); +}); + +test('a disabled entry keeps the disabled badge even when the runtime is not loaded', () => { + const c = { name: 'wiki', type: 'confluence', enabled: false }; + const { html: out } = render({ runtime: false, c, row: { name: 'wiki', status: 'disabled' }, type: 'confluence' }); + assert.strictEqual(badge(out), 'disabled'); + assert.ok(!out.includes('data-kb-not-loaded')); +}); + +// Drive the real loader with stubbed config + status fetches and return the +// runtime flag it stores, so the status-fetch failure path is covered. +const HTTP_OK = 200; +const HTTP_SERVER_ERROR = 500; +const loadRuntime = async statusRes => { + const cfgRes = { ok: true, status: HTTP_OK, json: async () => ({ connectors: [{ name: 'plaud-voice', type: 'plaud', enabled: true }], types: ['plaud'] }) }; + const fetch = async url => (url.endsWith('/status') ? statusRes : cfgRes); + const state = { layers: [] }; + const run = new Function('fetch', '_kcState', 'kbRenderIntegrationsModal', + "let _kbIntegrationSelected = 'plaud';\n" + fnSrc('kbLoadIntegrationsModal') + '\nreturn kbLoadIntegrationsModal("plaud").then(() => _kcState);'); + return (await run(fetch, state, () => {})).runtime; +}; + +test('a failed status fetch leaves runtime true, so cards keep pending rather than claiming not loaded', async () => { + assert.strictEqual(await loadRuntime({ ok: false, status: HTTP_SERVER_ERROR, json: async () => ({ error: 'boom' }) }), true); + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + assert.strictEqual(badge(render({ runtime: true, c, row: null }).html), 'pending'); +}); + +test('a status response without a runtime key is treated as loaded', async () => { + assert.strictEqual(await loadRuntime({ ok: true, status: HTTP_OK, json: async () => ({ connectors: [] }) }), true); +}); + +test('an explicit runtime:false in the status response is stored and renders not loaded', async () => { + const runtime = await loadRuntime({ ok: true, status: HTTP_OK, json: async () => ({ connectors: [], runtime: false }) }); + assert.strictEqual(runtime, false); + const c = { name: 'plaud-voice', type: 'plaud', enabled: true }; + assert.strictEqual(badge(render({ runtime, c, row: null }).html), NOT_LOADED); +}); + +test('non-owner view still shows the not-loaded badge and hint without action buttons', () => { + const c = { name: 'wiki', type: 'confluence', enabled: true }; + const { html: out } = render({ runtime: false, c, row: null, type: 'confluence', owner: false }); + assert.strictEqual(badge(out), NOT_LOADED); + assert.ok(out.includes('data-kb-not-loaded')); + assert.strictEqual(buttons(out).length, 0); +}); diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 8dc4184b81..c10e188d90 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -155,10 +155,10 @@ func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { for _, want := range []string{ `'plaud': { label: 'Plaud voice notes'`, `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, - `if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, + `else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, `data-action="kbPlaudPasteComplete"`, `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete'`, - `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, + `if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : '');`, `data-action="kbPlaudConnect"`, `data-action="kbPlaudDisconnect"`, `data-action="kbPlaudRefreshAuth"`, diff --git a/src/pkg/dashboard/static-src/css/knowledge.css b/src/pkg/dashboard/static-src/css/knowledge.css index 1f129c0545..8c4ff2e085 100644 --- a/src/pkg/dashboard/static-src/css/knowledge.css +++ b/src/pkg/dashboard/static-src/css/knowledge.css @@ -18,6 +18,7 @@ .kc-pill[data-status="ok"] { background: var(--green); } .kc-pill[data-status="syncing"] { background: var(--blue); } .kc-pill[data-status="error"] { background: var(--red); } + .kc-pill[data-status="not loaded"] { background: var(--orange); } .kc-publish { margin-top: var(--sp-4); border-top: var(--line-width) solid var(--border); padding-top: var(--sp-3); } .kc-publish-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: var(--sp-2); } .kc-publish-head h4 { margin: 0; } diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index 4cd1dfa12c..2438327e5c 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2367,7 +2367,7 @@ const cfg = await cfgRes.json().catch(() => ({})); if (!cfgRes.ok) throw new Error(cfg.error || ('HTTP ' + cfgRes.status)); const st = stRes.ok ? await stRes.json().catch(() => ({})) : {}; - _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: !!st.runtime, publish: st.publish || null }); + _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: st.runtime !== false, publish: st.publish || null }); } catch (e) { _kcState = Object.assign(_kcState, { loaded: true, error: e.message }); } @@ -2423,23 +2423,40 @@ return body; } + // Badge shown when the hive's connector syncer is not running (status + // response `runtime: false`): the entry is saved but this process never + // constructed it, so sign-in, Check and Sync would all 404. Distinct from + // `pending`, which is a loaded connector awaiting sign-in or first sync. + // Only an explicit `runtime: false` counts: a failed or absent status + // response must not tell the owner to restart the hive + // (hivecommons/hive#18093). + const KC_STATUS_NOT_LOADED = 'not loaded'; + const KC_NOT_LOADED_HINT = 'Connector runtime is not running on this hive — restart the hive after enabling knowledge connectors.'; + function kbConnectorNotLoaded(c) { return !!c.enabled && !_kcState.runtime; } + function kbConnectorNotLoadedHtml() { + return '<div class="muted-xs" data-kb-not-loaded>' + escapeHtml(KC_NOT_LOADED_HINT) + ' <a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener">Docs ↗</a></div>'; + } + function kbConnectorRow(type, c, owner) { const byName = {}; (_kcState.rows || []).forEach(r => { byName[r.name] = r; }); const r = byName[c.name] || {}; - const status = r.status || (c.enabled ? 'pending' : 'disabled'); + const notLoaded = kbConnectorNotLoaded(c); + const status = notLoaded ? KC_STATUS_NOT_LOADED : (r.status || (c.enabled ? 'pending' : 'disabled')); const publish = _kcState.publish && _kcState.publish.connector === c.name ? ' · publish ' + (_kcState.publish.running ? 'syncing' : _kcState.publish.last_error ? 'error' : _kcState.publish.last_success ? 'ok' : 'pending') : ''; let html = '<div class="card-inset row-between mb-3" data-kb-connector-name="' + escapeHtml(c.name) + '"><div class="kb-integration-row-main">'; html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (notLoaded) html += kbConnectorNotLoadedHtml(); + else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { + const disabledAttr = ' disabled title="' + escapeHtml(KC_NOT_LOADED_HINT) + '"'; html += '<div class="kb-integration-actions">'; - if (type === 'plaud') html += kbPlaudAuthButtons(c.name); - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : ''); + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (notLoaded ? disabledAttr : status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; html += '</div>'; @@ -2577,13 +2594,15 @@ else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; } - function kbPlaudAuthButtons(name) { + // disabledAttr, when set, is appended to every button (connector not loaded, #18093). + function kbPlaudAuthButtons(name, disabledAttr) { const a = _kbPlaudAuth[name] || {}; const n = escapeHtml(name); + const off = disabledAttr || ''; let html = ''; - if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; - else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '"' + off + '>Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (off || (a.pending ? ' disabled' : '')) + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '"' + (off || ' title="Re-check sign-in with Plaud"') + '>Check</button>'; return html; } async function kbPlaudAuthLoad(name, refresh) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index af856dff8b..6359a27634 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5821,6 +5821,7 @@ .kc-pill[data-status="ok"] { background: var(--green); } .kc-pill[data-status="syncing"] { background: var(--blue); } .kc-pill[data-status="error"] { background: var(--red); } + .kc-pill[data-status="not loaded"] { background: var(--orange); } .kc-publish { margin-top: var(--sp-4); border-top: var(--line-width) solid var(--border); padding-top: var(--sp-3); } .kc-publish-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: var(--sp-2); } .kc-publish-head h4 { margin: 0; } @@ -46297,7 +46298,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const cfg = await cfgRes.json().catch(() => ({})); if (!cfgRes.ok) throw new Error(cfg.error || ('HTTP ' + cfgRes.status)); const st = stRes.ok ? await stRes.json().catch(() => ({})) : {}; - _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: !!st.runtime, publish: st.publish || null }); + _kcState = Object.assign(_kcState, { loaded: true, error: '', connectors: cfg.connectors || [], types: cfg.types || [], ctx7AutoImport: cfg.context7_auto_import || [], layers: cfg.layers || _kcState.layers, rows: st.connectors || [], runtime: st.runtime !== false, publish: st.publish || null }); } catch (e) { _kcState = Object.assign(_kcState, { loaded: true, error: e.message }); } @@ -46353,23 +46354,40 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> return body; } + // Badge shown when the hive's connector syncer is not running (status + // response `runtime: false`): the entry is saved but this process never + // constructed it, so sign-in, Check and Sync would all 404. Distinct from + // `pending`, which is a loaded connector awaiting sign-in or first sync. + // Only an explicit `runtime: false` counts: a failed or absent status + // response must not tell the owner to restart the hive + // (hivecommons/hive#18093). + const KC_STATUS_NOT_LOADED = 'not loaded'; + const KC_NOT_LOADED_HINT = 'Connector runtime is not running on this hive — restart the hive after enabling knowledge connectors.'; + function kbConnectorNotLoaded(c) { return !!c.enabled && !_kcState.runtime; } + function kbConnectorNotLoadedHtml() { + return '<div class="muted-xs" data-kb-not-loaded>' + escapeHtml(KC_NOT_LOADED_HINT) + ' <a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener">Docs ↗</a></div>'; + } + function kbConnectorRow(type, c, owner) { const byName = {}; (_kcState.rows || []).forEach(r => { byName[r.name] = r; }); const r = byName[c.name] || {}; - const status = r.status || (c.enabled ? 'pending' : 'disabled'); + const notLoaded = kbConnectorNotLoaded(c); + const status = notLoaded ? KC_STATUS_NOT_LOADED : (r.status || (c.enabled ? 'pending' : 'disabled')); const publish = _kcState.publish && _kcState.publish.connector === c.name ? ' · publish ' + (_kcState.publish.running ? 'syncing' : _kcState.publish.last_error ? 'error' : _kcState.publish.last_success ? 'ok' : 'pending') : ''; let html = '<div class="card-inset row-between mb-3" data-kb-connector-name="' + escapeHtml(c.name) + '"><div class="kb-integration-row-main">'; html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; - if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (notLoaded) html += kbConnectorNotLoadedHtml(); + else if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { + const disabledAttr = ' disabled title="' + escapeHtml(KC_NOT_LOADED_HINT) + '"'; html += '<div class="kb-integration-actions">'; - if (type === 'plaud') html += kbPlaudAuthButtons(c.name); - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name, notLoaded ? disabledAttr : ''); + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (notLoaded ? disabledAttr : status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; html += '</div>'; @@ -46507,13 +46525,15 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; } - function kbPlaudAuthButtons(name) { + // disabledAttr, when set, is appended to every button (connector not loaded, #18093). + function kbPlaudAuthButtons(name, disabledAttr) { const a = _kbPlaudAuth[name] || {}; const n = escapeHtml(name); + const off = disabledAttr || ''; let html = ''; - if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; - else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; - html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '"' + off + '>Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (off || (a.pending ? ' disabled' : '')) + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '"' + (off || ' title="Re-check sign-in with Plaud"') + '>Check</button>'; return html; } async function kbPlaudAuthLoad(name, refresh) { From 2284f84eab1afdd27ab8ee760044ad72a4529f4e Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:09:44 -0500 Subject: [PATCH 155/166] =?UTF-8?q?=F0=9F=90=9B=20fix(ioscan):=20retry=20o?= =?UTF-8?q?ne=20transient=20classifier=20endpoint=20failure=20before=20fai?= =?UTF-8?q?ling=20open=20(#18106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single 429/5xx or dropped connection from the ioscan model endpoint made the scheduler apply its fail mode immediately: under fail-open the text passed on unclassified and the hive filed an ioscan_classifier_unavailable selfreport (#18105, previously #15077 on the same fingerprint). The HTTP chat client now retries such a transient failure once after 500ms. Timeouts and permanent statuses (4xx other than 429) are not retried, and a caller cancellation during the pause returns the original error. Hive-Run: hivecommons/hive#18105 Hive-Plan: selfreport ioscan_classifier_unavailable fix Hive-Spec: ioscan-classifier#fail-mode Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../fixed-18105-ioscan-classifier-retry.md | 1 + src/docs/fleet-report.md | 2 +- src/pkg/ioscan/classifier.go | 80 ++++++- src/pkg/ioscan/classifier_retry_test.go | 208 ++++++++++++++++++ 4 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fixed-18105-ioscan-classifier-retry.md create mode 100644 src/pkg/ioscan/classifier_retry_test.go diff --git a/changelog.d/fixed-18105-ioscan-classifier-retry.md b/changelog.d/fixed-18105-ioscan-classifier-retry.md new file mode 100644 index 0000000000..133036f418 --- /dev/null +++ b/changelog.d/fixed-18105-ioscan-classifier-retry.md @@ -0,0 +1 @@ +- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 7c0b9a20dc..f0b350b191 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -150,7 +150,7 @@ title is `[selfreport] <class>: <short message>`. | `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | | `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | -| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | +| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). The HTTP model endpoint retries one transient failure (429, 500/502/503/504, or a connection dropped before a reply; not a timeout) before either fail mode applies, so a single gateway blip is neither withheld nor passed on unclassified. Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/pkg/ioscan/classifier.go b/src/pkg/ioscan/classifier.go index c09b20591f..523980f8e7 100644 --- a/src/pkg/ioscan/classifier.go +++ b/src/pkg/ioscan/classifier.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -27,6 +28,7 @@ const ( DefaultClassifierCacheEntries = 1024 DefaultClassifierBudgetPerEval = 64 classifierTimeout = 20 * time.Second + classifierRetryDelay = 500 * time.Millisecond classifierMaxTokens = 180 classifierMaxRationaleChars = 500 SemanticClassifierRule = "injection.semantic_classifier" @@ -202,10 +204,11 @@ func NewLLMClassifier(c LLMClassifierConfig) (*LLMClassifier, error) { return nil, fmt.Errorf("ioscan classifier: no model endpoint resolved") } client = &openAIChatClient{ - endpoint: endpoint, - apiKey: c.APIKey, - model: model, - http: &http.Client{Timeout: classifierTimeout}, + endpoint: endpoint, + apiKey: c.APIKey, + model: model, + http: &http.Client{Timeout: classifierTimeout}, + retryDelay: classifierRetryDelay, } } return &LLMClassifier{client: client}, nil @@ -287,8 +290,24 @@ type openAIChatClient struct { apiKey string model string http *http.Client + // retryDelay is the pause before the single retry of a transient + // failure (see transientClassifierErr). + retryDelay time.Duration } +// transientClassifierErr marks a failure the endpoint is likely to recover +// from on the next request: a rate limit, a gateway/upstream 5xx, or a +// connection dropped before a reply. Timeouts are deliberately not transient +// so a hung backend costs one classifierTimeout per string, not two. +type transientClassifierErr struct{ err error } + +func (e *transientClassifierErr) Error() string { return e.err.Error() } +func (e *transientClassifierErr) Unwrap() error { return e.err } + +// Complete sends one chat completion. A transient failure is retried once +// after retryDelay so a single gateway blip does not make the scheduler pass +// untrusted text on unclassified (fail-open) or withhold it (fail-closed) +// (hivecommons/hive#18105). func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, maxTokens int) (string, error) { body, err := json.Marshal(chatRequest{ Model: c.model, @@ -299,6 +318,48 @@ func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, if err != nil { return "", err } + reply, err := c.completeOnce(ctx, body) + var transient *transientClassifierErr + if err == nil || !errors.As(err, &transient) { + return reply, err + } + if c.retryDelay > 0 { + t := time.NewTimer(c.retryDelay) + select { + case <-ctx.Done(): + t.Stop() + return "", err + case <-t.C: + } + } + reply, retryErr := c.completeOnce(ctx, body) + if retryErr != nil { + return "", fmt.Errorf("%w (after 1 retry)", retryErr) + } + return reply, nil +} + +// retryableStatus reports whether an HTTP status from the classifier model +// endpoint is worth one retry. +func retryableStatus(code int) bool { + switch code { + case http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, + http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + } + return false +} + +// isTimeoutErr reports whether err is a request or dial timeout. +func isTimeoutErr(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var te interface{ Timeout() bool } + return errors.As(err, &te) && te.Timeout() +} + +func (c *openAIChatClient) completeOnce(ctx context.Context, body []byte) (string, error) { reqCtx, cancel := context.WithTimeout(ctx, classifierTimeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, c.endpoint+"/v1/chat/completions", bytes.NewReader(body)) @@ -311,11 +372,18 @@ func (c *openAIChatClient) Complete(ctx context.Context, messages []chatMessage, } resp, err := c.http.Do(req) if err != nil { - return "", err + if ctx.Err() != nil || isTimeoutErr(err) { + return "", err + } + return "", &transientClassifierErr{err: err} } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("classifier model returned %d", resp.StatusCode) + statusErr := fmt.Errorf("classifier model returned %d", resp.StatusCode) + if retryableStatus(resp.StatusCode) { + return "", &transientClassifierErr{err: statusErr} + } + return "", statusErr } respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { diff --git a/src/pkg/ioscan/classifier_retry_test.go b/src/pkg/ioscan/classifier_retry_test.go new file mode 100644 index 0000000000..3c89ee9f0f --- /dev/null +++ b/src/pkg/ioscan/classifier_retry_test.go @@ -0,0 +1,208 @@ +package ioscan + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +const benignReply = `{"choices":[{"message":{"content":"{\"score\":0.1,\"category\":\"benign\",\"rationale\":\"ok\"}"}}]}` + +// A single gateway blip must not reach the scheduler's fail mode: the +// transient statuses are retried once and the second reply is used +// (hivecommons/hive#18105). +func TestOpenAIChatClientRetriesTransientStatusOnce(t *testing.T) { + for _, status := range []int{ + http.StatusTooManyRequests, + http.StatusInternalServerError, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + } { + t.Run(fmt.Sprint(status), func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(status) + return + } + fmt.Fprint(w, benignReply) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + reply, err := client.Complete(context.Background(), nil, 10) + if err != nil { + t.Fatalf("Complete: %v, want the retry to succeed", err) + } + if !strings.Contains(reply, `"category":"benign"`) { + t.Fatalf("reply = %q, want the retried reply", reply) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2 (one failure, one retry)", got) + } + }) + } +} + +// A persistent transient failure is retried exactly once, then surfaces with +// the original status so the fail-mode WARN still names it. +func TestOpenAIChatClientGivesUpAfterOneRetry(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + _, err := client.Complete(context.Background(), nil, 10) + if err == nil || !strings.Contains(err.Error(), "returned 503") || !strings.Contains(err.Error(), "after 1 retry") { + t.Fatalf("Complete error = %v, want the 503 noted as retried", err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2", got) + } +} + +// Client errors (bad key, unknown model) will not fix themselves: no retry. +func TestOpenAIChatClientDoesNotRetryPermanentStatus(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusNotFound} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(status) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + _, err := client.Complete(context.Background(), nil, 10) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("returned %d", status)) { + t.Fatalf("Complete error = %v, want the %d status", err, status) + } + if strings.Contains(err.Error(), "retry") { + t.Fatalf("Complete error = %v, want no retry for a permanent status", err) + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1", got) + } + }) + } +} + +// A connection dropped before any reply (gateway restart) is retried. +func TestOpenAIChatClientRetriesDroppedConnection(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("response writer cannot hijack") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + _ = conn.Close() + return + } + fmt.Fprint(w, benignReply) + })) + defer srv.Close() + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: srv.Client()} + if _, err := client.Complete(context.Background(), nil, 10); err != nil { + t.Fatalf("Complete: %v, want the retry after a dropped connection to succeed", err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2", got) + } +} + +// A timeout is not retried: a hung backend must cost one timeout per string, +// not two. +func TestOpenAIChatClientDoesNotRetryTimeout(t *testing.T) { + var calls atomic.Int32 + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + select { + case <-release: + case <-r.Context().Done(): + } + })) + defer srv.Close() + defer close(release) + httpClient := srv.Client() + httpClient.Timeout = 50 * time.Millisecond + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: httpClient} + if _, err := client.Complete(context.Background(), nil, 10); err == nil { + t.Fatal("Complete: expected a timeout error") + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1 (timeouts are not retried)", got) + } +} + +// cancelAfterTransport cancels the caller's context as soon as the first +// response arrives, so the cancellation lands in the retry pause. +type cancelAfterTransport struct { + next http.RoundTripper + cancel context.CancelFunc +} + +func (c cancelAfterTransport) RoundTrip(r *http.Request) (*http.Response, error) { + resp, err := c.next.RoundTrip(r) + c.cancel() + return resp, err +} + +// A caller that cancels during the retry pause gets the original error back +// without a second request. +func TestOpenAIChatClientRetryHonoursCancellation(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusBadGateway) + })) + defer srv.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + httpClient := srv.Client() + httpClient.Transport = cancelAfterTransport{next: httpClient.Transport, cancel: cancel} + client := &openAIChatClient{endpoint: srv.URL, model: "judge-1", http: httpClient, retryDelay: time.Hour} + done := make(chan error, 1) + go func() { + _, err := client.Complete(ctx, nil, 10) + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "returned 502") { + t.Fatalf("Complete error = %v, want the original 502", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Complete did not return after cancellation during the retry pause") + } + if got := calls.Load(); got != 1 { + t.Fatalf("calls = %d, want 1 (no retry after cancellation)", got) + } +} + +func TestNewLLMClassifierSetsRetryDelay(t *testing.T) { + c, err := NewLLMClassifier(LLMClassifierConfig{Endpoint: "https://models.example"}) + if err != nil { + t.Fatalf("NewLLMClassifier: %v", err) + } + client, ok := c.client.(*openAIChatClient) + if !ok { + t.Fatalf("client = %T, want the built-in OpenAI-compatible client", c.client) + } + if client.retryDelay != classifierRetryDelay { + t.Fatalf("retryDelay = %v, want %v", client.retryDelay, classifierRetryDelay) + } +} From 86679e5a24bd55ce886f2ab8c1ca3a601c040464 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:19:42 -0500 Subject: [PATCH 156/166] =?UTF-8?q?=F0=9F=90=9B=20fix(automerge):=20warn?= =?UTF-8?q?=20once=20per=20author=20and=20repo=20for=20unsearchable=20cont?= =?UTF-8?q?ributor=20search=20(#18089)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix(automerge): warn once per author and repo for unsearchable contributor search The contributor-lane 422 warning was keyed per (author, repository) pair, so one repository the App cannot see warned once for every contributor, and one unsearchable author warned once for every hive repository. TunaOS logged it 15 times in 10m and self-reported it as unclassified_error_repeated. Refusals are still remembered per pair for the TTL, but WARN now fires only when neither the author nor the searched repository has warned before; everything else logs at Debug. Fixes #18059 Hive-Run: hivecommons/hive#18059 Hive-Plan: selfreport-unclassified-error-repeated Hive-Spec: contributor-lane-auto-promotion#unsearchable-warn-once Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * 🧪 automerge: pin the 4096-entry cap on the unsearchable maps The mutation run on #18089 left both `len(...) > 4096` resets in noteContributorUnsearchable surviving as `>=`. Add a boundary test: a map holding exactly 4096 entries is kept, and one past the cap is reset. Each `>=` mutant now fails the test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...xed-18059-contributor-unsearchable-warn.md | 1 + .../automerge_contributor_unsearchable.go | 29 ++++++-- ...automerge_contributor_unsearchable_test.go | 67 +++++++++++++++++++ src/pkg/github/automerge/automerge_sweep.go | 8 ++- 4 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixed-18059-contributor-unsearchable-warn.md diff --git a/changelog.d/fixed-18059-contributor-unsearchable-warn.md b/changelog.d/fixed-18059-contributor-unsearchable-warn.md new file mode 100644 index 0000000000..75dd2ae587 --- /dev/null +++ b/changelog.d/fixed-18059-contributor-unsearchable-warn.md @@ -0,0 +1 @@ +- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). diff --git a/src/pkg/github/automerge/automerge_contributor_unsearchable.go b/src/pkg/github/automerge/automerge_contributor_unsearchable.go index ee9714616b..eab10b38e9 100644 --- a/src/pkg/github/automerge/automerge_contributor_unsearchable.go +++ b/src/pkg/github/automerge/automerge_contributor_unsearchable.go @@ -17,8 +17,12 @@ import ( // TTL lets a repaired repository or account come back without a restart. const contributorUnsearchableTTL = time.Hour -// contributorUnsearchableMessage is logged once per (author, repository) -// pair per process at WARN, then at Debug. +// contributorUnsearchableMessage is logged at WARN only for a refusal whose +// author and searched repository have both not been warned for yet in this +// process, then at Debug. Keying the WARN by pair let one invisible +// repository warn once for every contributor (or one unsearchable author once +// for every hive repository), which the self-report classifier counted as a +// repeated unclassified error (hivecommons/hive#18059). const contributorUnsearchableMessage = "contributor-lane auto-promotion search cannot see repository for this author; leaving it out of the hive-wide count" // searchQueryUnsearchable reports whether err is GitHub's 422 Validation @@ -45,20 +49,31 @@ func (c *Engine) contributorSearchSkips(author string, ref contributorRepoRef) b } // noteContributorUnsearchable records that GitHub refused to search ref for -// author, so the next contributorUnsearchableTTL of sweeps leave it out. The -// first refusal for a pair logs at WARN; a refusal again after the TTL logs -// at Debug, so a permanently unsearchable pair is not a repeated warning. +// author, so the next contributorUnsearchableTTL of sweeps leave it out. A +// refusal logs at WARN only when neither its author nor its searched +// repository has warned before in this process; every other refusal — +// another author for the same invisible repository, another repository for +// the same unsearchable author, or the same pair again after the TTL — logs +// at Debug, so the warnings stay bounded by the distinct causes rather than +// growing with authors × repositories. func (c *Engine) noteContributorUnsearchable(owner, repo, author string, ref contributorRepoRef, err error) { key := contributorUnsearchableKey(author, ref) + authorKey := "author:" + strings.ToLower(author) + repoKey := "repo:" + strings.ToLower(ref.Owner+"/"+ref.Repo) c.contributorUnsearchableMu.Lock() if c.contributorUnsearchable == nil || len(c.contributorUnsearchable) > 4096 { c.contributorUnsearchable = map[string]time.Time{} } - _, seen := c.contributorUnsearchable[key] + if c.contributorUnsearchableWarned == nil || len(c.contributorUnsearchableWarned) > 4096 { + c.contributorUnsearchableWarned = map[string]bool{} + } c.contributorUnsearchable[key] = c.clock().Add(contributorUnsearchableTTL) + warned := c.contributorUnsearchableWarned[authorKey] || c.contributorUnsearchableWarned[repoKey] + c.contributorUnsearchableWarned[authorKey] = true + c.contributorUnsearchableWarned[repoKey] = true c.contributorUnsearchableMu.Unlock() args := []any{"repo", owner + "/" + repo, "author", author, "searched_repo", ref.Owner + "/" + ref.Repo, "retry_after", contributorUnsearchableTTL, "error", err} - if seen { + if warned { c.debug(contributorUnsearchableMessage, args...) return } diff --git a/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go b/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go index 99d84bb503..c30cd3ee89 100644 --- a/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go +++ b/src/pkg/github/automerge/automerge_contributor_unsearchable_test.go @@ -3,6 +3,8 @@ package automerge import ( "bytes" "context" + "errors" + "fmt" "log/slog" "strings" "testing" @@ -72,3 +74,68 @@ func TestContributorMergedReposRemembersUnsearchableRepository(t *testing.T) { t.Fatalf("warning should name the refused repository:\n%s", out) } } + +// Regression for hivecommons/hive#18059: one author refused for several +// repositories, and several authors refused for the same repository, warn +// once in total instead of once per (author, repository) pair — the pair +// product flooded WARN (15 in 10m on one hive) and self-reported as an +// unclassified repeated error. Every refusal is still remembered per pair. +func TestContributorUnsearchableWarnsOncePerAuthorAndRepository(t *testing.T) { + fx := validContributorFixture() + fx.unsearchable = map[string]bool{"acme/docs": true, "acme/homebrew-hive": true} + api := newContributorLaneAPI(t, &fx) + defer api.Close() + var logs bytes.Buffer + clock := &contributorLaneClock{at: time.Date(2026, 11, 13, 3, 0, 0, 0, time.UTC)} + engine := newContributorLaneEngineRepos(api.URL, []string{"widget", "hive", "docs", "homebrew-hive"}, defaultContributorLanePolicy(), clock, Options{ + Logger: slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + authors := []string{contributorLaneTestAuthor, "thatguywob", "another-dev"} + for _, author := range authors { + engine.contributorMergedRepos(context.Background(), "acme", "widget", author) + } + + out := logs.String() + if n := strings.Count(out, contributorUnsearchableMessage); n != 1 { + t.Fatalf("unsearchable warning logged %d times for %d authors × 2 refused repositories, want 1:\n%s", n, len(authors), out) + } + for _, author := range authors { + for _, refused := range []string{"docs", "homebrew-hive"} { + if !engine.contributorSearchSkips(author, contributorRepoRef{Owner: "acme", Repo: refused}) { + t.Fatalf("refusal of acme/%s for %s was not remembered", refused, author) + } + } + } +} + +// The pair TTL map and the WARN-dedup map are each bounded at 4096 entries: +// a map holding exactly 4096 is kept, and only one past the cap is reset, so +// the cap is the documented size and not one entry fewer. +func TestNoteContributorUnsearchableResetsOnlyPastCap(t *testing.T) { + const capacity = 4096 + engine := &Engine{ + contributorUnsearchable: map[string]time.Time{}, + contributorUnsearchableWarned: map[string]bool{}, + } + for i := 0; i < capacity; i++ { + engine.contributorUnsearchable[fmt.Sprintf("filler-%d", i)] = time.Time{} + engine.contributorUnsearchableWarned[fmt.Sprintf("filler-%d", i)] = true + } + refusal := errors.New("422") + + engine.noteContributorUnsearchable("acme", "widget", "first-dev", contributorRepoRef{Owner: "acme", Repo: "docs"}, refusal) + if got := len(engine.contributorUnsearchable); got != capacity+1 { + t.Fatalf("pair map at the cap: len = %d, want %d (kept, new pair added)", got, capacity+1) + } + if got := len(engine.contributorUnsearchableWarned); got != capacity+2 { + t.Fatalf("warned map at the cap: len = %d, want %d (kept, author and repo added)", got, capacity+2) + } + + engine.noteContributorUnsearchable("acme", "widget", "second-dev", contributorRepoRef{Owner: "acme", Repo: "wiki"}, refusal) + if got := len(engine.contributorUnsearchable); got != 1 { + t.Fatalf("pair map past the cap: len = %d, want 1 (reset, new pair added)", got) + } + if got := len(engine.contributorUnsearchableWarned); got != 2 { + t.Fatalf("warned map past the cap: len = %d, want 2 (reset, author and repo added)", got) + } +} diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index e0afef2944..0f8d9375fe 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -297,8 +297,12 @@ type Engine struct { // contributorUnsearchable maps author|owner/repo to the time until which // contributorMergedRepos leaves that pair out of its search, after // GitHub refused to search it (see automerge_contributor_unsearchable.go). - contributorUnsearchableMu sync.Mutex - contributorUnsearchable map[string]time.Time + // contributorUnsearchableWarned holds the "author:<login>" and + // "repo:<owner/repo>" keys a refusal has already warned for, so the + // author × repository product logs WARN at most once per side. + contributorUnsearchableMu sync.Mutex + contributorUnsearchable map[string]time.Time + contributorUnsearchableWarned map[string]bool mergerAuthzMu sync.RWMutex // humanReturnChecked remembers, per PR key, the head whose worn From 16795e35bc69d941fc890d51de4f1ee460f5560f Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 18:21:28 +0000 Subject: [PATCH 157/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.244.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.244.0 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 10 ++++++++++ changelog.d/added-benchmarks-links.md | 1 - changelog.d/fixed-18092-knowledge-check-toast.md | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/added-benchmarks-links.md delete mode 100644 changelog.d/fixed-18092-knowledge-check-toast.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f6020238..b69a348f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.244.0) + +### Added + +- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). + +### Fixed + +- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). + ## 2026-11-13 (v5.243.0) ### Added diff --git a/changelog.d/added-benchmarks-links.md b/changelog.d/added-benchmarks-links.md deleted file mode 100644 index 26cf95b2bb..0000000000 --- a/changelog.d/added-benchmarks-links.md +++ /dev/null @@ -1 +0,0 @@ -- Hub header nav and My Hives now link to the fleet benchmarks page (hivecommons.dev/benchmarks), and the spoke dashboard FAQ points at it for model comparisons (#18034). diff --git a/changelog.d/fixed-18092-knowledge-check-toast.md b/changelog.d/fixed-18092-knowledge-check-toast.md deleted file mode 100644 index 1500f367c0..0000000000 --- a/changelog.d/fixed-18092-knowledge-check-toast.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge Integrations: the per-connector **Check** button now reports its result — a success/info toast with the sign-in state (and the card refreshed), or an error toast carrying the server's `error` string verbatim (for example `no knowledge connectors are configured`) instead of a "checking" toast followed by silence ([#18092](https://github.com/hivecommons/hive/issues/18092)). From f2c8d50d48f97cbae819f0b1f0e1d2688e6bacdd Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:38:13 -0500 Subject: [PATCH 158/166] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20make=20the?= =?UTF-8?q?=20narrow=20navbar=20agents=20pill=20a=20real,=20uncapped=20but?= =?UTF-8?q?ton=20(#18114)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The narrow top-bar "3 agents ▾" pill was a CSS ::after pseudo-element: its text counted the (at most 3) visible tiles rather than the agents in the run order, and a pseudo-element can never be a click target, so it did nothing when clicked. Replace it with a <button data-action="openAgentsUpNextPanel"> built in renderAgentNavbarUpNext from textContent, kept as the strip's last child so the tile diff's index-based insertBefore still orders tiles ahead of it. The label is the uncapped count of scheduled agents ("9 agents", "1 agent"), no caret, with a title/aria-label naming the action. The 340px container query shows the button; the 220px query swaps in a "Next" span instead of generated content. Hive-Run: hivecommons/hive#18110 Hive-Plan: issue-18110 "How to build it" Hive-Spec: hivecommons/hive#18110#acceptance-criteria Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...ixed-18110-navbar-agents-summary-button.md | 1 + .../dashboard/agent_collapsed_tiles_test.go | 7 +- .../dashboard/agent_navbar_summary_ui_test.go | 140 ++++++++++++++++++ src/pkg/dashboard/navbar_upgrade_bee_test.go | 4 +- .../static-src/css/sections-collapse.css | 10 +- src/pkg/dashboard/static-src/js/sections.js | 42 +++++- src/pkg/dashboard/static/index.html | 52 ++++++- 7 files changed, 241 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-18110-navbar-agents-summary-button.md create mode 100644 src/pkg/dashboard/agent_navbar_summary_ui_test.go diff --git a/changelog.d/fixed-18110-navbar-agents-summary-button.md b/changelog.d/fixed-18110-navbar-agents-summary-button.md new file mode 100644 index 0000000000..7c48d4aa50 --- /dev/null +++ b/changelog.d/fixed-18110-navbar-agents-summary-button.md @@ -0,0 +1 @@ +- Dashboard top bar: when the centre column is narrow, the agent run-order strip's summary pill is now a real button that opens the Agents section (click, Enter or Space), and it shows how many scheduled agents are in the run order ("9 agents", "1 agent") instead of a count capped at 3; the misleading "▾" caret is gone ([#18110](https://github.com/hivecommons/hive/issues/18110)). diff --git a/src/pkg/dashboard/agent_collapsed_tiles_test.go b/src/pkg/dashboard/agent_collapsed_tiles_test.go index 1895afb12a..c4acfd6db0 100644 --- a/src/pkg/dashboard/agent_collapsed_tiles_test.go +++ b/src/pkg/dashboard/agent_collapsed_tiles_test.go @@ -46,7 +46,8 @@ func TestAgentsSidebarLinkAndNavbarUpNextContracts(t *testing.T) { `id="agent-navbar-upnext"`, `class="agent-navbar-upnext"`, `function renderAgentNavbarUpNext(agents, nowMs)`, - `agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3)`, + `const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now));`, + `const list = scheduled.slice(0, 3);`, `function renderAgentNavbarTileDiff(wrap, list, now)`, `data-agent-key`, `function openAgentsUpNextPanel()`, @@ -54,8 +55,8 @@ func TestAgentsSidebarLinkAndNavbarUpNextContracts(t *testing.T) { `.agent-navbar-tile { flex: 0 0 auto; min-width: 136px;`, `.agent-navbar-tile .agent-tile-name { grid-column: 1; grid-row: 1; min-width: max-content; overflow: visible; text-overflow: clip; }`, `@container navbar-center (max-width: 560px)`, - `.agent-navbar-upnext::after { content: attr(data-agent-summary);`, - `wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : '';`, + `.agent-navbar-summary { flex: 0 0 auto; display: none;`, + `const summary = renderAgentNavbarSummary(wrap, scheduled.length);`, `renderAgentNavbarUpNext(agents, Date.now())`, `refreshAgentNavbarUpNextCountdowns(now);`, `@media (max-width: 900px) { .agent-navbar-upnext { display: none !important; } }`, diff --git a/src/pkg/dashboard/agent_navbar_summary_ui_test.go b/src/pkg/dashboard/agent_navbar_summary_ui_test.go new file mode 100644 index 0000000000..3c3c2df114 --- /dev/null +++ b/src/pkg/dashboard/agent_navbar_summary_ui_test.go @@ -0,0 +1,140 @@ +package dashboard + +import ( + "os/exec" + "strings" + "testing" +) + +// When the navbar centre column is narrow the up-next strip collapses to a +// single summary control. It used to be a CSS ::after pseudo-element whose +// text was capped at 3 (the tile slice) and which could not receive clicks +// (#18110). These tests execute the real render functions against a tiny fake +// DOM so the count, the action wiring and the "no button when empty" rule are +// exercised, not just pattern-matched. +func TestAgentNavbarSummaryButtonCountAndAction(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node unavailable: agent navbar summary behavior was not executed") + } + html := indexHTML(t) + block := jsFunc(t, html, "agentNavbarSummaryLabel") + "\n" + + jsFunc(t, html, "renderAgentNavbarSummary") + "\n" + + jsFunc(t, html, "renderAgentNavbarUpNext") + + script := ` +class El { + constructor(tag) { this.tagName = tag.toUpperCase(); this.children = []; this.attrs = {}; this.className = ''; this._text = ''; this.parentNode = null; this.hidden = false; this.dataset = {}; } + setAttribute(k, v) { if (k === 'class') this.className = String(v); else this.attrs[k] = String(v); } + getAttribute(k) { if (k === 'class') return this.className; return Object.prototype.hasOwnProperty.call(this.attrs, k) ? this.attrs[k] : null; } + appendChild(c) { if (c.parentNode) c.parentNode.removeChild(c); c.parentNode = this; this.children.push(c); return c; } + insertBefore(c, ref) { if (!ref) return this.appendChild(c); if (c.parentNode) c.parentNode.removeChild(c); const i = this.children.indexOf(ref); c.parentNode = this; this.children.splice(i, 0, c); return c; } + removeChild(c) { this.children = this.children.filter(x => x !== c); c.parentNode = null; return c; } + remove() { if (this.parentNode) this.parentNode.removeChild(this); } + get lastElementChild() { return this.children[this.children.length - 1] || null; } + get textContent() { return this.children.length ? this.children.map(c => c.textContent).join('') : this._text; } + set textContent(v) { this.children = []; this._text = String(v); } + querySelector(sel) { const cls = sel.replace(/^\./, ''); for (const c of this.children) { if (c.className.split(/\s+/).includes(cls)) return c; const d = c.querySelector(sel); if (d) return d; } return null; } + querySelectorAll(sel) { const cls = sel.replace(/^\./, ''); const out = []; const walk = el => el.children.forEach(c => { if (c.className.split(/\s+/).includes(cls)) out.push(c); walk(c); }); walk(this); return out; } +} +const wrap = new El('span'); +wrap.setAttribute('id', 'agent-navbar-upnext'); +const document = { getElementById: id => id === 'agent-navbar-upnext' ? wrap : null, createElement: tag => new El(tag) }; +const window = { _lastStatus: { governor: {} } }; +function agentTileStates(agents) { return agents.map(a => ({ id: a.name, agent: a })); } +function agentNavbarHasSchedule(a) { return a.scheduled !== false; } +// Mirror the real diff's index-based insertBefore so tile placement relative +// to the summary button is exercised. +function renderAgentNavbarTileDiff(w, list) { + w.querySelectorAll('.agent-navbar-tile').forEach(t => t.remove()); + list.forEach((item, i) => { const tile = new El('button'); tile.className = 'agent-navbar-tile'; tile.setAttribute('data-agent-key', item.id); w.insertBefore(tile, w.children[i] || null); }); +} +function ensureAgentsCollapsedCountdownTicker() {} +` + block + ` +function agents(n, extraUnscheduled) { + const out = []; + for (let i = 0; i < n; i++) out.push({ name: 'a' + i }); + for (let i = 0; i < (extraUnscheduled || 0); i++) out.push({ name: 'off' + i, scheduled: false }); + return out; +} +function fail(msg) { throw new Error(msg); } +function summary() { return wrap.querySelector('.agent-navbar-summary'); } +function check(name, n, extra, wantText) { + renderAgentNavbarUpNext(agents(n, extra), 0); + const btn = summary(); + if (!btn) fail(name + ': summary button missing'); + if (btn.tagName !== 'BUTTON' || btn.getAttribute('type') !== 'button') fail(name + ': summary is not a type=button <button>'); + if (btn.getAttribute('data-action') !== 'openAgentsUpNextPanel') fail(name + ': data-action = ' + btn.getAttribute('data-action')); + const full = btn.querySelector('.agent-navbar-summary-full'); + if (!full || full.textContent !== wantText) fail(name + ': text = ' + (full && full.textContent) + ', want ' + wantText); + if (!btn.querySelector('.agent-navbar-summary-short')) fail(name + ': narrow "Next" span missing'); + if (btn.textContent.includes('▾')) fail(name + ': summary still shows a dropdown caret'); + const title = btn.getAttribute('title') || ''; + if (!title.startsWith(wantText + ' in the run order') || btn.getAttribute('aria-label') !== title) fail(name + ': title/aria-label = ' + title); + if (wrap.hidden) fail(name + ': strip hidden with scheduled agents'); + if (wrap.lastElementChild !== btn) fail(name + ': summary is not the last child of the strip'); + const tiles = wrap.querySelectorAll('.agent-navbar-tile'); + if (tiles.length !== Math.min(n, 3)) fail(name + ': tiles = ' + tiles.length); + if (wrap.querySelectorAll('.agent-navbar-summary').length !== 1) fail(name + ': duplicate summary buttons'); +} +check('nine scheduled agents are counted past the 3-tile cap', 9, 2, '9 agents'); +check('a single scheduled agent reads singular', 1, 0, '1 agent'); +check('two scheduled agents read plural', 2, 3, '2 agents'); +renderAgentNavbarUpNext(agents(0, 4), 0); +if (summary()) fail('no scheduled agents must not render a summary button'); +if (!wrap.hidden) fail('no scheduled agents must hide the strip'); +check('the button comes back after an empty render', 4, 0, '4 agents'); +` + cmd := exec.Command(node, "-e", script) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("agent navbar summary JS failed: %v\n%s", err, out) + } +} + +func TestAgentNavbarSummaryStaticCSS(t *testing.T) { + html := indexHTML(t) + for _, gone := range []string{ + `content: attr(data-agent-summary)`, + `.agent-navbar-upnext::after`, + `agents ▾`, + } { + if strings.Contains(html, gone) { + t.Fatalf("static/index.html still contains the unclickable pseudo-element pill %q", gone) + } + } + base := cssRule(t, html, ".agent-navbar-summary") + for _, want := range []string{"display: none", "cursor: pointer", "min-height: calc(var(--nav-chip-h) + var(--sp-4))"} { + if !strings.Contains(base, want) { + t.Fatalf(".agent-navbar-summary rule missing %q: %s", want, base) + } + } + if !strings.Contains(html, ".agent-navbar-summary:hover, .agent-navbar-summary:focus-visible {") { + t.Fatal(".agent-navbar-summary has no :focus-visible style") + } + containerBlock := func(query string) string { + i := strings.Index(html, "@container navbar-center ("+query+") {") + if i < 0 { + t.Fatalf("static/index.html missing @container navbar-center (%s)", query) + } + end := strings.Index(html[i:], "\n }") + if end < 0 { + t.Fatalf("unterminated @container navbar-center (%s)", query) + } + return html[i : i+end] + } + if q := containerBlock("max-width: 340px"); !strings.Contains(q, ".agent-navbar-upnext .agent-navbar-summary { display: inline-flex; }") { + t.Fatalf("340px container query does not show the summary button:\n%s", q) + } + q := containerBlock("max-width: 220px") + for _, want := range []string{ + ".agent-navbar-summary .agent-navbar-summary-full { display: none; }", + ".agent-navbar-summary .agent-navbar-summary-short { display: inline; }", + } { + if !strings.Contains(q, want) { + t.Fatalf("220px container query missing %q:\n%s", want, q) + } + } + if strings.Contains(q, "content:") { + t.Fatalf("220px container query must toggle a span, not generate content:\n%s", q) + } +} diff --git a/src/pkg/dashboard/navbar_upgrade_bee_test.go b/src/pkg/dashboard/navbar_upgrade_bee_test.go index 835c09ecf7..dc686033cd 100644 --- a/src/pkg/dashboard/navbar_upgrade_bee_test.go +++ b/src/pkg/dashboard/navbar_upgrade_bee_test.go @@ -101,13 +101,13 @@ func TestNavbarUpgradePillCannotCoverCenterAgents(t *testing.T) { `.oc-topbar-right { position: relative; z-index: 2;`, `flex-wrap: nowrap;`, `.oc-navbar-upgrade { --oc-version-bee-lane: 56px; flex: 0 1 min(260px, 24vw); min-width: 0; max-width: min(260px, 24vw);`, - `.agent-navbar-upnext::after { content: attr(data-agent-summary);`, + `.agent-navbar-summary { flex: 0 0 auto; display: none;`, `@container navbar-center (max-width: 560px)`, `.agent-navbar-tile .agent-tile-next { display: none; }`, `@container navbar-center (max-width: 340px)`, `.agent-navbar-upnext .agent-navbar-tile { display: none; }`, `@media (max-width: 900px) { .agent-navbar-upnext { display: none !important; } }`, - `wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : '';`, + `const summary = renderAgentNavbarSummary(wrap, scheduled.length);`, `const detail = versionUpgradeProgressStatus(v || {}, progress);`, } { if !strings.Contains(html, want) { diff --git a/src/pkg/dashboard/static-src/css/sections-collapse.css b/src/pkg/dashboard/static-src/css/sections-collapse.css index 793af8e0d4..cb73116104 100644 --- a/src/pkg/dashboard/static-src/css/sections-collapse.css +++ b/src/pkg/dashboard/static-src/css/sections-collapse.css @@ -127,7 +127,9 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } + .agent-navbar-summary-short { display: none; } @container navbar-center (max-width: 560px) { .agent-navbar-tile { min-width: 104px; } .agent-navbar-tile .agent-tile-next { display: none; } @@ -140,10 +142,12 @@ @container navbar-center (max-width: 340px) { .agent-navbar-upnext { overflow: hidden; mask-image: none; } .agent-navbar-upnext .agent-navbar-tile { display: none; } - .agent-navbar-upnext::after { display: inline-flex; } + .agent-navbar-upnext .agent-navbar-summary { display: inline-flex; } } @container navbar-center (max-width: 220px) { - .agent-navbar-upnext::after { content: 'Next'; padding-inline: var(--sp-1); } + .agent-navbar-upnext .agent-navbar-summary { padding-inline: var(--sp-1); } + .agent-navbar-summary .agent-navbar-summary-full { display: none; } + .agent-navbar-summary .agent-navbar-summary-short { display: inline; } } @keyframes navbar-up-next-pulse { 0% { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--amber) 72%, transparent), 0 0 0 var(--sp-0) color-mix(in srgb, var(--amber) 0%, transparent); } diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index 35e32dd311..85b6a94e2b 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -1163,14 +1163,52 @@ next kick: ${agentCollapsedTimestampDetail(nextMs, a && (a.nextKickAt || a.nextK wrap.dataset.upNextKey = nextItem ? nextItem.id : ''; animateAgentNavbarReorder(wrap, beforeRects); } + function agentNavbarSummaryLabel(count) { + return String(count) + (count === 1 ? ' agent' : ' agents'); + } + // The narrow-strip summary is a real button (not a CSS pseudo-element) so + // it can carry data-action and receive clicks and keyboard activation. It + // stays the strip's LAST child so renderAgentNavbarTileDiff's index-based + // insertBefore always places tiles ahead of it. + function renderAgentNavbarSummary(wrap, count) { + let btn = wrap.querySelector('.agent-navbar-summary'); + if (!count) { + if (btn) btn.remove(); + return null; + } + if (!btn) { + btn = document.createElement('button'); + btn.setAttribute('type', 'button'); + btn.className = 'agent-navbar-summary'; + btn.setAttribute('data-action', 'openAgentsUpNextPanel'); + btn.setAttribute('data-stop', '1'); + const full = document.createElement('span'); + full.className = 'agent-navbar-summary-full'; + const short = document.createElement('span'); + short.className = 'agent-navbar-summary-short'; + short.textContent = 'Next'; + btn.appendChild(full); + btn.appendChild(short); + } + const label = agentNavbarSummaryLabel(count); + const title = label + ' in the run order — open the Agents section'; + const full = btn.querySelector('.agent-navbar-summary-full'); + if (full && full.textContent !== label) full.textContent = label; + btn.setAttribute('title', title); + btn.setAttribute('aria-label', title); + if (wrap.lastElementChild !== btn) wrap.appendChild(btn); + return btn; + } function renderAgentNavbarUpNext(agents, nowMs) { const wrap = document.getElementById('agent-navbar-upnext'); if (!wrap) return; const now = Number.isFinite(nowMs) ? nowMs : Date.now(); - const list = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3); + const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)); + const list = scheduled.slice(0, 3); wrap.hidden = list.length === 0; - wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : ''; + const summary = renderAgentNavbarSummary(wrap, scheduled.length); renderAgentNavbarTileDiff(wrap, list, now); + if (summary && wrap.lastElementChild !== summary) wrap.appendChild(summary); if (list.length) ensureAgentsCollapsedCountdownTicker(); } function refreshAgentNavbarUpNextCountdowns(nowMs) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 6359a27634..1cccd9750b 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -6119,7 +6119,9 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary { flex: 0 0 auto; display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); cursor: pointer; font: inherit; font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-summary:hover, .agent-navbar-summary:focus-visible { border-color: var(--border); background: var(--surface-2); outline: none; } + .agent-navbar-summary-short { display: none; } @container navbar-center (max-width: 560px) { .agent-navbar-tile { min-width: 104px; } .agent-navbar-tile .agent-tile-next { display: none; } @@ -6132,10 +6134,12 @@ @container navbar-center (max-width: 340px) { .agent-navbar-upnext { overflow: hidden; mask-image: none; } .agent-navbar-upnext .agent-navbar-tile { display: none; } - .agent-navbar-upnext::after { display: inline-flex; } + .agent-navbar-upnext .agent-navbar-summary { display: inline-flex; } } @container navbar-center (max-width: 220px) { - .agent-navbar-upnext::after { content: 'Next'; padding-inline: var(--sp-1); } + .agent-navbar-upnext .agent-navbar-summary { padding-inline: var(--sp-1); } + .agent-navbar-summary .agent-navbar-summary-full { display: none; } + .agent-navbar-summary .agent-navbar-summary-short { display: inline; } } @keyframes navbar-up-next-pulse { 0% { box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--amber) 72%, transparent), 0 0 0 var(--sp-0) color-mix(in srgb, var(--amber) 0%, transparent); } @@ -11582,14 +11586,52 @@ <h3 class="faq-group-title">Where to get help</h3> wrap.dataset.upNextKey = nextItem ? nextItem.id : ''; animateAgentNavbarReorder(wrap, beforeRects); } + function agentNavbarSummaryLabel(count) { + return String(count) + (count === 1 ? ' agent' : ' agents'); + } + // The narrow-strip summary is a real button (not a CSS pseudo-element) so + // it can carry data-action and receive clicks and keyboard activation. It + // stays the strip's LAST child so renderAgentNavbarTileDiff's index-based + // insertBefore always places tiles ahead of it. + function renderAgentNavbarSummary(wrap, count) { + let btn = wrap.querySelector('.agent-navbar-summary'); + if (!count) { + if (btn) btn.remove(); + return null; + } + if (!btn) { + btn = document.createElement('button'); + btn.setAttribute('type', 'button'); + btn.className = 'agent-navbar-summary'; + btn.setAttribute('data-action', 'openAgentsUpNextPanel'); + btn.setAttribute('data-stop', '1'); + const full = document.createElement('span'); + full.className = 'agent-navbar-summary-full'; + const short = document.createElement('span'); + short.className = 'agent-navbar-summary-short'; + short.textContent = 'Next'; + btn.appendChild(full); + btn.appendChild(short); + } + const label = agentNavbarSummaryLabel(count); + const title = label + ' in the run order — open the Agents section'; + const full = btn.querySelector('.agent-navbar-summary-full'); + if (full && full.textContent !== label) full.textContent = label; + btn.setAttribute('title', title); + btn.setAttribute('aria-label', title); + if (wrap.lastElementChild !== btn) wrap.appendChild(btn); + return btn; + } function renderAgentNavbarUpNext(agents, nowMs) { const wrap = document.getElementById('agent-navbar-upnext'); if (!wrap) return; const now = Number.isFinite(nowMs) ? nowMs : Date.now(); - const list = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)).slice(0, 3); + const scheduled = agentTileStates(agents, now, window._lastStatus && window._lastStatus.governor).filter(item => agentNavbarHasSchedule(item.agent, now)); + const list = scheduled.slice(0, 3); wrap.hidden = list.length === 0; - wrap.dataset.agentSummary = list.length ? String(list.length) + ' agents ▾' : ''; + const summary = renderAgentNavbarSummary(wrap, scheduled.length); renderAgentNavbarTileDiff(wrap, list, now); + if (summary && wrap.lastElementChild !== summary) wrap.appendChild(summary); if (list.length) ensureAgentsCollapsedCountdownTicker(); } function refreshAgentNavbarUpNextCountdowns(nowMs) { From ea24e8adff96128deb19be9fa4bd75ad6bc48561 Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:46:56 +0000 Subject: [PATCH 159/166] feat: ship spektacular in the hive image (#18104) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/added-17888-ship-spektacular.md | 1 + src/Dockerfile | 27 +++++++++++++++++++++ src/Dockerfile.contributor | 27 +++++++++++++++++++++ src/docs/cli-pins.md | 1 + 4 files changed, 56 insertions(+) create mode 100644 changelog.d/added-17888-ship-spektacular.md diff --git a/changelog.d/added-17888-ship-spektacular.md b/changelog.d/added-17888-ship-spektacular.md new file mode 100644 index 0000000000..db2fb2c358 --- /dev/null +++ b/changelog.d/added-17888-ship-spektacular.md @@ -0,0 +1 @@ +- The hive image now ships the `spektacular` CLI (pinned 0.31.0, checksum-verified, amd64 and arm64), so the Inbox → RFC destination runs `spektacular voice --publish` in-pod without an agent fallback (#17888). diff --git a/src/Dockerfile b/src/Dockerfile index e802b98fed..fd60ca9443 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -381,6 +381,33 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ echo "WARN: Goose CLI download failed — skipping (backend: goose will be unavailable)"; \ fi +# --- Layer 8.1: spektacular CLI (Inbox -> RFC destination) --- +# Pinned hivecommons/spektacular release (includes `spektacular voice`, +# hivecommons/spektacular#123), SHA-256-verified before extraction, installed +# to /usr/local/bin/spektacular so inbox.spek.binary / runs.spektacular.binary +# resolve on PATH with no config. Download failures are tolerated (the Inbox +# falls back to inbox.spek.agent); checksum mismatches hard-fail the build. +# Bump note: see src/docs/cli-pins.md (spektacular row); update the version and +# both digests together from the release's checksums.txt. +ARG SPEKTACULAR_VERSION=0.31.0 +ARG SPEKTACULAR_SHA256_AMD64=21bd077b6723de41185fa657c0e54ea5726c1c2928794548ada4a8776a7b4746 +ARG SPEKTACULAR_SHA256_ARM64=81d3b92c907bd6d66e4e1db79df24be8b64803dd26e0de1d7147f1ff5fc9b8d0 +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) SPEK_ARCH=x86_64; SPEK_SHA256="$SPEKTACULAR_SHA256_AMD64" ;; \ + arm64) SPEK_ARCH=arm64; SPEK_SHA256="$SPEKTACULAR_SHA256_ARM64" ;; \ + *) echo "unsupported arch for spektacular: $ARCH" >&2; exit 1 ;; \ + esac && \ + mkdir -p /tmp/spek-dl && \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-connrefused "https://github.com/hivecommons/spektacular/releases/download/${SPEKTACULAR_VERSION}/spektacular_${SPEKTACULAR_VERSION}_linux_${SPEK_ARCH}.tar.gz" -o /tmp/spek-dl/spektacular.tar.gz; then \ + echo "${SPEK_SHA256} /tmp/spek-dl/spektacular.tar.gz" | sha256sum -c - && \ + tar xz -C /usr/local/bin -f /tmp/spek-dl/spektacular.tar.gz spektacular && \ + chmod +x /usr/local/bin/spektacular && \ + rm -rf /tmp/spek-dl; \ + else \ + echo "WARN: spektacular download failed - skipping (Inbox falls back to inbox.spek.agent)"; \ + fi + # --- Layer 8.25: Google Antigravity CLI (backend: agy) --- # The vendor distributes a self-contained Linux executable, not an npm package. # Pin the version, build ID, and SHA-512 from its public updater manifest; do diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index bc5e3349ad..3e6703fdf8 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -390,6 +390,33 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ echo "Goose install skipped"; \ fi +# spektacular CLI (Inbox -> RFC destination) --- +# Pinned hivecommons/spektacular release (includes `spektacular voice`, +# hivecommons/spektacular#123), SHA-256-verified before extraction, installed +# to /usr/local/bin/spektacular so inbox.spek.binary / runs.spektacular.binary +# resolve on PATH with no config. Download failures are tolerated (the Inbox +# falls back to inbox.spek.agent); checksum mismatches hard-fail the build. +# Bump note: see src/docs/cli-pins.md (spektacular row); update the version and +# both digests together from the release's checksums.txt. +ARG SPEKTACULAR_VERSION=0.31.0 +ARG SPEKTACULAR_SHA256_AMD64=21bd077b6723de41185fa657c0e54ea5726c1c2928794548ada4a8776a7b4746 +ARG SPEKTACULAR_SHA256_ARM64=81d3b92c907bd6d66e4e1db79df24be8b64803dd26e0de1d7147f1ff5fc9b8d0 +RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$ARCH" in \ + amd64) SPEK_ARCH=x86_64; SPEK_SHA256="$SPEKTACULAR_SHA256_AMD64" ;; \ + arm64) SPEK_ARCH=arm64; SPEK_SHA256="$SPEKTACULAR_SHA256_ARM64" ;; \ + *) echo "unsupported arch for spektacular: $ARCH" >&2; exit 1 ;; \ + esac && \ + mkdir -p /tmp/spek-dl && \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-connrefused "https://github.com/hivecommons/spektacular/releases/download/${SPEKTACULAR_VERSION}/spektacular_${SPEKTACULAR_VERSION}_linux_${SPEK_ARCH}.tar.gz" -o /tmp/spek-dl/spektacular.tar.gz; then \ + echo "${SPEK_SHA256} /tmp/spek-dl/spektacular.tar.gz" | sha256sum -c - && \ + tar xz -C /usr/local/bin -f /tmp/spek-dl/spektacular.tar.gz spektacular && \ + chmod +x /usr/local/bin/spektacular && \ + rm -rf /tmp/spek-dl; \ + else \ + echo "WARN: spektacular download failed - skipping (Inbox falls back to inbox.spek.agent)"; \ + fi + # Install agy (Google Antigravity CLI). Not an npm package — Google ships it # as a standalone binary from a versioned, checksummed GCS URL (the same URL # Homebrew's antigravity-cli cask resolves to, confirmed against diff --git a/src/docs/cli-pins.md b/src/docs/cli-pins.md index e53751befa..4a430bed07 100644 --- a/src/docs/cli-pins.md +++ b/src/docs/cli-pins.md @@ -10,6 +10,7 @@ Every agent CLI hive drives is baked into the images at a fixed version: | pi | `PI_VERSION` (hub), `PI_CODING_AGENT_VERSION` (contributor) | npm `@earendil-works/pi-coding-agent` | npm | | Goose | `GOOSE_VERSION`, `GOOSE_SHA256_{AMD64,ARM64}` | GitHub releases `aaif-goose/goose` | SHA-256 per arch | | Antigravity (agy) | `AGY_VERSION` + `AGY_BUILD`, `AGY_SHA512_*` (hub); `AGY_VERSION=<ver>-<build>`, `AGY_SHA256_*` (contributor) | Google's updater artifact, as tracked by the Homebrew `antigravity-cli` cask | SHA-512 (hub) / SHA-256 (contributor) per arch | +| spektacular | `SPEKTACULAR_VERSION`, `SPEKTACULAR_SHA256_{AMD64,ARM64}` | GitHub releases `hivecommons/spektacular` | SHA-256 per arch, from the release `checksums.txt` (manual bump for now; not yet in `cli-pin-bump.sh`) | | Oh My Pi (omp) | `OMP_VERSION`, `OMP_SHA256_*` | GitHub releases `can1357/oh-my-pi` | SHA-256 per arch, cross-checked with `SHA256SUMS.txt` | | Muse Code | `MUSE_VERSION`, `MUSE_SHA256_*` | `api.meta.ai/muse-code/channels/muse-stable` and its release manifest | SHA-256 per arch, cross-checked with the manifest | | bobshell (bob) | `BOBSHELL_VERSION`, `BOBSHELL_SHA256_*` | `bobshell2-version.txt` on IBM COS | SHA-256, cross-checked with the vendor `.tgz.sha256` | From 2805fe41f2ead93017bc450f0b1c4a3b94538e51 Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 13:47:47 -0500 Subject: [PATCH 160/166] =?UTF-8?q?=F0=9F=90=9B=20fleetreport:=20stop=20se?= =?UTF-8?q?lf-reporting=20completed=20Claude=20credential=20re-bridges=20(?= =?UTF-8?q?#18108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code renames a fresh file over each agent's shared-login symlink on every token refresh, every agent refreshes within the same minute or two, and the watchdog re-bridges each link within 5s. That burst of successful repairs (already audited) crossed the unclassified_error_repeated threshold and filed a fleet self-report for designed, self-healed behaviour. Exclude exactly that completed-repair WARN from the slogfeed tee; it stays in local logs. Failed or refused repairs log other messages and remain reportable. Fixes #18107 Hive-Run: hivecommons/hive#18107 Hive-Plan: selfreport-triage Hive-Spec: fleetreport-slogfeed#expected-recovery-not-reported Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .../fixed-18107-claude-rebridge-selfreport.md | 1 + src/crux/agent/claude_credential_bridge.go | 3 +++ src/pkg/fleetreport/slogfeed/slogfeed.go | 19 ++++++++++++++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-18107-claude-rebridge-selfreport.md diff --git a/changelog.d/fixed-18107-claude-rebridge-selfreport.md b/changelog.d/fixed-18107-claude-rebridge-selfreport.md new file mode 100644 index 0000000000..ecdbbba2d5 --- /dev/null +++ b/changelog.d/fixed-18107-claude-rebridge-selfreport.md @@ -0,0 +1 @@ +- Completed Claude credential re-bridges (a token refresh replacing an agent's shared-login symlink, repaired automatically) no longer file `unclassified_error_repeated` fleet self-reports; failed repairs still do. diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go index 1f36cb2096..4257ec3dde 100644 --- a/src/crux/agent/claude_credential_bridge.go +++ b/src/crux/agent/claude_credential_bridge.go @@ -567,6 +567,9 @@ func (m *Manager) rebridgeClaudeCredential(agentName, link, target, trigger stri if rep == nil { return nil } + // The fleet self-report feed deliberately ignores this exact message (a + // completed repair, expected in a burst at every token refresh): + // claudeCredentialRebridged in pkg/fleetreport/slogfeed. Keep them in sync. m.logger.Warn("claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged", "agent", agentName, "trigger", trigger, diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 5a9507e578..3375f2902e 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -115,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) || claudeCredentialRebridged(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -143,6 +143,23 @@ func signedCommitOwnershipSkip(r slog.Record) bool { return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == "signed-commit reconciler: skipped; the PR keeps an unsigned commit" } +// claudeCredentialRebridgedMessage is the WARN crux/agent logs after it has +// SUCCESSFULLY restored a per-agent Claude credential symlink that a token +// refresh replaced (src/crux/agent/claude_credential_bridge.go). Keep the two +// strings in sync. +const claudeCredentialRebridgedMessage = "claude credential bridge: a private credential file had replaced the shared-login symlink; re-bridged" + +// claudeCredentialRebridged is the credential bridge's completed repair. Claude +// Code renames a fresh file over the shared-login symlink on every token +// refresh, every agent shares one access token and so refreshes within the same +// minute or two, and the watchdog checks each link every 5s: a burst of one +// re-bridge per agent is the designed recovery, already recorded in the audit +// log, not a defect (hivecommons/hive#18107). The WARN stays in local logs; a +// failed or refused repair logs a different message and stays reportable. +func claudeCredentialRebridged(r slog.Record) bool { + return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == claudeCredentialRebridgedMessage +} + // canceledError applies the existing cancellation exclusion to structured // errors too. An accompanying different error must remain reportable. func canceledError(r slog.Record, bound []slog.Attr) bool { diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index d21d195f56..1c1a9c8131 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -84,6 +84,24 @@ func TestSignedCommitOwnershipSkipStaysInLogsWithoutFleetReport(t *testing.T) { } } +func TestClaudeCredentialRebridgeStaysInLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + for range 6 { + logger.Warn(claudeCredentialRebridgedMessage, "agent", "reviewer", "trigger", "watchdog", "kept", "shared") + } + if len(*seen) != 0 { + t.Fatalf("completed re-bridges produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), "re-bridged"); n != 6 { + t.Fatalf("local logs lost the re-bridge warning: got %d, want 6: %s", n, out.String()) + } + logger.Warn("claude credential bridge: repair failed", "agent", "reviewer", "error", "restore credential symlink: file exists") + logger.Error(claudeCredentialRebridgedMessage, "agent", "reviewer") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("failed repairs must remain reportable: %+v", *seen) + } +} + func TestCanceledStageDigestStaysLocal(t *testing.T) { logger, seen, out := newFeed(t) const msg = "stage labels: check digest failed" From e993b5709f8ac3450d05978442de0281464041da Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 13:53:53 -0500 Subject: [PATCH 161/166] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20omp=201?= =?UTF-8?q?8.19.0=20->=2018.20.0=20(#18064)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=18.20.0 SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- src/Dockerfile | 6 +++--- src/Dockerfile.contributor | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index fd60ca9443..05b954dbe8 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -438,9 +438,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.19.0 -ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 -ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 +ARG OMP_VERSION=18.20.0 +ARG OMP_SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 +ARG OMP_SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 3e6703fdf8..dc3d62e858 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -492,9 +492,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.19.0 -ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 -ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 +ARG OMP_VERSION=18.20.0 +ARG OMP_SHA256_AMD64=c8694be8c6e6a1f45e3636dd8a8e9eaa78878ad629a89000b403ec5e456d3f89 +ARG OMP_SHA256_ARM64=8b5e197db7c58b7ebabf83c6f03597f89bae3a766a84c6481b3a942a125f5e22 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ From 3f8208f808afd9e652c20119c30c0a15ba760942 Mon Sep 17 00:00:00 2001 From: hive-release-bot <actions@github.com> Date: Fri, 13 Nov 2026 18:53:57 +0000 Subject: [PATCH 162/166] =?UTF-8?q?=F0=9F=94=96=20release:=20v5.244.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated release commit. Compiles changelog.d/ fragments and moves the CHANGELOG.md Unreleased section into a dated v5.244.1 entry. See src/docs/releases.md. Signed-off-by: hive-release-bot <actions@github.com> --- CHANGELOG.md | 8 ++++++++ changelog.d/fixed-18059-contributor-unsearchable-warn.md | 1 - .../fixed-18093-knowledge-connector-not-loaded-badge.md | 1 - changelog.d/fixed-18105-ioscan-classifier-retry.md | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/fixed-18059-contributor-unsearchable-warn.md delete mode 100644 changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md delete mode 100644 changelog.d/fixed-18105-ioscan-classifier-retry.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b69a348f0e..ae387773f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ Hive did not historically maintain a complete changelog. This file starts a prag ## Unreleased +## 2026-11-13 (v5.244.1) + +### Fixed + +- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). +- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). +- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). + ## 2026-11-13 (v5.244.0) ### Added diff --git a/changelog.d/fixed-18059-contributor-unsearchable-warn.md b/changelog.d/fixed-18059-contributor-unsearchable-warn.md deleted file mode 100644 index 75dd2ae587..0000000000 --- a/changelog.d/fixed-18059-contributor-unsearchable-warn.md +++ /dev/null @@ -1 +0,0 @@ -- Contributor-lane auto-promotion now warns once per refused author and once per refused repository when GitHub's search cannot see a repository, instead of once per author × repository pair, so one invisible repository or one unsearchable author no longer floods WARN and self-reports as a repeated unclassified error (#18059). diff --git a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md b/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md deleted file mode 100644 index 319a50e047..0000000000 --- a/changelog.d/fixed-18093-knowledge-connector-not-loaded-badge.md +++ /dev/null @@ -1 +0,0 @@ -- Knowledge Integrations: when the connector syncer is not running (`runtime: false`, hive booted with no `knowledge.connectors`), connector cards now show a `not loaded` badge with a restart hint and docs link instead of `pending`, and Connect / Check / Sync are disabled with the same hint rather than failing with 404 toasts; `pending` stays reserved for a loaded connector awaiting sign-in ([#18093](https://github.com/hivecommons/hive/issues/18093)). diff --git a/changelog.d/fixed-18105-ioscan-classifier-retry.md b/changelog.d/fixed-18105-ioscan-classifier-retry.md deleted file mode 100644 index 133036f418..0000000000 --- a/changelog.d/fixed-18105-ioscan-classifier-retry.md +++ /dev/null @@ -1 +0,0 @@ -- The ioscan classifier now retries a single transient model-endpoint failure (HTTP 429/5xx or a dropped connection) before applying its fail mode, so one gateway blip no longer passes untrusted text on unclassified or withholds it ([#18105](https://github.com/hivecommons/hive/issues/18105)). From 1ff54108ab759fc3e93c8e2298f12b0e48b5723a Mon Sep 17 00:00:00 2001 From: Danathar <doug.baggett@gmail.com> Date: Fri, 13 Nov 2026 14:01:47 -0500 Subject: [PATCH 163/166] =?UTF-8?q?=F0=9F=90=9B=20governor:=20hold=20kicks?= =?UTF-8?q?=20for=20agents=20out=20of=20provider=20quota=20(#18117)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(governor): hold kicks for agents out of provider quota The provider spend gate only fires for agents behind the hive's inference gateway, so an agent whose own provider refuses it for quota (Copilot's "You have exceeded your monthly quota") kept being kicked and sent review work every cycle, each turn ending at once, until review dispatches expired and PRs were parked for a human. Add a per-agent quota hold applied to the fully assembled kick list right after the spend gate: an agent whose QuotaExhausted flag is set has its kicks withheld, except one probe kick per governor.provider_budget.probe_interval_s. The probe time is stamped only when the kick is delivered, and forgotten once the flag clears. Healthy agents are never touched. The quota alert now says kicks are held and how often they are retried. Fixes #18112 Hive-Run: hivecommons/hive#18112 Hive-Plan: issue-18112 How to build it Hive-Spec: issue-18112#wanted-behavior Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> * test(governor): cover the per-agent quota hold guard Mutation testing found the guard around gateKickMessagesForAgentQuota in runEvalCycle unreached by any test (main_eval_cycle.go:1111). Move it into agentQuotaHoldState.gate and test the empty-set, no-message and gating paths. The delivery hook now stamps probe times from the returned probe set, which is the same set of agents as before: a held agent's only kept kick is its probe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Danathar <Danathar@users.noreply.github.com> --------- Signed-off-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Danathar <Danathar@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- ...fixed-quota-exhausted-agents-not-kicked.md | 1 + src/cmd/hive/eval_cycle_quota_hold.go | 137 +++++++++ src/cmd/hive/eval_cycle_quota_hold_test.go | 272 ++++++++++++++++++ src/cmd/hive/eval_cycle_seams.go | 9 +- src/cmd/hive/main_eval_cycle.go | 30 +- src/cmd/hive/provider_budget_alert_test.go | 19 +- .../sweeps-and-operations.md | 1 + 7 files changed, 459 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixed-quota-exhausted-agents-not-kicked.md create mode 100644 src/cmd/hive/eval_cycle_quota_hold.go create mode 100644 src/cmd/hive/eval_cycle_quota_hold_test.go diff --git a/changelog.d/fixed-quota-exhausted-agents-not-kicked.md b/changelog.d/fixed-quota-exhausted-agents-not-kicked.md new file mode 100644 index 0000000000..cce7aae2e1 --- /dev/null +++ b/changelog.d/fixed-quota-exhausted-agents-not-kicked.md @@ -0,0 +1 @@ +- **Agents out of provider quota are no longer kicked every cycle** ([#18112](https://github.com/hivecommons/hive/issues/18112)). The provider spending-limit gate only saw agents behind the hive's inference gateway, so an agent whose own provider refused it — GitHub Copilot's "You have exceeded your monthly quota" — kept being kicked and sent review work all night, each turn ending at once, until its review dispatches expired and pull requests were parked for a human. The governor now holds kicks per agent while that agent's `QuotaExhausted` flag is set, lets one probe kick through per `governor.provider_budget.probe_interval_s` (default 30 minutes) to notice when the quota resets, and resumes normal kicking as soon as the flag clears. Other agents are never held, and the "provider quota exhausted" alert now says the kicks are held and how often they are retried. diff --git a/src/cmd/hive/eval_cycle_quota_hold.go b/src/cmd/hive/eval_cycle_quota_hold.go new file mode 100644 index 0000000000..f7b552c562 --- /dev/null +++ b/src/cmd/hive/eval_cycle_quota_hold.go @@ -0,0 +1,137 @@ +package main + +import ( + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/scheduler" +) + +// PER-AGENT PROVIDER QUOTA HOLD (hivecommons/hive#18112). +// +// The provider spend gate (gateKickMessagesForProviderBudget) only fires for +// agents behind the hive's own inference gateway: its latch is set by the +// gateway proxy, which never sees a backend that calls its provider directly +// (GitHub Copilot, for one). Those agents still raise crux.Process.QuotaExhausted +// from the pane ("You have exceeded your monthly quota"), but nothing withheld +// their kicks, so the governor kept kicking them — and sending them review +// work — every cycle, each turn ending at once on the same quota error. +// +// The hold is PER AGENT, unlike the spend gate, because the limit is on that +// agent's provider account, not on a key every agent shares: one agent out of +// quota must never hold any other agent. A held agent still gets one probe +// kick per governor.provider_budget.probe_interval_s, so the hive notices when +// the quota resets (the flag clears once the pane stops showing the message). + +// agentQuotaHoldState remembers, per quota-exhausted agent, when the last +// probe kick was actually delivered. Package-level (agentQuotaHold) for the +// same reason as providerBudgetProbe: runEvalCycle has no state of its own. +type agentQuotaHoldState struct { + mu sync.Mutex + lastProbe map[string]time.Time +} + +// MarkProbeDelivered records that a probe kick reached agent at the given +// time. Call it only after a kick is delivered: a withheld, deferred or failed +// kick learned nothing about the quota and must not push the next probe out. +func (s *agentQuotaHoldState) MarkProbeDelivered(agent string, at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastProbe == nil { + s.lastProbe = map[string]time.Time{} + } + s.lastProbe[agent] = at +} + +// Forget drops agent's probe time, so a later quota hit starts fresh. +func (s *agentQuotaHoldState) Forget(agent string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.lastProbe, agent) +} + +// LastProbe reports when a probe kick was last delivered to agent; the zero +// time means never, which makes the agent due a probe at once. +func (s *agentQuotaHoldState) LastProbe(agent string) time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return s.lastProbe[agent] +} + +// quotaExhaustedAgents reads the per-agent quota flag from one status snapshot +// and forgets the probe time of every agent whose flag is clear, so its kicks +// flow normally and its next quota hit starts a fresh interval. nil entries +// are skipped. The returned set holds only the exhausted agents. +func (s *agentQuotaHoldState) quotaExhaustedAgents(statuses map[string]*crux.Process) map[string]bool { + exhausted := map[string]bool{} + for name, proc := range statuses { + if proc == nil { + continue + } + if proc.QuotaExhausted { + exhausted[name] = true + continue + } + s.Forget(name) + } + return exhausted +} + +// gate applies the per-agent quota hold to one cycle's kicks. It always reads +// the flags from statuses (so cleared agents forget their probe time), then +// returns messages unchanged when no agent is held or there is nothing to +// gate; otherwise it defers to gateKickMessagesForAgentQuota. +func (s *agentQuotaHoldState) gate(messages []scheduler.KickMessage, statuses map[string]*crux.Process, now time.Time, probeInterval time.Duration) (kept []scheduler.KickMessage, withheld []string, probes []string) { + exhausted := s.quotaExhaustedAgents(statuses) + if len(exhausted) == 0 || len(messages) == 0 { + return messages, nil, nil + } + return gateKickMessagesForAgentQuota(messages, func(agent string) bool { return exhausted[agent] }, s.LastProbe, now, probeInterval) +} + +// gateKickMessagesForAgentQuota withholds kicks for agents whose provider +// quota is exhausted, except for one probe kick per held agent once +// probeInterval has passed since that agent's last delivered probe (a zero +// lastProbe is due at once). Only the FIRST message for a probing agent is +// kept; the rest of that agent's messages are withheld this cycle. Agents +// that are not exhausted pass through untouched. Message order is preserved. +// +// withheld and probes name agents in message order; an agent appears in +// withheld once per withheld message, and in probes at most once. +func gateKickMessagesForAgentQuota(messages []scheduler.KickMessage, quotaExhausted func(agent string) bool, lastProbe func(agent string) time.Time, now time.Time, probeInterval time.Duration) (kept []scheduler.KickMessage, withheld []string, probes []string) { + decided := map[string]bool{} + for _, msg := range messages { + if !quotaExhausted(msg.Agent) { + kept = append(kept, msg) + continue + } + if !decided[msg.Agent] { + decided[msg.Agent] = true + if last := lastProbe(msg.Agent); last.IsZero() || now.Sub(last) >= probeInterval { + probes = append(probes, msg.Agent) + kept = append(kept, msg) + continue + } + } + withheld = append(withheld, msg.Agent) + } + return kept, withheld, probes +} + +// compactDuration renders an operator-facing interval without the zero tails +// of time.Duration.String: 30m, 2h, 1h30m, 45s. +func compactDuration(d time.Duration) string { + if d <= 0 { + return "0s" + } + s := d.Round(time.Second).String() + if strings.HasSuffix(s, "m0s") { + s = strings.TrimSuffix(s, "0s") + } + if strings.HasSuffix(s, "h0m") { + s = strings.TrimSuffix(s, "0m") + } + return s +} diff --git a/src/cmd/hive/eval_cycle_quota_hold_test.go b/src/cmd/hive/eval_cycle_quota_hold_test.go new file mode 100644 index 0000000000..77c938fa8f --- /dev/null +++ b/src/cmd/hive/eval_cycle_quota_hold_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "errors" + "log/slog" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/scheduler" +) + +// Tests for the per-agent provider quota hold (hivecommons/hive#18112). + +var quotaHoldNow = time.Date(2026, 11, 13, 9, 0, 0, 0, time.UTC) + +func quotaSet(agents ...string) func(string) bool { + set := map[string]bool{} + for _, a := range agents { + set[a] = true + } + return func(agent string) bool { return set[agent] } +} + +func probesAt(at map[string]time.Time) func(string) time.Time { + return func(agent string) time.Time { return at[agent] } +} + +func kickAgents(msgs []scheduler.KickMessage) []string { + out := []string{} + for _, m := range msgs { + out = append(out, m.Agent+":"+m.Message) + } + return out +} + +func TestQuotaHoldGate_NoHeldAgentsPassesThrough(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "a", Message: "1"}, {Agent: "b", Message: "2"}} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet(), probesAt(nil), quotaHoldNow, 30*time.Minute) + if !reflect.DeepEqual(kept, msgs) { + t.Fatalf("kept = %v, want every message unchanged", kickAgents(kept)) + } + if len(withheld) != 0 || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want none", withheld, probes) + } +} + +func TestQuotaHoldGate_HeldAgentNotDueIsWithheld(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "copilot", Message: "gov"}} + last := map[string]time.Time{"copilot": quotaHoldNow.Add(-10 * time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("copilot"), probesAt(last), quotaHoldNow, 30*time.Minute) + if len(kept) != 0 { + t.Fatalf("kept = %v, want the held agent's kick withheld", kickAgents(kept)) + } + if !reflect.DeepEqual(withheld, []string{"copilot"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want [copilot] and no probe", withheld, probes) + } +} + +// A held agent that is due a probe gets exactly one kick; its other messages +// this cycle (for example a fix kick ahead of its governor kick) stay held. +func TestQuotaHoldGate_HeldAgentDueGetsOneProbe(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "copilot", Message: "fix"}, + {Agent: "copilot", Message: "gov"}, + } + last := map[string]time.Time{"copilot": quotaHoldNow.Add(-30 * time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("copilot"), probesAt(last), quotaHoldNow, 30*time.Minute) + if want := []string{"copilot:fix"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v (first message only)", kickAgents(kept), want) + } + if !reflect.DeepEqual(probes, []string{"copilot"}) || !reflect.DeepEqual(withheld, []string{"copilot"}) { + t.Fatalf("probes=%v withheld=%v, want one probe and one withheld", probes, withheld) + } +} + +// One agent out of quota must not hold any other agent; order is kept. +func TestQuotaHoldGate_MixedHeldAndHealthyKeepsOrder(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "healthy-1", Message: "a"}, + {Agent: "held", Message: "review"}, + {Agent: "healthy-2", Message: "b"}, + {Agent: "held", Message: "gov"}, + } + last := map[string]time.Time{"held": quotaHoldNow.Add(-time.Minute)} + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("held"), probesAt(last), quotaHoldNow, 30*time.Minute) + if want := []string{"healthy-1:a", "healthy-2:b"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v", kickAgents(kept), want) + } + if !reflect.DeepEqual(withheld, []string{"held", "held"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want [held held] and no probe", withheld, probes) + } +} + +func TestQuotaHoldGate_ZeroProbeTimeIsDueAtOnce(t *testing.T) { + msgs := []scheduler.KickMessage{ + {Agent: "healthy", Message: "a"}, + {Agent: "held", Message: "gov"}, + } + kept, withheld, probes := gateKickMessagesForAgentQuota(msgs, quotaSet("held"), probesAt(nil), quotaHoldNow, 30*time.Minute) + if want := []string{"healthy:a", "held:gov"}; !reflect.DeepEqual(kickAgents(kept), want) { + t.Fatalf("kept = %v, want %v", kickAgents(kept), want) + } + if !reflect.DeepEqual(probes, []string{"held"}) || len(withheld) != 0 { + t.Fatalf("probes=%v withheld=%v, want [held] probed at once", probes, withheld) + } +} + +func TestQuotaHoldState_MarkThenLastProbeThenForget(t *testing.T) { + var s agentQuotaHoldState + if got := s.LastProbe("a"); !got.IsZero() { + t.Fatalf("LastProbe on empty state = %v, want zero", got) + } + s.MarkProbeDelivered("a", quotaHoldNow) + if got := s.LastProbe("a"); !got.Equal(quotaHoldNow) { + t.Fatalf("LastProbe = %v, want %v", got, quotaHoldNow) + } + if got := s.LastProbe("b"); !got.IsZero() { + t.Fatalf("another agent's probe time = %v, want zero", got) + } + s.Forget("a") + if got := s.LastProbe("a"); !got.IsZero() { + t.Fatalf("LastProbe after Forget = %v, want zero", got) + } +} + +// When an agent's flag clears its probe time is forgotten, so its kicks flow +// normally and a later quota hit starts a fresh interval. +func TestQuotaHoldState_ClearedFlagForgetsProbe(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("held", quotaHoldNow) + s.MarkProbeDelivered("recovered", quotaHoldNow) + got := s.quotaExhaustedAgents(map[string]*crux.Process{ + "held": {QuotaExhausted: true}, + "recovered": {QuotaExhausted: false}, + "gone": nil, + }) + if !reflect.DeepEqual(got, map[string]bool{"held": true}) { + t.Fatalf("exhausted = %v, want only held", got) + } + if !s.LastProbe("held").Equal(quotaHoldNow) { + t.Fatal("a still-held agent must keep its probe time") + } + if !s.LastProbe("recovered").IsZero() { + t.Fatal("a recovered agent's probe time must be forgotten") + } +} + +// gate is the eval cycle's entry point: it must return the cycle's messages +// untouched (same slice, no copy) when no agent is held or there is nothing to +// kick, still forget recovered agents, and gate only when both are non-empty. +func TestQuotaHoldState_GateShortCircuitsAndGates(t *testing.T) { + msgs := []scheduler.KickMessage{{Agent: "held", Message: "1"}, {Agent: "ok", Message: "2"}, {Agent: "held", Message: "3"}} + + t.Run("no held agent returns the same slice", func(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("ok", quotaHoldNow) + kept, withheld, probes := s.gate(msgs, map[string]*crux.Process{"ok": {}}, quotaHoldNow, 30*time.Minute) + if len(kept) != len(msgs) || &kept[0] != &msgs[0] { + t.Fatalf("kept = %v, want the input slice itself", kickAgents(kept)) + } + if withheld != nil || probes != nil { + t.Fatalf("withheld=%v probes=%v, want nil", withheld, probes) + } + if !s.LastProbe("ok").IsZero() { + t.Fatal("a recovered agent's probe time must be forgotten even when nothing is gated") + } + }) + + t.Run("no messages returns the empty input", func(t *testing.T) { + var s agentQuotaHoldState + empty := []scheduler.KickMessage{} + kept, withheld, probes := s.gate(empty, map[string]*crux.Process{"held": {QuotaExhausted: true}}, quotaHoldNow, 30*time.Minute) + if kept == nil || len(kept) != 0 || withheld != nil || probes != nil { + t.Fatalf("kept=%v withheld=%v probes=%v, want the empty input and nothing else", kept, withheld, probes) + } + }) + + t.Run("one held agent is gated", func(t *testing.T) { + var s agentQuotaHoldState + s.MarkProbeDelivered("held", quotaHoldNow.Add(-time.Minute)) + kept, withheld, probes := s.gate(msgs, map[string]*crux.Process{"held": {QuotaExhausted: true}, "ok": {}}, quotaHoldNow, 30*time.Minute) + if got := kickAgents(kept); !reflect.DeepEqual(got, []string{"ok:2"}) { + t.Fatalf("kept = %v, want only ok:2", got) + } + if !reflect.DeepEqual(withheld, []string{"held", "held"}) || len(probes) != 0 { + t.Fatalf("withheld=%v probes=%v, want held twice and no probe", withheld, probes) + } + }) +} + +// The full cycle: withheld, then one probe after the interval, then withheld +// again on the next cycle once that probe was delivered. +func TestQuotaHoldGate_ProbeOncePerInterval(t *testing.T) { + var s agentQuotaHoldState + held := quotaSet("held") + msgs := []scheduler.KickMessage{{Agent: "held", Message: "gov"}} + interval := 30 * time.Minute + + s.MarkProbeDelivered("held", quotaHoldNow) + if kept, _, _ := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, quotaHoldNow.Add(5*time.Minute), interval); len(kept) != 0 { + t.Fatalf("inside the interval kept = %v, want none", kickAgents(kept)) + } + at := quotaHoldNow.Add(interval) + kept, _, probes := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, at, interval) + if len(kept) != 1 || !reflect.DeepEqual(probes, []string{"held"}) { + t.Fatalf("after the interval kept=%v probes=%v, want exactly one probe", kickAgents(kept), probes) + } + s.MarkProbeDelivered("held", at) + if kept, _, _ := gateKickMessagesForAgentQuota(msgs, held, s.LastProbe, at.Add(2*time.Minute), interval); len(kept) != 0 { + t.Fatalf("next cycle kept = %v, want the agent held again", kickAgents(kept)) + } +} + +// The probe time is recorded on delivery only: a probe whose send fails must +// not advance it, so the next cycle may probe again. +func TestQuotaHoldDispatch_ProbeRecordedOnDeliveryNotOnSendError(t *testing.T) { + for _, tc := range []struct { + name string + sendErr error + want bool + }{ + {"delivered", nil, true}, + {"send error", errors.New("agent CLI did not reach input prompt"), false}, + {"busy deferral", crux.ErrAgentBusy, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var s agentQuotaHoldState + exhausted := map[string]bool{"held": true} + r := newKickDispatchRecorder() + r.sendErr["held"] = tc.sendErr + deps := r.deps() + inner := deps.onDelivered + deps.onDelivered = func(msg scheduler.KickMessage, deliveredAt time.Time) { + inner(msg, deliveredAt) + if exhausted[msg.Agent] { + s.MarkProbeDelivered(msg.Agent, deliveredAt) + } + } + msgs, _, probes := gateKickMessagesForAgentQuota( + []scheduler.KickMessage{{Agent: "held", Message: "gov"}, {Agent: "healthy", Message: "gov"}}, + func(a string) bool { return exhausted[a] }, s.LastProbe, quotaHoldNow, 30*time.Minute) + if !reflect.DeepEqual(probes, []string{"held"}) { + t.Fatalf("probes = %v, want [held]", probes) + } + dispatchAgentKicks(msgs, false, deps, slog.Default()) + if got := !s.LastProbe("held").IsZero(); got != tc.want { + t.Fatalf("probe recorded = %v, want %v", got, tc.want) + } + if !s.LastProbe("healthy").IsZero() { + t.Fatal("a healthy agent's delivery must never record a probe time") + } + }) + } +} + +func TestCompactDuration(t *testing.T) { + for d, want := range map[time.Duration]string{ + 30 * time.Minute: "30m", + 2 * time.Hour: "2h", + 90 * time.Minute: "1h30m", + 45 * time.Second: "45s", + 90 * time.Second: "1m30s", + 0: "0s", + time.Hour + 30*time.Second: "1h0m30s", + } { + if got := compactDuration(d); got != want { + t.Errorf("compactDuration(%v) = %q, want %q", d, got, want) + } + } +} diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index 91925bc20e..e5d0e0af08 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -417,7 +417,11 @@ type providerBudgetAlert struct { // consulted the agent manager on the NOT-latched branch, and collecting agent // statuses every cycle while the provider is clipped would be work the old // code never did. Passing a thunk keeps that laziness exactly. -func decideProviderBudgetAlert(latched, suppress bool, cause string, since time.Time, rebuffs int, quotaReason func() string) providerBudgetAlert { +// +// quotaProbeInterval is how often an agent held for exhausted provider quota +// is probed (#18112); the quota banner names it so the operator knows the +// hive is holding those kicks and when it will retry them. +func decideProviderBudgetAlert(latched, suppress bool, cause string, since time.Time, rebuffs int, quotaProbeInterval time.Duration, quotaReason func() string) providerBudgetAlert { if latched { state := "agent kicks suspended" if !suppress { @@ -431,7 +435,8 @@ func decideProviderBudgetAlert(latched, suppress bool, cause string, since time. return providerBudgetAlert{Message: msg, Cause: msg} } if reason := quotaReason(); reason != "" { - return providerBudgetAlert{Message: "provider quota exhausted — " + reason} + return providerBudgetAlert{Message: fmt.Sprintf("provider quota exhausted — %s; their kicks are held and retried every %s", + reason, compactDuration(quotaProbeInterval))} } return providerBudgetAlert{Clear: true} } diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 0c56eb0086..26e660c119 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -91,6 +91,11 @@ var providerBudgetNotify governor.ProviderBudgetNotifyState // providerBudgetNotify: runEvalCycle has no state of its own. var providerBudgetProbe governor.ProviderBudgetProbeState +// agentQuotaHold remembers, per agent out of provider quota, when its last +// probe kick was delivered (hivecommons/hive#18112). Package-level for the +// same reason as providerBudgetProbe. +var agentQuotaHold agentQuotaHoldState + // governorKickFairness remembers, per agent, how many cycles in a row the // governor kick was deferred behind a delivered review-fix kick // (hivecommons/hive#15166). Package-level for the same reason as @@ -886,7 +891,7 @@ func runEvalCycle( suppressKicks := governor.ProviderBudgetSuppresses(providerBudgetLatched, providerBudgetProbe.Freshest(providerBudgetLastRebuff), time.Now(), providerBudgetProbeInterval) budgetAlert := decideProviderBudgetAlert(providerBudgetLatched, suppressKicks, - providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, + providerBudgetCause, providerBudgetSince, providerBudgetRebuffs, providerBudgetProbeInterval, func() string { return hub.QuotaExhaustedAgentReason(hub.QuotaExhaustedProcessCount(agentMgr.AllStatuses())) }) @@ -1093,6 +1098,23 @@ func runEvalCycle( "last_rebuff", providerBudgetLastRebuff, "probe_interval", providerBudgetProbeInterval) } messages = kickGate.Kept + // PER-AGENT PROVIDER QUOTA HOLD (#18112). The spend gate above only sees + // agents behind the hive's inference gateway; an agent whose own provider + // refuses it for quota (Copilot's "monthly quota") is held here, per + // agent, from the QuotaExhausted flag every backend reports. Review-swarm + // kicks are in this list too, so a held reviewer gets no review work. + quotaProbeNow := map[string]bool{} + { + kept, quotaWithheld, quotaProbes := agentQuotaHold.gate(messages, agentMgr.AllStatuses(), time.Now(), providerBudgetProbeInterval) + if len(quotaWithheld) > 0 || len(quotaProbes) > 0 { + logger.Warn("provider quota exhausted: withholding agent kicks", + "withheld", quotaWithheld, "probe", quotaProbes, "probe_interval", providerBudgetProbeInterval) + } + for _, agent := range quotaProbes { + quotaProbeNow[agent] = true + } + messages = kept + } // Self-upgrade drain (#11876): a hub-instructed upgrade is waiting for // in-flight kicks to end, so starting new turns would only feed the // restart. Withheld kicks stay due and go out from the new process. @@ -1207,6 +1229,12 @@ func runEvalCycle( }, onDelivered: func(msg scheduler.KickMessage, deliveredAt time.Time) { deliveredKicks = append(deliveredKicks, msg) + if quotaProbeNow[msg.Agent] { + // The only kick a held agent can receive is its probe; + // stamp it on delivery so a failed or deferred probe + // is retried next cycle (#18112). + agentQuotaHold.MarkProbeDelivered(msg.Agent, deliveredAt) + } gov.RecordKickForRepoAt(msg.Agent, msg.Repo, deliveredAt) dashSrv.AuditLog("governor", "kick", "trigger=governor-eval", msg.Agent) diff --git a/src/cmd/hive/provider_budget_alert_test.go b/src/cmd/hive/provider_budget_alert_test.go index bcfb10685f..878d96bb04 100644 --- a/src/cmd/hive/provider_budget_alert_test.go +++ b/src/cmd/hive/provider_budget_alert_test.go @@ -23,7 +23,7 @@ func neverCalled(t *testing.T) func() string { } func TestDecideProviderBudgetAlertLatchedSuspended(t *testing.T) { - got := decideProviderBudgetAlert(true, true, "spend cap hit", time.Now(), 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", time.Now(), 1, 30*time.Minute, neverCalled(t)) if got.Clear { t.Fatal("a held latch must not clear the banner") @@ -40,7 +40,7 @@ func TestDecideProviderBudgetAlertLatchedSuspended(t *testing.T) { } func TestDecideProviderBudgetAlertLatchedProbing(t *testing.T) { - got := decideProviderBudgetAlert(true, false, "spend cap hit", time.Now(), 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, false, "spend cap hit", time.Now(), 1, 30*time.Minute, neverCalled(t)) if !strings.Contains(got.Message, "probing with a single agent kick") { t.Fatalf("a stale latch probes rather than staying muted, got %q", got.Message) @@ -54,7 +54,7 @@ func TestDecideProviderBudgetAlertLatchedProbing(t *testing.T) { // between "one call was refused" and "this has been refusing since Tuesday". func TestDecideProviderBudgetAlertRepeatedRebuffsAreQuantified(t *testing.T) { since := time.Date(2026, 9, 16, 14, 30, 0, 0, time.UTC) - got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 47, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 47, 30*time.Minute, neverCalled(t)) if !strings.Contains(got.Message, "47 refused calls") { t.Fatalf("repeated rebuffs must be counted, got %q", got.Message) @@ -66,7 +66,7 @@ func TestDecideProviderBudgetAlertRepeatedRebuffsAreQuantified(t *testing.T) { func TestDecideProviderBudgetAlertSingleRebuffIsNotQuantified(t *testing.T) { since := time.Date(2026, 9, 16, 14, 30, 0, 0, time.UTC) - got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 1, neverCalled(t)) + got := decideProviderBudgetAlert(true, true, "spend cap hit", since, 1, 30*time.Minute, neverCalled(t)) if strings.Contains(got.Message, "refused calls") { t.Fatalf("a single rebuff must not be reported as a run of them, got %q", got.Message) @@ -80,7 +80,7 @@ func TestDecideProviderBudgetAlertSingleRebuffIsNotQuantified(t *testing.T) { // branch: the provider is not refusing on money, but agents are individually // out of quota, which is a different cause and must not be silent. func TestDecideProviderBudgetAlertQuotaExhaustedFallback(t *testing.T) { - got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, + got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, 30*time.Minute, func() string { return "3 agents out of quota" }) if got.Clear { @@ -89,13 +89,18 @@ func TestDecideProviderBudgetAlertQuotaExhaustedFallback(t *testing.T) { if !strings.Contains(got.Message, "provider quota exhausted") || !strings.Contains(got.Message, "3 agents out of quota") { t.Fatalf("quota exhaustion must name itself and its reason, got %q", got.Message) } + // #18112: the banner says the hive is holding those agents' kicks and + // when it retries them, not just that the quota ran out. + if !strings.Contains(got.Message, "their kicks are held and retried every 30m") { + t.Fatalf("quota exhaustion must say kicks are held and the probe interval, got %q", got.Message) + } if got.Cause != "" { t.Fatalf("only the latched branch replaces the cause, got %q", got.Cause) } } func TestDecideProviderBudgetAlertHealthyClears(t *testing.T) { - got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, func() string { return "" }) + got := decideProviderBudgetAlert(false, false, "", time.Now(), 0, 30*time.Minute, func() string { return "" }) if !got.Clear { t.Fatal("a healthy provider with no exhausted quota must clear the banner, or it sticks forever") @@ -121,7 +126,7 @@ func TestDecideProviderBudgetAlertNeverRaisesAndClears(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := decideProviderBudgetAlert(tc.latched, tc.suppress, "cause", time.Now(), 1, + got := decideProviderBudgetAlert(tc.latched, tc.suppress, "cause", time.Now(), 1, 30*time.Minute, func() string { return tc.quota }) if got.Message != "" && got.Clear { t.Fatalf("must never raise and clear in the same cycle: %+v", got) diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index e9e558eb20..1adeee5c39 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -916,6 +916,7 @@ governor: ``` Lower it to resume sooner after a reset at the cost of a rebuffed run per probe; raise it to waste less while noticing later. +- An agent whose **own** provider refuses it for quota — for example GitHub Copilot's "You have exceeded your monthly quota", which never passes through the hive's inference gateway — is held per agent ([#18112](https://github.com/hivecommons/hive/issues/18112)). Every backend reports the pane's quota message as the agent's `QuotaExhausted` flag; while it is set, the governor withholds that agent's kicks, including review-swarm review and fix kicks, so a held reviewer is not sent review work that would only expire. Once per `governor.provider_budget.probe_interval_s` (the same setting as above) one kick is let through as a probe, and the interval restarts only when that probe is actually delivered; a failed or deferred probe is retried on the next cycle. When the pane stops showing the quota message the flag clears and the agent's kicks flow normally from the next cycle. One agent out of quota never holds another, and manual kicks from the dashboard are not held. The dashboard alert reads `provider quota exhausted — N agent(s) out of provider quota; their kicks are held and retried every 30m`, and the log shows `provider quota exhausted: withholding agent kicks` with the withheld and probed agents. ## Merge-hotspot advisor (`governor.hotspot_*`) From 099d40104eb66f540740b94dba47ef98326d360d Mon Sep 17 00:00:00 2001 From: "hivecommons-hive[bot]" <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:08:58 +0000 Subject: [PATCH 164/166] fix: stop repeating close-on-merge warning for unreadable PRs (#18103) Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- ...ixed-18084-close-on-merge-unreadable-pr.md | 1 + src/pkg/github/close_on_merge.go | 16 ++++++++++++++ src/pkg/github/close_on_merge_paths_test.go | 21 +++++++++++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18084-close-on-merge-unreadable-pr.md diff --git a/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md b/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md new file mode 100644 index 0000000000..76d3978235 --- /dev/null +++ b/changelog.d/fixed-18084-close-on-merge-unreadable-pr.md @@ -0,0 +1 @@ +- Close-on-merge no longer re-warns every tick for merged PRs it cannot read (404/403); it logs once per recheck window. diff --git a/src/pkg/github/close_on_merge.go b/src/pkg/github/close_on_merge.go index 239bc97fda..08cf0ecbb8 100644 --- a/src/pkg/github/close_on_merge.go +++ b/src/pkg/github/close_on_merge.go @@ -192,6 +192,17 @@ func closeOnMergeClaimSettled(claim IssueClaim, now time.Time, recheck time.Dura return now.Sub(claim.CloseOnMergeSettledAt) < recheck } +// closeOnMergePRUnreadable reports a terminal read failure: the PR is gone or +// this identity cannot see it (404/403), so retrying every tick is noise. +func closeOnMergePRUnreadable(err error) bool { + var ghErr *gh.ErrorResponse + if !errors.As(err, &ghErr) || ghErr.Response == nil { + return false + } + code := ghErr.Response.StatusCode + return code == http.StatusNotFound || code == http.StatusForbidden +} + // closeOnMergeClaimMergedAt is the instant the sweep treats as the claim's // merge time for MergedAfter filtering: MergedAt, else the first observation. func closeOnMergeClaimMergedAt(claim IssueClaim) time.Time { @@ -255,6 +266,11 @@ func (c *Client) CloseOnMergeLedgerSweep(ctx context.Context, ledger *ClaimLedge if opts.Logger != nil { opts.Logger.Warn("close-on-merge: could not read merged PR", "repo", claim.PRRepo, "pr", claim.PRNumber, "error", err) } + // 404/403 will not heal on the next tick: remember it so the + // warning repeats at most once per recheck window. + if closeOnMergePRUnreadable(err) && ledger.MarkCloseOnMergeSettled(claim.Repo, claim.Issue, claim.PRNumber, now) { + sweep.Marked++ + } continue } res := c.closeIssueForMergedPRClaim(ctx, claim.Repo, claim.Issue, pr, true, opts) diff --git a/src/pkg/github/close_on_merge_paths_test.go b/src/pkg/github/close_on_merge_paths_test.go index eacbbe2ba5..d9cec8f187 100644 --- a/src/pkg/github/close_on_merge_paths_test.go +++ b/src/pkg/github/close_on_merge_paths_test.go @@ -107,6 +107,10 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) return } + if strings.HasSuffix(r.URL.Path, "/404") { + w.WriteHeader(http.StatusNotFound) + return + } _ = json.NewEncoder(w).Encode(map[string]any{ "number": 42, "title": "fix", "body": "Fixes #7", "merged": true, "merged_at": "2026-10-09T12:00:00Z", "merge_commit_sha": "abcdef0123456789", @@ -143,6 +147,8 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { {Repo: "o/r", Issue: 9, PRNumber: 44, PRRepo: "o/r", MergedPR: true, Reference: true}, // Merged claim whose PR cannot be read: logged and skipped. {Repo: "o/r", Issue: 10, PRNumber: 500, PRRepo: "o/r", MergedPR: true}, + // Merged claim whose PR is 404: warned once, then settled. + {Repo: "o/r", Issue: 12, PRNumber: 404, PRRepo: "o/r", MergedPR: true}, // Merged claim missing a PR number: skipped. {Repo: "o/r", Issue: 11, PRNumber: 0, PRRepo: "o/r", MergedPR: true}, } { @@ -170,8 +176,19 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { if len(patched) != 1 { t.Fatalf("patched = %v, want one close", patched) } - if len(prGets) != 2 { - t.Fatalf("PR GETs = %v, want /pulls/42 and /pulls/500 only", prGets) + if len(prGets) != 3 { + t.Fatalf("PR GETs = %v, want /pulls/42, /pulls/500 and /pulls/404 only", prGets) + } + // Second sweep: the 404 claim is settled and not re-read; the 500 one is. + prGets = nil + c.CloseOnMergeForLedger(context.Background(), ledger, CloseOnMergeOptions{ + Identity: HiveIdentity{AppLogin: "hive[bot]"}, + Logger: logger, + }) + for _, p := range prGets { + if strings.HasSuffix(p, "/404") { + t.Fatalf("404 PR re-read on second sweep: %v", prGets) + } } if !strings.Contains(logBuf.String(), "could not read merged PR") { t.Fatalf("unreadable PR was not logged: %s", logBuf.String()) From f899e1ff7aeeeb879d078723b33d9b6bb440d51c Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Fri, 13 Nov 2026 14:11:09 -0500 Subject: [PATCH 165/166] =?UTF-8?q?=F0=9F=8C=B1=20images:=20bump=20codex?= =?UTF-8?q?=200.167.0=20->=200.168.0=20(#18062)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated pin bump (#8419). Digests recomputed from the downloaded release artifacts by src/scripts/cli-pin-bump.sh: VERSION=0.168.0 Signed-off-by: hive-release[bot] <hive-release@hive.kubestellar.io> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: hive-release[bot] <hive-release@hive.kubestellar.io> Co-authored-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> --- src/Dockerfile | 2 +- src/Dockerfile.contributor | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 05b954dbe8..27866c4f76 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -324,7 +324,7 @@ RUN timeout 120s npm install -g --ignore-scripts --no-fund --no-audit \ # dashboard's codex model discovery (`codex app-server` model/list — pure # local stdio, no network, unaffected by the proxy CA) falls back to the # static list, and non-codex backends are unaffected. -ARG CODEX_VERSION=0.167.0 +ARG CODEX_VERSION=0.168.0 ARG CLINE_VERSION=3.0.74 # npm can briefly advertise AWS SDK versions before all registry replicas serve # them. Wait at most three minutes between attempts and revalidate metadata. diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index dc3d62e858..9013899341 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -262,7 +262,7 @@ ARG KILO_CLI_VERSION=7.5.6 # Same ARG name and value as src/Dockerfile so bin/test_backend_smoke.sh can # assert the two pins agree — this line was previously an inline hardcode that # could (and did) drift from the main image's CODEX_VERSION unnoticed. -ARG CODEX_VERSION=0.167.0 +ARG CODEX_VERSION=0.168.0 RUN npm install -g --ignore-scripts --no-fund --no-audit \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ @github/copilot@${COPILOT_VERSION} \ From 85a2b42c0054f09922712b9afb2a5ae198481672 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:45:08 -0500 Subject: [PATCH 166/166] =?UTF-8?q?=F0=9F=8C=B1=20Fix=20CI=20on=20the=20v5?= =?UTF-8?q?=20->=20v6=20top-up=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Resync api-reference citations (334 drifted line refs after the merge). - dashboard: the merged-in v5 navbar summary button left v6's base `.agent-navbar-upnext::after { content: attr(data-agent-summary) }` rule; scope it to the v6 "+N" overflow count ([data-agent-more]) and stub v6's fitAgentNavbarTiles/ensureAgentNavbarFitObserver in the v5-authored test. - dashboard: declare v5's hold-shift guard globals in the v6-only your-turn fake browser, and stub v5's webhookGovStatHtml in the v6-only standby tile test. - dashboard: list GET /api/config/github/webhooks (v5) as a settings_read exclusion; it probes App hook permissions live on every read. - hub: classify v6's HeartbeatResponse.UpgradeChannelTag in v5's self-service credential fence (fenced, rides with UpgradeTo). - github: v5's 404 close-on-merge claim reused issue 12, which v6's test lists as an open PR; give it its own open issue (14). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/admin-mcp.md | 3 + src/docs/api-reference.md | 344 +++++++++--------- src/docs/api-reference/agents-and-controls.md | 28 +- src/docs/api-reference/auth.md | 6 +- src/docs/api-reference/beads.md | 10 +- src/docs/api-reference/configuration.md | 26 +- src/docs/api-reference/dashboard-misc.md | 26 +- .../api-reference/health-status-events.md | 22 +- src/docs/api-reference/inception.md | 28 +- src/docs/api-reference/knowledge.md | 92 ++--- src/docs/api-reference/nous.md | 50 +-- src/docs/api-reference/packs-and-acmm.md | 36 +- src/docs/api-reference/snapshots-and-style.md | 2 +- src/pkg/dashboard/admin_mcp_settings_test.go | 1 + .../dashboard/agent_navbar_summary_ui_test.go | 3 + .../dashboard/standby_tile_visible.test.js | 1 + .../static-src/css/sections-collapse.css | 2 +- src/pkg/dashboard/static/index.html | 2 +- src/pkg/dashboard/your_turn_ui_11566_test.go | 2 +- src/pkg/github/close_on_merge_paths_test.go | 3 +- src/pkg/hub/heartbeat_selfservice_fence.go | 13 +- 21 files changed, 355 insertions(+), 345 deletions(-) diff --git a/src/docs/admin-mcp.md b/src/docs/admin-mcp.md index a98a86d0f1..00a38b43c0 100644 --- a/src/docs/admin-mcp.md +++ b/src/docs/admin-mcp.md @@ -495,6 +495,9 @@ Not reachable: credentials (`config_effective_read` is the vetted view of the effective configuration and its provenance); - an agent's export bundle (the `agent` and `agent_prompt` sections hold it); +- the GitHub webhook setup card, which probes the GitHub App's hook + permissions live on every read (webhook verification and health are in + `/api/status`); - the theme and sidebar, which are purely presentational. A test fails when the dashboard serves a new `GET /api/config` route that is diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 594fe1d9dc..68adce78d3 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -40,17 +40,17 @@ summary and the `health_deep_read` diagnostic tool. | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/<version>+<short>` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1849` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1850` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1851` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1857` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts; `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1862` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1863` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1866` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1867` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1868` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1869` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1870` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1864` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1865` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1866` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1872` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts; `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1877` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1878` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1881` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1882` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1883` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1884` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1885` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -93,7 +93,7 @@ summary and the `health_deep_read` diagnostic tool. | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1944` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1959` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | @@ -121,9 +121,9 @@ summary and the `health_deep_read` diagnostic tool. | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1871` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1872` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1873` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1886` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1887` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1888` | ## Configuration @@ -196,17 +196,17 @@ summary and the `health_deep_read` diagnostic tool. | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:332` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:333` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:336` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:396` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:397` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:398` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:388` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:389` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:390` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:391` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:392` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:393` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:573` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:399` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:400` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:401` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:391` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:392` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:393` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:395` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:396` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:397` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:576` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:234` | @@ -215,8 +215,8 @@ summary and the `health_deep_read` diagnostic tool. | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:243` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:246` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:247` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:569` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:570` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:572` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:573` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:254` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:257` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features; `writebackKinds` replaces the whole `writeback.kinds` map (a kind sent as `null` or `{}` is dropped). Wrapped by the admin MCP `governor.feature_settings` and `writeback.kind_policy` operations | `pkg/dashboard/api.go:263` | @@ -270,21 +270,21 @@ summary and the `health_deep_read` diagnostic tool. | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:188` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:189` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:256` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:543` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:546` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:547` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:551` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:546` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:549` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:550` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:554` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:555` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:556` | -| `GET` | `/api/search` | Merger or owner | Portfolio search over this spoke's local index (no hub fan-out). Query `q` (full text), `kind` and `repo` (`owner/name`; both repeatable or comma-separated), `from` (inclusive) and `to` (exclusive; a `YYYY-MM-DD` date covers that day) as RFC 3339 or `YYYY-MM-DD`, and `limit` (1–100, default 20); `probe=1` runs no query and is not audited, answering `204` when the index is ready. Returns `total`, highlighted `hits` (snippet plus byte `spans`) and `facets` by `kind`, `repo`, `verdict` and `sentinel.rule`; `coverage` names repos whose index could not be read. Hit text passes through ioscan, so injected text appears only as a visible redaction marker (`redacted: true`). The query string, actor and role are audited as `search_query`; result bodies are not. `404` unless `search.enabled`; `400` for bad parameters and for the reserved `semantic=true`; `503` while the index is not yet available | `pkg/dashboard/api.go:559` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:561` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:564` | -| `GET` | `/api/repos/{owner}/{repo}/health` | Dashboard auth/session | Report-only repo health score from the search index (`search.enabled`; `503` when off, `404` until the repo has indexed review evidence, `400` for a bad owner/repo). Recomputed on every index rebuild. Returns `score` (0–100, or `null` when no component is reported), `status`, `window_days` (90), `min_n` (10, the hub `minEvidenceN` rule), `computed_at` and `components[]` in order `ci_pass_rate`, `time_to_merge`, `revert_followup_rate`, `review_verdict_mix`, `stale_issue_ratio`, `signoff_rate`, `dependency_freshness`, `security_doc_rot`, each `{name, status, score (0–1), weight, contribution, n, inputs, reason}`. A component with `n` below `min_n` or without local data is `not_reported` (never zero) and the remaining weights are renormalised; weights and ages come from `search.health`. Spoke-only; never an ACMM input and never read by merge decisions or Governor priority. | `pkg/dashboard/api.go:565` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:338` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:339` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:340` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:341` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:558` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:559` | +| `GET` | `/api/search` | Merger or owner | Portfolio search over this spoke's local index (no hub fan-out). Query `q` (full text), `kind` and `repo` (`owner/name`; both repeatable or comma-separated), `from` (inclusive) and `to` (exclusive; a `YYYY-MM-DD` date covers that day) as RFC 3339 or `YYYY-MM-DD`, and `limit` (1–100, default 20); `probe=1` runs no query and is not audited, answering `204` when the index is ready. Returns `total`, highlighted `hits` (snippet plus byte `spans`) and `facets` by `kind`, `repo`, `verdict` and `sentinel.rule`; `coverage` names repos whose index could not be read. Hit text passes through ioscan, so injected text appears only as a visible redaction marker (`redacted: true`). The query string, actor and role are audited as `search_query`; result bodies are not. `404` unless `search.enabled`; `400` for bad parameters and for the reserved `semantic=true`; `503` while the index is not yet available | `pkg/dashboard/api.go:562` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:564` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:567` | +| `GET` | `/api/repos/{owner}/{repo}/health` | Dashboard auth/session | Report-only repo health score from the search index (`search.enabled`; `503` when off, `404` until the repo has indexed review evidence, `400` for a bad owner/repo). Recomputed on every index rebuild. Returns `score` (0–100, or `null` when no component is reported), `status`, `window_days` (90), `min_n` (10, the hub `minEvidenceN` rule), `computed_at` and `components[]` in order `ci_pass_rate`, `time_to_merge`, `revert_followup_rate`, `review_verdict_mix`, `stale_issue_ratio`, `signoff_rate`, `dependency_freshness`, `security_doc_rot`, each `{name, status, score (0–1), weight, contribution, n, inputs, reason}`. A component with `n` below `min_n` or without local data is `not_reported` (never zero) and the remaining weights are renormalised; weights and ages come from `search.health`. Spoke-only; never an ACMM input and never read by merge decisions or Governor priority. | `pkg/dashboard/api.go:568` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:341` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:342` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:343` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:344` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:139` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:143` | @@ -313,26 +313,26 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:343` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:344` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:345` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:346` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:347` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:348` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:366`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage, job_id}` (`job_id` is the let-hive-decide job a record is for); `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:368`, `pkg/dashboard/api_repo_item_decision.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:370`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:371`, `pkg/dashboard/api_repo_item_decision.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `<!-- hive:delegated-decision … -->` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park}, notes, error}`); 409 with the running job while one is in flight; `GET` reads it, from memory while this process holds the job and otherwise from the operator-action ledger (`from_ledger: true`; status, requester, start, verdict and rule, with the outcome's words in `notes` or `error`), so the last result outlives the 30-minute in-memory eviction and a restart. A job still running when the hive restarts is never resumed: the next process marks its ledger record `failed` (*interrupted by a hive restart before it finished, and not resumed …*) with a `repo_item_decision_delegated_failed` audit entry, and the operator asks again. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:594`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:385` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`, and on the needs-human advice `why` (plain-language reasons with counts) and `help`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](advisory.md#owner-advice) | `pkg/dashboard/api.go:387`, `pkg/dashboard/api_hive_advice.go` | -| `POST` | `/api/repos/{owner}/{repo}/issues/{number}/release` | Owner or repo write/maintain/admin | Execute the typed `kind=release` owner step the hub proposed on a parked issue, with the same live preconditions as `/hive approve` ([#12635](https://github.com/hivecommons/hive/issues/12635)). Tag and target are resolved from the trusted owner-step, never from the request. Returns 503 when the GitHub client or config is unavailable; 409 when the repository is `label_driven` (the same refusal `/hive approve` gets there), when `release_on_approve` is off, or when a live precondition fails (`error` carries the refusal reason, which is also posted on the issue); 502 when the release write itself fails. Success returns `{ok, repo, number, tag, url, minStatusSeq}` plus `warning` if the follow-up issue comment failed after the release was created. | `pkg/dashboard/api.go:376`, `pkg/dashboard/api_issue_release.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:597`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:386`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:387` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:388` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`, and on the needs-human advice `why` (plain-language reasons with counts) and `help`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](advisory.md#owner-advice) | `pkg/dashboard/api.go:390`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/issues/{number}/release` | Owner or repo write/maintain/admin | Execute the typed `kind=release` owner step the hub proposed on a parked issue, with the same live preconditions as `/hive approve` ([#12635](https://github.com/hivecommons/hive/issues/12635)). Tag and target are resolved from the trusted owner-step, never from the request. Returns 503 when the GitHub client or config is unavailable; 409 when the repository is `label_driven` (the same refusal `/hive approve` gets there), when `release_on_approve` is off, or when a live precondition fails (`error` carries the refusal reason, which is also posted on the issue); 502 when the release write itself fails. Success returns `{ok, repo, number, tag, url, minStatusSeq}` plus `warning` if the follow-up issue comment failed after the release was created. | `pkg/dashboard/api.go:379`, `pkg/dashboard/api_issue_release.go` | ## Cost, tokens, telemetry @@ -356,37 +356,37 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `tag` (comma-separated; a fact must carry every tag), `state` (comma-separated lifecycle states or `all`; only facts in those states are listed, and any state other than `approved` requires a verified owner and returns 403 otherwise), `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:403` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:404` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](public-knowledge-mcp.md)) | `pkg/dashboard/api.go:413` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:405` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:406` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:407` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:408` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:414` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:415` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:416` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:417` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:418` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:419` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:420` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:421` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:427` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:428` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:429` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:430` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:431` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:432` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:433` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:434` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:435` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:436` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:437` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:438` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:439` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:440` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:441` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:442` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `tag` (comma-separated; a fact must carry every tag), `state` (comma-separated lifecycle states or `all`; only facts in those states are listed, and any state other than `approved` requires a verified owner and returns 403 otherwise), `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:406` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:407` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](public-knowledge-mcp.md)) | `pkg/dashboard/api.go:416` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:408` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:409` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:410` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:411` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:417` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:418` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:419` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:420` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:421` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:422` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:423` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:424` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:430` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:431` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:432` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:433` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:434` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:436` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:437` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:438` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:439` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:440` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:441` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:442` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:443` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:444` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:445` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -394,36 +394,36 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | `POST` | `/api/config/knowledge/connectors/{name}/sync` | Owner only | Start an immediate background sync of one connector | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/agent-scopes` | Dashboard auth/session | Per-agent knowledge scopes (`knowledge.agent_scopes`: layers, repos, types, tags, include_states) ([docs](knowledge-toc.md#per-agent-scopes)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:452` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:453` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:454` | -| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:455` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:456` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:457` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:458` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:459` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:460` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:461` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:462` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:463` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:425` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:426` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:455` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:456` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:457` | +| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:458` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:459` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:460` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:461` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:462` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:463` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:464` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:465` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:466` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:428` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:429` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:452` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:453` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:454` | -| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:455` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:456` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:457` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:458` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:459` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:460` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:461` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:462` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:463` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:425` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:426` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:455` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:456` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:457` | +| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:458` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:459` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:460` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:461` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:462` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:463` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:464` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:465` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:466` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:428` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:429` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | @@ -534,70 +534,70 @@ always resolved server-side from the validated token. | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:498` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:499` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:500` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:501` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:502` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:504` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:505` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:506` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:507` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:508` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:509` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:510` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:511` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:512` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:513` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:514` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:515` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:516` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:517` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:518` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:519` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:522` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:523` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:524` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:525` | -| `POST` | `/api/nous/promote` | Owner only | Nous Promote (manual verdict promotion to an issue or Spektacular run; see [experiment-lab.md](experiment-lab.md#verdict-card-and-manual-promote)) | `pkg/dashboard/api.go:503` | -| `POST` | `/api/nous/artifacts/sync` | Owner only | Nous Artifact Sync | `pkg/dashboard/api.go:520` | -| `POST` | `/api/nous/reset` | Owner only | Nous Reset (wipe baseline snapshots, learned knowledge or everything; see [experiment-lab.md](experiment-lab.md#resetting-data)) | `pkg/dashboard/api.go:521` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:501` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:502` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:503` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:504` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:505` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:507` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:508` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:509` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:510` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:511` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:512` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:513` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:514` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:515` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:516` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:517` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:518` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:519` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:520` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:521` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:522` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:525` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:526` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:527` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:528` | +| `POST` | `/api/nous/promote` | Owner only | Nous Promote (manual verdict promotion to an issue or Spektacular run; see [experiment-lab.md](experiment-lab.md#verdict-card-and-manual-promote)) | `pkg/dashboard/api.go:506` | +| `POST` | `/api/nous/artifacts/sync` | Owner only | Nous Artifact Sync | `pkg/dashboard/api.go:523` | +| `POST` | `/api/nous/reset` | Owner only | Nous Reset (wipe baseline snapshots, learned knowledge or everything; see [experiment-lab.md](experiment-lab.md#resetting-data)) | `pkg/dashboard/api.go:524` | ## Inception | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:468` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:469` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:470` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:471` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:472` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:473` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:474` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:475` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:476` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:477` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:478` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:479` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:480` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:481` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:471` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:472` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:473` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:474` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:475` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:476` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:477` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:478` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:479` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:480` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:481` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:482` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:483` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:484` | ## Beads | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:527` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:528` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:529` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:530` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:531` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:530` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:531` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:532` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:533` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:534` | ## Dashboard miscellaneous | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:64` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:538` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:541` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:73` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:74` | | `POST` | `/api/self-upgrade` | Owner only | Self Upgrade | `pkg/dashboard/api.go:75` | @@ -611,11 +611,11 @@ always resolved server-side from the validated token. | `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:281` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:486` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:489` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:490` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:492` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:493` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:489` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:492` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:493` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:495` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:496` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:219` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:220` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:221` | @@ -631,11 +631,11 @@ always resolved server-side from the validated token. | `GET` | `/api/summaries` | Dashboard auth/session | Summaries | `pkg/dashboard/api.go:203` | | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `finisher` / `finisher_verb` / `finisher_reason` (who can finish the item, from the your-turn decision: `hive`, `person` when it is the operator's turn, `elsewhere` when it waits on someone else (a hold, a covering PR, a reporter, an open blocker, or its author on a returned PR or a person's draft), absent when the snapshot has no decision; also on each `linked_prs` entry), `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`). Standing meta issues (bot Dependency Dashboards and the Hive Advisory Report) are in `items` with `bucket` `dependency_dashboard` / `hive_advisory`, so `open_issues` equals the Overview tile, but the monitor never draws them as rows; `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; `merged` is the durable merged ledger's summary — `{today, total_7d, total_30d, all_time, recent: [≤20 × {repo, number, title, url, merged_at, merged_by (`hive` or a login), lane, issue}], tz, day_start}`, `today` rolling at midnight in the spoke's zone — behind the monitor's Merged chip. Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=<remaining>`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:204` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:206` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:401` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:465` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:466` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat. JSON body `{"query": "<text>", "history": [...]}`; `message` is accepted as an alias for `query` (`query` wins when both are set). A missing or blank field returns 400 `query is required`. `!`-prefixed commands go to the chat spine (`{"accepted":true,"seq":N}`, read-write); while an inception is in `clarify` with unanswered questions, an owner's unprefixed reply also goes to the spine, which submits it as the answer to the next unanswered question. | `pkg/dashboard/api.go:495` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:533` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:404` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:468` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:469` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat. JSON body `{"query": "<text>", "history": [...]}`; `message` is accepted as an alias for `query` (`query` wins when both are set). A missing or blank field returns 400 `query is required`. `!`-prefixed commands go to the chat spine (`{"accepted":true,"seq":N}`, read-write); while an inception is in `clarify` with unanswered questions, an owner's unprefixed reply also goes to the spine, which submits it as the answer to the next unanswered question. | `pkg/dashboard/api.go:498` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:536` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:362` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:363` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:364` | @@ -663,7 +663,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1878` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1893` | ## Hub SaaS 102 routes — see [api-reference/hub-saas.md](api-reference/hub-saas.md). diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index adb6c03e2a..896334c6b3 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -23,21 +23,21 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:188` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:189` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:256` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:543` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:546` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:547` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:551` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:546` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:549` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:550` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:554` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:15` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:555` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:556` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:561` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:564` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:338` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:339` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:340` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:341` | - -| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:566` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:558` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:559` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:564` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `<evidence-root>/<owner>/<repo>/issues/<YYYY-Www>.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:567` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:341` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:342` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:343` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:344` | + +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:569` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:139` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:143` | | `GET` | `/api/agents/{name}/kicks` | Dashboard auth/session | Agent Kick Log List | `pkg/dashboard/api.go:146` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index e5e2eb8e3e..c7c182fa25 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1871` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1872` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1873` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1886` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1887` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1888` | diff --git a/src/docs/api-reference/beads.md b/src/docs/api-reference/beads.md index 047ca85b2a..42b122f466 100644 --- a/src/docs/api-reference/beads.md +++ b/src/docs/api-reference/beads.md @@ -5,8 +5,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:527` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:528` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:529` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:530` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:531` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:530` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:531` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:532` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:533` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:534` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 9641a1e78d..c085e7ac01 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -75,17 +75,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:332` | | `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:333` | | `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:336` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:396` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:397` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:398` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:388` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:389` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:390` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:391` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:392` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:393` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:573` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:399` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:400` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:401` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:391` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:392` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:393` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:395` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:396` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:397` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:576` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:234` | @@ -94,8 +94,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:243` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:246` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:247` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:569` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:570` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:572` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:573` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:254` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:257` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:263` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 8ea3fbddee..9c4de907e9 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -25,7 +25,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](../audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:64` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:538` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:541` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:73` | | `POST` | `/api/upgrade/announce` | Owner role | Announce a pod roll before it happens ([#15788](https://github.com/hivecommons/hive/issues/15788)): body `{"target": "<sha|tag>", "source": "hub"|"operator", "reason": "..."}` (`version` is an alias of `target`; all fields optional). Marks `/api/status`/`/api/version` `upgradeProgress` as `state: pulling, phase: announced` with the target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation; a SIGTERM afterwards (or with no announce at all) publishes `phase: restarting` from the same record. The hub sends the same announce through the heartbeat response (`upgrade_announce`) for hub-managed hives; an announce with no kill expires after 15 minutes. Returns `{ok, upgradeProgress}` | `pkg/dashboard/api.go:76` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:74` | @@ -40,18 +40,18 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:281` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:486` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:489` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:490` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:492` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:493` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:489` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:492` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:493` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:495` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:496` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:219` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:220` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:221` | | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:324` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {<repo>: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:325` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:583` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:586` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:131` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:136` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:150` | @@ -62,11 +62,11 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, `claimant: {kind: hive-agent|clanker|person, login, agent, hive, since, source: claim-ledger|claim-comment|assignee|kick-history|author}` naming who holds a claimed / in-progress issue or a PR under review or rework (absent when nobody is named; mirrored onto `linked_prs[]`), (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), (PRs) `lineage` on the hive's copy of an adopted PR — `{kind: adopted|superseded|rebased-by-hive, from_number, from_url, from_author, from_author_kind, to_number, to_url, at, reason}`, derived from the stage store's adoption record or the `hive/adopt-<n>` head branch, mirrored onto the issue's `linked_prs[]` entry; absent on every other row — and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; a PR in a conflict-aware merge train carries `merge_train` `{id, state, front, position, size, attempt, max_attempts, overlap_files, ahead, length, head_number, head_state, head_reason, head_since, eta_s, eta_at, eta_basis, late, stalled, stalled_s}` — `head_state` one of `needs-human` / `ci` / `soaking` / `ready` / `merging` / `fixing` / `unknown` with `head_reason` in plain words; `eta_s` / `eta_at` the estimated wait for this member's own merge attempt (own remaining soak + each PR ahead's remaining soak or one sweep + the next sweep tick; absent while the head waits on a person or CI), `eta_at` anchored to the member's first estimate at its position so a countdown survives polls, `eta_basis` its short reading ("2 ahead · head soaking 12m · next sweep in 40s"), `late` once now has overrun the first estimate twice over, `stalled` / `stalled_s` once the head has sat longer than `governor.merge_train_stall_warn_s` ([#15395](https://github.com/hivecommons/hive/issues/15395)); a PR the queue holds because another open PR declares the same `Fixes #N` carries no `merge_train` and a `merge_gate` of `{state: "duplicate-of:<winner>", duplicate_of, duplicate_issue}` plus the `needs-decision` modifier (the queue keeps one PR per issue: green first, then the earlier-opened); a PR GitHub reports conflicting on its own, or one under repair (stage/070, 071, 072, 074, 085, or a red required check), carries no `merge_train` and is not counted in any member's `size` / `length` — a repaired member the view dropped carries `left_train: {why: fixing|needs-author|returned-to-contributor|adopted-by-hive|needs-decision|checks-failing, note}` instead (mirrored onto the issue's `linked_prs[]` entry), and a train left with one member reports no `merge_train` at all; a PR short of the merge queue (below stage/080) the planner ordered rides no train either — it carries `overlap: {behind, with, files, note}` ("overlaps #15637 (sidebar.html, index.html) — will queue behind it") and no `merge_train`, and counts in no member's `size` — the state slugs (`front`, `queued-behind`) are payload values only, the SPA renders ordinal places ("Queued for merge · 1st of 2", with the train's tag when more than one train is on the page). Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=<remaining>`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:204` | | `GET` | `/api/matrix/lines` | Dashboard auth/session | Matrix Lines (deprecated name, see [Published-checkpoint regions](#published-checkpoint-regions)): serve bounded backlog of pushed lines behind dashboard.matrix with repo allowlist filtering. `max` counts only lines the repo allowlist admits, so a page is never emptied by filtered-out lines. A client paging with `max` should use the last returned line's `seq` as its next `since`, not the top-level `seq`, which is the source's latest sequence number and would skip lines still past the page. `more` is true only while another line the caller may see remains past the page; stop paging when it is false. | `pkg/dashboard/api.go:205` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:206` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:401` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:465` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:466` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:495` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:533` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:404` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:468` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:469` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:498` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:536` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:362` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:363` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:364` | @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1878` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1893` | ### Activity observation provenance diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index c49f7afa4f..526275537b 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=<sha>&to=<channel-branch-tag-or-sha>` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1849` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1850` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1851` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1857` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count); `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1862` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1863` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1866` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1867` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1868` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1869` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1870` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1864` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1865` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1866` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1872` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[<repo>]`, `writeback.per_repo[<repo>].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count); `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1877` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1878` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1881` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1882` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1883` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1884` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1885` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/inception.md b/src/docs/api-reference/inception.md index ce9f4ce88e..261263799c 100644 --- a/src/docs/api-reference/inception.md +++ b/src/docs/api-reference/inception.md @@ -5,17 +5,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:468` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:469` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:470` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:471` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:472` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:473` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:474` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:475` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:476` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:477` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:478` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:479` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:480` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:481` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:471` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:472` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:473` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:474` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:475` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:476` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:477` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:478` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:479` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:480` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:481` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:482` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:483` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:484` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 391b7c4287..886110f84d 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -5,38 +5,38 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:403` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:404` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:413` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:405` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:406` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:407` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:408` | -| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:410` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:414` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:415` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:416` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:417` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:418` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:419` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:420` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:421` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:427` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:428` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:429` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:430` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:431` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:432` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:433` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:434` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:435` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:436` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:437` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:438` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:439` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:440` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:441` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:442` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:406` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:407` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; otherwise 404 to a signed-in caller and 401 to one who is not | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:416` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:408` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:409` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:410` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:411` | +| `POST` | `/api/knowledge/cli-memory/drain-legacy` | Owner only | One-shot drain of the shared home's legacy Claude memory (`/data/home/.claude/projects/*/memory`) into `state: draft` entries in `bead-synth-wiki`, tagged `cli-memory` and `agent:shared`, deduplicated by content hash. Body `{"truncate": bool}` (default false) empties only files whose every fact is now stored. Returns `store`, `agent`, `sources`, `drained`, `dupes`, `skipped`, `truncate`, `truncated`, `kept`, `errors`; 409 when the vault is not connected. Audited as `knowledge_drain_legacy_memory` ([docs](../knowledge-lifecycle.md#auto-memory-drafts)) | `pkg/dashboard/api.go:413` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:417` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:418` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:419` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:420` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:421` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:422` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Create one fact (`title`, `body`, `type`, `tags`, `layer`, `confidence`, `state`). `state` is `approved` (default) or `draft`; a draft is written with `state: draft` into the local channel named by `layer` (400 when there is none, 409 when its title already names an entry). A request tagged `cli-memory` must be a draft carrying only `cli-memory`, `agent:<name>`, `cli:<claude\|codex\|gemini>`, `memhash:<hex>` and `repo:<org>/<repo>` tags (400 otherwise) | `pkg/dashboard/api.go:423` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:424` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:430` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:431` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:432` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:433` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:434` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:436` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:437` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:438` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:439` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:440` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:441` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:442` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:443` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:444` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:445` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -46,20 +46,20 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](../settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:452` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:453` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:454` | -| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:455` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:456` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:457` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:458` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:459` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:460` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:461` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:462` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:463` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:425` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:426` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:455` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:456` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:457` | +| `PUT` | `/api/knowledge/git-sources` | Owner only | Git Sources Update | `pkg/dashboard/api.go:458` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:459` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:460` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:461` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:462` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:463` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:464` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:465` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:466` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:428` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:429` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | diff --git a/src/docs/api-reference/nous.md b/src/docs/api-reference/nous.md index ffafe72ff7..e03925f1c6 100644 --- a/src/docs/api-reference/nous.md +++ b/src/docs/api-reference/nous.md @@ -5,28 +5,28 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:498` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:499` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:500` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:501` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:502` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:504` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:505` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:506` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:507` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:508` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:509` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:510` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:511` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:512` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:513` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:514` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:515` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:516` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:517` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:518` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:519` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:522` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:523` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:524` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:525` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:501` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:502` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:503` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:504` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:505` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:507` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:508` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:509` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:510` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:511` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:512` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:513` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:514` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:515` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:516` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:517` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:518` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:519` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:520` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:521` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:522` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:525` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:526` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:527` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:528` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 22b5c77ed8..a0d1888f83 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -5,29 +5,29 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:343` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:344` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:345` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:346` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:347` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:348` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](../repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:366`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage}`; `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. Rows also carry `rebase_merge: {state: approved|rebasing|ci-pending|merged|failed, actor, base, text, fail_reason, conflict_files[], armed_at}` while a *Rebase, then merge as maintainer* follow-through is live or just settled (persisted in `/data/rebase-merge-followups.json`; `fail_reason: foreign-head` when the head moved after the approval by someone other than the hive, so nothing merged), and `rebase: {attempt, max, exhausted, head, text}` on a conflicting PR the fixer is rebasing under the two-attempt loop guard (`/data/rebase-attempts.json`). | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:368`, `pkg/dashboard/api_repo_item_decision.go` | -| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:372`, `pkg/dashboard/api_maintainer_notes.go` | -| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:373`, `pkg/dashboard/api_maintainer_notes.go` | -| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:374`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/<hive-id>` label (shortened with a hash suffix when the plain spelling would exceed GitHub's 50-character label cap, [#17037](https://github.com/hivecommons/hive/issues/17037)); `{"held": false}` removes exactly the labels causing the hold (`hive-pause/<hive-id>` and/or generic hold labels) and never removes `hive/<hive-id>` provenance. A GitHub failure answers `424` with the step and GitHub's reason — never `502`/`503`, which the hosted Ingress replaces with its generic "hive unavailable" page. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-<hive-id>.json` with audit-backed, provenance-only, and ambiguous legacy `hive/<hive-id>` classifications. | `pkg/dashboard/api.go:370`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"close-read"|"close-issue"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": "<login>"|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision <text>` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants first post a required comment naming the operator (and the superseding PR when `superseded_by` is given); a failed comment prevents closure. `close-superseded` and `close-wontfix` close as **not planned**; `close-read` acknowledges an advisory issue and closes it **as completed**. `close-issue` (`type: "issue"` only) closes the single named open issue **as completed**, at any stage, leaving linked PRs and other issues untouched; it returns 400 for a PR (including one sent as an issue), 404 when the issue is not found, and 409 when already closed. It retains the existing permission and mutation gates, including the paused-repo `status: "refused"` result without writes. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": "<clanker login>"|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `<!-- hive:rerouted … -->` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:371`, `pkg/dashboard/api_repo_item_decision.go` | +| `GET` | `/api/notes` | Signed-in GitHub user (or verified owner) | Maintainer notes — the private per-item notes maintainers leave each other on Activity Monitor rows ([#maintainer-notes](../operator-reference.md#maintainer-notes)). `?repo=owner/name&number=N` lists one item's live notes oldest first; with no query, every live note in the hive (the SPA's count badges). `{notes: [{id, repo, number, author, text, created_at}]}`. Never read from or written to GitHub. | `pkg/dashboard/api.go:375`, `pkg/dashboard/api_maintainer_notes.go` | +| `POST` | `/api/notes` | Owner or repo write/maintain/admin, signed-in GitHub user | Append a maintainer note. Body `{repo, number, text}` — `text` is plain text up to 2000 characters; the author is the dashboard session's GitHub login, never a body field. `401` signed out, `403` without write on the repo, `400` for an empty/oversized note or a bad item. `201 {ok, note}`; appended to `/data/maintainer-notes.jsonl`, audited as `maintainer_note_add`, and broadcast as the `maintainer-notes` SSE event `{seq, repo, number, count, action}` so other maintainers' browsers refresh the row. | `pkg/dashboard/api.go:376`, `pkg/dashboard/api_maintainer_notes.go` | +| `DELETE` | `/api/notes/{id}` | The note's author, or a verified owner | Tombstone a maintainer note (append-only store: a `deleted_at` record, never a rewrite). `404` unknown or already deleted, `403` for anyone else. `{ok, note}`; audited as `maintainer_note_delete`, broadcast as `maintainer-notes`. | `pkg/dashboard/api.go:377`, `pkg/dashboard/api_maintainer_notes.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `<!-- hive:delegated-decision … -->` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:378`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:379` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:381` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:594`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:369`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:385` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:387`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:381`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:382` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:384` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:597`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — `<!-- hive:sentinel-confirmed -->`, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:372`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:386`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:387` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:388` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:390`, `pkg/dashboard/api_hive_advice.go` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 5bddc23ff2..3e82c88beb 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1944` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1959` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | diff --git a/src/pkg/dashboard/admin_mcp_settings_test.go b/src/pkg/dashboard/admin_mcp_settings_test.go index 9b56c4093f..02fc9387a7 100644 --- a/src/pkg/dashboard/admin_mcp_settings_test.go +++ b/src/pkg/dashboard/admin_mcp_settings_test.go @@ -26,6 +26,7 @@ var settingsReadExcludedRoutes = map[string]string{ "/api/config/agent/{name}/export": "a download bundle of the agent and agent_prompt sections", "/api/config/dashboard/theme": "purely presentational (the exclusion of presentational endpoints)", "/api/config/sidebar": "purely presentational (the exclusion of presentational endpoints)", + "/api/config/github/webhooks": "an owner setup card that probes the GitHub App's hook permissions live on every read; webhook verification and health are in /api/status.webhooks", } // settingsPageOtherRoutes are the GET routes outside /api/config that Settings diff --git a/src/pkg/dashboard/agent_navbar_summary_ui_test.go b/src/pkg/dashboard/agent_navbar_summary_ui_test.go index 3c3c2df114..2de20abeaf 100644 --- a/src/pkg/dashboard/agent_navbar_summary_ui_test.go +++ b/src/pkg/dashboard/agent_navbar_summary_ui_test.go @@ -50,6 +50,9 @@ function renderAgentNavbarTileDiff(w, list) { list.forEach((item, i) => { const tile = new El('button'); tile.className = 'agent-navbar-tile'; tile.setAttribute('data-agent-key', item.id); w.insertBefore(tile, w.children[i] || null); }); } function ensureAgentsCollapsedCountdownTicker() {} +// v6 measures the strip after every render; layout is out of scope here. +function fitAgentNavbarTiles() { return 0; } +function ensureAgentNavbarFitObserver() {} ` + block + ` function agents(n, extraUnscheduled) { const out = []; diff --git a/src/pkg/dashboard/standby_tile_visible.test.js b/src/pkg/dashboard/standby_tile_visible.test.js index 28a911ead2..ec28439af4 100644 --- a/src/pkg/dashboard/standby_tile_visible.test.js +++ b/src/pkg/dashboard/standby_tile_visible.test.js @@ -30,6 +30,7 @@ const stubs = { govPRModelsWindow: '7d', loadReviewerAccuracy: () => {}, fleetRunStatHtml: () => '', + webhookGovStatHtml: () => '', formatGovernorNextRun: () => '', governorModeGradient: () => '', renderBudgetBar: () => '', diff --git a/src/pkg/dashboard/static-src/css/sections-collapse.css b/src/pkg/dashboard/static-src/css/sections-collapse.css index 0f85b4a38b..345c579391 100644 --- a/src/pkg/dashboard/static-src/css/sections-collapse.css +++ b/src/pkg/dashboard/static-src/css/sections-collapse.css @@ -139,7 +139,7 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-upnext[data-agent-more]::after { display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } /* The "+N" count of tiles that did not fit. While the fit is measured it sits first in the row, so its own width is reserved. */ @container navbar-center (width > 340px) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 9688ef14b0..bae091d534 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -6386,7 +6386,7 @@ .agent-navbar-tile .agent-up-next.running-now { color: var(--green); } .agent-navbar-tile.active .agent-tile-dot { animation: green-pulse 2.4s ease-in-out infinite; } .agent-navbar-tile.stopped .agent-tile-dot, .agent-navbar-tile.needs-login .agent-tile-dot { background: var(--red); box-shadow: 0 0 0 1px color-mix(in srgb, var(--red) 40%, transparent); } - .agent-navbar-upnext::after { content: attr(data-agent-summary); display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } + .agent-navbar-upnext[data-agent-more]::after { display: none; align-items: center; justify-content: center; min-height: calc(var(--nav-chip-h) + var(--sp-4)); padding: var(--sp-2) var(--nav-chip-px); border: 1px solid var(--line-subtle); border-radius: var(--r); background: color-mix(in srgb, var(--surface-2) 64%, transparent); color: var(--text); font-size: var(--fs-xs); font-weight: 800; white-space: nowrap; } /* The "+N" count of tiles that did not fit. While the fit is measured it sits first in the row, so its own width is reserved. */ @container navbar-center (width > 340px) { diff --git a/src/pkg/dashboard/your_turn_ui_11566_test.go b/src/pkg/dashboard/your_turn_ui_11566_test.go index 6ffc366b98..a25f29c75b 100644 --- a/src/pkg/dashboard/your_turn_ui_11566_test.go +++ b/src/pkg/dashboard/your_turn_ui_11566_test.go @@ -273,7 +273,7 @@ const location = { search: '', hash: '' }; function getComputedStyle() { return { getPropertyValue() { return ''; } }; } const historyData = []; function getHistoryTimes() { return []; } function sparkSvg() { return ''; } let _yourTurnMenu = null; -let _repoResizeDrag = null, _repoOrderDrag = null, _overviewProjectFilterKey = '', _overviewProjectFilterCount = null, _overviewLastRepos = []; +let _repoResizeDrag = null, _repoOrderDrag = null, _overviewProjectFilterKey = '', _overviewProjectFilterCount = null, _overviewLastRepos = [], _repoHoldLayoutSig = null, _repoHoldShiftAt = 0; function marker(repo, kind, number, variant, tip) { const el = mkEl('marker-' + variant + '-' + repo + number, 'button'); el.dataset = { ytRepo: repo, ytKind: kind, ytNumber: String(number), ytVariant: variant, ytTip: tip }; diff --git a/src/pkg/github/close_on_merge_paths_test.go b/src/pkg/github/close_on_merge_paths_test.go index 40f6595ed8..db7bddda1f 100644 --- a/src/pkg/github/close_on_merge_paths_test.go +++ b/src/pkg/github/close_on_merge_paths_test.go @@ -125,6 +125,7 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { _ = json.NewEncoder(w).Encode([]map[string]any{ {"number": 7, "state": "open", "body": "body", "labels": []map[string]string{}}, {"number": 10, "state": "open", "body": "body", "labels": []map[string]string{}}, + {"number": 14, "state": "open", "body": "body", "labels": []map[string]string{}}, // An open PR is listed by the issues API but never matches a claim. {"number": 12, "state": "open", "pull_request": map[string]string{"url": "x"}}, }) @@ -163,7 +164,7 @@ func TestCloseOnMergeForLedgerFiltersAndCloses(t *testing.T) { // Merged claim whose PR cannot be read: logged and skipped. {Repo: "o/r", Issue: 10, PRNumber: 500, PRRepo: "o/r", MergedPR: true}, // Merged claim whose PR is 404: warned once, then settled. - {Repo: "o/r", Issue: 12, PRNumber: 404, PRRepo: "o/r", MergedPR: true}, + {Repo: "o/r", Issue: 14, PRNumber: 404, PRRepo: "o/r", MergedPR: true}, // Merged claim missing a PR number: skipped. {Repo: "o/r", Issue: 11, PRNumber: 0, PRRepo: "o/r", MergedPR: true}, } { diff --git a/src/pkg/hub/heartbeat_selfservice_fence.go b/src/pkg/hub/heartbeat_selfservice_fence.go index 191ce9d28c..3ce7a31122 100644 --- a/src/pkg/hub/heartbeat_selfservice_fence.go +++ b/src/pkg/hub/heartbeat_selfservice_fence.go @@ -73,12 +73,13 @@ var heartbeatResponseFieldFence = map[string]heartbeatFieldClass{ "GitHubAppConfig": heartbeatFieldFenced, // App, cluster and secondary keys, webhook relay key "PendingGateway": heartbeatFieldFenced, // funded gateway API key // Upgrade and lifecycle management. - "UpgradeTo": heartbeatFieldFenced, - "UpgradeInterrupt": heartbeatFieldFenced, - "SwitchToTag": heartbeatFieldFenced, - "RestartSpoke": heartbeatFieldFenced, - "UpgradeAnnounce": heartbeatFieldFenced, - "UpgradePolicy": heartbeatFieldFenced, // the owner, not the hub, sets upgrade posture + "UpgradeTo": heartbeatFieldFenced, + "UpgradeInterrupt": heartbeatFieldFenced, + "UpgradeChannelTag": heartbeatFieldFenced, // rides with UpgradeTo + "SwitchToTag": heartbeatFieldFenced, + "RestartSpoke": heartbeatFieldFenced, + "UpgradeAnnounce": heartbeatFieldFenced, + "UpgradePolicy": heartbeatFieldFenced, // the owner, not the hub, sets upgrade posture // Operational actions. "ResetAgentRestarts": heartbeatFieldFenced, "Breaker": heartbeatFieldFenced,