diff --git a/CHANGELOG.md b/CHANGELOG.md index 808183540..73c2f3973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.33] - 2026-08-14 + +### Added + +- mcp: report re-warm progress against a real record denominator +- daemon: report promotion_ms so a cold query stops looking instant + +### Changed + +- watchdog: cheap liveness probe and an interval that breathes + +### Fixed + +- mcp: cold-index queries return a warming contract instead of hanging +- mcp: expose per-drive tier so the warm-retry contract is followable +- mcp: report residency numbers beside the tier map in uffs_status +- dev: stop use-local killing AI-host MCP sessions to unlock binaries +- cli: name the real tier for a demoted drive, not always 'parked' +- dev: drop let-chains from install-bins (rust-script default edition) +- cli: make the MCP stdio session scan work on Windows +- cli: separate gateway, supervisor and worker in the stdio session list +- cli: show resident-vs-expected records while drives are demoted +- mcp: one readiness field, and name the drive that is loading +- version: re-stamp the -dirty flag when the tree changes, not just HEAD +- ci: clamp RLIMIT_NOFILE before zig links, unblocking toolchain-sync +- ci: clamp the fd limit when ulimit reports "unlimited" too +- ci: raise the fd limit for zig links — macOS defaults to 256 + ## [0.6.32] - 2026-08-13 ### Added @@ -2761,7 +2789,8 @@ thin clients over a unified `uffsd` process. ### Fixed - Various MFT parsing edge cases -[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.32...HEAD +[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.33...HEAD +[0.6.33]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.32...v0.6.33 [0.6.32]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.31...v0.6.32 [0.6.31]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.30...v0.6.31 [0.6.30]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.29...v0.6.30 diff --git a/Cargo.lock b/Cargo.lock index eff8ab87e..03654bebe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4334,7 +4334,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "uffs-bench" -version = "0.6.32" +version = "0.6.33" dependencies = [ "chrono", "clap", @@ -4351,7 +4351,7 @@ dependencies = [ [[package]] name = "uffs-broker" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "serde", @@ -4369,14 +4369,14 @@ dependencies = [ [[package]] name = "uffs-broker-protocol" -version = "0.6.32" +version = "0.6.33" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "uffs-ci-pipeline" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "chrono", @@ -4395,7 +4395,7 @@ dependencies = [ [[package]] name = "uffs-cli" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "assert_cmd", @@ -4418,7 +4418,7 @@ dependencies = [ [[package]] name = "uffs-client" -version = "0.6.32" +version = "0.6.33" dependencies = [ "dirs-next", "libc", @@ -4438,7 +4438,7 @@ dependencies = [ [[package]] name = "uffs-core" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "bytemuck", @@ -4469,7 +4469,7 @@ dependencies = [ [[package]] name = "uffs-daemon" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "clap", @@ -4502,7 +4502,7 @@ dependencies = [ [[package]] name = "uffs-diag" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "chrono", @@ -4517,7 +4517,7 @@ dependencies = [ [[package]] name = "uffs-fetch" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "hex", @@ -4528,7 +4528,7 @@ dependencies = [ [[package]] name = "uffs-format" -version = "0.6.32" +version = "0.6.33" dependencies = [ "chrono", "itoa", @@ -4539,7 +4539,7 @@ dependencies = [ [[package]] name = "uffs-gen-hooks" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "clap", @@ -4551,7 +4551,7 @@ dependencies = [ [[package]] name = "uffs-gen-workflow" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "clap", @@ -4564,7 +4564,7 @@ dependencies = [ [[package]] name = "uffs-manifest-audit" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "clap", @@ -4576,7 +4576,7 @@ dependencies = [ [[package]] name = "uffs-mcp" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "axum", @@ -4600,7 +4600,7 @@ dependencies = [ [[package]] name = "uffs-mft" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "bitflags", @@ -4641,14 +4641,14 @@ dependencies = [ [[package]] name = "uffs-polars" -version = "0.6.32" +version = "0.6.33" dependencies = [ "polars", ] [[package]] name = "uffs-security" -version = "0.6.32" +version = "0.6.33" dependencies = [ "aes-gcm", "dirs-next", @@ -4663,22 +4663,22 @@ dependencies = [ [[package]] name = "uffs-statusfmt" -version = "0.6.32" +version = "0.6.33" [[package]] name = "uffs-text" -version = "0.6.32" +version = "0.6.33" dependencies = [ "bytemuck", ] [[package]] name = "uffs-time" -version = "0.6.32" +version = "0.6.33" [[package]] name = "uffs-update" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "dirs-next", @@ -4695,11 +4695,11 @@ dependencies = [ [[package]] name = "uffs-version" -version = "0.6.32" +version = "0.6.33" [[package]] name = "uffs-vss-requestor" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "cc", @@ -4711,7 +4711,7 @@ dependencies = [ [[package]] name = "uffs-watchdog" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "dirs-next", @@ -4720,7 +4720,7 @@ dependencies = [ [[package]] name = "uffs-winsvc" -version = "0.6.32" +version = "0.6.33" dependencies = [ "anyhow", "windows", diff --git a/Cargo.toml b/Cargo.toml index d11df3860..c68b062b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ members = [ # Workspace Package Metadata (inherited by all crates) # ───────────────────────────────────────────────────────────────────────────── [workspace.package] -version = "0.6.32" +version = "0.6.33" edition = "2024" # No `rust-version` claim: the workspace is structurally nightly-only. # `crates/uffs-polars` enables `polars/nightly` unconditionally, which @@ -135,36 +135,36 @@ publish = false # proposed-plan output for 12 days because `release-plz update` # failed at `cargo package` with this very error. See # `release-automation-baseline.md` §10 for the diagnostic trail. -uffs-polars = { path = "crates/uffs-polars", version = "0.6.32" } -uffs-security = { path = "crates/uffs-security", version = "0.6.32" } -uffs-text = { path = "crates/uffs-text", version = "0.6.32" } -uffs-time = { path = "crates/uffs-time", version = "0.6.32" } -uffs-version = { path = "crates/uffs-version", version = "0.6.32" } -uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.32" } -uffs-mft = { path = "crates/uffs-mft", version = "0.6.32" } -uffs-format = { path = "crates/uffs-format", version = "0.6.32" } -uffs-core = { path = "crates/uffs-core", version = "0.6.32" } -uffs-client = { path = "crates/uffs-client", version = "0.6.32" } +uffs-polars = { path = "crates/uffs-polars", version = "0.6.33" } +uffs-security = { path = "crates/uffs-security", version = "0.6.33" } +uffs-text = { path = "crates/uffs-text", version = "0.6.33" } +uffs-time = { path = "crates/uffs-time", version = "0.6.33" } +uffs-version = { path = "crates/uffs-version", version = "0.6.33" } +uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.33" } +uffs-mft = { path = "crates/uffs-mft", version = "0.6.33" } +uffs-format = { path = "crates/uffs-format", version = "0.6.33" } +uffs-core = { path = "crates/uffs-core", version = "0.6.33" } +uffs-client = { path = "crates/uffs-client", version = "0.6.33" } # `uffs-broker-protocol` carries the wire-protocol types shared between # `uffs-broker` (the elevated handle vendor, Windows-only binary) and # `uffs-daemon::broker_client` (the handle consumer). Pure-logic # Layer-0 lib — cross-platform tests run on every CI lane. Added in # F5 (issue #205) so neither side duplicates `BROKER_PIPE_NAME` / # wire-format byte literals. -uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.32" } +uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.33" } # `uffs-winsvc` — native Windows service control (SCM query/start/stop) + # the non-connecting broker-pipe readiness probe. Layer-0 leaf: its only # dependency is the `windows` crate (windows-target), with non-Windows # stubs so cross-platform consumers (uffs-update, uffs-cli) compile. # Single source of truth for the `sc`/SCM mechanics previously duplicated # across uffs-broker, uffs-update, and uffs-cli. -uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.32" } +uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.33" } # `uffs-fetch` — hardened release-asset transport (blocking reqwest + # rustls with retry/timeout/byte-cap, plus `SHA256SUMS` verification), # extracted from `uffs-update` as a small public lib so external products # can reuse it. Cross-platform pure-logic leaf; keeps the HTTP/TLS stack # out of the lean `uffs` CLI exactly as before. -uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.32" } +uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.33" } # NOTE: no `uffs-broker` workspace dependency alias on purpose — # `uffs-broker` is a binary-only crate (the only `[lib]` it carries is # this protocol module's now-extracted sibling); no other workspace diff --git a/crates/uffs-cli/Cargo.toml b/crates/uffs-cli/Cargo.toml index 1bbb3cab2..2889d8628 100644 --- a/crates/uffs-cli/Cargo.toml +++ b/crates/uffs-cli/Cargo.toml @@ -69,7 +69,7 @@ path = "src/main.rs" # `version = "0.5.90"` is required for `cargo package` validation — # see root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-client = { path = "../uffs-client", version = "0.6.32", default-features = false } +uffs-client = { path = "../uffs-client", version = "0.6.33", default-features = false } # Canonical CSV / parity / legacy-footer writer. Direct dep (not a # re-export chain through `uffs-client`) so the CLI and the daemon @@ -77,7 +77,7 @@ uffs-client = { path = "../uffs-client", version = "0.6.32", default-features = # `version = "0.5.90"` is required for `cargo package` validation — # see root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-format = { path = "../uffs-format", version = "0.6.32" } +uffs-format = { path = "../uffs-format", version = "0.6.33" } # Typed drive-letter newtype. Direct dep so the CLI command signatures # (`daemon_load`, `daemon_tiering`, etc.) name `DriveLetter` natively diff --git a/crates/uffs-cli/src/args_help.rs b/crates/uffs-cli/src/args_help.rs index f2947f0b5..45e91487e 100644 --- a/crates/uffs-cli/src/args_help.rs +++ b/crates/uffs-cli/src/args_help.rs @@ -263,6 +263,10 @@ OPTIONS: -v, --verbose Expand every section (build, broker mode, live-update, memory, paths; broker binary + uptime on Windows) --json Machine-readable superset of all sections + --brief With --json: liveness booleans only. Skips the broker + pipe probe and the MCP process scan, so it costs a + socket connect instead of several subprocesses — for + supervisors and scripts that poll on a timer. "; /// Print status help. diff --git a/crates/uffs-cli/src/client_profile.rs b/crates/uffs-cli/src/client_profile.rs new file mode 100644 index 000000000..4595cdd15 --- /dev/null +++ b/crates/uffs-cli/src/client_profile.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Client-side `--profile` / `--benchmark` timing block. +//! +//! Split from `main.rs` to keep that file under the workspace 800-LOC +//! ceiling. Purely presentational: it owns the `ClientProfile` bundle +//! and renders it to stderr, with no I/O or daemon knowledge of its +//! own. + +/// Packaging these into a struct keeps `run_search` under the +/// `clippy::too-many-lines` cap and lets the profile helper take one +/// argument instead of six. +pub(crate) struct ClientProfile<'a> { + /// Wall-clock time spent in `UffsClientSync::connect_with_args`. + pub(crate) connect_ms: u128, + /// Wall-clock time spent in `await_ready` (daemon warm-up). + pub(crate) ready_ms: u128, + /// Wall-clock time spent in the `search_cli` IPC round-trip. + pub(crate) ipc_ms: u128, + /// Daemon-reported SCAN duration (from the response envelope) — + /// excludes index warm-up, reported separately below. + pub(crate) duration_ms: u64, + /// Daemon-reported milliseconds spent paging parked/cold drives + /// back in before the scan. `0` on a warm index; tens of seconds + /// on a cold one, where it is the entire wall-clock story. + pub(crate) promotion_ms: u64, + /// Payload delivery channel the daemon picked for this response. + /// Used by [`print_client_profile`] to show the transport name + /// and to pick the cheapest authoritative row-count source. + pub(crate) payload: &'a uffs_client::protocol::response::SearchPayload, + /// Total row count reported by the daemon, independent of which + /// transport carries the payload. Used to display the "Total + /// matches:" line when the transport is a shmem blob — counting + /// newlines in the mmap would consume the file before the stdout + /// write and double the syscall cost. + pub(crate) total_count: u64, + /// Daemon-side `profile` object from the response envelope. When + /// populated, its `scan_ms` / `sort_ms` / `path_resolve_ms` / + /// `write_ms` fields are rendered as a sub-phase breakdown inside + /// the daemon block so the `--profile` output pinpoints where the + /// per-query cost sits (scan vs sort vs path resolution vs disk + /// write). + pub(crate) daemon_profile: Option<&'a uffs_client::protocol::response::SearchProfile>, +} + +/// Print the `--profile` / `--benchmark` client-side timing block to +/// stderr (matches the daemon-side profile formatting). +#[expect( + clippy::print_stderr, + reason = "intentional --profile output to stderr" +)] +pub(crate) fn print_client_profile(prof: &ClientProfile<'_>) { + use uffs_client::protocol::response::SearchPayload; + + eprintln!("=== PROFILE: Client → Daemon ==="); + eprintln!(" Connect: {:>6} ms", prof.connect_ms); + eprintln!(" Await ready: {:>6} ms", prof.ready_ms); + eprintln!( + " Search (IPC): {:>6} ms (daemon: {} ms)", + prof.ipc_ms, prof.duration_ms + ); + // Printed only when it happened: a warm index promotes nothing, and + // a zero line every run would train the eye to skip it. + if prof.promotion_ms > 0 { + eprintln!( + " Index warm-up: {:>6} ms (paged parked/cold drives back in)", + prof.promotion_ms + ); + } + // Sub-phase breakdown from the daemon profile. Any non-zero + // component is printed; all-zero (regex/trigram paths, legacy + // daemons) collapses to a single-line total. + if let Some(dp) = prof.daemon_profile { + let scan = dp.scan_ms; + let sort = dp.sort_ms; + let resolve = dp.path_resolve_ms; + let write = dp.write_ms; + if (scan | sort | resolve | write) > 0 { + eprintln!( + " scan={scan} ms sort={sort} ms path_resolve={resolve} ms write={write} ms" + ); + } + // Deep-profile breakdown: only present when the numeric-sort + // branch populated the `path_*` sub-counters. Prints per- + // record averages derived from ns totals so the user can see + // immediately whether the bottleneck is path-walking or + // row-building, and whether the DirCache hit rate is high + // enough to warrant a locality optimisation. + let candidates = dp.path_candidates; + let cache_entries = dp.path_cache_entries; + let resolve_ns = dp.path_resolve_fn_ns; + let build_ns = dp.path_build_row_ns; + if candidates > 0 { + let hits = candidates.saturating_sub(cache_entries); + // Integer-math hit rate in permille (0–1000) to avoid + // float arithmetic — clippy::float_arithmetic is banned + // in production lints. `permille / 10 . permille % 10` + // prints as "99.7" for 997. + let hit_permille = hits.saturating_mul(1000) / candidates; + let hit_whole = hit_permille / 10; + let hit_frac = hit_permille % 10; + let avg_resolve_ns = resolve_ns / candidates; + let avg_build_ns = build_ns / candidates; + eprintln!( + " deep: candidates={candidates} unique_parents={cache_entries} \ + hit_rate={hit_whole}.{hit_frac}%" + ); + eprintln!( + " resolve_fn={} ms ({} ns/rec) build_row={} ms ({} ns/rec)", + resolve_ns / 1_000_000, + avg_resolve_ns, + build_ns / 1_000_000, + avg_build_ns, + ); + } + } + // Row count resolution — pick the cheapest authoritative source + // depending on which payload variant the daemon used: + // 1. `ShmemBlob` → mmap'd file; counting newlines would read every page just to + // discard the count, so use the daemon's pre- computed `total_count` + // instead. + // 2. `InlineBlob` → inline string already in memory; scanning for `\n` is ~5 + // GB/s, cheap. + // 3. Rows variants (`InlineRows`, `ShmemRows`) → `row_count_hint()` is O(1) — + // `Vec::len` or the daemon's pre-computed count. + // 4. `Empty` → zero rows, nothing to count. + let row_count = match prof.payload { + SearchPayload::ShmemBlob(_) => { + // `try_from` instead of `as` to preserve correctness on + // hypothetical 32-bit targets where `u64` would truncate + // (clippy::cast_possible_truncation). `u64::MAX` is a + // strictly larger fallback than any realistic row count. + usize::try_from(prof.total_count).unwrap_or(usize::MAX) + } + SearchPayload::InlineBlob(blob) => blob.bytes().filter(|byte| *byte == b'\n').count(), + SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { + prof.payload.row_count_hint().unwrap_or(0) + } + }; + // Label the count by what it actually measures per transport: blob + // variants carry rendered text (newline count includes header/footer + // lines) or the daemon's pre-limit total, NOT the post-`--limit` page + // (2026-06-12 dry run: `--limit 5` printed "Rows returned: 7"). + match prof.payload { + SearchPayload::ShmemBlob(_) => { + eprintln!(" Total matches: {row_count:>6}"); + } + SearchPayload::InlineBlob(_) => { + eprintln!(" Output lines: {row_count:>6}"); + } + SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { + eprintln!(" Rows returned: {row_count:>6}"); + } + } + match prof.payload { + SearchPayload::ShmemBlob(_) => { + eprintln!(" Transport: shmem_blob (mmap + write_all, binary)"); + } + SearchPayload::InlineBlob(_) => { + eprintln!(" Transport: inline_blob (single write_all)"); + } + SearchPayload::ShmemRows { .. } => { + eprintln!(" Transport: shmem_rows (mmap + per-row format)"); + } + SearchPayload::InlineRows(_) | SearchPayload::Empty => { + // inline_rows is the default — no extra line needed. + // empty responses skip the transport line entirely. + } + } +} diff --git a/crates/uffs-cli/src/commands/daemon_status.rs b/crates/uffs-cli/src/commands/daemon_status.rs index 28bed91bf..9e31f6e7f 100644 --- a/crates/uffs-cli/src/commands/daemon_status.rs +++ b/crates/uffs-cli/src/commands/daemon_status.rs @@ -175,11 +175,52 @@ fn print_drive_headline(palette: Palette, drives: &[DriveInfo], width: usize) { return; } let records: usize = drives.iter().map(|dr| dr.records).sum(); - let value = format!( - "{} loaded \u{b7} {} records", - drives.len(), - uffs_client::format::format_number_commas(records as u64) - ); + // Resident vs expected. "7 loaded · 0 records" is true and unreadable: + // it looks like an empty index rather than a demoted one, and gives no + // way to watch a re-warm progress. When some drives are demoted, say + // how far along the load is against the count they held when warm. + let expected: u64 = drives + .iter() + .map(|dr| dr.records_when_warm.unwrap_or(dr.records as u64)) + .sum(); + let demoted = drives.iter().any(|dr| { + matches!( + dr.tier, + Some(ShardTier::Parked | ShardTier::Cold | ShardTier::Evicting) + ) + }); + let value = if demoted && expected > 0 { + let pct = (records as u64) + .saturating_mul(100) + .checked_div(expected) + .unwrap_or(0); + // Name the drive in flight: a re-warm lands one drive at a time, + // so the count sits still for tens of seconds while a large one + // loads. Without this, repeated looks show identical numbers and + // a working warm is indistinguishable from a stalled one. + let loading: Vec = drives + .iter() + .filter(|dr| dr.loading == Some(true)) + .map(|dr| dr.letter.to_string()) + .collect(); + let in_flight = if loading.is_empty() { + String::new() + } else { + format!(" \u{b7} loading {}", loading.join(", ")) + }; + format!( + "{} loaded \u{b7} {} of {} records resident ({pct}% warmed){in_flight}", + drives.len(), + uffs_client::format::format_number_commas(records as u64), + uffs_client::format::format_number_commas(expected), + ) + } else { + format!( + "{} loaded \u{b7} {} records", + drives.len(), + uffs_client::format::format_number_commas(records as u64) + ) + }; println!("{}", field(palette, "Drives", &value, width)); } @@ -481,12 +522,40 @@ fn print_physical_drive_line(palette: Palette, drive: &PhysicalDrive, loaded: &[ .map_or_else( || (Glyph::Off, format!(" \u{b7} {}", palette.dim("not loaded"))), |info| { - ( - Glyph::Up, + // A parked/cold shard reports 0 records because its body + // is released, not because the drive is empty. Rendering + // that as "indexed (0 records)" reads as a broken index — + // say "parked" and the count returns on the next query. + let parked = matches!( + info.tier, + Some(ShardTier::Parked | ShardTier::Cold | ShardTier::Evicting) + ); + let note = if parked { + // Name the actual tier: a hibernated drive is `cold`, + // not `parked`, and calling both "parked" hides which + // rung of the ladder a drive is actually on. + let label = match info.tier { + Some(ShardTier::Cold) => "cold", + Some(ShardTier::Evicting) => "evicting", + _ => "parked", + }; + format!( + " \u{b7} {}", + palette.dim(&format!("{label:<8} (re-warms on next query)")) + ) + } else { format!( " \u{b7} indexed ({:>11} records)", format_number_commas(info.records as u64) - ), + ) + }; + ( + if parked { + tier_glyph(info.tier) + } else { + Glyph::Up + }, + note, ) }, ); diff --git a/crates/uffs-cli/src/commands/system_status/mod.rs b/crates/uffs-cli/src/commands/system_status/mod.rs index 540af8f50..ba6f9962b 100644 --- a/crates/uffs-cli/src/commands/system_status/mod.rs +++ b/crates/uffs-cli/src/commands/system_status/mod.rs @@ -28,9 +28,22 @@ const BROKER_PIPE_PROBE_MS: u32 = 1_000; /// One mebibyte, for the `bytes → MB` display conversions. const MIB: u64 = 1024 * 1024; -/// `uffs --status [-v] [--json]` — show combined system status. +/// `uffs --status [-v] [--json] [--brief]` — show combined system status. +/// +/// `brief` exists for the watchdog, which polls this on a timer and +/// needs only "is each service alive". The full view costs an SCM +/// query, a named-pipe probe with a timeout, and a process enumeration +/// that spawns a shell (plus one more per session found) — roughly four +/// PowerShell launches per call on Windows. At a 5 s poll that is +/// ~69 000 shell spawns a day to answer two booleans, and a health +/// check that expensive can fail *because* the machine is loaded, which +/// is exactly when a supervisor must not. #[expect(clippy::print_stdout, reason = "CLI user-facing output")] -pub(crate) fn system_status(verbose: bool, json: bool) { +pub(crate) fn system_status(verbose: bool, json: bool, brief: bool) { + if brief { + render_brief_json(); + return; + } let daemon = gather_daemon(); if json { render_json(daemon.as_ref()); @@ -92,6 +105,27 @@ fn render_json(daemon: Option<&DaemonSnapshot>) { } } +/// Liveness-only JSON: the two booleans a supervisor polls for, and +/// nothing that costs a subprocess. +/// +/// Deliberately does NOT call `broker_json` (SCM query + pipe probe) or +/// `mcp_stdio_json` (process enumeration). The daemon check is a +/// socket connect; the gateway check is a PID-file read plus a liveness +/// test — both sub-millisecond. +#[expect(clippy::print_stdout, reason = "CLI --json output")] +fn render_brief_json() { + let daemon_running = UffsClientSync::connect_raw().is_ok(); + let mcp_running = uffs_client::mcp_pid::is_mcp_server_running().is_some(); + let doc = serde_json::json!({ + "daemon": { "running": daemon_running }, + "mcp_http": { "running": mcp_running }, + }); + match serde_json::to_string(&doc) { + Ok(text) => println!("{text}"), + Err(err) => println!("{{\"error\":\"{err}\"}}"), + } +} + /// JSON for the daemon section (status + drives + stats, or `running:false`). fn daemon_json(daemon: Option<&DaemonSnapshot>) -> serde_json::Value { daemon.map_or_else( diff --git a/crates/uffs-cli/src/commands/system_status/process_scan.rs b/crates/uffs-cli/src/commands/system_status/process_scan.rs index dd82b4c4e..c855744d0 100644 --- a/crates/uffs-cli/src/commands/system_status/process_scan.rs +++ b/crates/uffs-cli/src/commands/system_status/process_scan.rs @@ -44,8 +44,11 @@ pub(super) fn print_mcp_stdio_section(palette: Palette) { if any_stale { println!( " {}", - palette - .dim("Older binary; the AI host that launched them refreshes on its next start.") + palette.dim( + "Serving a superseded binary (renamed aside by an installer, or older \ + than the one on disk). Still functional; the AI host that launched \ + them picks up the new one on its next start." + ) ); } } @@ -129,6 +132,105 @@ fn capture_with_timeout( /// The `ps` calls run under [`PS_TIMEOUT`] (see [`capture_with_timeout`]), so a /// process with a huge argument vector can never hang `--status`. Returns /// `None` when the scan could not complete (spawn failed or timed out). +#[cfg(windows)] +fn find_mcp_stdio_processes() -> Option> { + // Windows has no `ps`, so the Unix path below returned nothing and the + // section rendered "(none)" — indistinguishable from "no sessions" on + // the one platform UFFS actually ships to. It reported "(none)" on a + // box that was demonstrably serving a session. + // + // `Get-CimInstance Win32_Process` is the supported enumeration; CSV + // keeps parsing trivial. Matching on the image name rather than the + // command line matters twice over: AI hosts spawn the supervisor as a + // bare `uffsmcp` with no arguments (so a `--mcp run` cmdline match + // finds nothing), and after an installer rename-swap a surviving + // supervisor runs as `uffsmcp.old0` — still serving, and still worth + // showing. + let script = "Get-CimInstance Win32_Process -Filter \"Name LIKE 'uffsmcp%'\" | \ + ForEach-Object { \ + $s = $_.CreationDate; \ + $age = if ($s) { [int]((Get-Date) - $s).TotalSeconds } else { 0 }; \ + \"$($_.ProcessId),$($_.ParentProcessId),$age,$($_.Name)\" }"; + let stdout = capture_with_timeout( + "powershell", + &["-NoProfile", "-NonInteractive", "-Command", script], + PS_TIMEOUT, + )?; + let text = String::from_utf8_lossy(&stdout); + let my_pid = std::process::id(); + let current_mtime = std::env::current_exe() + .ok() + .and_then(|path| std::fs::metadata(path).ok()) + .and_then(|meta| meta.modified().ok()); + // The HTTP gateway is a `uffsmcp` process too, so an image-name + // filter catches it — but it has its own section above, and listing + // it here as a "stdio session" is simply wrong. + let gateway_pid = uffs_client::mcp_pid::is_mcp_server_running(); + // Collect PIDs first so a supervisor's worker child can be told + // apart from a supervisor: the worker's parent is another `uffsmcp`. + let rows: Vec<(u32, u32, u64)> = text + .lines() + .filter_map(|line| { + let mut fields = line.trim().split(','); + let pid = fields.next()?.trim().parse::().ok()?; + let parent = fields.next()?.trim().parse::().unwrap_or(0); + let age = fields.next()?.trim().parse::().unwrap_or(0); + Some((pid, parent, age)) + }) + .collect(); + let is_supervisor_pid = |pid: u32| rows.iter().any(|&(other, _, _)| other == pid); + + let mut sessions = Vec::new(); + for line in text.lines() { + let mut fields = line.trim().split(','); + let Some(pid) = fields.next().and_then(|val| val.trim().parse::().ok()) else { + continue; + }; + if pid == my_pid || gateway_pid == Some(pid) { + continue; + } + let parent_pid: u32 = fields + .next() + .and_then(|val| val.trim().parse().ok()) + .unwrap_or(0); + let age_secs: u64 = fields + .next() + .and_then(|val| val.trim().parse().ok()) + .unwrap_or(0); + let _image = fields.next().unwrap_or(""); + // A `uffsmcp` whose parent is another `uffsmcp` is a supervisor's + // worker child, not a session of its own. Worth showing — a + // worker younger than its supervisor is the visible proof that a + // hot-swap happened — but labelled, not counted as a session. + let is_worker = is_supervisor_pid(parent_pid); + let parent_name = if is_worker { + Some(format!("worker of PID {parent_pid}")) + } else { + resolve_parent_name(parent_pid) + }; + // Staleness by *age*, not by image name: Windows keeps reporting + // the original name after a rename-swap, so the name says nothing. + // A supervisor older than the installed binary is serving a + // superseded image — harmless once it has hot-swapped its worker, + // which is exactly why the worker is listed too. + let is_stale = !is_worker + && current_mtime.is_some_and(|bin_mtime| { + std::time::SystemTime::now() + .checked_sub(core::time::Duration::from_secs(age_secs)) + .is_some_and(|started| started < bin_mtime) + }); + sessions.push(StdioSession { + pid, + uptime: core::time::Duration::from_secs(age_secs), + parent_name, + is_stale, + }); + } + Some(sessions) +} + +/// Unix: enumerate via `ps`, matching the `uffs --mcp run` command line. +#[cfg(not(windows))] fn find_mcp_stdio_processes() -> Option> { let stdout = capture_with_timeout("ps", &["-eo", "pid,ppid,etime,args"], PS_TIMEOUT)?; @@ -152,6 +254,7 @@ fn find_mcp_stdio_processes() -> Option> { } /// Parse one `ps` line into a [`StdioSession`] if it is an MCP stdio process. +#[cfg(not(windows))] fn parse_stdio_line( line: &str, my_pid: u32, @@ -190,6 +293,7 @@ fn parse_stdio_line( } /// Parse `ps` elapsed time format: `[[dd-]hh:]mm:ss`. +#[cfg(not(windows))] fn parse_ps_etime(etime: &str) -> core::time::Duration { let mut total_secs: u64 = 0; let (days_part, time_part) = if let Some((days, rest)) = etime.split_once('-') { @@ -217,6 +321,20 @@ fn resolve_parent_name(ppid: u32) -> Option { return None; } let ppid_str = ppid.to_string(); + // Windows: `ps` does not exist, so this silently returned `None` and + // every session rendered without the "(parent: …)" tag that tells you + // WHICH host owns it — the most useful field in the list. + #[cfg(windows)] + let stdout = { + let script = + format!("(Get-CimInstance Win32_Process -Filter 'ProcessId = {ppid_str}').Name"); + capture_with_timeout( + "powershell", + &["-NoProfile", "-NonInteractive", "-Command", &script], + PS_TIMEOUT, + )? + }; + #[cfg(not(windows))] let stdout = capture_with_timeout("ps", &["-p", &ppid_str, "-o", "comm="], PS_TIMEOUT)?; // Strict decode: this process name is returned and used for a // comparison/targeting decision, so invalid UTF-8 fails closed (None) diff --git a/crates/uffs-cli/src/dispatch.rs b/crates/uffs-cli/src/dispatch.rs index ab611acc3..a03a48a33 100644 --- a/crates/uffs-cli/src/dispatch.rs +++ b/crates/uffs-cli/src/dispatch.rs @@ -129,9 +129,10 @@ pub(crate) fn dispatch_command(command: Command, args: &[String]) -> Result<()> } } -/// `--status [-v] [--json]` — combined daemon + broker + MCP status (never -/// fails). `-v`/`--verbose` expands every section; `--json` emits the -/// machine-readable superset. +/// `--status [-v] [--json] [--brief]` — combined daemon + broker + MCP +/// status (never fails). `-v`/`--verbose` expands every section; +/// `--json` emits the machine-readable superset; `--brief` trims it to +/// the liveness booleans a supervisor needs. fn run_status(args: &[String]) { if args.iter().any(|arg| arg == "--help" || arg == "-h") { crate::args::print_status_help(); @@ -139,7 +140,8 @@ fn run_status(args: &[String]) { } let verbose = args.iter().any(|arg| arg == "-v" || arg == "--verbose"); let json = args.iter().any(|arg| arg == "--json"); - commands::system_status::system_status(verbose, json); + let brief = args.iter().any(|arg| arg == "--brief"); + commands::system_status::system_status(verbose, json, brief); } #[cfg(test)] diff --git a/crates/uffs-cli/src/main.rs b/crates/uffs-cli/src/main.rs index 8b2fad8f4..81d327ece 100644 --- a/crates/uffs-cli/src/main.rs +++ b/crates/uffs-cli/src/main.rs @@ -108,155 +108,9 @@ fn run() -> Result<()> { } /// Timing + payload summary forwarded to [`print_client_profile`]. -/// -/// Packaging these into a struct keeps `run_search` under the -/// `clippy::too-many-lines` cap and lets the profile helper take one -/// argument instead of six. -struct ClientProfile<'a> { - /// Wall-clock time spent in `UffsClientSync::connect_with_args`. - connect_ms: u128, - /// Wall-clock time spent in `await_ready` (daemon warm-up). - ready_ms: u128, - /// Wall-clock time spent in the `search_cli` IPC round-trip. - ipc_ms: u128, - /// Daemon-reported search duration (from the response envelope). - duration_ms: u64, - /// Payload delivery channel the daemon picked for this response. - /// Used by [`print_client_profile`] to show the transport name - /// and to pick the cheapest authoritative row-count source. - payload: &'a uffs_client::protocol::response::SearchPayload, - /// Total row count reported by the daemon, independent of which - /// transport carries the payload. Used to display the "Total - /// matches:" line when the transport is a shmem blob — counting - /// newlines in the mmap would consume the file before the stdout - /// write and double the syscall cost. - total_count: u64, - /// Daemon-side `profile` object from the response envelope. When - /// populated, its `scan_ms` / `sort_ms` / `path_resolve_ms` / - /// `write_ms` fields are rendered as a sub-phase breakdown inside - /// the daemon block so the `--profile` output pinpoints where the - /// per-query cost sits (scan vs sort vs path resolution vs disk - /// write). - daemon_profile: Option<&'a uffs_client::protocol::response::SearchProfile>, -} - -/// Print the `--profile` / `--benchmark` client-side timing block to -/// stderr (matches the daemon-side profile formatting). -#[expect( - clippy::print_stderr, - reason = "intentional --profile output to stderr" -)] -fn print_client_profile(prof: &ClientProfile<'_>) { - use uffs_client::protocol::response::SearchPayload; - - eprintln!("=== PROFILE: Client → Daemon ==="); - eprintln!(" Connect: {:>6} ms", prof.connect_ms); - eprintln!(" Await ready: {:>6} ms", prof.ready_ms); - eprintln!( - " Search (IPC): {:>6} ms (daemon: {} ms)", - prof.ipc_ms, prof.duration_ms - ); - // Sub-phase breakdown from the daemon profile. Any non-zero - // component is printed; all-zero (regex/trigram paths, legacy - // daemons) collapses to a single-line total. - if let Some(dp) = prof.daemon_profile { - let scan = dp.scan_ms; - let sort = dp.sort_ms; - let resolve = dp.path_resolve_ms; - let write = dp.write_ms; - if (scan | sort | resolve | write) > 0 { - eprintln!( - " scan={scan} ms sort={sort} ms path_resolve={resolve} ms write={write} ms" - ); - } - // Deep-profile breakdown: only present when the numeric-sort - // branch populated the `path_*` sub-counters. Prints per- - // record averages derived from ns totals so the user can see - // immediately whether the bottleneck is path-walking or - // row-building, and whether the DirCache hit rate is high - // enough to warrant a locality optimisation. - let candidates = dp.path_candidates; - let cache_entries = dp.path_cache_entries; - let resolve_ns = dp.path_resolve_fn_ns; - let build_ns = dp.path_build_row_ns; - if candidates > 0 { - let hits = candidates.saturating_sub(cache_entries); - // Integer-math hit rate in permille (0–1000) to avoid - // float arithmetic — clippy::float_arithmetic is banned - // in production lints. `permille / 10 . permille % 10` - // prints as "99.7" for 997. - let hit_permille = hits.saturating_mul(1000) / candidates; - let hit_whole = hit_permille / 10; - let hit_frac = hit_permille % 10; - let avg_resolve_ns = resolve_ns / candidates; - let avg_build_ns = build_ns / candidates; - eprintln!( - " deep: candidates={candidates} unique_parents={cache_entries} \ - hit_rate={hit_whole}.{hit_frac}%" - ); - eprintln!( - " resolve_fn={} ms ({} ns/rec) build_row={} ms ({} ns/rec)", - resolve_ns / 1_000_000, - avg_resolve_ns, - build_ns / 1_000_000, - avg_build_ns, - ); - } - } - // Row count resolution — pick the cheapest authoritative source - // depending on which payload variant the daemon used: - // 1. `ShmemBlob` → mmap'd file; counting newlines would read every page just to - // discard the count, so use the daemon's pre- computed `total_count` - // instead. - // 2. `InlineBlob` → inline string already in memory; scanning for `\n` is ~5 - // GB/s, cheap. - // 3. Rows variants (`InlineRows`, `ShmemRows`) → `row_count_hint()` is O(1) — - // `Vec::len` or the daemon's pre-computed count. - // 4. `Empty` → zero rows, nothing to count. - let row_count = match prof.payload { - SearchPayload::ShmemBlob(_) => { - // `try_from` instead of `as` to preserve correctness on - // hypothetical 32-bit targets where `u64` would truncate - // (clippy::cast_possible_truncation). `u64::MAX` is a - // strictly larger fallback than any realistic row count. - usize::try_from(prof.total_count).unwrap_or(usize::MAX) - } - SearchPayload::InlineBlob(blob) => blob.bytes().filter(|byte| *byte == b'\n').count(), - SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { - prof.payload.row_count_hint().unwrap_or(0) - } - }; - // Label the count by what it actually measures per transport: blob - // variants carry rendered text (newline count includes header/footer - // lines) or the daemon's pre-limit total, NOT the post-`--limit` page - // (2026-06-12 dry run: `--limit 5` printed "Rows returned: 7"). - match prof.payload { - SearchPayload::ShmemBlob(_) => { - eprintln!(" Total matches: {row_count:>6}"); - } - SearchPayload::InlineBlob(_) => { - eprintln!(" Output lines: {row_count:>6}"); - } - SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { - eprintln!(" Rows returned: {row_count:>6}"); - } - } - match prof.payload { - SearchPayload::ShmemBlob(_) => { - eprintln!(" Transport: shmem_blob (mmap + write_all, binary)"); - } - SearchPayload::InlineBlob(_) => { - eprintln!(" Transport: inline_blob (single write_all)"); - } - SearchPayload::ShmemRows { .. } => { - eprintln!(" Transport: shmem_rows (mmap + per-row format)"); - } - SearchPayload::InlineRows(_) | SearchPayload::Empty => { - // inline_rows is the default — no extra line needed. - // empty responses skip the transport line entirely. - } - } -} +#[path = "client_profile.rs"] +mod client_profile; +use client_profile::{ClientProfile, print_client_profile}; /// Forward raw search args to the daemon via `search_cli` RPC. pub(crate) fn run_search(args: &[String]) -> Result<()> { @@ -346,6 +200,7 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { ready_ms, ipc_ms, duration_ms: response.duration_ms, + promotion_ms: response.promotion_ms.unwrap_or(0), payload: &response.payload, total_count: response.total_count, daemon_profile: response.profile.as_ref(), diff --git a/crates/uffs-client/src/protocol/response.rs b/crates/uffs-client/src/protocol/response.rs index ccdcbc2f8..5fcc73886 100644 --- a/crates/uffs-client/src/protocol/response.rs +++ b/crates/uffs-client/src/protocol/response.rs @@ -335,8 +335,30 @@ pub struct SearchResponse { pub total_count: u64, /// Total records scanned. pub records_scanned: usize, - /// Search duration in milliseconds. + /// Search duration in milliseconds — **scan time only**. + /// + /// Explicitly excludes the tier promotion this query may have paid + /// for: see [`Self::promotion_ms`]. A query that spent 21 s paging + /// a cold index back in and 1 ms scanning it reports + /// `duration_ms: 1`, which is true and, on its own, badly + /// misleading. pub duration_ms: u64, + /// Milliseconds spent promoting parked/cold shards back to warm + /// before the scan could run — `0` when everything was already + /// resident, which is the steady state. + /// + /// Split out rather than folded into `duration_ms` because they + /// answer different questions: `duration_ms` is "how fast is the + /// index", `promotion_ms` is "what did I pay to have an index at + /// all". Merging them would make a warm search look slow; omitting + /// it (the original behaviour) hides the expensive case entirely — + /// a client watching `duration_ms: 1` cannot tell it just waited + /// 21 s, so the cost is invisible exactly when it matters. + /// + /// Absent from an older daemon, which reads as "not reported" + /// rather than "was free". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promotion_ms: Option, /// Whether results were truncated by limit. pub truncated: bool, /// Detailed timing breakdown from the daemon (only when diff --git a/crates/uffs-client/src/protocol/response_status.rs b/crates/uffs-client/src/protocol/response_status.rs index 49e8c0fed..72211c388 100644 --- a/crates/uffs-client/src/protocol/response_status.rs +++ b/crates/uffs-client/src/protocol/response_status.rs @@ -74,6 +74,27 @@ pub struct DriveInfo { /// as "no tier marker available" and renders accordingly. #[serde(default, skip_serializing_if = "Option::is_none")] pub tier: Option, + /// Records this drive held the last time it was warm — the + /// denominator for re-warm progress. + /// + /// `records` above is `0` for a demoted shard, so on its own it + /// cannot say how far a re-warm has to go. Drive-count progress + /// is not a substitute: four of seven drives warm was only 24 % of + /// the records, because the three still cold were the large ones. + /// `0` (or absent, from an older daemon) means this drive has not + /// been warm in the current daemon's lifetime, so no denominator + /// exists yet. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub records_when_warm: Option, + /// `true` while this drive's body is actively being paged in. + /// + /// A re-warm is stepwise per drive, so the record count plateaus + /// for tens of seconds while one large drive loads. Without this, + /// repeated polls showing identical numbers are indistinguishable + /// from a stalled warm — and giving up is the natural response to + /// what looks like no progress. Absent from an older daemon. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loading: Option, } /// Response for the `status` method. diff --git a/crates/uffs-client/src/protocol/tests.rs b/crates/uffs-client/src/protocol/tests.rs index 06e95bbeb..02f606ed1 100644 --- a/crates/uffs-client/src/protocol/tests.rs +++ b/crates/uffs-client/src/protocol/tests.rs @@ -289,6 +289,7 @@ fn search_response_inline_rows_round_trip() { total_count: 1, records_scanned: 1_000_000, duration_ms: 8, + promotion_ms: None, truncated: false, profile: None, applied_sorts: vec![SearchSortSpec { @@ -405,6 +406,7 @@ fn search_response_shmem_blob_round_trip() { total_count: 168_295, records_scanned: 3_571_389, duration_ms: 60, + promotion_ms: None, truncated: false, profile: None, applied_sorts: vec![], @@ -465,6 +467,7 @@ fn search_response_inline_blob_round_trip() { total_count: 2, records_scanned: 250_000, duration_ms: 5, + promotion_ms: None, truncated: false, profile: None, applied_sorts: vec![], @@ -537,6 +540,7 @@ fn search_response_shmem_rows_round_trip() { total_count: 250_000, records_scanned: 5_000_000, duration_ms: 42, + promotion_ms: None, truncated: false, profile: None, applied_sorts: vec![], @@ -878,6 +882,7 @@ fn search_response_with_aggregations_round_trip() { total_count: 0, records_scanned: 500_000, duration_ms: 12, + promotion_ms: None, truncated: false, profile: None, applied_sorts: vec![], diff --git a/crates/uffs-client/src/shmem.rs b/crates/uffs-client/src/shmem.rs index e1f78dddd..472afa278 100644 --- a/crates/uffs-client/src/shmem.rs +++ b/crates/uffs-client/src/shmem.rs @@ -454,6 +454,10 @@ pub fn read_search_results(path: &Path) -> io::Result { records_scanned: usize::try_from(header.records_scanned) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?, duration_ms: header.duration_ms, + // The shmem header carries no promotion field; the caller that + // issued the search already has it from the RPC response this + // blob was announced in. + promotion_ms: None, truncated: header.truncated != 0, profile: None, applied_sorts: Vec::new(), diff --git a/crates/uffs-daemon/src/cache/shard/drive_stats.rs b/crates/uffs-daemon/src/cache/shard/drive_stats.rs index 245ce4400..e7b1e9273 100644 --- a/crates/uffs-daemon/src/cache/shard/drive_stats.rs +++ b/crates/uffs-daemon/src/cache/shard/drive_stats.rs @@ -122,6 +122,24 @@ pub(crate) struct DriveStats { /// /// [`StatusDrivesResponse`]: uffs_client::protocol::response::StatusDrivesResponse promotions_total: AtomicU64, + /// Record count this drive had the last time its body was + /// resident — the denominator for warm progress. + /// + /// A `Parked`/`Cold` shard drops its body and therefore reports + /// `records: 0`, which makes "6.07 M loaded" un-interpretable: an + /// agent waiting out a re-warm can see how far it has come but not + /// how far there is to go, and drive-count progress lies badly + /// (four of seven drives warm was only 24 % of the records, + /// because the three still cold were the big ones). + /// + /// `DriveStats` is the right home because the registry preserves + /// its `Arc` across every tier transition — the count survives + /// demotion for free, with no new plumbing through the + /// `new_parked` / `new_cold` rebuild constructors. Zero means + /// "never been warm in this daemon's lifetime", which is honest: + /// a first-ever load genuinely has no denominator until it + /// finishes. + last_known_records: AtomicU64, } impl DriveStats { @@ -135,9 +153,23 @@ impl DriveStats { last_decay_queries_total: AtomicU64::new(0), last_query_at_ms: AtomicU64::new(0), promotions_total: AtomicU64::new(0), + last_known_records: AtomicU64::new(0), } } + /// Remember how many records this drive held while warm, so a + /// later re-warm can report progress against a real denominator. + pub(crate) fn set_last_known_records(&self, records: u64) { + self.last_known_records.store(records, Ordering::Relaxed); + } + + /// Records this drive held when last resident; `0` if it has never + /// been warm in this daemon's lifetime. + #[must_use] + pub(crate) fn last_known_records(&self) -> u64 { + self.last_known_records.load(Ordering::Relaxed) + } + /// Lock-free increment of the total query counter. /// /// Phase 3 prefer [`Self::mark_query_at`] which also bumps @@ -404,6 +436,14 @@ pub(crate) struct DriveStatsSnapshot { /// preserves backward compat with on-disk persisted stats. #[serde(default)] pub promotions_total: u64, + /// Records held when last warm — see + /// [`DriveStats::last_known_records`]. Persisting it means a + /// freshly restarted daemon can report re-warm progress against a + /// real denominator on its very first load, instead of only after + /// it has been warm once. Defaults to `0` (no denominator yet) + /// for snapshots written before this field existed. + #[serde(default)] + pub last_known_records: u64, } impl From<&DriveStats> for DriveStatsSnapshot { @@ -415,6 +455,7 @@ impl From<&DriveStats> for DriveStatsSnapshot { last_query_at_ms: stats.last_query_at_ms.load(Ordering::Relaxed), last_decay_queries_total: stats.last_decay_queries_total.load(Ordering::Relaxed), promotions_total: stats.promotions_total.load(Ordering::Relaxed), + last_known_records: stats.last_known_records.load(Ordering::Relaxed), } } } @@ -428,6 +469,7 @@ impl From for DriveStats { last_decay_queries_total: AtomicU64::new(snap.last_decay_queries_total), last_query_at_ms: AtomicU64::new(snap.last_query_at_ms), promotions_total: AtomicU64::new(snap.promotions_total), + last_known_records: AtomicU64::new(snap.last_known_records), } } } diff --git a/crates/uffs-daemon/src/handler_csv_blob_tests.rs b/crates/uffs-daemon/src/handler_csv_blob_tests.rs index 3058da6ce..75ec023d5 100644 --- a/crates/uffs-daemon/src/handler_csv_blob_tests.rs +++ b/crates/uffs-daemon/src/handler_csv_blob_tests.rs @@ -37,6 +37,7 @@ fn bare_response(rows: Vec) -> SearchResponse { total_count: u64::try_from(row_count).unwrap_or(u64::MAX), records_scanned: row_count, duration_ms: 0, + promotion_ms: None, truncated: false, profile: None, applied_sorts: Vec::new(), diff --git a/crates/uffs-daemon/src/handler_paths_blob_tests.rs b/crates/uffs-daemon/src/handler_paths_blob_tests.rs index d77cbf841..265a3b873 100644 --- a/crates/uffs-daemon/src/handler_paths_blob_tests.rs +++ b/crates/uffs-daemon/src/handler_paths_blob_tests.rs @@ -56,6 +56,7 @@ fn bare_response(rows: Vec) -> SearchResponse { total_count: u64::try_from(row_count).unwrap_or(u64::MAX), records_scanned: row_count, duration_ms: 0, + promotion_ms: None, truncated: false, profile: None, applied_sorts: Vec::new(), diff --git a/crates/uffs-daemon/src/index/drives.rs b/crates/uffs-daemon/src/index/drives.rs index 65caf8bb7..92b45e0d3 100644 --- a/crates/uffs-daemon/src/index/drives.rs +++ b/crates/uffs-daemon/src/index/drives.rs @@ -98,6 +98,15 @@ impl IndexManager { // where a guard's scope outlives the block that actually // needs it, since concurrent writers (demote / promote) // would block longer than necessary. + // Letters with a promote in flight right now. A re-warm is + // stepwise per drive, so `records` plateaus while one large + // drive loads; without this a caller cannot tell a working warm + // from a hung one, and three identical polls read as "hung". + let loading: Vec = + self.in_flight_promotes.lock().map_or_else( + |poisoned| poisoned.into_inner().keys().copied().collect(), + |map| map.keys().copied().collect(), + ); let drives: Vec = { let guard = self.index.read().await; guard @@ -108,11 +117,19 @@ impl IndexManager { || (0_usize, tier_source_label(tier).to_owned()), |body| (body.records.len(), describe_index_source(&body.source)), ); + // While the body is resident, park the count in the + // Arc-preserved stats so a later re-warm has a real + // denominator to report progress against. + if records > 0 { + shard.stats.set_last_known_records(records as u64); + } DriveInfo { letter: shard.drive, records, source, tier: Some(tier), + records_when_warm: Some(shard.stats.last_known_records()), + loading: Some(loading.contains(&shard.drive)), } }) .collect() diff --git a/crates/uffs-daemon/src/index/search.rs b/crates/uffs-daemon/src/index/search.rs index ad1fa483b..38c8366c9 100644 --- a/crates/uffs-daemon/src/index/search.rs +++ b/crates/uffs-daemon/src/index/search.rs @@ -72,19 +72,8 @@ impl IndexManager { // cap is saturated. Return a no-payload response with // the remaining metadata fields at their zero defaults // so the client still sees a valid (if empty) shape. - return SearchResponse { - payload: SearchPayload::Empty, - total_count: 0, - records_scanned: 0, - duration_ms: 0, - truncated: false, - profile: None, - applied_sorts: Vec::new(), - applied_projection: Vec::new(), - response_mode: None, - projected_rows: None, - aggregations: vec![], - }; + // Rejected before any promote was attempted. + return empty_response(0, None); }; let query_start = Instant::now(); @@ -193,10 +182,18 @@ impl IndexManager { // behaviour. // Registry warm-up only applies to live shards; a diff searches the // caller's baseline index, which is not in the registry. - if !is_diff { + // Timed unconditionally, not just under `--profile`: paging a + // cold index back in is the largest cost a query can incur, and + // a client that cannot see it reads `duration_ms: 1` and + // concludes the search was instant. + let promotion_ms = if is_diff { + 0 + } else { + let t_promote = Instant::now(); self.ensure_warm_for_dispatch(&effective_params.drives, &filters.extensions) .await; - } + u64::try_from(t_promote.elapsed().as_millis()).unwrap_or(u64::MAX) + }; // ── Snapshot the index (< 1 μs) ──────────────────────────── let t_lock = profiling.then(Instant::now); @@ -302,38 +299,19 @@ impl IndexManager { Ok(Ok(res)) => res, Ok(Err(_join_err)) => { tracing::error!("search task panicked"); - return SearchResponse { - payload: SearchPayload::Empty, - total_count: 0, - records_scanned: 0, - duration_ms: 0, - truncated: false, - profile: None, - applied_sorts: Vec::new(), - applied_projection: Vec::new(), - response_mode: None, - projected_rows: None, - aggregations: vec![], - }; + // Promotion already happened; report what it cost even + // though the scan then failed. + return empty_response(0, Some(promotion_ms)); } Err(_timeout) => { tracing::warn!( pattern = %effective_params.pattern, "search timed out after 30s" ); - return SearchResponse { - payload: SearchPayload::Empty, - total_count: 0, - records_scanned: 0, - duration_ms: 30_000, - truncated: false, - profile: None, - applied_sorts: Vec::new(), - applied_projection: Vec::new(), - response_mode: None, - projected_rows: None, - aggregations: vec![], - }; + // A timeout that spent most of its budget paging the + // index in is a different diagnosis from one that spent + // it scanning — say which. + return empty_response(30_000, Some(promotion_ms)); } }; let search_us = if profiling { @@ -429,6 +407,7 @@ impl IndexManager { output = output_path, rows = rows_written, duration_ms, + promotion_ms, "daemon wrote results directly to file" ); // Update perf counters. @@ -469,6 +448,7 @@ impl IndexManager { total_count, records_scanned: result.records_scanned, duration_ms, + promotion_ms: Some(promotion_ms), truncated: false, profile, applied_sorts: Vec::new(), @@ -627,6 +607,7 @@ impl IndexManager { total_count, records_scanned: result.records_scanned, duration_ms, + promotion_ms: Some(promotion_ms), truncated, profile, applied_sorts, @@ -732,6 +713,27 @@ impl IndexManager { } } +/// A response carrying no rows — the shape every early-out path returns +/// (permit exhaustion, scan panic, scan timeout). Factored out because +/// the three literals differed in two fields, so every new +/// `SearchResponse` field had to be threaded through all of them. +const fn empty_response(duration_ms: u64, promotion_ms: Option) -> SearchResponse { + SearchResponse { + payload: SearchPayload::Empty, + total_count: 0, + records_scanned: 0, + duration_ms, + promotion_ms, + truncated: false, + profile: None, + applied_sorts: Vec::new(), + applied_projection: Vec::new(), + response_mode: None, + projected_rows: None, + aggregations: Vec::new(), + } +} + /// Decide the backend scan limit (the cap applied *before* the daemon's /// final truncate-to-`user_limit`). /// diff --git a/crates/uffs-mcp/src/schemas.rs b/crates/uffs-mcp/src/schemas.rs index 5acfb7baf..41dd1a71b 100644 --- a/crates/uffs-mcp/src/schemas.rs +++ b/crates/uffs-mcp/src/schemas.rs @@ -21,8 +21,14 @@ pub(crate) struct SearchOutput { pub total_count: u64, /// Total records scanned across all drives. pub records_scanned: usize, - /// Query execution time in milliseconds. + /// Scan time in milliseconds — excludes index warm-up, which is + /// reported separately as `promotion_ms`. pub duration_ms: u64, + /// Milliseconds spent paging parked/cold drives back in before the + /// scan could run; `0` on a warm index. Without this a query that + /// warmed for 21 s and scanned for 1 ms reports `duration_ms: 1`, + /// hiding the expensive case entirely. + pub promotion_ms: u64, /// Whether more results exist beyond this page. pub truncated: bool, /// Opaque cursor for fetching the next page (null when no more pages). @@ -112,10 +118,17 @@ pub(crate) struct DriveOutput { /// `char`-typed schema. JSON schema is `char` (see [`SearchRowOutput`]). #[schemars(with = "char")] pub letter: uffs_mft::platform::DriveLetter, - /// Number of records in the compact index. + /// Number of records in the compact index. **`0` for a `parked` or + /// `cold` drive** — the body is released, not empty; the count + /// returns when the drive re-warms. Read `tier` before concluding + /// a drive holds nothing. pub records: usize, /// Data source (`"cache"`, `"live"`, `"mft_file"`). pub source: String, + /// Memory tier: `"hot"` / `"warm"` (searchable now) or + /// `"parked"` / `"cold"` (body released — a query re-warms it, + /// taking 30–120 s). `null` from a pre-tiering daemon. + pub tier: Option, } // ── uffs_status ───────────────────────────────────────────────────── @@ -123,8 +136,52 @@ pub(crate) struct DriveOutput { /// Structured output for `uffs_status`. #[derive(Debug, Serialize, JsonSchema)] pub(crate) struct StatusOutput { - /// Current daemon status object. - pub status: serde_json::Value, + /// Daemon **process** state: `"running"`, `"loading (3/7 drives)"`, + /// or `"refreshing (C, D)"`. + /// + /// Deliberately never says "ready" — that word belongs to + /// `index_ready` alone. A running daemon can have every drive + /// parked, and when both fields said "ready"/"false" the payload + /// answered the same apparent question two ways; the wrong one was + /// found first. + pub daemon_process: String, + /// `true` when every loaded drive is `hot`/`warm`, i.e. a query + /// answers immediately. `false` means at least one drive is + /// parked/cold and a query against it triggers a 30–120 s re-warm. + /// This is the field to poll while waiting out a warm. + pub index_ready: bool, + /// Per-drive tier, `"C"` → `"warm"`. The authoritative answer to + /// "is the index actually ready", which the lifecycle `status` + /// field above does not give. + pub drives: alloc::collections::BTreeMap, + /// Records resident across all loaded drives. **`0` while every + /// drive is parked** — bodies released, not an empty index. Reads + /// as the corroborating signal for `index_ready`. + pub total_records: usize, + /// Index bytes resident in the heap, in MB. `0` while parked; + /// several GB when warm. `null` from a daemon that does not + /// report it. + pub index_heap_mb: Option, + /// Drives currently being paged in, e.g. `["E"]`. + /// + /// A re-warm is stepwise per drive, so `total_records` plateaus for + /// tens of seconds while one large drive loads. Without knowing a + /// load is in flight, three identical polls read as "hung" and the + /// natural response is to give up — the one remaining way a caller + /// bails early on a warm that is working fine. Empty means nothing + /// is loading right now. + pub currently_loading: Vec, + /// Records expected once every drive is warm — the denominator for + /// `total_records`. `null` when no drive has been warm yet, so + /// there is genuinely nothing to measure against. + pub records_when_warm: Option, + /// Re-warm progress, 0–100, as `total_records / records_when_warm`. + /// `null` when the denominator is unknown. + /// + /// Measured in **records, not drives**: drive-count progress + /// misleads badly, since drives differ in size by three orders of + /// magnitude (four of seven warm was only 24 % of records). + pub warming_progress_pct: Option, /// Daemon uptime in seconds. pub uptime_secs: u64, /// Number of active connections. diff --git a/crates/uffs-mcp/src/tools/aggregate.rs b/crates/uffs-mcp/src/tools/aggregate.rs index 161f06f15..4b94e87ac 100644 --- a/crates/uffs-mcp/src/tools/aggregate.rs +++ b/crates/uffs-mcp/src/tools/aggregate.rs @@ -61,6 +61,17 @@ fn default_pattern() -> String { "*".to_owned() } +/// Parse the tool's drive-name strings (`"C"`, `"C:"`, …) into typed +/// letters, silently dropping anything unparseable — a bad drive arg +/// scopes to nothing rather than failing the whole aggregation. +fn parse_drive_letters(drives: &[String]) -> Vec { + drives + .iter() + .filter_map(|drv| drv.chars().next()) + .filter_map(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) + .collect() +} + /// Execute the aggregate tool. /// /// # Errors @@ -100,12 +111,7 @@ pub(crate) async fn run( }); } - let drives: Vec = args - .drives - .iter() - .filter_map(|drv| drv.chars().next()) - .filter_map(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) - .collect(); + let drives = parse_drive_letters(&args.drives); let mut params = SearchParams { pattern: args.pattern, @@ -128,6 +134,10 @@ pub(crate) async fn run( // Apply roots-based scoping (drive + path prefix) when no explicit drives. roots::apply_roots_scope(roots_state, &mut params); + // Cold-index contract: return a retryable "warming" error instead of + // blocking silently for the length of a re-warm (see `tools::warm`). + super::warm::warm_gate(client, ¶ms.drives).await?; + // Log the exact RPC payload for debugging parity with API validation. tracing::info!( params_json = %serde_json::to_string(¶ms).unwrap_or_default(), diff --git a/crates/uffs-mcp/src/tools/drives.rs b/crates/uffs-mcp/src/tools/drives.rs index c22a5db6b..748d405db 100644 --- a/crates/uffs-mcp/src/tools/drives.rs +++ b/crates/uffs-mcp/src/tools/drives.rs @@ -31,6 +31,7 @@ pub(crate) async fn run(client: &mut UffsClient) -> Result Result " ← body released; a query re-warms it (30-120 s)", + _ => "", + }; _ = writeln!( output, - " {}: {:>10} records ({})", - drive.letter, drive.records, drive.source + " {}: {:>10} records ({}, {}){}", + drive.letter, drive.records, drive.source, tier, note ); } diff --git a/crates/uffs-mcp/src/tools/facet_values.rs b/crates/uffs-mcp/src/tools/facet_values.rs index 33ab28e49..36b2913b1 100644 --- a/crates/uffs-mcp/src/tools/facet_values.rs +++ b/crates/uffs-mcp/src/tools/facet_values.rs @@ -103,6 +103,10 @@ pub(crate) async fn run( // Apply roots-based scoping (drive + path prefix). roots::apply_roots_scope(roots_state, &mut params); + // Cold-index contract: return a retryable "warming" error instead of + // blocking silently for the length of a re-warm (see `tools::warm`). + super::warm::warm_gate(client, ¶ms.drives).await?; + let mut response = client .search(¶ms) .await diff --git a/crates/uffs-mcp/src/tools/info.rs b/crates/uffs-mcp/src/tools/info.rs index e02b23e98..105ab3aa1 100644 --- a/crates/uffs-mcp/src/tools/info.rs +++ b/crates/uffs-mcp/src/tools/info.rs @@ -30,6 +30,17 @@ pub(crate) async fn run( return Err(BridgeError::MissingParam("path")); } + // Cold-index contract, scoped to the path's own drive: an info + // lookup on a parked drive must not block for the re-warm either. + let scope: Vec = args + .path + .chars() + .next() + .and_then(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) + .into_iter() + .collect(); + super::warm::warm_gate(client, &scope).await?; + let response = client .info(&args.path) .await diff --git a/crates/uffs-mcp/src/tools/mod.rs b/crates/uffs-mcp/src/tools/mod.rs index cff1e909d..6d5c92e46 100644 --- a/crates/uffs-mcp/src/tools/mod.rs +++ b/crates/uffs-mcp/src/tools/mod.rs @@ -20,3 +20,5 @@ pub(crate) mod info; pub(crate) mod search; /// `uffs_status` — daemon health and loading progress. pub(crate) mod status; +/// Cold-index gate shared by the query tools. +pub(crate) mod warm; diff --git a/crates/uffs-mcp/src/tools/search.rs b/crates/uffs-mcp/src/tools/search.rs index 10eae29e0..f39a6811c 100644 --- a/crates/uffs-mcp/src/tools/search.rs +++ b/crates/uffs-mcp/src/tools/search.rs @@ -364,6 +364,10 @@ pub(crate) async fn run( // Apply roots-based scoping (drive + path prefix) when no explicit drives. roots::apply_roots_scope(roots_state, &mut search_params); + // Cold-index contract: return a retryable "warming" error instead of + // blocking silently for the length of a re-warm (see `tools::warm`). + super::warm::warm_gate(client, &search_params.drives).await?; + tracing::debug!( params_json = %serde_json::to_string(&search_params).unwrap_or_default(), "uffs_search: sending search request to daemon" @@ -422,6 +426,7 @@ pub(crate) async fn run( total_count, records_scanned: response.records_scanned, duration_ms: response.duration_ms, + promotion_ms: response.promotion_ms.unwrap_or(0), has_more, effective_limit, next_cursor: next_cursor.clone(), @@ -446,6 +451,7 @@ pub(crate) async fn run( total_count, records_scanned: response.records_scanned, duration_ms: response.duration_ms, + promotion_ms: response.promotion_ms.unwrap_or(0), truncated: has_more, next_cursor, warnings, @@ -496,8 +502,15 @@ struct FormatContext<'a> { total_count: u64, /// Number of MFT records scanned. records_scanned: usize, - /// Wall-clock duration of the search in milliseconds. + /// Wall-clock duration of the SCAN in milliseconds — excludes any + /// tier promotion, which is reported separately. duration_ms: u64, + /// Milliseconds spent paging parked/cold drives back in before the + /// scan. Surfaced whenever non-zero: without it a query that spent + /// 21 s warming and 1 ms scanning reports "1ms", and the cost of + /// the slow case is invisible precisely when someone is wondering + /// where the time went. + promotion_ms: u64, /// Whether there are more results beyond this page. has_more: bool, /// The effective per-page limit (after clamping). @@ -506,24 +519,39 @@ struct FormatContext<'a> { next_cursor: Option, } +/// Render the promotion cost, or nothing when there was none. +/// +/// Silent in the steady state (a warm index promotes nothing), loud +/// when a query just paid tens of seconds to page the index in. +fn warm_note(promotion_ms: u64) -> String { + if promotion_ms == 0 { + String::new() + } else { + format!(" + {promotion_ms}ms index warm-up") + } +} + /// Render the human-readable text summary for the search results. fn format_text_output(ctx: &FormatContext<'_>) -> String { let mut output = String::new(); if ctx.page_rows.is_empty() { _ = write!( output, - "0 matches ({} scanned in {}ms)\n\n", - ctx.records_scanned, ctx.duration_ms, + "0 matches ({} scanned in {}ms{})\n\n", + ctx.records_scanned, + ctx.duration_ms, + warm_note(ctx.promotion_ms), ); } else { _ = write!( output, - "Showing {}-{} of {} matches ({} scanned in {}ms)\n\n", + "Showing {}-{} of {} matches ({} scanned in {}ms{})\n\n", ctx.offset + 1, ctx.end_offset, ctx.total_count, ctx.records_scanned, ctx.duration_ms, + warm_note(ctx.promotion_ms), ); } for warning in ctx.warnings { diff --git a/crates/uffs-mcp/src/tools/status.rs b/crates/uffs-mcp/src/tools/status.rs index ad04011b9..26f679a35 100644 --- a/crates/uffs-mcp/src/tools/status.rs +++ b/crates/uffs-mcp/src/tools/status.rs @@ -5,10 +5,117 @@ use rmcp::model::{CallToolResult, ContentBlock}; use uffs_client::connect::UffsClient; +use uffs_client::protocol::response::ShardTier; use crate::error::BridgeError; use crate::schemas::StatusOutput; +/// Thousands-separate a count so `0` versus `24,988,343` is legible at +/// a glance — the difference between a parked and a warm index. +fn commas(value: u64) -> String { + let digits = value.to_string(); + let mut out = String::with_capacity(digits.len() + digits.len() / 3); + for (idx, ch) in digits.chars().enumerate() { + if idx > 0 && (digits.len() - idx).is_multiple_of(3) { + out.push(','); + } + out.push(ch); + } + out +} + +/// Describe the daemon **process** state without ever using the word +/// "ready". +/// +/// The daemon's lifecycle enum has a `Ready` variant meaning "the +/// process is up and accepting connections". Serialised verbatim it +/// put `"state": "ready"` in the payload beside `index_ready: false` — +/// two fields answering what looks like the same question with +/// different answers, and a model skimming for a readiness signal finds +/// the wrong one first. That is not hypothetical: it is the exact +/// misread that produced a confident "the daemon is not cold" against +/// seven parked shards. +/// +/// So the process state is rendered as `running` / `loading` / +/// `refreshing`, which is what it actually means, and the only "ready" +/// left anywhere in the payload is `index_ready` — the field that +/// really answers it. +fn daemon_state_label(status: &uffs_client::protocol::response::DaemonStatus) -> String { + use uffs_client::protocol::response::DaemonStatus; + match status { + DaemonStatus::Ready => "running".to_owned(), + DaemonStatus::Loading { + drives_loaded, + drives_total, + } => format!("loading ({drives_loaded}/{drives_total} drives)"), + DaemonStatus::Refreshing { drives } => { + let list: Vec = drives.iter().map(ToString::to_string).collect(); + format!("refreshing ({})", list.join(", ")) + } + } +} + +/// Render a shard tier as the lowercase name agents match on. +/// +/// `None` (pre-tiering daemon) reads as `warm`: those daemons never +/// demote, so their shards are searchable by construction. +pub(crate) const fn tier_name(tier: Option) -> &'static str { + match tier { + Some(ShardTier::Hot) => "hot", + None | Some(ShardTier::Warm) => "warm", + Some(ShardTier::Parked) => "parked", + Some(ShardTier::Cold) => "cold", + Some(ShardTier::Evicting) => "evicting", + Some(ShardTier::Unknown) => "unknown", + } +} + +/// `Drives: C=warm D=cold …`, or a note that tiers are unavailable. +fn render_tier_line(tiers: &alloc::collections::BTreeMap) -> String { + if tiers.is_empty() { + return "Drives: (tier state unavailable)".to_owned(); + } + let rendered: Vec = tiers + .iter() + .map(|(letter, tier)| format!("{letter}={tier}")) + .collect(); + format!("Drives: {}", rendered.join(" ")) +} + +/// The one line that says whether a query will answer or warm. +const fn readiness_line(index_ready: bool) -> &'static str { + if index_ready { + "Index: READY — every drive is warm; queries answer immediately." + } else { + "Index: WARMING NEEDED — one or more drives are parked/cold. A query \ + against them triggers a 30-120 s re-warm and returns a retry hint. \ + Poll this tool until index_ready is true." + } +} + +/// Name the drive being paged in, so a plateau reads as "S is loading" +/// rather than "hung" — the difference between waiting and giving up. +fn render_loading_line(loading_now: &[String]) -> String { + if loading_now.is_empty() { + return String::new(); + } + format!( + "\nLoading now: {} (counts plateau until it lands)", + loading_now.join(", ") + ) +} + +/// Progress against the expected total — shown only mid-warm, where at +/// 100 % it is noise and without a denominator it would be a guess. +fn render_progress(pct: Option, expected: Option) -> String { + match (pct, expected) { + (Some(percent), Some(total)) if percent < 100 => { + format!(" of {} expected ({percent}% warmed)", commas(total)) + } + _ => String::new(), + } +} + /// Execute the status tool (no arguments). /// /// # Errors @@ -20,19 +127,95 @@ pub(crate) async fn run(client: &mut UffsClient) -> Result = drives + .as_ref() + .map(|list| { + list.iter() + .map(|drv| (drv.letter.to_string(), tier_name(drv.tier).to_owned())) + .collect() + }) + .unwrap_or_default(); + let loading_now: Vec = drives + .as_ref() + .map(|list| { + list.iter() + .filter(|drv| drv.loading == Some(true)) + .map(|drv| drv.letter.to_string()) + .collect() + }) + .unwrap_or_default(); + let index_ready = drives.as_ref().is_some_and(|list| { + list.iter() + .all(|drv| matches!(drv.tier, None | Some(ShardTier::Warm | ShardTier::Hot))) + }); // The running `uffsmcp` build version — a read-only freshness signal the // agent can surface (UFFS self-updates via `uffs --update`). let server_version = env!("CARGO_PKG_VERSION"); + // Residency corroboration: records + heap. Both read 0 while every + // drive is parked, which is the same fact the tier map states — but + // an agent that distrusts one number has the other to check it + // against, and "0 records / 0 MB" is unmistakable. + let total_records = client.stats().await.map_or(0, |stats| stats.total_records); + let index_heap_mb = response.index_heap_bytes.map(|bytes| bytes / (1024 * 1024)); + + // Re-warm progress against a real denominator: the records each + // drive held when last resident, summed. Deliberately record-based + // — drive-count progress reads 57 % when only 24 % of the records + // are in, because the drives still cold are the big ones. + let records_when_warm: Option = drives.as_ref().and_then(|list| { + let expected: u64 = list + .iter() + .map(|drv| drv.records_when_warm.unwrap_or(0)) + .sum(); + (expected > 0).then_some(expected) + }); + let warming_progress_pct = records_when_warm.map(|expected| { + let loaded = u64::try_from(total_records).unwrap_or(u64::MAX); + let pct = loaded + .saturating_mul(100) + .checked_div(expected) + .unwrap_or(0); + u8::try_from(pct.min(100)).unwrap_or(100) + }); + + let tier_line = render_tier_line(&tiers); + let readiness_line = readiness_line(index_ready); + let loading_line = render_loading_line(&loading_now); + let heap_str = index_heap_mb.map_or_else(|| "n/a".to_owned(), |mb| format!("{mb} MB")); + let progress_str = render_progress(warming_progress_pct, records_when_warm); + let text = format!( - "Daemon Status: {status_str}\nUptime: {}s\nConnections: {}\nPID: {}\nUFFS server version: {server_version}\n", - response.uptime_secs, response.connections, response.pid + "Daemon process: {daemon_process}\n{readiness_line}\n{tier_line}{loading_line}\n\ + Resident: {} records{progress_str}, index heap {heap_str}\n\ + Uptime: {}s (process uptime — NOT how long the index has been warm)\n\ + Connections: {}\nPID: {}\nUFFS server version: {server_version}\n", + commas(u64::try_from(total_records).unwrap_or(u64::MAX)), + response.uptime_secs, + response.connections, + response.pid ); let structured = StatusOutput { - status: serde_json::to_value(&response.status)?, + daemon_process: daemon_process.clone(), + index_ready, + drives: tiers, + total_records, + index_heap_mb, + currently_loading: loading_now.clone(), + records_when_warm, + warming_progress_pct, uptime_secs: response.uptime_secs, connections: response.connections, pid: response.pid, @@ -43,3 +226,38 @@ pub(crate) async fn run(client: &mut UffsClient) -> Result Vec { + drives + .iter() + .filter(|info| scope.is_empty() || scope.contains(&info.letter)) + .filter(|info| { + matches!( + info.tier, + Some(ShardTier::Parked | ShardTier::Cold | ShardTier::Unknown) + ) + }) + .map(|info| info.letter) + .collect() +} + +/// Gate a query tool on index warmth: `Ok(())` when every scoped drive +/// is `Warm`/`Hot`; otherwise kick a detached re-warm and return the +/// retry-shaped error described in the module docs. +/// +/// # Errors +/// +/// Returns [`BridgeError::Daemon`] when scoped drives are re-warming +/// (the retry contract) or when the tier probe itself fails. +pub(crate) async fn warm_gate( + client: &mut UffsClient, + scope: &[uffs_mft::platform::DriveLetter], +) -> Result<(), BridgeError> { + let drives = client + .drives() + .await + .map_err(|err| BridgeError::Daemon(format!("warm check failed: {err}")))?; + + let cold = not_ready(&drives.drives, scope); + if cold.is_empty() { + return Ok(()); + } + + // Detached warm trigger: its own connection, its own lifetime. + // Once the request reaches the daemon it runs to completion there, + // so the warm finishes even if this MCP session dies; per-letter + // single-flight dedup in the daemon makes a repeat trigger (an + // agent retrying early) join the in-flight load instead of + // duplicating it. + let trigger_drives = cold.clone(); + drop(tokio::spawn(async move { + let Ok(mut warm_client) = UffsClient::connect_raw().await else { + tracing::warn!("warm trigger: daemon connect failed — retry will re-trigger"); + return; + }; + let mut params = SearchParams { + pattern: WARM_TRIGGER_PATTERN.to_owned(), + drives: trigger_drives.clone(), + limit: Some(1), + ..Default::default() + }; + params.populate_canonical_fields(); + match warm_client.search(¶ms).await { + Ok(_) => tracing::info!(drives = ?trigger_drives, "warm trigger completed"), + Err(err) => tracing::warn!(%err, "warm trigger search failed"), + } + })); + + let list: Vec = cold.iter().map(ToString::to_string).collect(); + Err(BridgeError::Daemon(format!( + "⏳ Index warming — drive(s) {} were parked/cold and are being re-warmed now \ + (typically 30–120 s; HDDs are the slow end). This call returned early instead \ + of blocking; nothing is wrong with your query. Poll uffs_status until its \ + `index_ready` field is true — NOT the `status` field, which reads \"Ready\" \ + for the daemon process even while every drive is parked — then retry this \ + exact query unchanged; it will answer in milliseconds.", + list.join(", ") + ))) +} + +#[cfg(test)] +mod tests { + use uffs_client::protocol::response::{DriveInfo, ShardTier}; + use uffs_mft::platform::DriveLetter; + + use super::not_ready; + + /// Build a `DriveInfo` with the given letter and tier. + fn info(letter: char, tier: Option) -> DriveInfo { + DriveInfo { + letter: DriveLetter::parse(letter).unwrap_or(DriveLetter::C), + records: 0, + source: "test".to_owned(), + tier, + records_when_warm: None, + loading: None, + } + } + + /// The exact field shape: warm C answers, parked D and cold S gate. + #[test] + fn parked_and_cold_drives_gate_warm_ones_do_not() { + let drives = vec![ + info('C', Some(ShardTier::Warm)), + info('D', Some(ShardTier::Parked)), + info('S', Some(ShardTier::Cold)), + ]; + let cold = not_ready(&drives, &[]); + assert_eq!(cold, vec![ + DriveLetter::parse('D').unwrap_or(DriveLetter::C), + DriveLetter::parse('S').unwrap_or(DriveLetter::C), + ]); + } + + /// A query scoped to a warm drive must NOT gate on some other + /// drive being cold — scoping is the whole point of the check. + #[test] + fn scope_limits_the_gate_to_touched_drives() { + let drives = vec![ + info('C', Some(ShardTier::Warm)), + info('D', Some(ShardTier::Cold)), + ]; + let scope = vec![DriveLetter::parse('C').unwrap_or(DriveLetter::C)]; + assert_eq!( + not_ready(&drives, &scope), + Vec::::new(), + "warm-scoped query must pass while another drive is cold" + ); + } + + /// Hot counts as ready; Unknown (never loaded) does not. + #[test] + fn hot_is_ready_unknown_is_not() { + let drives = vec![ + info('C', Some(ShardTier::Hot)), + info('E', Some(ShardTier::Unknown)), + ]; + let cold = not_ready(&drives, &[]); + assert_eq!(cold, vec![ + DriveLetter::parse('E').unwrap_or(DriveLetter::C) + ]); + } + + /// A pre-tiering daemon (no tier field) never demotes — treat its + /// shards as warm rather than gating every query forever. + #[test] + fn missing_tier_reads_as_ready() { + let drives = vec![info('C', None)]; + assert_eq!(not_ready(&drives, &[]), Vec::::new()); + } +} diff --git a/crates/uffs-version/src/lib.rs b/crates/uffs-version/src/lib.rs index 9d69fe84a..a00d33ecc 100644 --- a/crates/uffs-version/src/lib.rs +++ b/crates/uffs-version/src/lib.rs @@ -161,6 +161,36 @@ pub fn emit_build_env() { // movement, so it is the reliable trigger. println!("cargo:rerun-if-changed=../../.git/HEAD"); println!("cargo:rerun-if-changed=../../.git/logs/HEAD"); + // …and `.git/index` for the `-dirty` half, which the two above do not + // cover at all: cleanliness is not a property of HEAD. + // + // Observed: a build made while the tree was dirty kept reporting + // `-dirty` long after the tree was clean, because HEAD had not + // moved since, so this script never re-ran. It took an empty commit + // to shake the stamp loose. The same staleness runs the other way and + // is worse — a build from a clean tree goes on claiming clean after + // the tree is edited, which is precisely the false reassurance the + // suffix exists to prevent. + // + // `.git/index` is rewritten by `add`, `commit`, `checkout` and `reset`, + // so it fires on those transitions. + // + // It does NOT cover a bare unstaged edit, which is the common case. + // Measured, not assumed: appending to a tracked file and running + // `git status` left `.git/index`'s mtime unchanged, so the trigger + // never fired and the build kept the previous stamp — `git status` + // only rewrites the index when it refreshes racily-clean stat + // entries, not when it merely observes a modified file. + // + // So the suffix is best-effort and skewed safe-ish: it goes stale in + // the "says clean while edited" direction until some git command + // touches the index. No cargo trigger can be airtight — "working + // tree cleanliness" is not a file — and the alternatives trade worse: + // forcing a re-run on every build spends two `git` subprocesses per + // cargo invocation, and dropping the suffix loses the signal + // entirely. Treat `-dirty` as informative when present, not as proof + // of cleanliness when absent. + println!("cargo:rerun-if-changed=../../.git/index"); println!("cargo:rerun-if-env-changed=RUSTC"); } diff --git a/crates/uffs-watchdog/src/main.rs b/crates/uffs-watchdog/src/main.rs index f433552e7..5d6eead8a 100644 --- a/crates/uffs-watchdog/src/main.rs +++ b/crates/uffs-watchdog/src/main.rs @@ -48,11 +48,30 @@ //! reading the MFT read as dead and was liable to be respawned on top of //! itself. //! -//! Liveness now comes from `uffs --status --json`, which reports every -//! service under its own key, so one service's state can never be -//! mistaken for another's. An unreadable probe means *unknown*, and -//! unknown is always left alone — a supervisor that restarts things -//! because it could not see them is worse than none. +//! Liveness now comes from `uffs --status --json --brief`, which +//! reports every service under its own key, so one service's state can +//! never be mistaken for another's. An unreadable probe means +//! *unknown*, and unknown is always left alone — a supervisor that +//! restarts things because it could not see them is worse than none. +//! +//! # Cost, and why the interval breathes +//! +//! A supervisor spends almost all of its life confirming that two +//! healthy processes are still healthy. Two things keep that from +//! being wasteful: +//! +//! * `--brief` trims the probe to a socket connect and a PID-file read. The +//! full `--status --json` also does an SCM query, a named-pipe probe with a +//! timeout, and a process enumeration that spawns a shell (plus one per +//! session found) — about four shell launches per call on Windows, which at a +//! fixed 5 s poll is ~69 000 a day to answer two booleans. +//! * The interval backs off from 5 s toward 60 s once nothing has changed for a +//! while, and snaps back the moment anything does. +//! +//! Both matter for correctness, not just tidiness: a health check +//! expensive enough to be affected by system load can fail *because* +//! the machine is struggling, which is precisely when a supervisor +//! must not. //! //! # Why a separate binary //! @@ -69,14 +88,35 @@ use core::time::Duration; use supervise::{Action, RespawnLedger, decide}; -/// How often liveness is polled. +/// How often liveness is polled while anything is changing. /// /// Seconds-scale: a respawn that lands within a few seconds of a crash /// is indistinguishable from "never went away" for an interactive -/// search, and the probe is one cheap status call covering every -/// supervised service. +/// search. const POLL_INTERVAL: Duration = Duration::from_secs(5); +/// Ceiling the interval backs off to once nothing has changed. +/// +/// A supervisor spends essentially all of its life watching two +/// processes that are fine. Paying the full poll rate for that is +/// wasted work — and worse, a health check that is expensive can fail +/// *because* the machine is loaded, which is exactly when it must not. +/// After [`STABLE_TICKS_BEFORE_BACKOFF`] uneventful ticks the interval +/// doubles each time up to this ceiling, and any event at all snaps it +/// straight back to [`POLL_INTERVAL`]. +/// +/// Worst-case detection latency becomes this value instead of +/// `POLL_INTERVAL`; for a crashed background service that is not a +/// difference anyone can perceive. +const MAX_POLL_INTERVAL: Duration = Duration::from_secs(60); + +/// Uneventful ticks tolerated before the interval starts growing. +/// +/// Non-zero so a flapping service is still watched at full rate: a +/// crash resets the counter, so backoff only happens during genuine +/// calm. +const STABLE_TICKS_BEFORE_BACKOFF: u32 = 6; + /// Environment override for [`POLL_INTERVAL`], in seconds (tests, and /// operators who want a tighter or looser loop). const POLL_ENV: &str = "UFFS_WATCHDOG_POLL_SECS"; @@ -129,27 +169,51 @@ fn main() -> anyhow::Result<()> { }, ]; - eprintln!("uffs-watchdog armed (poll {}s)", poll.as_secs()); + eprintln!( + "uffs-watchdog armed (poll {}s, backing off to {}s while stable)", + poll.as_secs(), + MAX_POLL_INTERVAL.as_secs() + ); + let mut interval = poll; + let mut calm_ticks: u32 = 0; loop { // One snapshot per tick, shared by every service: the probe is a // single subprocess rather than one per service, and every // decision in a tick is taken against the same instant. let snapshot = status_snapshot(); + let mut eventful = false; for service in &mut services { - tick(service, snapshot.as_deref()); + eventful |= tick(service, snapshot.as_deref()); + } + // Anything happening at all — a respawn, a stop honoured, a + // service first seen — restores full rate. Calm compounds. + if eventful { + calm_ticks = 0; + interval = poll; + } else { + calm_ticks = calm_ticks.saturating_add(1); + if calm_ticks > STABLE_TICKS_BEFORE_BACKOFF { + interval = (interval * 2).min(MAX_POLL_INTERVAL); + } } - std::thread::sleep(poll); + std::thread::sleep(interval); } } /// Take one machine-readable snapshot of every service's liveness. /// -/// `uffs --status --json` connects with `connect_raw`, which never -/// auto-spawns anything, so probing has no side effects — an important -/// property for something that runs every few seconds forever. +/// `--brief` is the point: the full `--status --json` additionally does +/// an SCM query, a named-pipe probe with a timeout, and a process +/// enumeration that spawns a shell — plus another per session found. +/// On Windows that is ~4 shell launches per call, and at this poll rate +/// ~69 000 a day, to answer two booleans. `--brief` is a socket +/// connect and a PID-file read. +/// +/// `connect_raw` never auto-spawns anything, so probing stays free of +/// side effects — essential for something that runs forever. fn status_snapshot() -> Option { let out = std::process::Command::new(uffs_exe()) - .args(["--status", "--json"]) + .args(["--status", "--json", "--brief"]) .output() .ok()?; Some(String::from_utf8_lossy(&out.stdout).into_owned()) @@ -201,25 +265,35 @@ fn lifecycle_dir() -> std::path::PathBuf { } /// Evaluate and act on one service for this tick. +/// +/// Returns `true` when something happened worth reacting to — a +/// service seen for the first time, or found down. The caller uses +/// that to keep the poll interval tight while things are moving and +/// let it back off during calm. #[expect( clippy::print_stderr, reason = "a supervisor's log IS its user interface; it has no other channel" )] -fn tick(service: &mut Service, snapshot: Option<&str>) { +fn tick(service: &mut Service, snapshot: Option<&str>) -> bool { // Unknown liveness is not death: leave the service exactly as it is - // and try again next tick. + // and try again next tick. Also not an event — an unreadable probe + // must not hold the loop at full rate forever. let Some(alive) = snapshot.and_then(|doc| running(doc, service.status_key)) else { - return; + return false; }; if alive { + // First sighting is an event: supervision has just begun for + // this service, and the next few ticks are the ones worth + // watching closely. + let first_sighting = !service.seen_running; service.seen_running = true; - return; + return first_sighting; } // Never *introduce* a service the user has not run themselves: a // machine that never starts the MCP gateway should not acquire one // because a watchdog is present. if !service.seen_running { - return; + return false; } let now = std::time::Instant::now(); let recent = service.ledger.recent(now); @@ -261,6 +335,10 @@ fn tick(service: &mut Service, snapshot: Option<&str>) { } } } + // A service being down is an event regardless of what we decided to + // do about it — including honouring a stop, since the operator is + // evidently touching things right now. + true } /// The `uffs` CLI to drive, resolved next to this binary so a watchdog diff --git a/docs/user-manual/installation.md b/docs/user-manual/installation.md index 07c629967..13841f387 100644 --- a/docs/user-manual/installation.md +++ b/docs/user-manual/installation.md @@ -262,6 +262,16 @@ them: * **The watchdog is stopped first and restarted last.** If it kept running during the install it would dutifully restart the daemon mid-teardown — the supervisor fighting the installer. +* **AI-host MCP sessions survive.** Teardown force-kills only the + daemon and the HTTP gateway *by PID*, never `uffsmcp` by image name — + that would also kill the stdio supervisors your AI hosts spawned, and + a session killed mid-request looks exactly like a hung tool call. A + binary that is locked by a running process is **renamed aside** rather + than deleted, so the new one lands at the canonical path and the + [stdio supervisor](mcp.md#zero-downtime-upgrades-stdio-supervisor) + hot-swaps its worker without dropping the host's connection. The + renamed `*.oldN` sidecars are swept by the next install, once the old + process has exited. * **The daemon and MCP server are restarted only if they were running before**, and the daemon comes back through the normal start path, so a resident daemon returns resident (`--no-retire`). diff --git a/docs/user-manual/mcp.md b/docs/user-manual/mcp.md index 9678f999a..27018aa05 100644 --- a/docs/user-manual/mcp.md +++ b/docs/user-manual/mcp.md @@ -215,6 +215,29 @@ The MCP server exposes six read-only tools. All are annotated as | `uffs_drives` | — | List all indexed drives with record counts | | `uffs_status` | — | Daemon health, uptime, memory, loading progress | +### Cold-index behaviour — the warming contract + +Drives tier down when idle ([Parked after 30 min, Cold after +24 h](daemon.md#memory-tiers--and-why-you-never-see-hot)), and the +re-warm on next touch can take 30–120 s on HDD-heavy systems. A query +tool never blocks for that. When a query would touch a drive that is +not `Warm`/`Hot`, the tool: + +1. **starts the re-warm immediately** in a detached request that + survives whatever the host does to the tool call, and +2. **returns an error right away**: *"⏳ Index warming — drive(s) D, S + were parked/cold … Poll `uffs_status` until every drive reports + 'warm', then retry this exact query."* + +So an agent's loop is: query → warming error → poll `uffs_status` +(never gated) → retry → millisecond answer. Repeat triggers are free — +the daemon's single-flight per-drive dedup joins the in-flight load +instead of starting another. Queries scoped to warm drives are never +gated by some *other* drive being cold. + +This mirrors the startup gate (`⏳ Daemon is starting up — X/Y drives +loaded`), which covers initial load the same way. + ### `uffs_search` The primary query tool. Accepts a `pattern` and any combination of filters diff --git a/just/shared.just b/just/shared.just index e47ca61a9..1dfca9396 100644 --- a/just/shared.just +++ b/just/shared.just @@ -192,6 +192,38 @@ toolchain-sync: exit 1 fi + # Raise the file-descriptor limit before any zig link. + # + # macOS ships a soft `RLIMIT_NOFILE` of 256. A zig link of this + # workspace passes ~240 objects plus ~50 rlibs and needs more than + # that, so it dies with: + # + # error: unable to search for static library …/libaddr2line-….rlib: + # ProcessFdQuotaExceeded + # + # toolchain-sync reads that as "this nightly is broken", steps back a + # day, and repeats until it hits the floor — aborting the ship with + # "current pin no longer compiles for x86_64-unknown-linux-gnu" when + # nothing is wrong with any toolchain. Every candidate passes clippy + # and the Windows gate; only the Linux probe fails, always on fds. + # + # Measured on this host: at soft 256 the link fails, and the same + # command at 4096 exits 0 — package-scoped and for the full + # `--workspace --all-features`. + # + # Only the SOFT limit moves; the hard limit is the ceiling and + # lowering it would be irreversible for this shell. If the hard + # limit is below the target we take whatever it allows. The result + # is printed: a guard that silently does nothing looks exactly like + # one that worked, and an earlier version of this block did nothing + # for two commits because of it. + FD_BEFORE=$(ulimit -Sn 2>/dev/null || echo unknown) + if [ "$FD_BEFORE" != "unlimited" ] && [ "$FD_BEFORE" -lt 4096 ] 2>/dev/null; then + ulimit -S -n 4096 2>/dev/null || ulimit -S -n "$(ulimit -Hn)" 2>/dev/null || true + fi + printf " \033[0;36mfd limit: %s → %s (zig linker needs headroom)\033[0m\n" \ + "$FD_BEFORE" "$(ulimit -Sn)" + START_DATE=$(date -u +%Y-%m-%d) printf "\033[0;34m Start date: nightly-%s (today UTC)\033[0m\n\n" "$START_DATE" diff --git a/just/test.just b/just/test.just index 93864678e..2eff7d901 100644 --- a/just/test.just +++ b/just/test.just @@ -194,6 +194,14 @@ lint-ci-linux-zig: # subcommand args; the binary itself exposes `clippy` / `check` / # `test` / etc. as proper subcommands that wire up the same zig-cc # linker shim cargo-zigbuild's `zigbuild` does. + # Raise RLIMIT_NOFILE first: macOS defaults the soft limit to 256, + # and a zig link of this workspace needs more, failing with + # `ProcessFdQuotaExceeded`. Measured: fails at 256, exits 0 at + # 4096. Soft limit only — see the long comment in `toolchain-sync`. + FD_BEFORE=$(ulimit -Sn 2>/dev/null || echo unknown); \ + if [ "$FD_BEFORE" != "unlimited" ] && [ "$FD_BEFORE" -lt 4096 ] 2>/dev/null; then \ + ulimit -S -n 4096 2>/dev/null || ulimit -S -n "$(ulimit -Hn)" 2>/dev/null || true; \ + fi; \ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C target-cpu=x86-64-v3 -C link-arg=-Wl,--gc-sections" \ cargo-zigbuild clippy --workspace --all-targets --all-features --target x86_64-unknown-linux-gnu --no-deps -- -D warnings @printf "\033[0;32m✅ Linux clippy gate (zigbuild) passed\033[0m\n" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e6cacc5fe..60c26d537 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -35,7 +35,7 @@ # Run `just toolchain-sync` to re-attempt a channel bump; the CI # pipeline auto-refreshes on `ship --fresh` unless `--skip-toolchain-sync` # is passed. -channel = "nightly-2026-08-13" +channel = "nightly-2026-08-14" # Specify components that should always be available components = [ diff --git a/scripts/dev/install-bins.rs b/scripts/dev/install-bins.rs index 4d37ee294..f1e80c5bd 100755 --- a/scripts/dev/install-bins.rs +++ b/scripts/dev/install-bins.rs @@ -95,6 +95,10 @@ fn main() { let mut skipped = 0_u32; let mut unchanged = 0_u32; eprintln!(); + // Clear sidecars left by earlier rename-swaps whose old process has + // since exited. Ones still held stay put and are swept next time. + sweep_sidecars(&bin_dir); + eprintln!("📦 Installing {} binaries to {}", executables.len(), bin_dir.display()); for src in &executables { let Some(file_name) = src.file_name() else { @@ -136,7 +140,18 @@ fn main() { if dest.is_dir() { let _ = std::fs::remove_dir_all(&dest); } - let _ = std::fs::remove_file(&dest); + // A running image cannot be deleted on Windows — but it can be + // renamed, which frees the path without touching the live + // process. That is how a binary gets replaced under a running + // stdio supervisor: it notices its image changed and hot-swaps + // its worker, so the AI host's session never drops. + if dest.exists() && std::fs::remove_file(&dest).is_err() { + if rename_aside(&dest).is_none() { + eprintln!(" ❌ {name:<28} in use and could not be moved aside"); + skipped += 1; + continue; + } + } match std::fs::copy(src, &dest) { Ok(bytes) => { #[cfg(unix)] @@ -189,6 +204,104 @@ fn main() { } } +/// Move a locked binary aside so a fresh one can take its canonical +/// path, returning where it went. +/// +/// Windows refuses to delete or overwrite a running image but happily +/// **renames** it: the running process keeps its handle to the old +/// inode while the path is freed. That is the whole mechanism behind +/// the zero-downtime upgrade the MCP stdio supervisor implements — it +/// watches its own image path and hot-swaps its worker when the bytes +/// there change, so a rename-swap upgrades a live agent session +/// without dropping the host's connection. +/// +/// Before this existed the installer took the blunt route and +/// force-killed every `uffsmcp.exe` by image name to unlock the file, +/// which killed the stdio supervisors belonging to interactive agent +/// sessions — the exact failure the supervisor was written to prevent, +/// and a plausible source of "the MCP call was accepted and then +/// vanished" reports. +/// +/// Returns `None` when even the rename fails, which means the caller +/// must skip that binary rather than corrupt the install. +fn rename_aside(dest: &std::path::Path) -> Option { + for attempt in 0..64_u32 { + let sidecar = dest.with_extension(format!("old{attempt}")); + if sidecar.exists() { + // A previous run's sidecar still held by a live process. + continue; + } + if std::fs::rename(dest, &sidecar).is_ok() { + return Some(sidecar); + } + } + None +} + +/// Delete leftover `*.oldN` sidecars from previous rename-swaps. +/// +/// A sidecar stays on disk while the old process still has it open; +/// once that process exits the file is deletable, so every later +/// install sweeps them. Failures are ignored — a sidecar still in use +/// simply waits for the next run. +fn sweep_sidecars(bin_dir: &std::path::Path) { + let Ok(entries) = std::fs::read_dir(bin_dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let is_sidecar = path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.starts_with("old") && ext[3..].chars().all(|c| c.is_ascii_digit())); + if is_sidecar { + let _ = std::fs::remove_file(&path); + } + } +} + +/// PID of a service reported by `uffs --status --json`, if it is running. +/// +/// Used to scope teardown kills to the process we actually mean. +/// Killing `uffsmcp.exe` by image name also kills every stdio +/// supervisor an AI host has spawned; those are other people's +/// sessions, not ours to end. +fn service_pid(key: &str) -> Option { + let out = Command::new("uffs") + .args(["--status", "--json"]) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let section = text.split(&format!("\"{key}\":")).nth(1)?; + // Sections are small objects; the first `"pid":` after the key is + // that service's own. + let after = section.split("\"pid\":").nth(1)?; + after + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect::() + .parse() + .ok() +} + +/// Force-kill one PID (last-resort backstop after a graceful stop). +fn kill_pid(pid: u32) { + let _ = if cfg!(windows) { + Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/F"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + } else { + Command::new("kill") + .args(["-9", &pid.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + }; +} + /// True when `src` and `dest` are byte-identical, so the copy can be /// skipped entirely. Compares length first (cheap, rejects almost every /// changed binary) and only then the contents. A missing or unreadable @@ -414,6 +527,9 @@ fn restart_mcp(bin_dir: &std::path::Path) { fn stop_running_services() { eprintln!(); eprintln!("🔪 Stopping daemon + MCP (best effort)..."); + // Capture PIDs before the graceful stops clear the PID files, so the + // force-kill backstops below can target exactly these processes. + let daemon_pid = service_pid("daemon"); if let Ok(mut child) = Command::new("uffs") .args(["--daemon", "kill"]) .stdout(Stdio::null()) @@ -434,17 +550,31 @@ fn stop_running_services() { } } } - // Ask the MCP gateway to stop cleanly first. The `taskkill /F` - // below is a `/F` by image name: it kills every `uffsmcp` process - // outright, so the gateway never removes its PID file and the next - // `--mcp status` reports `not running (stale PID file, PID …)`. - // A graceful stop leaves no such litter; the force-kill stays as the - // backstop for a wedged process. + // Ask the MCP gateway to stop cleanly. Capture its PID first: if it + // ignores the request we force-kill THAT process and nothing else. + // + // This used to be `taskkill /IM uffsmcp.exe /F`, which killed every + // `uffsmcp` on the box — including the stdio supervisors that AI + // hosts spawn for interactive agent sessions. Those belong to + // other people's sessions; ending them mid-request produces exactly + // the "call accepted, then silently lost, never returns" signature + // that got reported against the MCP bridge. Binary replacement no + // longer needs the kill either: `rename_aside` frees the path + // without touching the running process, which is the zero-downtime + // upgrade path the supervisor already implements. + let gateway_pid = service_pid("mcp_http"); let _ = Command::new("uffs") .args(["--mcp", "stop"]) .stdout(Stdio::null()) .stderr(Stdio::null()) .status(); + // Nested rather than a `let`-chain: rust-script compiles this file + // on its default edition, where `if let … && …` is not available. + if let Some(pid) = gateway_pid { + if mcp_is_running() { + kill_pid(pid); + } + } // The watchdog is stopped FIRST: if it kept running while we tear // the daemon down, it would dutifully restart it mid-install — the // supervisor fighting the installer. It is restarted at the end. @@ -463,21 +593,19 @@ fn stop_running_services() { .status() }; } - for name in ["uffsd", "uffsmcp"] { - let status = if cfg!(windows) { - Command::new("taskkill") - .args(["/IM", &format!("{name}.exe"), "/F"]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - } else { - Command::new("pkill") - .args(["-x", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - }; - let _ = status; + // The daemon: `uffs --daemon kill` above already targets the PID in + // the PID file and cleans up after itself. This is the backstop for + // the case where that failed, still scoped to the one PID the daemon + // reports rather than every `uffsd` image on the box. + // + // `uffsmcp` is deliberately absent from any image-name kill: see the + // gateway block above. Stdio supervisors owned by AI hosts survive + // an install now, and `rename_aside` replaces their binary underneath + // them so they hot-swap instead of dying. + if let Some(pid) = daemon_pid { + if daemon_is_running() { + kill_pid(pid); + } } }