From a5326d97f7f6651fe2b767804ebdb974c4a330e9 Mon Sep 17 00:00:00 2001 From: Benjamin Arntzen Date: Fri, 26 Jun 2026 18:38:22 +0100 Subject: [PATCH] feat(server): add /metrics endpoint + HTTP request metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concurrent PXE imaging can starve Dragonfly's HTTP serving pipeline, but there was no visibility into it (no /metrics, only TraceLayer logs). Add a Prometheus /metrics endpoint plus a request middleware that records, per bounded endpoint label: in-flight gauge, request counter (method/status), and latency histogram. Endpoint labels are a coarse classification of the path (classify), not the raw path — /boot/{mac} is high-cardinality; collapsing to "boot"/"image"/"os_image"/ etc. keeps the histogram meaningful under a boot storm. Uses metrics + metrics-exporter-prometheus (axum-prometheus was avoided: it dragged tower 0.5 + tower-http 0.6 alongside the workspace's 0.4/0.5). The recorder handle is cached in a OnceLock so /metrics needs no AppState change. Wire: install the recorder at server start (run), register GET /metrics, and apply the middleware as the outermost router layer. Tests: classify truth-table + cardinality bound, and an end-to-end test driving a request through the instrumented router and scraping /metrics for the counter/histogram/labels. Full dragonfly-server suite green (193). Co-Authored-By: Claude --- Cargo.lock | 195 ++++++++++++++++++- crates/dragonfly-server/Cargo.toml | 4 +- crates/dragonfly-server/src/lib.rs | 10 + crates/dragonfly-server/src/metrics_mw.rs | 220 ++++++++++++++++++++++ 4 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 crates/dragonfly-server/src/metrics_mw.rs diff --git a/Cargo.lock b/Cargo.lock index 3c5cd314..4c70caa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,7 +1660,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1951,6 +1951,8 @@ dependencies = [ "lazy_static", "libc", "listenfd", + "metrics", + "metrics-exporter-prometheus", "mime_guess", "minijinja", "minijinja-autoreload", @@ -2161,7 +2163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2201,6 +2203,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "expanduser" version = "1.2.2" @@ -2505,6 +2518,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if 1.0.4", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result 0.4.1", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2667,6 +2695,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3130,6 +3164,7 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "rustls 0.23.36", + "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -3182,7 +3217,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -3791,6 +3826,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "left-right" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libc" version = "0.2.182" @@ -3977,6 +4023,19 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if 1.0.4", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.16.3" @@ -4075,6 +4134,56 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64 0.22.1", + "evmap", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-util", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "rustls 0.23.36", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.2", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -4530,7 +4639,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5453,6 +5562,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.9" @@ -5466,7 +5590,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.36", - "socket2 0.5.10", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -5503,9 +5627,9 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5588,6 +5712,24 @@ version = "0.10.0-rc-3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f66ee92bc15280519ef199a274fe0cafff4245d31bc39aaa31c011ad56cb1f05" +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +dependencies = [ + "rustversion", +] + [[package]] name = "ratatui" version = "0.30.0" @@ -5673,6 +5815,15 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "rayon" version = "1.11.0" @@ -6264,7 +6415,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6320,6 +6471,18 @@ dependencies = [ "security-framework 2.11.1", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.6.0", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -6468,6 +6631,12 @@ dependencies = [ "syn 2.0.116", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -6867,6 +7036,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -7377,7 +7552,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix 1.1.3", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8583,7 +8758,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/dragonfly-server/Cargo.toml b/crates/dragonfly-server/Cargo.toml index e40ae4b8..f04e4ae1 100644 --- a/crates/dragonfly-server/Cargo.toml +++ b/crates/dragonfly-server/Cargo.toml @@ -149,6 +149,8 @@ http = "1" # Embed static assets in release builds rust-embed = { version = "8", features = ["interpolate-folder-path"] } mime_guess = "2" +metrics = "0.24.6" +metrics-exporter-prometheus = "0.18.3" [build-dependencies] minijinja-embed = "2.3.0" @@ -157,4 +159,4 @@ minijinja-embed = "2.3.0" tempfile = "3.8" serial_test = "3.0" # WebSocket client for integration tests (agent push + events stream). -tokio-tungstenite = { version = "0.27" } \ No newline at end of file +tokio-tungstenite = { version = "0.27" } diff --git a/crates/dragonfly-server/src/lib.rs b/crates/dragonfly-server/src/lib.rs index d8caa78b..b3d26757 100644 --- a/crates/dragonfly-server/src/lib.rs +++ b/crates/dragonfly-server/src/lib.rs @@ -1,5 +1,6 @@ use anyhow::{Context, anyhow}; use axum::extract::{MatchedPath, Path}; +use axum::middleware::from_fn; use axum::{Router, extract::Extension, http::StatusCode, response::IntoResponse, routing::get}; use axum_login::AuthManagerLayerBuilder; use listenfd::ListenFd; @@ -50,6 +51,7 @@ mod filters; // Uncomment unused module pub mod ha; pub mod handlers; pub mod image_cache; +pub mod metrics_mw; pub mod mode; pub mod network_detect; pub mod os_templates; @@ -794,6 +796,10 @@ pub async fn run() -> anyhow::Result<()> { .with(fmt::layer()); // --- Logging Initialized --- + // Install the Prometheus recorder (global default) before serving traffic so + // /metrics and the request middleware have somewhere to record from the start. + metrics_mw::install().context("install prometheus recorder")?; + // Determine modes SECOND (after logging is set up) let is_installation_server = std::env::var("DRAGONFLY_INSTALL_SERVER_MODE").is_ok(); let is_explicit_demo_mode = std::env::var("DRAGONFLY_DEMO_MODE").is_ok(); @@ -1325,6 +1331,8 @@ pub async fn run() -> anyhow::Result<()> { .merge(auth_router()) .merge(ui::ui_router()) .route("/favicon.ico", get(handle_favicon)) + // Prometheus metrics for the HTTP serving pipeline (see metrics_mw). + .route("/metrics", get(metrics_mw::serve_metrics)) // Boot endpoints - /boot/{mac} for iPXE scripts, /boot/{arch}/{asset} for kernel/initramfs .route("/boot/{mac}", get(api::ipxe_script)) // Spark ELF - bare metal discovery agent (loaded by GRUB via multiboot2) @@ -1402,6 +1410,8 @@ pub async fn run() -> anyhow::Result<()> { tracing::error!(parent: span, latency = ?latency, error = ?error, "Request failed"); }) ) + // Per-request Prometheus metrics: count, latency, in-flight (see metrics_mw). + .layer(from_fn(metrics_mw::middleware)) .with_state(app_state.clone()); // State applied here // Start handoff listener and network services diff --git a/crates/dragonfly-server/src/metrics_mw.rs b/crates/dragonfly-server/src/metrics_mw.rs new file mode 100644 index 00000000..119cbffd --- /dev/null +++ b/crates/dragonfly-server/src/metrics_mw.rs @@ -0,0 +1,220 @@ +// Dragonfly +// Copyright (C) Riff Labs Limited +// +// Prometheus `/metrics` endpoint + HTTP request middleware. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +//! Prometheus metrics for the HTTP serving pipeline. +//! +//! Exposes an in-flight request gauge, a per-endpoint request counter, and a +//! per-endpoint latency histogram at `GET /metrics` (text/plain; version=0.0.4) +//! so the serving pipeline — the surface that starves under concurrent PXE +//! imaging — can be scraped and bisected. +//! +//! Endpoint labels are a coarse, *bounded* classification of the request path +//! ([`classify`]). The raw path (e.g. `/boot/{mac}`) is high-cardinality; the +//! label groups by logical serve surface so the latency histogram stays +//! meaningful under load. + +use axum::extract::Request; +use axum::http::{StatusCode, header}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use metrics_exporter_prometheus::{BuildError, PrometheusBuilder, PrometheusHandle}; +use std::sync::OnceLock; +use std::time::Instant; + +static METRICS_HANDLE: OnceLock = OnceLock::new(); + +/// Install the Prometheus recorder as the process-global default and cache its +/// render handle. Call once at server start; a second call leaves the cached +/// handle in place (the recorder itself is process-global and rejects re-install +/// by returning an error, which we ignore after the first success). +pub fn install() -> Result<(), BuildError> { + let handle = PrometheusBuilder::new().install_recorder()?; + let _ = METRICS_HANDLE.set(handle); + Ok(()) +} + +/// Map a request path to a bounded endpoint label. +/// +/// Coarse on purpose: `/boot/{mac}`, `/boot/{arch}/{asset}`, and the static +/// boot blobs all collapse to `boot`; the big OS image (`/images/...`) and the +/// JIT-converted `/os/...` are distinguished because they are the bandwidth- and +/// CPU-bound surfaces. This keeps label cardinality small (a handful) regardless +/// of how many distinct MACs/IPs boot. +fn classify(path: &str) -> &'static str { + if path.starts_with("/images/") { + "image" + } else if path.starts_with("/os/") { + "os_image" + } else if path.starts_with("/boot-debian/") { + "boot_debian_asset" + } else if path.starts_with("/boot/pxelinux.cfg") { + "pxelinux_config" + } else if path.starts_with("/boot/") { + "boot" + } else if path.starts_with("/ipxe/") { + "ipxe_artifact" + } else if path.starts_with("/api/") { + "api" + } else if path == "/metrics" || path == "/favicon.ico" { + "infra" + } else { + "other" + } +} + +/// Map an HTTP method to a bounded `&'static str` label (avoids borrowing the +/// request method into the metrics label macros). +fn method_label(method: &axum::http::Method) -> &'static str { + match *method { + axum::http::Method::GET => "GET", + axum::http::Method::POST => "POST", + axum::http::Method::PUT => "PUT", + axum::http::Method::DELETE => "DELETE", + axum::http::Method::PATCH => "PATCH", + axum::http::Method::HEAD => "HEAD", + axum::http::Method::OPTIONS => "OPTIONS", + _ => "other", + } +} + +/// axum middleware: count requests, time them, and track in-flight, all labelled +/// by the bounded endpoint classification. +pub async fn middleware(req: Request, next: Next) -> Response { + let endpoint = classify(req.uri().path()); + let method = method_label(req.method()); + metrics::gauge!("http_requests_in_flight", "endpoint" => endpoint).increment(1.0); + let start = Instant::now(); + let resp = next.run(req).await; + metrics::gauge!("http_requests_in_flight", "endpoint" => endpoint).decrement(1.0); + let status = resp.status().as_u16(); + metrics::counter!( + "http_requests_total", + "endpoint" => endpoint, + "method" => method, + "status" => status.to_string() + ) + .increment(1); + metrics::histogram!("http_request_duration_seconds", "endpoint" => endpoint) + .record(start.elapsed().as_secs_f64()); + resp +} + +/// `GET /metrics` — render recorded metrics in Prometheus text format. +pub async fn serve_metrics() -> Response { + let body = METRICS_HANDLE + .get() + .map(PrometheusHandle::render) + .unwrap_or_default(); + ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/plain; version=0.0.4")], + body, + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_routes_to_bounded_labels() { + assert_eq!(classify("/images/debian-13-amd64.raw"), "image"); + assert_eq!(classify("/os/debian-13/amd64"), "os_image"); + assert_eq!(classify("/boot-debian/amd64/vmlinuz"), "boot_debian_asset"); + assert_eq!(classify("/boot/aa:bb:cc:dd:ee:ff"), "boot"); + assert_eq!(classify("/boot/amd64/initrd.img"), "boot"); + assert_eq!(classify("/boot/pxelinux.cfg/default"), "pxelinux_config"); + assert_eq!(classify("/ipxe/debian/13"), "ipxe_artifact"); + assert_eq!(classify("/api/machines"), "api"); + assert_eq!(classify("/metrics"), "infra"); + assert_eq!(classify("/favicon.ico"), "infra"); + assert_eq!(classify("/something/else"), "other"); + } + + #[test] + fn classify_is_bounded_under_high_cardinality_inputs() { + // Distinct MACs must NOT each become a distinct label. + let mut labels = std::collections::HashSet::new(); + for i in 0..1000 { + labels.insert(classify(&format!("/boot/aa:bb:cc:dd:ee:{i:02x}"))); + } + assert_eq!(labels.len(), 1); + assert_eq!(labels.into_iter().next().unwrap(), "boot"); + } + + #[tokio::test] + async fn metrics_endpoint_reports_recorded_requests() { + use axum::body::{self, Body}; + use axum::http::Request; + use axum::middleware::from_fn as test_from_fn; + use axum::routing::get; + use tower::ServiceExt; + + async fn ok_handler() -> &'static str { + "ok" + } + + // Install the process-global recorder (ignore a re-install from a prior test). + let _ = install(); + + let app = axum::Router::new() + .route("/boot/test", get(ok_handler)) + .route("/metrics", get(serve_metrics)) + .layer(test_from_fn(middleware)); + + // Drive one request through the instrumented boot path. + let probe = app + .clone() + .oneshot( + Request::builder() + .uri("/boot/test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(probe.status(), axum::http::StatusCode::OK); + + // Scrape /metrics. + let resp = app + .oneshot( + Request::builder() + .uri("/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + let ct = resp + .headers() + .get(axum::http::header::CONTENT_TYPE) + .expect("content-type") + .to_str() + .unwrap() + .to_owned(); + assert!(ct.starts_with("text/plain"), "content-type: {ct}"); + let bytes = body::to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let body = String::from_utf8(bytes.to_vec()).unwrap(); + assert!( + body.contains("http_requests_total"), + "missing counter: {body}" + ); + assert!( + body.contains("http_request_duration_seconds"), + "missing histogram: {body}" + ); + assert!( + body.contains("endpoint=\"boot\""), + "expected boot label: {body}" + ); + } +}