diff --git a/.config/test-budgets.yaml b/.config/test-budgets.yaml index 2cbc1e0ba61..cfd0e29bdee 100644 --- a/.config/test-budgets.yaml +++ b/.config/test-budgets.yaml @@ -3,9 +3,10 @@ # Wall-clock budgets for each test step, in seconds, enforced on the Linux CI # runners; other platforms report their times without failing. wall_seconds -# bounds a step's run, or each lifecycle partition; test_seconds bounds any one -# test in it. Lower a budget when a change makes the step faster; raising one -# needs the reason in the pull request. See docs/contributing/testing.md. +# bounds a step's nextest run, the live-docker step's nextest commands together, +# or each lifecycle partition; test_seconds bounds any one test in it. Lower a +# budget when a change makes the step faster; raising one needs the reason in +# the pull request. See docs/contributing/testing.md. # Unit and integration tests: 27 s on linux_amd64, slowest test 5.1 s. test: @@ -16,3 +17,19 @@ test: lifecycle: wall_seconds: 360 test_seconds: 120 + +# Live / Docker, its four nextest commands together: at most 255 s on +# linux_amd64 and 210 s on linux_arm64 in 11 v1 runs. Its slowest test, +# retained_runtime_rebuilds_offline_without_protobuf_or_a_dependency_cache, +# took 110 s on linux_amd64 in a full report; in the v1 runs, whose reports +# kept only the last command, it ran past 120 s in 5 of 11 linux_amd64 runs +# and, by its command's wall time, under about 150 s. +live-docker: + wall_seconds: 360 + test_seconds: 180 + +# Live / Kind: at most 199 s on linux_amd64 and 171 s on linux_arm64 in 11 v1 +# runs; slowest test 83 s. +live-kind: + wall_seconds: 270 + test_seconds: 120 diff --git a/.github/workflows/native-platform.yml b/.github/workflows/native-platform.yml index ffde68f8c5c..90a56a080ee 100644 --- a/.github/workflows/native-platform.yml +++ b/.github/workflows/native-platform.yml @@ -158,6 +158,36 @@ jobs: if-no-files-found: ignore retention-days: 14 + # Each partition reports and enforces its own budget; this job reports the + # platform's whole lifecycle suite in its summary, from the partitions' + # uploaded timings, including failed partitions'. It never fails the platform. + lifecycle-timing: + name: Lifecycle timing + needs: [build, lifecycle] + if: ${{ !cancelled() && needs.build.result == 'success' && needs.lifecycle.result != 'skipped' }} + continue-on-error: true + runs-on: ${{ inputs.runner }} + timeout-minutes: 10 + steps: + # The build tool comes from the lifecycle inputs; this job compiles nothing. + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.build.outputs.artifact }} + path: .build/ci + # Every attempt's timings; the report reads each shard's latest attempt. + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: lifecycle-${{ inputs.platform }}-* + path: .build/ci/timings + - name: Report the whole lifecycle suite + env: + SHARDS: ${{ inputs.lifecycle-shards }} + run: | + tool=.build/ci/nemoclaw-build + if [ "$RUNNER_OS" = Windows ]; then tool="$tool.exe"; fi + tar -xf .build/ci/lifecycle-inputs.tar "$tool" + "$tool" lifecycle-timing .build/ci/timings --platform "$TEST_PLATFORM" --shards "$SHARDS" + # The live suites reuse this platform's bundle instead of building their own, # so they start when it is ready. They are not required checks: a failure # shows on its job without failing the platform's Test / PLATFORM check. diff --git a/crates/nemoclaw-build/src/ci/timing.rs b/crates/nemoclaw-build/src/ci/timing.rs index 03cbb5de618..e60c84773fb 100644 --- a/crates/nemoclaw-build/src/ci/timing.rs +++ b/crates/nemoclaw-build/src/ci/timing.rs @@ -1,6 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Where test time goes, read from a nextest JUnit report. +//! Where test time goes, read from nextest JUnit reports. use serde::Deserialize; use std::{collections::BTreeMap, fmt::Write}; @@ -115,13 +115,28 @@ impl Run { /// each binary, then the `rows` slowest tests. #[must_use] pub fn report(&self, profile: &str, rows: usize) -> String { - let summed: f64 = self.tests.iter().map(|test| test.seconds).sum(); - let mut report = format!( - "### {profile}: {} tests, {:.1} s wall, {:.1} s summed\n\n", + let mut report = self.headline(profile); + report.push('\n'); + report.push_str(&self.tables(rows)); + report + } + + fn summed_seconds(&self) -> f64 { + self.tests.iter().map(|test| test.seconds).sum() + } + + fn headline(&self, profile: &str) -> String { + format!( + "### {profile}: {} tests, {:.1} s wall, {:.1} s summed\n", self.tests.len(), self.wall_seconds, - summed - ); + self.summed_seconds() + ) + } + + /// The slowest modules and tests, without the headline. + fn tables(&self, rows: usize) -> String { + let mut report = String::new(); let mut groups: BTreeMap<(&str, &str), (usize, f64, f64)> = BTreeMap::new(); for test in &self.tests { let module = test.name.split("::").next().unwrap_or(&test.name); @@ -158,3 +173,118 @@ impl Run { report } } + +/// Join the JUnit reports of nextest runs made one after another, such as the +/// live-docker step's, into one report whose run time is their sum. +/// +/// # Errors +/// Returns an error when there are no reports, or one is not a nextest JUnit +/// report with its run time. +pub fn join_consecutive(reports: &[String]) -> Result { + let mut joined: Option = None; + for xml in reports { + let report = quick_junit::Report::deserialize_from_str(xml) + .map_err(|error| format!("invalid JUnit report: {error}"))?; + let time = report.time.ok_or("JUnit report lacks its run time")?; + match &mut joined { + None => joined = Some(report), + Some(joined) => { + let total = joined.time.unwrap_or_default() + time; + joined.add_test_suites(report.test_suites).set_time(total); + } + } + } + joined + .ok_or("no JUnit reports to join")? + .to_string() + .map_err(|error| format!("cannot write the joined JUnit report: {error}")) +} + +/// A Markdown report of a platform's whole lifecycle suite, from the JUnit +/// reports its partitions uploaded: the artifacts +/// `lifecycle-PLATFORM-SHARD-ATTEMPT`, downloaded as subdirectories of +/// `directory`. Each shard's latest attempt counts, and each of `shards` without +/// a report is named. Partitions run in parallel on separate runners, so the +/// suite's wall time is its slowest partition's. +/// +/// # Errors +/// Returns an error when `directory` cannot be read or a partition's report +/// cannot be parsed. +pub fn partitioned_report( + directory: &std::path::Path, + platform: &str, + shards: &[u32], + rows: usize, +) -> Result { + let step = super::Step::Lifecycle; + let (_, file) = step + .junit() + .ok_or("the lifecycle step writes no JUnit report")?; + let prefix = format!("{}-{platform}-", step.name()); + // Shard to its latest attempt and artifact name. + let mut latest: BTreeMap = BTreeMap::new(); + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries.collect::, _>>(), + // No partition uploaded a report. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(error), + } + .map_err(|error| format!("cannot read {}: {error}", directory.display()))?; + for entry in entries { + let name = entry.file_name().to_string_lossy().into_owned(); + let Some((shard, attempt)) = name + .strip_prefix(&prefix) + .and_then(|rest| rest.split_once('-')) + .and_then(|(shard, attempt)| Some((shard.parse().ok()?, attempt.parse().ok()?))) + else { + continue; + }; + if latest.get(&shard).is_none_or(|(seen, _)| *seen < attempt) { + latest.insert(shard, (attempt, name)); + } + } + let mut all: Vec = shards + .iter() + .copied() + .chain(latest.keys().copied()) + .collect(); + all.sort_unstable(); + all.dedup(); + + let mut whole = Run { + wall_seconds: 0.0, + tests: Vec::new(), + }; + let mut table = + String::from("| Partition | Attempt | Tests | Wall | Summed |\n|---|---|---|---|---|\n"); + for shard in &all { + let Some((attempt, name)) = latest.get(shard) else { + let _ = writeln!(table, "| {shard} | | | | no JUnit report |"); + continue; + }; + let path = directory.join(name).join(file); + let run = std::fs::read_to_string(&path) + .map_err(|error| error.to_string()) + .and_then(|xml| Run::parse(&xml)) + .map_err(|error| format!("cannot read {}: {error}", path.display()))?; + let _ = writeln!( + table, + "| {shard} | {attempt} | {} | {:.1} s | {:.1} s |", + run.tests.len(), + run.wall_seconds, + run.summed_seconds() + ); + whole.wall_seconds = whole.wall_seconds.max(run.wall_seconds); + whole.tests.extend(run.tests); + } + let mut report = whole.headline(&format!( + "{} on {platform}, {} partitions", + step.name(), + all.len() + )); + report.push_str("\nPartitions run in parallel; the wall time is the slowest partition's.\n\n"); + report.push_str(&table); + report.push('\n'); + report.push_str(&whole.tables(rows)); + Ok(report) +} diff --git a/crates/nemoclaw-build/src/main.rs b/crates/nemoclaw-build/src/main.rs index fccfbe4ff2f..d7ca642c5b1 100644 --- a/crates/nemoclaw-build/src/main.rs +++ b/crates/nemoclaw-build/src/main.rs @@ -86,6 +86,18 @@ enum Action { #[arg(long)] base: String, }, + /// Report a platform's whole lifecycle suite from its partitions' JUnit + /// artifacts, and add the report to the GitHub job summary. + LifecycleTiming { + /// Directory holding the downloaded lifecycle-PLATFORM-SHARD-ATTEMPT + /// artifacts, one subdirectory each. + directory: PathBuf, + #[arg(long)] + platform: String, + /// The shards CI ran, as a JSON list such as [1, 2]. + #[arg(long)] + shards: String, + }, } #[derive(Clone, clap::ValueEnum)] enum ChangedWorkflow { @@ -477,6 +489,19 @@ async fn main() -> Result<()> { } .map_err(Into::into); } + if let Action::LifecycleTiming { + directory, + platform, + shards, + } = &cli.command + { + let shards: Vec = serde_json::from_str(shards) + .map_err(|error| format!("--shards must be a JSON list of numbers: {error}"))?; + let report = + nemoclaw_build::ci::timing::partitioned_report(directory, platform, &shards, 15)?; + run_ci::publish(&report); + return Ok(()); + } if let Action::Changes { workflow, base } = &cli.command { let (workflow, name) = match workflow { ChangedWorkflow::Images => (nemoclaw_build::changes::Workflow::Images, "images"), @@ -513,8 +538,11 @@ async fn main() -> Result<()> { Action::Schema { .. } | Action::Docs { .. } | Action::Fern { .. } => { unreachable!("documentation generation returned before build tool checks") } - Action::Ci { .. } | Action::Images { .. } | Action::Changes { .. } => { - unreachable!("CI, image, and change commands returned before build tool checks") + Action::Ci { .. } + | Action::Images { .. } + | Action::Changes { .. } + | Action::LifecycleTiming { .. } => { + unreachable!("CI, image, change, and timing commands returned before build tool checks") } #[cfg(feature = "sdk")] Action::Bundle { platform } => { diff --git a/crates/nemoclaw-build/src/run_ci.rs b/crates/nemoclaw-build/src/run_ci.rs index ed9147581dc..194a268eeb0 100644 --- a/crates/nemoclaw-build/src/run_ci.rs +++ b/crates/nemoclaw-build/src/run_ci.rs @@ -336,6 +336,12 @@ fn report_timing(step: Step, partition: Option<&str>, begun: Instant) -> Vec, begun: Instant) -> Vec| -> Result<()> { + // Each nextest command rewrites the profile's JUnit report; keep each one, + // so the step reports and uploads all of its tests. + let Some((profile, file)) = Step::LiveDocker.junit() else { + unreachable!("live-docker writes a JUnit report") + }; + let junit = Path::new("target").join("nextest").join(profile).join(file); + let mut reports = Vec::new(); + let mut nextest = |filter: &str, document: Option<&str>| -> Result<()> { let mut command = cargo(); configure(&mut command); command @@ -281,17 +288,33 @@ pub(super) fn run_live_docker( if let Some(document) = document { command.env("NEMOCLAW_TEST_GATEWAY_DOCUMENT", document); } - run(&mut command) + // A command that writes no report must not repeat the previous one. + let _ = fs::remove_file(&junit); + let result = run(&mut command); + if let Ok(xml) = fs::read_to_string(&junit) { + reports.push(xml); + } + result }; let gateway = GATEWAY_TESTS .iter() .map(|name| format!("test({name})")) .collect::>() .join(" | "); - // `-E` narrows the profile's default filter rather than replacing it. - nextest(&format!("not ({gateway})"), None)?; - for (name, document) in GATEWAY_TESTS.iter().zip(&documents) { - nextest(&format!("test({name})"), Some(document))?; + let result = (|| { + // `-E` narrows the profile's default filter rather than replacing it. + nextest(&format!("not ({gateway})"), None)?; + for (name, document) in GATEWAY_TESTS.iter().zip(&documents) { + nextest(&format!("test({name})"), Some(document))?; + } + Ok(()) + })(); + if !reports.is_empty() { + let joined = ci::timing::join_consecutive(&reports) + .and_then(|xml| fs::write(&junit, xml).map_err(|error| error.to_string())); + if let Err(error) = joined { + eprintln!("cannot join the live-docker JUnit reports: {error}"); + } } - Ok(()) + result } diff --git a/crates/nemoclaw-build/tests/ci.rs b/crates/nemoclaw-build/tests/ci.rs index bb706655cbb..464afa52a8d 100644 --- a/crates/nemoclaw-build/tests/ci.rs +++ b/crates/nemoclaw-build/tests/ci.rs @@ -326,8 +326,99 @@ fn budgets_name_each_step_over_its_wall_time_and_each_test_over_its_limit() { ); assert!(budgets.get("live-kind").is_none()); assert!(ci::timing::Budgets::parse("test:\n wall_seconds: -1\n").is_err()); - // The repository's budgets parse and cover the test and lifecycle steps. + // The repository's budgets parse and cover every step that writes a JUnit + // report, so each such step is enforced on the Linux runners. let repository = ci::timing::Budgets::parse(include_str!("../../../.config/test-budgets.yaml")).unwrap(); - assert!(repository.get("test").is_some() && repository.get("lifecycle").is_some()); + for step in Step::ALL + .into_iter() + .chain([Step::Archive, Step::LiveDocker, Step::LiveKind]) + .filter(|step| step.junit().is_some()) + { + assert!( + repository.get(step.name()).is_some(), + "{} has no budget in .config/test-budgets.yaml", + step.name() + ); + } +} + +const RERUN_JUNIT: &str = r#" + + + + + +"#; + +#[test] +fn junit_reports_of_consecutive_runs_join_into_one_report_of_their_summed_time() { + let joined = ci::timing::join_consecutive(&[JUNIT.to_owned(), RERUN_JUNIT.to_owned()]).unwrap(); + let run = ci::timing::Run::parse(&joined).unwrap(); + assert_eq!(run.wall_seconds, 15.5); + assert_eq!(run.tests.len(), 5); + assert!(run.tests[1].failed); + assert_eq!(run.tests[4].name, "deployment::rerun"); + assert_eq!(run.tests[4].seconds, 2.5); + + let single = ci::timing::join_consecutive(&[RERUN_JUNIT.to_owned()]).unwrap(); + assert_eq!(ci::timing::Run::parse(&single).unwrap().tests.len(), 1); + assert!(ci::timing::join_consecutive(&[]).is_err()); + assert!(ci::timing::join_consecutive(&[JUNIT.to_owned(), "".to_owned()]).is_err()); +} + +#[test] +fn a_partitioned_suite_is_reported_whole_from_the_latest_attempt_of_each_partition() { + let directory = tempfile::tempdir().unwrap(); + for (artifact, xml) in [ + // Shard 1 was rerun: its second attempt replaces the first. + ("lifecycle-linux_amd64-1-1", JUNIT), + ("lifecycle-linux_amd64-1-2", RERUN_JUNIT), + ("lifecycle-linux_amd64-2-1", JUNIT), + // Other platforms and artifacts are not this suite's partitions. + ("lifecycle-linux_arm64-3-1", JUNIT), + ("lifecycle-inputs-linux_amd64-abc-1", JUNIT), + ] { + let artifact = directory.path().join(artifact); + std::fs::create_dir(&artifact).unwrap(); + std::fs::write(artifact.join("lifecycle.xml"), xml).unwrap(); + } + + let report = + ci::timing::partitioned_report(directory.path(), "linux_amd64", &[1, 2, 3], 3).unwrap(); + // Partitions run in parallel, so the suite's wall time is its slowest partition's. + assert!( + report.contains( + "### lifecycle on linux_amd64, 3 partitions: 5 tests, 12.5 s wall, 15.0 s summed" + ), + "{report}" + ); + assert!(report.contains("| 1 | 2 | 1 | 3.0 s | 2.5 s |"), "{report}"); + assert!( + report.contains("| 2 | 1 | 4 | 12.5 s | 12.5 s |"), + "{report}" + ); + // A partition that uploaded no report is named, so the totals cannot mislead. + assert!(report.contains("| 3 | | | | no JUnit report |"), "{report}"); + assert!( + report.contains("| 9.2 s | nemoclaw-e2e::integration deployment::slow_scenario |"), + "{report}" + ); + assert!( + report.contains("| 2.5 s | nemoclaw-sdk::integration deployment::rerun |"), + "{report}" + ); + + std::fs::write( + directory + .path() + .join("lifecycle-linux_amd64-2-1/lifecycle.xml"), + "", + ) + .unwrap(); + assert!( + ci::timing::partitioned_report(directory.path(), "linux_amd64", &[1, 2], 3) + .unwrap_err() + .contains("lifecycle-linux_amd64-2-1") + ); } diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 5f1b1ecf65d..5a61e145345 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -86,7 +86,20 @@ Each platform starts its workers after its own jobs; it does not wait for other The existing `Test / PLATFORM` required checks succeed only when that platform's build, bundle, and lifecycle jobs succeed. After each test step, `cargo ci` prints where the time went: the step's test count, wall time, and summed test time, the time per test binary and module, and the 15 slowest tests; a lifecycle partition reports its own tests. In GitHub Actions, the report is added to the job summary. -The `test-` and `lifecycle-` timing artifacts contain the JUnit reports with per-test durations, with the partition in each lifecycle artifact name. +Each platform's `Lifecycle timing` job then reports the whole lifecycle suite from the partitions' JUnit reports, using each partition's latest attempt and naming any partition without one; its wall time is the slowest partition's. +That job never fails the platform. +The `test-`, `lifecycle-`, `live-docker-`, and `live-kind-` timing artifacts contain the JUnit reports with per-test durations, with the partition in each lifecycle artifact name; the `live-docker` report covers all four of its nextest commands. + +[`.config/test-budgets.yaml`](../../.config/test-budgets.yaml) gives each test step a wall-clock budget and a limit for any one test: + +| Step | Wall budget | Per-test limit | +|---|---|---| +| `test` | 45 s | 8 s | +| `lifecycle`, each partition | 360 s | 120 s | +| `live-docker` | 360 s | 180 s | +| `live-kind` | 270 s | 120 s | + +On the Linux CI runners, a step over its budget, or with a test over its limit, fails and names each offender with its time and limit; macOS, Windows, and local runs only report them. Both profiles finish the remaining tests after a failure. Use the [fixture prerequisites](integration-tests.md#opentofu-and-bundle-lifecycle) before selecting ignored tests; the profiles do not configure a bundle or authorize live resources. Nextest does not run doctests, so the separate Cargo command remains required.