ci: report each platform's whole lifecycle suite and budget the live suites

.config/test-budgets.yaml

@@ -3,9 +3,10 @@
 
 # Wall-clock budgets for each test step, in seconds, enforced on the Linux CI
 # runners; other platforms report their times without failing. wall_seconds
-# bounds a step's run, or each lifecycle partition; test_seconds bounds any one
-# test in it. Lower a budget when a change makes the step faster; raising one
-# needs the reason in the pull request. See docs/contributing/testing.md.
+# bounds a step's nextest run, the live-docker step's nextest commands together,
+# or each lifecycle partition; test_seconds bounds any one test in it. Lower a
+# budget when a change makes the step faster; raising one needs the reason in
+# the pull request. See docs/contributing/testing.md.
 
 # Unit and integration tests: 27 s on linux_amd64, slowest test 5.1 s.
 test:
@@ -16,3 +17,19 @@ test:
 lifecycle:
   wall_seconds: 360
   test_seconds: 120
+
+# Live / Docker, its four nextest commands together: at most 255 s on
+# linux_amd64 and 210 s on linux_arm64 in 11 v1 runs. Its slowest test,
+# retained_runtime_rebuilds_offline_without_protobuf_or_a_dependency_cache,
+# took 110 s on linux_amd64 in a full report; in the v1 runs, whose reports
+# kept only the last command, it ran past 120 s in 5 of 11 linux_amd64 runs
+# and, by its command's wall time, under about 150 s.
+live-docker:
+  wall_seconds: 360
+  test_seconds: 180
+
+# Live / Kind: at most 199 s on linux_amd64 and 171 s on linux_arm64 in 11 v1
+# runs; slowest test 83 s.
+live-kind:
+  wall_seconds: 270
+  test_seconds: 120

.github/workflows/native-platform.yml

@@ -158,6 +158,36 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  # Each partition reports and enforces its own budget; this job reports the
+  # platform's whole lifecycle suite in its summary, from the partitions'
+  # uploaded timings, including failed partitions'. It never fails the platform.
+  lifecycle-timing:
+    name: Lifecycle timing
+    needs: [build, lifecycle]
+    if: ${{ !cancelled() && needs.build.result == 'success' && needs.lifecycle.result != 'skipped' }}
+    continue-on-error: true
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 10
+    steps:
+      # The build tool comes from the lifecycle inputs; this job compiles nothing.
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ needs.build.outputs.artifact }}
+          path: .build/ci
+      # Every attempt's timings; the report reads each shard's latest attempt.
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: lifecycle-${{ inputs.platform }}-*
+          path: .build/ci/timings
+      - name: Report the whole lifecycle suite
+        env:
+          SHARDS: ${{ inputs.lifecycle-shards }}
+        run: |
+          tool=.build/ci/nemoclaw-build
+          if [ "$RUNNER_OS" = Windows ]; then tool="$tool.exe"; fi
+          tar -xf .build/ci/lifecycle-inputs.tar "$tool"
+          "$tool" lifecycle-timing .build/ci/timings --platform "$TEST_PLATFORM" --shards "$SHARDS"
+
   # The live suites reuse this platform's bundle instead of building their own,
   # so they start when it is ready. They are not required checks: a failure
   # shows on its job without failing the platform's Test / PLATFORM check.

crates/nemoclaw-build/src/ci/timing.rs

@@ -1,6 +1,6 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-//! Where test time goes, read from a nextest JUnit report.
+//! Where test time goes, read from nextest JUnit reports.
 
 use serde::Deserialize;
 use std::{collections::BTreeMap, fmt::Write};
@@ -115,13 +115,28 @@ impl Run {
     /// each binary, then the `rows` slowest tests.
     #[must_use]
     pub fn report(&self, profile: &str, rows: usize) -> String {
-        let summed: f64 = self.tests.iter().map(|test| test.seconds).sum();
-        let mut report = format!(
-            "### {profile}: {} tests, {:.1} s wall, {:.1} s summed\n\n",
+        let mut report = self.headline(profile);
+        report.push('\n');
+        report.push_str(&self.tables(rows));
+        report
+    }
+
+    fn summed_seconds(&self) -> f64 {
+        self.tests.iter().map(|test| test.seconds).sum()
+    }
+
+    fn headline(&self, profile: &str) -> String {
+        format!(
+            "### {profile}: {} tests, {:.1} s wall, {:.1} s summed\n",
             self.tests.len(),
             self.wall_seconds,
-            summed
-        );
+            self.summed_seconds()
+        )
+    }
+
+    /// The slowest modules and tests, without the headline.
+    fn tables(&self, rows: usize) -> String {
+        let mut report = String::new();
         let mut groups: BTreeMap<(&str, &str), (usize, f64, f64)> = BTreeMap::new();
         for test in &self.tests {
             let module = test.name.split("::").next().unwrap_or(&test.name);
@@ -158,3 +173,118 @@ impl Run {
         report
     }
 }
+
+/// Join the JUnit reports of nextest runs made one after another, such as the
+/// live-docker step's, into one report whose run time is their sum.
+///
+/// # Errors
+/// Returns an error when there are no reports, or one is not a nextest JUnit
+/// report with its run time.
+pub fn join_consecutive(reports: &[String]) -> Result<String, String> {
+    let mut joined: Option<quick_junit::Report> = None;
+    for xml in reports {
+        let report = quick_junit::Report::deserialize_from_str(xml)
+            .map_err(|error| format!("invalid JUnit report: {error}"))?;
+        let time = report.time.ok_or("JUnit report lacks its run time")?;
+        match &mut joined {
+            None => joined = Some(report),
+            Some(joined) => {
+                let total = joined.time.unwrap_or_default() + time;
+                joined.add_test_suites(report.test_suites).set_time(total);
+            }
+        }
+    }
+    joined
+        .ok_or("no JUnit reports to join")?
+        .to_string()
+        .map_err(|error| format!("cannot write the joined JUnit report: {error}"))
+}
+
+/// A Markdown report of a platform's whole lifecycle suite, from the JUnit
+/// reports its partitions uploaded: the artifacts
+/// `lifecycle-PLATFORM-SHARD-ATTEMPT`, downloaded as subdirectories of
+/// `directory`. Each shard's latest attempt counts, and each of `shards` without
+/// a report is named. Partitions run in parallel on separate runners, so the
+/// suite's wall time is its slowest partition's.
+///
+/// # Errors
+/// Returns an error when `directory` cannot be read or a partition's report
+/// cannot be parsed.
+pub fn partitioned_report(
+    directory: &std::path::Path,
+    platform: &str,
+    shards: &[u32],
+    rows: usize,
+) -> Result<String, String> {
+    let step = super::Step::Lifecycle;
+    let (_, file) = step
+        .junit()
+        .ok_or("the lifecycle step writes no JUnit report")?;
+    let prefix = format!("{}-{platform}-", step.name());
+    // Shard to its latest attempt and artifact name.
+    let mut latest: BTreeMap<u32, (u32, String)> = BTreeMap::new();
+    let entries = match std::fs::read_dir(directory) {
+        Ok(entries) => entries.collect::<Result<Vec<_>, _>>(),
+        // No partition uploaded a report.
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()),
+        Err(error) => Err(error),
+    }
+    .map_err(|error| format!("cannot read {}: {error}", directory.display()))?;
+    for entry in entries {
+        let name = entry.file_name().to_string_lossy().into_owned();
+        let Some((shard, attempt)) = name
+            .strip_prefix(&prefix)
+            .and_then(|rest| rest.split_once('-'))
+            .and_then(|(shard, attempt)| Some((shard.parse().ok()?, attempt.parse().ok()?)))
+        else {
+            continue;
+        };
+        if latest.get(&shard).is_none_or(|(seen, _)| *seen < attempt) {
+            latest.insert(shard, (attempt, name));
+        }
+    }
+    let mut all: Vec<u32> = shards
+        .iter()
+        .copied()
+        .chain(latest.keys().copied())
+        .collect();
+    all.sort_unstable();
+    all.dedup();
+
+    let mut whole = Run {
+        wall_seconds: 0.0,
+        tests: Vec::new(),
+    };
+    let mut table =
+        String::from("| Partition | Attempt | Tests | Wall | Summed |\n|---|---|---|---|---|\n");
+    for shard in &all {
+        let Some((attempt, name)) = latest.get(shard) else {
+            let _ = writeln!(table, "| {shard} | | | | no JUnit report |");
+            continue;
+        };
+        let path = directory.join(name).join(file);
+        let run = std::fs::read_to_string(&path)
+            .map_err(|error| error.to_string())
+            .and_then(|xml| Run::parse(&xml))
+            .map_err(|error| format!("cannot read {}: {error}", path.display()))?;
+        let _ = writeln!(
+            table,
+            "| {shard} | {attempt} | {} | {:.1} s | {:.1} s |",
+            run.tests.len(),
+            run.wall_seconds,
+            run.summed_seconds()
+        );
+        whole.wall_seconds = whole.wall_seconds.max(run.wall_seconds);
+        whole.tests.extend(run.tests);
+    }
+    let mut report = whole.headline(&format!(
+        "{} on {platform}, {} partitions",
+        step.name(),
+        all.len()
+    ));
+    report.push_str("\nPartitions run in parallel; the wall time is the slowest partition's.\n\n");
+    report.push_str(&table);
+    report.push('\n');
+    report.push_str(&whole.tables(rows));
+    Ok(report)
+}

crates/nemoclaw-build/src/main.rs

@@ -86,6 +86,18 @@ enum Action {
         #[arg(long)]
         base: String,
     },
+    /// Report a platform's whole lifecycle suite from its partitions' JUnit
+    /// artifacts, and add the report to the GitHub job summary.
+    LifecycleTiming {
+        /// Directory holding the downloaded lifecycle-PLATFORM-SHARD-ATTEMPT
+        /// artifacts, one subdirectory each.
+        directory: PathBuf,
+        #[arg(long)]
+        platform: String,
+        /// The shards CI ran, as a JSON list such as [1, 2].
+        #[arg(long)]
+        shards: String,
+    },
 }
 #[derive(Clone, clap::ValueEnum)]
 enum ChangedWorkflow {
@@ -477,6 +489,19 @@ async fn main() -> Result<()> {
         }
         .map_err(Into::into);
     }
+    if let Action::LifecycleTiming {
+        directory,
+        platform,
+        shards,
+    } = &cli.command
+    {
+        let shards: Vec<u32> = serde_json::from_str(shards)
+            .map_err(|error| format!("--shards must be a JSON list of numbers: {error}"))?;
+        let report =
+            nemoclaw_build::ci::timing::partitioned_report(directory, platform, &shards, 15)?;
+        run_ci::publish(&report);
+        return Ok(());
+    }
     if let Action::Changes { workflow, base } = &cli.command {
         let (workflow, name) = match workflow {
             ChangedWorkflow::Images => (nemoclaw_build::changes::Workflow::Images, "images"),
@@ -513,8 +538,11 @@ async fn main() -> Result<()> {
         Action::Schema { .. } | Action::Docs { .. } | Action::Fern { .. } => {
             unreachable!("documentation generation returned before build tool checks")
         }
-        Action::Ci { .. } | Action::Images { .. } | Action::Changes { .. } => {
-            unreachable!("CI, image, and change commands returned before build tool checks")
+        Action::Ci { .. }
+        | Action::Images { .. }
+        | Action::Changes { .. }
+        | Action::LifecycleTiming { .. } => {
+            unreachable!("CI, image, change, and timing commands returned before build tool checks")
         }
         #[cfg(feature = "sdk")]
         Action::Bundle { platform } => {

crates/nemoclaw-build/src/run_ci.rs

@@ -336,6 +336,12 @@ fn report_timing(step: Step, partition: Option<&str>, begun: Instant) -> Vec<Str
             let _ = writeln!(report, "- Over budget: {line}");
         }
     }
+    publish(&report);
+    if enforces_budgets() { over } else { Vec::new() }
+}
+
+/// Print a timing report, and add it to the GitHub job summary when there is one.
+pub(super) fn publish(report: &str) {
     eprintln!("{report}");
     if let Some(summary) = std::env::var_os("GITHUB_STEP_SUMMARY") {
         use std::io::Write;
@@ -348,7 +354,6 @@ fn report_timing(step: Step, partition: Option<&str>, begun: Instant) -> Vec<Str
             eprintln!("cannot write the job summary: {error}");
         }
     }
-    if enforces_budgets() { over } else { Vec::new() }
 }
 
 pub(super) async fn run_steps(

crates/nemoclaw-build/src/run_ci/live_docker.rs

@@ -271,7 +271,14 @@ pub(super) fn run_live_docker(
     let [args] = Step::LiveDocker.cargo_args() else {
         unreachable!("live-docker is one nextest command")
     };
-    let nextest = |filter: &str, document: Option<&str>| -> Result<()> {
+    // Each nextest command rewrites the profile's JUnit report; keep each one,
+    // so the step reports and uploads all of its tests.
+    let Some((profile, file)) = Step::LiveDocker.junit() else {
+        unreachable!("live-docker writes a JUnit report")
+    };
+    let junit = Path::new("target").join("nextest").join(profile).join(file);
+    let mut reports = Vec::new();
+    let mut nextest = |filter: &str, document: Option<&str>| -> Result<()> {
         let mut command = cargo();
         configure(&mut command);
         command
@@ -281,17 +288,33 @@ pub(super) fn run_live_docker(
         if let Some(document) = document {
             command.env("NEMOCLAW_TEST_GATEWAY_DOCUMENT", document);
         }
-        run(&mut command)
+        // A command that writes no report must not repeat the previous one.
+        let _ = fs::remove_file(&junit);
+        let result = run(&mut command);
+        if let Ok(xml) = fs::read_to_string(&junit) {
+            reports.push(xml);
+        }
+        result
     };
     let gateway = GATEWAY_TESTS
         .iter()
         .map(|name| format!("test({name})"))
         .collect::<Vec<_>>()
         .join(" | ");
-    // `-E` narrows the profile's default filter rather than replacing it.
-    nextest(&format!("not ({gateway})"), None)?;
-    for (name, document) in GATEWAY_TESTS.iter().zip(&documents) {
-        nextest(&format!("test({name})"), Some(document))?;
+    let result = (|| {
+        // `-E` narrows the profile's default filter rather than replacing it.
+        nextest(&format!("not ({gateway})"), None)?;
+        for (name, document) in GATEWAY_TESTS.iter().zip(&documents) {
+            nextest(&format!("test({name})"), Some(document))?;
+        }
+        Ok(())
+    })();
+    if !reports.is_empty() {
+        let joined = ci::timing::join_consecutive(&reports)
+            .and_then(|xml| fs::write(&junit, xml).map_err(|error| error.to_string()));
+        if let Err(error) = joined {
+            eprintln!("cannot join the live-docker JUnit reports: {error}");
+        }
     }
-    Ok(())
+    result
 }