diff --git a/baml_language/crates/bex_events/src/prof/consumer.rs b/baml_language/crates/bex_events/src/prof/consumer.rs index b8700b7cc90..b71a0c85c05 100644 --- a/baml_language/crates/bex_events/src/prof/consumer.rs +++ b/baml_language/crates/bex_events/src/prof/consumer.rs @@ -1015,6 +1015,22 @@ mod tests { const ENGINE: u64 = 0x50AC_0001; let rounds: u64 = if cfg!(miri) { 4 } else { 64 }; let per_round: u64 = if cfg!(miri) { 20 } else { 500 }; + // Per-round wedge bound, not a latency assertion. Under Miri's + // interpreter a 1 s wall-clock bound is machine-marginal and failed + // deterministically on some hosts, so allow a minute there. Natively + // that same minute across 64 rounds would let a wedge burn an hour as + // a bare job timeout instead of failing fast with the named panic. + let round_ack_timeout = if cfg!(miri) { + Duration::from_mins(1) + } else { + // 5 s was measured insufficient natively: under a full-fleet CI + // fan-out one round's ack took >15 s of wall clock (the whole + // suite ran ~50x slower than idle). 30 s keeps the fail-fast + // property - a wedged consumer still dies with this named panic + // inside the job timeout (64 rounds x 30 s = 32 min < 45 min) - + // with real headroom over the worst load observed. + Duration::from_secs(30) + }; let dir = temp_dir("soak"); let registry: &'static Registry = leak(Registry::new()); @@ -1063,7 +1079,9 @@ mod tests { ctl_tx.send(ControlMsg::Flush(ack_tx)).unwrap(); ctx.wake().force_wake(); ack_rx - .recv_timeout(Duration::from_secs(1)) + // The guarantee this test needs is the ack: the consumer + // pooled the dead ring before the next acquire. + .recv_timeout(round_ack_timeout) .expect("soak consumer did not flush before the next churn round"); } diff --git a/baml_language/sdk_tests/crates/java/function_calls/customizable/TestCancellation.java b/baml_language/sdk_tests/crates/java/function_calls/customizable/TestCancellation.java index 119f46c06a6..f36e99d3873 100644 --- a/baml_language/sdk_tests/crates/java/function_calls/customizable/TestCancellation.java +++ b/baml_language/sdk_tests/crates/java/function_calls/customizable/TestCancellation.java @@ -46,7 +46,9 @@ class TestCancellation { - private static final long MAX_CANCELLATION_MILLIS = 500; + private static final long MAX_CANCELLATION_MILLIS = 5000; + // The cancelled calls below sleep 60s: the operation must dwarf this bound, + // or a regression that ignored cancellation would finish inside it and pass. private static void assertCancelledPanic(BamlPanic exc) { assertInstanceOf(Cancelled.class, exc.value()); @@ -98,7 +100,7 @@ public void run() { try { BamlPanic exc = - assertThrows(BamlPanic.class, () -> Fns.SleepMs(2000L, ctx)); + assertThrows(BamlPanic.class, () -> Fns.SleepMs(60000L, ctx)); assertCancelledPanic(exc); } finally { timer.cancel(); @@ -111,7 +113,7 @@ public void run() { void test_cancellation_async_cancel_via_call_context() { long start = System.nanoTime(); BamlCallContext ctx = new BamlCallContext(); - CompletableFuture future = Fns.SleepMs_async(2000L, ctx); + CompletableFuture future = Fns.SleepMs_async(60000L, ctx); sleepMillis(50); ctx.abort(); @@ -128,7 +130,7 @@ void test_cancellation_async_cancel_via_call_context() { @Test void test_cancellation_async_cancel_via_task_cancel() { long start = System.nanoTime(); - CompletableFuture future = Fns.SleepMs_async(2000L); + CompletableFuture future = Fns.SleepMs_async(60000L); sleepMillis(50); future.cancel(true); @@ -141,7 +143,7 @@ void test_cancellation_async_cancel_via_task_cancel() { void test_cancellation_async_cancel_via_task_group_sibling() { long start = System.nanoTime(); - CompletableFuture sleep = Fns.SleepMs_async(2000L); + CompletableFuture sleep = Fns.SleepMs_async(60000L); CompletableFuture failSoon = CompletableFuture.runAsync( () -> { @@ -170,7 +172,7 @@ void test_cancellation_async_cancel_via_task_group_sibling() { @Test void test_cancellation_async_cancel_via_asyncio_timeout() { long start = System.nanoTime(); - CompletableFuture future = Fns.SleepMs_async(2000L); + CompletableFuture future = Fns.SleepMs_async(60000L); assertThrows(TimeoutException.class, () -> future.get(50, TimeUnit.MILLISECONDS)); diff --git a/baml_language/sdk_tests/crates/rust/function_calls/customizable/test_cancellation.rs b/baml_language/sdk_tests/crates/rust/function_calls/customizable/test_cancellation.rs index b895a57ab88..cacf3cf6662 100644 --- a/baml_language/sdk_tests/crates/rust/function_calls/customizable/test_cancellation.rs +++ b/baml_language/sdk_tests/crates/rust/function_calls/customizable/test_cancellation.rs @@ -12,7 +12,9 @@ use std::time::{Duration, Instant}; use baml_bridge::runtime::BamlCallContext; use baml_sdk::throws_test; -const _MAX_CANCELLATION_SECONDS: f64 = 0.5; +const _MAX_CANCELLATION_SECONDS: f64 = 5.0; +// The cancelled calls below sleep 60s: the operation must dwarf this bound, or a +// regression that ignored cancellation would still finish inside it and pass. /// python asserts `isinstance(exc.value, Cancelled)`; `baml_bridge::Error::Panic` /// carries only the rendered message + trace, so the class check adapts to @@ -69,7 +71,7 @@ fn test_cancellation_sync_cancel_via_call_context() { }); // PROVISIONAL: `_ctx=ctx` → the `_with_ctx` sibling. - let result = throws_test::SleepMs_with_ctx(2000, &ctx); + let result = throws_test::SleepMs_with_ctx(60000, &ctx); _assert_cancelled_panic(result.unwrap_err()); timer.join().unwrap(); }); @@ -86,7 +88,7 @@ async fn test_cancellation_async_cancel_via_call_context() { // here the call and the aborter run under `join!` and the aborted call // itself resolves to the cancellation error. // PROVISIONAL: `_ctx=ctx` → the `_with_ctx` sibling. - let (result, ()) = tokio::join!(throws_test::SleepMs_async_with_ctx(2000, &ctx), async { + let (result, ()) = tokio::join!(throws_test::SleepMs_async_with_ctx(60000, &ctx), async { tokio::time::sleep(Duration::from_millis(50)).await; ctx.abort(); }); @@ -98,7 +100,7 @@ async fn test_cancellation_async_cancel_via_call_context() { #[tokio::test] async fn test_cancellation_async_cancel_via_task_cancel() { let start = Instant::now(); - let task = tokio::spawn(throws_test::SleepMs_async(2000)); + let task = tokio::spawn(throws_test::SleepMs_async(60000)); tokio::time::sleep(Duration::from_millis(50)).await; task.abort(); @@ -127,7 +129,7 @@ async fn test_cancellation_async_cancel_via_task_group_sibling() { // `task.cancelled()`. let result = tokio::try_join!( async { - throws_test::SleepMs_async(2000) + throws_test::SleepMs_async(60000) .await .map_err(|_| "sleep failed") }, @@ -145,7 +147,7 @@ async fn test_cancellation_async_cancel_via_asyncio_timeout() { // elapsed error is the `TimeoutError`, and the timed-out call future is // dropped (cancelled). let result = - tokio::time::timeout(Duration::from_millis(50), throws_test::SleepMs_async(2000)).await; + tokio::time::timeout(Duration::from_millis(50), throws_test::SleepMs_async(60000)).await; assert!(result.is_err()); _assert_fast_cancellation(start); diff --git a/baml_language/sdk_tests/crates/typescript/function_calls/customizable/cancellation.test.ts b/baml_language/sdk_tests/crates/typescript/function_calls/customizable/cancellation.test.ts index fe74875746b..a2c96ec1929 100644 --- a/baml_language/sdk_tests/crates/typescript/function_calls/customizable/cancellation.test.ts +++ b/baml_language/sdk_tests/crates/typescript/function_calls/customizable/cancellation.test.ts @@ -14,7 +14,10 @@ import { call_with_callback_async } from "./baml_sdk/host_callable_tests/index.j const SLEEP_FQN = "user.throws_test.SleepMs"; const HOST_CALLBACK_FQN = "user.host_callable_tests.call_with_callback"; -const MAX_CANCELLATION_MS = 500; +// The cancelled calls below sleep 60s (or hang on a pending host callback): the +// operation must dwarf this bound, or a regression that ignored cancellation +// would still finish inside it and pass. +const MAX_CANCELLATION_MS = 5000; function expectAbortError(error: unknown): void { expect(error).toBeInstanceOf(Error); @@ -78,7 +81,7 @@ describe( callFunctionSync( getRuntime(), SLEEP_FQN, - { ms: 2000 }, + { ms: 60000 }, undefined, undefined, ctx,