From 6c39c608c2b24ec962d77d1c50610d4a9fbcfdea Mon Sep 17 00:00:00 2001 From: Matteo Visconti di Oleggio Castello Date: Thu, 20 Aug 2026 13:00:44 -0700 Subject: [PATCH] webgl: fix interpreter-exit deadlock in headless_viewer `headless_viewer` awaited the WebSocket "connect" on a `ThreadPoolExecutor` worker running `server.get_client()`, which blocks on a `threading.Event` with no timeout. `cancel_futures=True` cannot cancel an already-running future, so on a failed connection the worker parked forever -- and `concurrent.futures` joins every worker at interpreter exit, wedging the process after the work was already done. In CI this showed up as a job hitting its 25-minute `timeout-minutes` guard ~15 min after pytest had printed its summary. Await the client on a daemon thread instead. Daemon threads are never joined at exit, so a stuck getter can never hold the process hostage. The join is bounded by the same timeout, and teardown sets the connect event so the getter unblocks promptly rather than lingering across sessions. Co-Authored-By: Claude Opus 5 --- cortex/export/headless.py | 54 +++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/cortex/export/headless.py b/cortex/export/headless.py index d3e8f2337..a8eadb458 100644 --- a/cortex/export/headless.py +++ b/cortex/export/headless.py @@ -342,38 +342,64 @@ def headless_viewer( pw_thread = _PlaywrightThread() handle = None + # ------------------------------------------------------------------ + # 3. Begin waiting for the WebSocket "connect" message on a *daemon* + # thread *before* navigating, so we cannot miss it even if the browser + # connects before page.goto() returns. + # + # A daemon thread (rather than a ThreadPoolExecutor) is essential here: + # server.get_client() blocks on a threading.Event with no timeout, so if + # the browser never sends "connect" the getter parks forever. An + # executor's worker is non-daemon and concurrent.futures joins every + # worker at interpreter exit -- a parked getter would then wedge the + # whole process at shutdown (observed as a multi-minute CI hang after + # the tests have already finished). Daemon threads are never joined at + # exit, so a stuck getter can never hold the process hostage. + # ------------------------------------------------------------------ + connect_result: dict[str, Any] = {} + + def _await_client() -> None: + try: + connect_result["handle"] = server.get_client() + except BaseException as exc: # noqa: BLE001 - reported to main thread + connect_result["error"] = exc + + client_thread = threading.Thread( + target=_await_client, name="headless-await-client", daemon=True + ) + try: - # ------------------------------------------------------------------ - # 3. Begin waiting for the WebSocket "connect" message in a thread - # *before* navigating, so we cannot miss it even if the browser - # connects before page.goto() returns. - # ------------------------------------------------------------------ - pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - fut = pool.submit(server.get_client) + client_thread.start() try: # Launch the browser and navigate. python_interface.js runs on # load and sends "connect" over WebSocket, which unblocks # server.get_client(). pw_thread.start(url, timeout=timeout) - # Retrieve the handle; it should already be ready by this point, - # but the timeout guard surfaces hung state clearly. - handle = fut.result(timeout=timeout) + # Wait (bounded) for the getter to return; the timeout guard + # surfaces hung state clearly instead of blocking indefinitely. + client_thread.join(timeout=timeout) + if "error" in connect_result: + raise connect_result["error"] + if "handle" not in connect_result: + raise TimeoutError( + f"No WebSocket 'connect' received within {timeout:.0f}s" + ) + handle = connect_result["handle"] except Exception as e: - fut.cancel() browser_errors = pw_thread.browser_errors detail = ( "\nBrowser errors:\n" + "\n".join(browser_errors) if browser_errors else "\nNo browser errors were captured." ) - pool.shutdown(wait=False, cancel_futures=True) + # Unblock the (daemon) getter if it is still parked on the connect + # event so it exits promptly rather than lingering across sessions. + server.connect.set() raise RuntimeError( f"Failed to establish WebSocket connection with headless browser at {url} " f"within {timeout:.0f} seconds. {detail}" ) from e - else: - pool.shutdown(wait=False) assert not isinstance(handle, list) # type narrowing to JSMixer handle.server = server