diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..3a7afde0e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +cortex/tests/reference_images/**/*.webp filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/install_from_wheel.yml b/.github/workflows/install_from_wheel.yml index a9faf0460..731007fa9 100644 --- a/.github/workflows/install_from_wheel.yml +++ b/.github/workflows/install_from_wheel.yml @@ -21,6 +21,8 @@ jobs: max-parallel: 5 steps: + # Don't clone with LFS because the wheel will not include the reference + # images, so we can save some bandwidth. - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a2e7ee939..436155af5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,6 +15,10 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 # Required for setuptools-scm to get version from tags + # MANIFEST.in sweeps the LFS-tracked reference images into the sdist; + # without this it would ship pointer stubs instead, which twine check + # does not catch. + lfs: true - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index c0c04f4d9..60866b97c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -21,7 +21,11 @@ jobs: max-parallel: 5 steps: + # The visual-regression reference images are LFS-tracked; without this the + # checkout leaves pointer files and those tests skip. - uses: actions/checkout@v7 + with: + lfs: true - name: Set up Python uses: actions/setup-python@v7 with: diff --git a/cortex/export/headless.py b/cortex/export/headless.py index a8eadb458..998e3d7de 100644 --- a/cortex/export/headless.py +++ b/cortex/export/headless.py @@ -90,7 +90,7 @@ def _wait_for_viewer_loaded(handle, timeout: float = 60.0) -> None: raise RuntimeError( f"Viewer's .loaded deferred did not resolve within {timeout:.0f}s " f"(last response: {last_err!r}). The CTM mesh may have failed to " - "download or parse, or mriview.js failed to initialise." + "download or parse, or mriview.js failed to initialize." ) @@ -99,6 +99,35 @@ def _wait_for_viewer_loaded(handle, timeout: float = 60.0) -> None: # --------------------------------------------------------------------------- # +#: How often the worker thread calls into Playwright to dispatch queued browser +#: events; ``browser_errors`` is current to within this interval. +EVENT_POLL_INTERVAL = 0.25 + +#: Browser messages that mean WebGL itself failed, as opposed to unrelated +#: javascript a page may log. Deliberately narrow: a healthy viewer already logs +#: a console.error for the Leap Motion websocket it cannot reach +#: (ws://127.0.0.1:6437), so asserting on any error at all fails every run. +#: +#: A link failure arrives as a console.error rather than an exception, so +#: nothing raises and the render comes back blank -- how Vertex2D broke +#: (gh-714). three.js emits it alongside "gl.VALIDATE_STATUS false" and +#: "gl.getError() 0"; do not add those. Nothing here calls gl.validateProgram(), +#: so VALIDATE_STATUS is false for want of a run, and getError() 0 is the +#: absence of an error. Driver shader-info warnings are excluded likewise. +WEBGL_FAILURE_PATTERNS = ( + "THREE.WebGLProgram: Could not initialise shader", + "Error creating WebGL context", +) + + +def filter_webgl_failures(browser_errors: list[str]) -> list[str]: + """Return only those browser messages that indicate WebGL itself failed.""" + return [ + e for e in browser_errors + if any(pattern in e for pattern in WEBGL_FAILURE_PATTERNS) + ] + + class _PlaywrightThread: """Manages the Playwright lifecycle on a private daemon thread. @@ -230,7 +259,27 @@ def _worker(self) -> None: return # Keep the thread (and therefore Playwright) alive until shutdown. - self._shutdown_event.wait() + # + # Playwright's sync API dispatches queued events only while something is + # calling into it, so parking here for the viewer's lifetime would leave + # console messages undelivered until _cleanup(). Poll instead: the cheap + # round-trip is what makes Playwright dispatch them. + while not self._shutdown_event.wait(EVENT_POLL_INTERVAL): + try: + self._page.evaluate("0") + except Exception: + # Stop polling, but do not tear the viewer down underneath the + # caller: park until shutdown, as this thread did before + # polling existed. Falling through to _cleanup() here would + # close the browser mid-session, surfacing much later as the + # next getImage timing out. + logger.warning( + "event polling stopped; browser_errors will no longer " + "update for this viewer", + exc_info=True, + ) + self._shutdown_event.wait() + break self._cleanup() # -- Playwright event handlers (called on the worker thread) ---------- # @@ -407,7 +456,7 @@ def _await_client() -> None: # any point during the session (each call returns a fresh snapshot). handle._pw_thread = pw_thread - # Block until the WebGL viewer has finished initialising (CTM mesh + # Block until the WebGL viewer has finished initializing (CTM mesh # download + parse + first setData). Replaces ad-hoc time.sleep(10) # calls in tests and callers, and shortens the wait when the # browser is faster than the worst-case timeout. diff --git a/cortex/export/save_views.py b/cortex/export/save_views.py index e240afb90..2d1f6e59f 100644 --- a/cortex/export/save_views.py +++ b/cortex/export/save_views.py @@ -196,6 +196,20 @@ def save_3d_views( ) time.sleep(1) + if headless: + # Only check for WebGL failures in headless mode, since we don't + # capture console output in the interactive mode. + pw_thread = handle._pw_thread # `handle` is a `JSMixer` + from cortex.export.headless import filter_webgl_failures + + failures = filter_webgl_failures(pw_thread.browser_errors) + if failures: + raise RuntimeError( + f"WebGL failed while rendering {view_name!r}/{surface!r}; " + f"{file_name!r} is likely blank.\n " + + "\n ".join(sorted(set(failures))) + ) + # Trim transparent edges if trim: try: diff --git a/cortex/tests/conftest.py b/cortex/tests/conftest.py new file mode 100644 index 000000000..9ce793d71 --- /dev/null +++ b/cortex/tests/conftest.py @@ -0,0 +1,49 @@ +"""Pin ``cortex.db`` to the filestore bundled with pycortex. + +The filestore is a configured path (``basic.filestore`` in ``options.cfg``), +so on a machine with a real filestore the suite would otherwise run against +whatever subjects that machine happens to have. Every test here uses the demo +subject ``S1``, and the reference renders in ``reference_images/`` are pixel +comparisons against the bundled one; a lab filestore with its own ``S1`` would +fail them for reasons that have nothing to do with the code under test, and +would collect flatmap caches along the way. +""" +import os +import sys + +import cortex +from cortex import database, options + + +def _bundled_filestore(): + """The demo filestore shipped alongside the installed ``cortex`` package.""" + pkgdir = os.path.dirname(os.path.abspath(cortex.__file__)) + candidates = [ + # Source checkout or editable install: filestore/ sits beside cortex/. + os.path.join(pkgdir, os.pardir, "filestore", "db"), + # Installed: setup.py copies filestore/ to /share/pycortex. + os.path.join(sys.prefix, "share", "pycortex", "db"), + ] + for path in candidates: + path = os.path.realpath(path) + if os.path.isdir(path): + return path + raise RuntimeError( + "could not locate the filestore bundled with pycortex; looked in " + + ", ".join(candidates) + ) + + +FILESTORE = _bundled_filestore() + +options.config.set("basic", "filestore", FILESTORE) +database.default_filestore = FILESTORE +# The `filestore=default_filestore` defaults throughout database.py were bound +# at import, so the singleton has to be repointed by hand. Everything reached +# through it (SubjectDB and below) is passed `self.filestore` explicitly. +cortex.db.filestore = FILESTORE +cortex.db.reload_subjects() + + +def pytest_report_header(config): + return f"pycortex filestore: {FILESTORE}" diff --git a/cortex/tests/reference_images/README.md b/cortex/tests/reference_images/README.md new file mode 100644 index 000000000..9c898220c --- /dev/null +++ b/cortex/tests/reference_images/README.md @@ -0,0 +1,107 @@ +# Reference images + +Stored renders that `cortex/tests/test_visual_regression.py` asserts against. + +## Contents + +| directory | images | contents | +| --- | --- | --- | +| `alpha_dataviews/` | 10 | five of the six public dataview classes (`Volume`, `Vertex`, `Volume2D`, `VolumeRGB`, `VertexRGB`), both renderers | +| `nan_dataviews/` | 10 | the same five, with NaNs over roughly half the primary data channel | +| `nan_alpha_dataviews/` | 4 | `VolumeRGB`/`VertexRGB` only, with the NaNs in the `alpha=` map | +| `nonflat_views/` | 4 | `Volume`/`Vertex` on the inflated and fiducial surfaces at `lateral_pivot`, webgl only | + +Filenames are `quickflat_` and `webgl_`, except `nonflat_views/`, +which uses `webgl___`. + +`Vertex2D` is the sixth class and has no images: its webgl flatmap renders +blank (gh-714) and `save_3d_views` raises, so it cannot be tested through the +webgl path at all. The two `Vertex2D` tests are **xfailed** on that +`RuntimeError`, strictly — if the render ever succeeds the XPASS says so rather +than passing silently. + +## Render settings + +The three flatmap directories render `quickflat_*` with +`cortex.quickflat.make_png` and `webgl_*` with `save_3d_views`, both with +curvature **un-thresholded** (`curvature_threshold=False` and +`surface.{subject}.curvature.smoothness=1.0`). (This is to avoid failures from +differences in the renderers' anti-aliasing implementations.) +Everything else is at its default. + +`nonflat_views/` keeps pycortex's default thresholded curvature, unlike the +flatmap groups. + +The exact keyword arguments are in `_render_and_check_dataview` and +`_render_and_check_webgl_only`; change either and the references must be +regenerated. + +## Checks + +The three flatmap tests check each render twice: against its own stored +reference at a tight tolerance (`MAX_MEAN_ABS_DIFF`, `MAX_FRACTION_DIFFERING`, +`MAX_FRACTION_GROSSLY_DIFFERING`, `MAX_SSIM_LOSS`, all four of which must pass), +and against the other renderer's render of the same dataview at a loose one +(`CROSS_MAX_MEAN_ABS_DIFF`, `CROSS_MAX_FRACTION_DIFFERING`), with no stored +fixture. `test_visual_comparison_nonflat_views` runs the reference check only. + +Both renderers write their flatmap content-tight and transparent outside it, so +the cross-renderer check only resizes webgl to quickflat's size before diffing. +RGB under fully transparent pixels is normalized first: it is undefined there, +and matplotlib leaves white where the browser leaves black. + +## Provenance + +Generated on `main` (`3779f7ca`), from the demo subject `S1` in the filestore +bundled with pycortex, which is pinned by `cortex/tests/conftest.py`. + +| | | +| --- | --- | +| chromium | 151.0.7922.34 (headless shell, SwiftShader software rendering) | +| playwright | 1.62.0 (fixes the chromium build above) | +| matplotlib | 3.10.9 | + +Both are pinned in the `test` dependency group, and re-pinning is part of +regenerating. playwright fixes the chromium build, which determines the 16 webgl +references; matplotlib rasterizes the 12 quickflat ones. + +Update matplotlib beyond 3.10.9 once Python 3.10 is dropped. + +## Format + +Lossless WebP (`method=6`, `quality=100`, `exact=True`): bit-exact after decode, +and 59% the size of optimized PNG (1229 KiB versus 2061 KiB for the set of 28). + +## Storage + +Tracked with **git LFS**. If yours are 130-byte text files rather than images, +the clone has not fetched them: + +``` +git lfs install && git lfs pull +``` + +The tests skip on that, and on the images being absent altogether, rather than +failing. + +## Distribution + +Kept out of the wheel (`exclude_package_data` in `setup.py`) and kept in the +source tarball (`MANIFEST.in`'s `recursive-include cortex *`), so a run against +an installed wheel degrades gracefully. + +## Regenerating + +The renders are deterministic: repeated runs on one machine produce +bit-identical output, including the WebGL ones under software rendering. They +are coupled to the Chromium and matplotlib builds above, so an upgrade can shift +anti-aliasing and rasterization; the tolerances absorb small shifts. If a +failure exceeds them, inspect the `diff_*.png` files it writes, confirm the +change is cosmetic, then: + +``` +REGENERATE_REFERENCE_IMAGES=1 pytest cortex/tests/test_visual_regression.py +``` + +That rewrites all four directories in one run. Review the resulting diff before +committing. diff --git a/cortex/tests/reference_images/alpha_dataviews/quickflat_Vertex.webp b/cortex/tests/reference_images/alpha_dataviews/quickflat_Vertex.webp new file mode 100644 index 000000000..641c5f763 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/quickflat_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5752210343f483e196538c7d216797217c775126060201442dbb2af44788a139 +size 39082 diff --git a/cortex/tests/reference_images/alpha_dataviews/quickflat_VertexRGB.webp b/cortex/tests/reference_images/alpha_dataviews/quickflat_VertexRGB.webp new file mode 100644 index 000000000..850554eb8 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/quickflat_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee0c980265a26e57bfb7e05c24f890ae13b232c0b41d109ec3752ab51f8cd1c +size 65514 diff --git a/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume.webp b/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume.webp new file mode 100644 index 000000000..84792e199 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da394850f9a4bb52b8427c122f8cb3f91395402022920c5812231f6502aebd8 +size 31144 diff --git a/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume2D.webp b/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume2D.webp new file mode 100644 index 000000000..0e5a5dc00 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/quickflat_Volume2D.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea3d927b30146cd622c9459d96d819e87efa621a865fb967d6ce3d3defd4f56 +size 60234 diff --git a/cortex/tests/reference_images/alpha_dataviews/quickflat_VolumeRGB.webp b/cortex/tests/reference_images/alpha_dataviews/quickflat_VolumeRGB.webp new file mode 100644 index 000000000..a3343f37d --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/quickflat_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0d45a0a3818e32decfa28266949b738ebdbe53a01f1b79224bda8b38270fd9 +size 68036 diff --git a/cortex/tests/reference_images/alpha_dataviews/webgl_Vertex.webp b/cortex/tests/reference_images/alpha_dataviews/webgl_Vertex.webp new file mode 100644 index 000000000..49bc5965d --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/webgl_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61fd0bdd4f1e770da67844aca34fb1a25d0712fe81a5014e716c22596373c789 +size 41080 diff --git a/cortex/tests/reference_images/alpha_dataviews/webgl_VertexRGB.webp b/cortex/tests/reference_images/alpha_dataviews/webgl_VertexRGB.webp new file mode 100644 index 000000000..7fbe6ba98 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/webgl_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78488461e5a1e0d7d58fb3c7ce199343809c021998d8a6da431284de0e315899 +size 59918 diff --git a/cortex/tests/reference_images/alpha_dataviews/webgl_Volume.webp b/cortex/tests/reference_images/alpha_dataviews/webgl_Volume.webp new file mode 100644 index 000000000..549e5c51d --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/webgl_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33233ea2bd0eca6ecd3fa3feb1110abd8e61e66da54b125a3a90c57cde874310 +size 13956 diff --git a/cortex/tests/reference_images/alpha_dataviews/webgl_Volume2D.webp b/cortex/tests/reference_images/alpha_dataviews/webgl_Volume2D.webp new file mode 100644 index 000000000..bb1fb5cb5 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/webgl_Volume2D.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d612aca5419a8824c97dd77f37e32e27c45c984a30632c34ac9232163ad5615f +size 62224 diff --git a/cortex/tests/reference_images/alpha_dataviews/webgl_VolumeRGB.webp b/cortex/tests/reference_images/alpha_dataviews/webgl_VolumeRGB.webp new file mode 100644 index 000000000..a87150500 --- /dev/null +++ b/cortex/tests/reference_images/alpha_dataviews/webgl_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9614a34b710bdb36df2adaff7caa043b9df737e110e4d68560e0981483d3f2eb +size 67094 diff --git a/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VertexRGB.webp b/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VertexRGB.webp new file mode 100644 index 000000000..97c955757 --- /dev/null +++ b/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d872b9c53ae897df179a4b1d0290280d612b33d818bfd71c2e873029c40fe341 +size 49050 diff --git a/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VolumeRGB.webp b/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VolumeRGB.webp new file mode 100644 index 000000000..4555a4e46 --- /dev/null +++ b/cortex/tests/reference_images/nan_alpha_dataviews/quickflat_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e5451db9418972db01b7c22a8b72fbef81c3676174315c71a2292a468fb4bc +size 58538 diff --git a/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VertexRGB.webp b/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VertexRGB.webp new file mode 100644 index 000000000..c1a817f3a --- /dev/null +++ b/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e8767376652081f7c240359520dd7b142de47eea21c26930d0fff7eae5ce080 +size 44118 diff --git a/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VolumeRGB.webp b/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VolumeRGB.webp new file mode 100644 index 000000000..717bf686d --- /dev/null +++ b/cortex/tests/reference_images/nan_alpha_dataviews/webgl_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:053954c6c122d1b4472b1af81289abe9b7a3c36814a61c05b9c81d6b79545eb8 +size 53344 diff --git a/cortex/tests/reference_images/nan_dataviews/quickflat_Vertex.webp b/cortex/tests/reference_images/nan_dataviews/quickflat_Vertex.webp new file mode 100644 index 000000000..948fbdbc9 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/quickflat_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ab79f41b3bc35a57d193656dfb2a70c21c4f4c9dfe56da82440a0aa7f14d60e +size 38986 diff --git a/cortex/tests/reference_images/nan_dataviews/quickflat_VertexRGB.webp b/cortex/tests/reference_images/nan_dataviews/quickflat_VertexRGB.webp new file mode 100644 index 000000000..c04f5de5f --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/quickflat_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8027a1d881f77e552963f83a6f68b69e00663189e36ddff5d2c84aec54571be9 +size 30010 diff --git a/cortex/tests/reference_images/nan_dataviews/quickflat_Volume.webp b/cortex/tests/reference_images/nan_dataviews/quickflat_Volume.webp new file mode 100644 index 000000000..643932918 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/quickflat_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea04a2943639a505888b504264863c1ef23b5153e9d4ade3fc2d7c74eb8c032 +size 34472 diff --git a/cortex/tests/reference_images/nan_dataviews/quickflat_Volume2D.webp b/cortex/tests/reference_images/nan_dataviews/quickflat_Volume2D.webp new file mode 100644 index 000000000..9f50bd8f0 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/quickflat_Volume2D.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2489f82aaa532db103154120628782b5113483fd3d2f8d9b657bcd093461a7a1 +size 40774 diff --git a/cortex/tests/reference_images/nan_dataviews/quickflat_VolumeRGB.webp b/cortex/tests/reference_images/nan_dataviews/quickflat_VolumeRGB.webp new file mode 100644 index 000000000..05d5dad00 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/quickflat_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18aff91a34a7d41873ecd56da9d52d624874d32408251877955cca4768dc6109 +size 39054 diff --git a/cortex/tests/reference_images/nan_dataviews/webgl_Vertex.webp b/cortex/tests/reference_images/nan_dataviews/webgl_Vertex.webp new file mode 100644 index 000000000..769973307 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/webgl_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0560ca05cc76d825f0cb38bd4dd9de36c794edd40af94fdd5a31d3e896fce6cc +size 34368 diff --git a/cortex/tests/reference_images/nan_dataviews/webgl_VertexRGB.webp b/cortex/tests/reference_images/nan_dataviews/webgl_VertexRGB.webp new file mode 100644 index 000000000..7673f8a5d --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/webgl_VertexRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26e57dc88d969249c6c089630234d03825502657bfbad0417a1115d0c7b7cc24 +size 24764 diff --git a/cortex/tests/reference_images/nan_dataviews/webgl_Volume.webp b/cortex/tests/reference_images/nan_dataviews/webgl_Volume.webp new file mode 100644 index 000000000..06586c024 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/webgl_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19181281ed688aaed051424fd8f17e04bca74040d80f7a5e64fa70d35fa81fc0 +size 19984 diff --git a/cortex/tests/reference_images/nan_dataviews/webgl_Volume2D.webp b/cortex/tests/reference_images/nan_dataviews/webgl_Volume2D.webp new file mode 100644 index 000000000..2aaac2934 --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/webgl_Volume2D.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cfa8f9eeaefa822f4f83fd7255dd6419b1b9f97264322511ea7c906a7a650d5 +size 36210 diff --git a/cortex/tests/reference_images/nan_dataviews/webgl_VolumeRGB.webp b/cortex/tests/reference_images/nan_dataviews/webgl_VolumeRGB.webp new file mode 100644 index 000000000..54ae2edea --- /dev/null +++ b/cortex/tests/reference_images/nan_dataviews/webgl_VolumeRGB.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d27a3eb97200778e381e7a6255ea6b542e9d0a812ddcfd67d236c860344be6 +size 33174 diff --git a/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Vertex.webp b/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Vertex.webp new file mode 100644 index 000000000..e591dc2bf --- /dev/null +++ b/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a00e55b1e5eb8b471a93d0b56562e3adc546dc729b864fb4fb07bb76936ecf1 +size 183216 diff --git a/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Volume.webp b/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Volume.webp new file mode 100644 index 000000000..d279e03a7 --- /dev/null +++ b/cortex/tests/reference_images/nonflat_views/webgl_fiducial_lateral_pivot_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7382caed6471c48bab9bcd614fd9a614582dd5402a6e1bee5008daa4cd640ac1 +size 194610 diff --git a/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Vertex.webp b/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Vertex.webp new file mode 100644 index 000000000..f3a3ee4ed --- /dev/null +++ b/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Vertex.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e15254dbec20d5969e6c8e897b910f17e420f27e9d27f45b1fc01372aa2791 +size 103612 diff --git a/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Volume.webp b/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Volume.webp new file mode 100644 index 000000000..f07e2f1e1 --- /dev/null +++ b/cortex/tests/reference_images/nonflat_views/webgl_inflated_lateral_pivot_Volume.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c6397df6c68d926278915a4a5c0a25ae4b66210bcb6f829baba972bf55ac8e4 +size 114478 diff --git a/cortex/tests/test_export.py b/cortex/tests/test_export.py index cbb0aa9e4..e8bd3e32c 100644 --- a/cortex/tests/test_export.py +++ b/cortex/tests/test_export.py @@ -37,7 +37,7 @@ def test_save_3d_views_headless(): list_surfaces=["inflated"], size=(1024, 768), trim=False, - # The WebGL scene needs time to initialise surfaces before + # The WebGL scene needs time to initialize surfaces before # _set_view can succeed; sleep=10 (the default) is safe. sleep=10, headless=True, @@ -81,4 +81,29 @@ def test_plot_panels_headless(): # provided, should have written the file to disk. assert fig is not None assert os.path.isfile(save_name) - assert os.path.getsize(save_name) > 0 \ No newline at end of file + assert os.path.getsize(save_name) > 0 + + +def test_save_3d_views_raises_on_webgl_failure(monkeypatch): + """A reported WebGL failure aborts the render rather than writing a blank png. + + Injected rather than provoked, so it does not depend on a broken shader in + the tree under test. + """ + monkeypatch.setattr( + "cortex.export.headless.filter_webgl_failures", + lambda errors: ["[console.error] THREE.WebGLProgram: Could not initialise shader."], + ) + vol = cortex.Volume(np.random.randn(*volshape), subj, xfmname) + with tempfile.TemporaryDirectory() as tmpdir: + with pytest.raises(RuntimeError, match="WebGL failed while rendering"): + cortex.export.save_3d_views( + vol, + base_name=os.path.join(tmpdir, "boom"), + list_angles=["lateral_pivot"], + list_surfaces=["inflated"], + size=(512, 384), + trim=False, + sleep=10, + headless=True, + ) diff --git a/cortex/tests/test_headless.py b/cortex/tests/test_headless.py index 318882aaa..8496a6cf1 100644 --- a/cortex/tests/test_headless.py +++ b/cortex/tests/test_headless.py @@ -5,13 +5,16 @@ pip install playwright playwright install chromium """ +import os +import tempfile + import numpy as np import pytest import cortex from cortex.export.headless import _PlaywrightThread -from .testing_utils import has_playwright +from .testing_utils import has_playwright, wait_for_file pytestmark = pytest.mark.skipif( not has_playwright, @@ -154,3 +157,70 @@ def test_headless_viewer_in_notebook(): for out in last_outputs if out["output_type"] == "stream" ), f"Notebook cell did not produce expected output. Outputs: {last_outputs}" + + +def test_filter_webgl_failures_keeps_only_real_failures(): + """Only genuine WebGL failures match; ordinary console noise does not. + + The filter has to stay narrow: a healthy viewer always logs a console.error + for the Leap Motion websocket it cannot reach. + """ + from cortex.export.headless import filter_webgl_failures + + noise = [ + "[console.error] WebSocket connection to 'ws://127.0.0.1:6437/v6.json' " + "failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED", + "[console.warning] THREE.WebGLShader: gl.getShaderInfoLog() WARNING: 0:87", + "[console.warning] [.WebGL-0x21b400157a00]GL Driver Message (OpenGL, Perf)", + # Emitted alongside a link failure but meaningless alone: nothing calls + # gl.validateProgram(), so VALIDATE_STATUS is false for want of a run. + "[console.error] gl.VALIDATE_STATUS false", + "[console.error] gl.getError() 0", + ] + assert filter_webgl_failures(noise) == [] + + link_failure = "[console.error] THREE.WebGLProgram: Could not initialise shader." + context_failure = "[pageerror] Error creating WebGL context." + assert filter_webgl_failures(noise + [link_failure]) == [link_failure] + assert filter_webgl_failures(noise + [context_failure]) == [context_failure] + + +def test_browser_errors_are_delivered_before_teardown(): + """Console messages must arrive while the viewer is alive, not at teardown. + + Playwright's sync API dispatches queued events only while something is + calling into it, so without the polling loop in ``_PlaywrightThread._run`` + nothing reaches ``browser_errors`` until ``_cleanup()`` -- long after + ``save_3d_views`` reads it. Rendering therefore has to grow the count. + + Note the growth comes from console noise the viewer emits anyway + (leapmotion websocket errors), not from the render. This test fails once + leapmotion is removed, and it would blame the polling loop. A fix would be + to emit a known message from the page, making it self-contained. + """ + import time + + from cortex.export.headless import EVENT_POLL_INTERVAL + + vol = cortex.Volume(np.random.randn(*volshape), subj, xfmname) + with cortex.export.headless_viewer(vol, viewer_params={}) as handle: + after_load = len(handle._pw_thread.browser_errors) + with tempfile.TemporaryDirectory() as tmpdir: + outfile = os.path.join(tmpdir, "poll.png") + handle.getImage(outfile, (512, 384)) + wait_for_file(outfile) + # Poll rather than sleep a fixed interval; a few cycles is enough. + deadline = time.time() + 2.0 + while time.time() < deadline: + during_render = len(handle._pw_thread.browser_errors) + if during_render > after_load: + break + time.sleep(EVENT_POLL_INTERVAL) + else: + during_render = len(handle._pw_thread.browser_errors) + + assert during_render > after_load, ( + "browser_errors did not grow while the viewer was alive (%d -> %d); " + "queued console events are not being dispatched" + % (after_load, during_render) + ) diff --git a/cortex/tests/test_visual_regression.py b/cortex/tests/test_visual_regression.py new file mode 100644 index 000000000..6e57b0210 --- /dev/null +++ b/cortex/tests/test_visual_regression.py @@ -0,0 +1,818 @@ +"""Visual regression tests: quickflat and webgl renders vs stored references. + +Four suites. Three of them render flatmaps of the six public dataview classes +(``Volume``, ``Vertex``, ``Volume2D``, ``Vertex2D``, ``VolumeRGB``, +``VertexRGB``) through both matplotlib (``cortex.quickflat.make_png``) and the +headless WebGL viewer (``save_3d_views``), varying what the data carries: +alpha-bearing values, NaNs in the data channels, and NaNs in the alpha map. The +last covers only the two RGB classes, the only ones taking an explicit +``alpha=``. Every one of those renders is checked twice -- against its own +stored reference at a tight tolerance, and directly against the other renderer's +render of the same dataview at a loose one. + +The fourth suite renders non-flatmap views, ``Volume`` and ``Vertex`` on the +inflated and fiducial surfaces, through ``save_3d_views``. Those are +webgl-only and get the reference check alone: quickflat produces flatmaps and +nothing else, so there is nothing to diff them against. + +Every render is transparent outside the flatmap, so the two renderers are +directly comparable without compositing or a coordinate correction. + +See ``reference_images/README.md`` for how the references were produced and how +to regenerate them. + +All tests are skipped if playwright is not installed. +""" + +import os +from pathlib import Path +from typing import Optional + +import numpy as np +import numpy.typing as npt +import pytest + +import cortex +import cortex.export +import cortex.polyutils +from cortex.dataset import Dataview +from cortex.tests.testing_utils import has_playwright + +pytestmark = pytest.mark.skipif( + not has_playwright, reason="playwright and chromium are required" +) + +# Vertex2D cannot be tested through the webgl path: its flatmap renders blank +# (gh-714) and save_3d_views raises, so no reference can be generated. #679's +# lighting refactor ported the HASFLAT bump-displacement block into the vertex +# shader, which under headless/SwiftShader leaves that flatmap unrendered. The +# mark is strict and on RuntimeError specifically, so a render that starts +# succeeding does not quietly pass: it reaches the reference check, which fails +# with a mismatched exception type and tells you to regenerate. +DATAVIEW_NAMES = [ + "Volume", + "Vertex", + "Volume2D", + pytest.param( + "Vertex2D", + marks=pytest.mark.xfail( + raises=RuntimeError, + strict=True, + reason="gh-714: the Vertex2D flatmap shader fails to link", + ), + ), + "VolumeRGB", + "VertexRGB", +] + +subj = "S1" +xfmname = "fullhead" + +#: Stored renders this test asserts against. See that directory's README for how +#: they were produced and how to regenerate them. +REFERENCE_ROOT = Path(__file__).parent / "reference_images" + +REFERENCE_DIR = REFERENCE_ROOT / "alpha_dataviews" + +#: As REFERENCE_DIR, but for dataviews whose source data contains NaNs. +NAN_REFERENCE_DIR = REFERENCE_ROOT / "nan_dataviews" + +#: As NAN_REFERENCE_DIR, but with the NaNs in the *alpha map* rather than in the +#: data. Only the RGB dataviews take an explicit ``alpha=``, so only those two. +NAN_ALPHA_REFERENCE_DIR = REFERENCE_ROOT / "nan_alpha_dataviews" + +#: Dataviews that accept an explicit alpha map, and so can carry NaNs in it. +NAN_ALPHA_DATAVIEW_NAMES = ["VolumeRGB", "VertexRGB"] + +#: Non-flatmap views, checked against a webgl reference only. quickflat renders +#: nothing but flatmaps, so these have no counterpart to diff against and no +#: cross-renderer leg -- see test_visual_comparison_nonflat_views. +NONFLAT_REFERENCE_DIR = REFERENCE_ROOT / "nonflat_views" + +#: (surface, angle, dataview). Volume and Vertex cover both shader paths, which +#: matters because the flatmap suite exercises them under conditions that turn +#: out to be a different regime: the two known webgl lighting bugs reproduce on +#: flatmaps only. +NONFLAT_VIEWS = [ + ("inflated", "lateral_pivot", "Volume"), + ("inflated", "lateral_pivot", "Vertex"), + ("fiducial", "lateral_pivot", "Volume"), + ("fiducial", "lateral_pivot", "Vertex"), +] + +#: Lossless WebP: bit-exact after decode and appreciably smaller than optimized +#: PNG. +REFERENCE_SUFFIX = ".webp" + +#: First bytes of a git-lfs pointer. The reference images are LFS-tracked, so +#: if LFS hasn't been properly initialized, these 130-byte text stubs exist in +#: place of the images. +LFS_POINTER_MAGIC = b"version https://git-lfs.github.com/spec/v1" + +#: Rewrite the references from this run instead of comparing against them. +REGENERATE_REFERENCES = bool(os.environ.get("REGENERATE_REFERENCE_IMAGES")) + +# Tolerances. The renders are deterministic -- repeated runs on one machine are +# bit-identical -- so these are not absorbing noise. They exist because the +# references are coupled to the Chromium and matplotlib builds that produced them, +# and an upgrade can shift anti-aliasing and rasterization slightly. They are far +# tighter than any real regression: a wrong colormap, a dropped alpha channel or +# swapped color channels all move large areas of the image by much more. +MAX_MEAN_ABS_DIFF = 2.0 # mean |difference| over all pixels/channels, of 255 +DIFF_THRESHOLD = 16 # a pixel "differs" if any channel moves by more +MAX_FRACTION_DIFFERING = 0.02 # at most this fraction of pixels may differ + +# The two limits above are both weak against a change that moves a *small* +# number of pixels by a *large* amount, which is what a geometry or contour +# shift looks like: the mean is diluted by the pixels that did not move. So two +# further criteria, each covering what the others miss: +# - mean and fraction>16 catch broad, low-amplitude shifts, which the gross +# fraction misses entirely. +# - fraction>32 catches sparse, high-amplitude ones. 32 rather than 64 because +# gh-695's premultiplied-alpha change scores 0% at 64 -- the suite would miss +# it -- and its mean sits below the cosmetic floor, so no tighter mean helps. +# - SSIM catches structural change, but is computed on luminance and so is +# blind to a channel permutation. It adds sensitivity alongside the others; +# it cannot replace them. +# +# The limits sit well above measured cosmetic drift rather than at it, since a +# real toolchain bump changes anti-aliasing and does show up in fraction>32. +GROSS_DIFF_THRESHOLD = 32 # a pixel differs "grossly" if any channel moves by more +MAX_FRACTION_GROSSLY_DIFFERING = 0.001 +MAX_SSIM_LOSS = 0.01 # 1 - mean SSIM over the luminance channel + +# Cross-renderer tolerances: quickflat vs webgl for the *same* dataview, rather +# than each against its own reference. Looser than the within-renderer ones +# above, since the two renderers genuinely differ in anti-aliasing and colormap +# sampling. Re-measure after any change to either renderer's output size. +CROSS_MAX_MEAN_ABS_DIFF = 3.5 +CROSS_DIFF_THRESHOLD = 32 # same as GROSS_DIFF_THRESHOLD. +CROSS_MAX_FRACTION_DIFFERING = 0.032 + +# Render settings chosen to minimize cross-renderer disagreement, from a +# factorial sweep of both renderers' settings. Only curvature thresholding was +# worth changing from the defaults: a thresholded curvature puts a hard binary +# edge at curvature=0 that each rasterizer resolves differently, where smooth +# curvature is low-frequency and resamples cleanly. On curvature-only content +# it costs 1.8x on the mean -- passing, now that the renders are the same size, +# but eating most of the headroom for no benefit. quickflat's +# ``curvature_threshold`` and webgl's ``curvature.smoothness`` are the same knob +# from opposite ends -- smoothness 0.0 *is* thresholded -- so both have to move +# together. +# +# Lighting, sampler, depth and thick/layers all stayed at their defaults; none +# moved the disagreement measurably on a flatmap, whose normals face the camera. +# +# NB this is not pycortex's default appearance, so these references do not cover +# the default curvature path. That is the trade for a tighter floor; +# nonflat_views/ keeps the default and recovers the coverage. +#: Height of the quickflat render. make_png scales the width to the subject's +#: flatmap aspect, giving roughly 490x256. +QUICKFLAT_HEIGHT = 256 + +#: Browser canvas for every webgl render. After trimming, lands at roughly +# quickflat's 490x256 (from QUICKFLAT_HEIGHT). +WEBGL_CANVAS = (925, 695) + +# Don't threshold curvature to reduce cross-renderer disagreement due to +# anti-aliasing implementations. +QUICKFLAT_CURVATURE_THRESHOLD = False +WEBGL_CURVATURE_SMOOTHNESS = 1.0 + + +def _normalize_transparent(rgba: npt.NDArray) -> npt.NDArray: + """Zero the RGB of fully transparent pixels, which is undefined there.""" + out = rgba.astype(np.int16).copy() + out[out[..., 3] == 0, :3] = 0 + return out + + +def _ssim(a: npt.NDArray, b: npt.NDArray) -> float: + """Mean structural similarity between two RGBA images, over luminance. + + Standard SSIM with an 11x11 Gaussian window (sigma 1.5) and the usual + stabilising constants, implemented on scipy because scikit-image, which + would otherwise supply it, is not a pycortex dependency. Returns 1.0 for + identical input. + + Computed on the channel mean, so it is invariant to a channel permutation -- + see the note on MAX_SSIM_LOSS. It is a structural check, not a color one. + """ + from scipy.ndimage import gaussian_filter + + x = a[..., :3].mean(axis=-1).astype(np.float64) + y = b[..., :3].mean(axis=-1).astype(np.float64) + c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2 + + blur = lambda img: gaussian_filter(img, sigma=1.5, truncate=(11 - 1) / 2 / 1.5) + mu_x, mu_y = blur(x), blur(y) + var_x = blur(x * x) - mu_x**2 + var_y = blur(y * y) - mu_y**2 + cov = blur(x * y) - mu_x * mu_y + + num = (2 * mu_x * mu_y + c1) * (2 * cov + c2) + den = (mu_x**2 + mu_y**2 + c1) * (var_x + var_y + c2) + return float((num / den).mean()) + + +def _unusable_reference(ref_path: Path) -> Optional[str]: + """Why ``ref_path`` cannot be compared against, or None if it can. + + Absent and unfetched-from-LFS are separate cases with separate remedies, and + neither should fail the run: an installed wheel legitimately has no + references, and a pointer means the clone simply has not fetched them. + """ + if not ref_path.exists(): + return ( + f"No reference {ref_path.name} in {ref_path.parent}. " + "See that directory's README for regeneration." + ) + with open(ref_path, "rb") as handle: + if handle.read(len(LFS_POINTER_MAGIC)) == LFS_POINTER_MAGIC: + return ( + f"{ref_path.name} is an unfetched git-lfs pointer, not an " + "image. Run `git lfs pull`." + ) + return None + + +def _reference_store_unusable() -> Optional[str]: + """Why the whole reference store is unusable, or None if it is fine. + + Absent-entirely and pointers-everywhere are properties of the checkout, not + of one render, so they are settled once at import. Leaving them to the + per-file check would render all eighteen views before skipping -- about 90 + seconds to produce eighteen skips. The per-file check still runs, for the + case this cannot see: a populated directory missing one render. + """ + stored = sorted(REFERENCE_ROOT.glob(f"*/*{REFERENCE_SUFFIX}")) + if not stored: + return ( + f"No reference images under {REFERENCE_ROOT}. " + "See that directory's README for regeneration." + ) + return _unusable_reference(stored[0]) + + +if not REGENERATE_REFERENCES: + _store_problem = _reference_store_unusable() + if _store_problem is not None: + pytest.skip(_store_problem, allow_module_level=True) + + +def _check_against_reference( + name: str, + actual_path: Path, + debug_dir: Path, + reference_dir: Path, +) -> Optional[str]: + """Compare one render to its reference. + + Checks four criteria, all of which must pass -- see ``MAX_MEAN_ABS_DIFF`` + and the tolerances below it for why they are complementary rather than + redundant. Returns a description of every breach, or None if the render + matches. + + A shape mismatch is not an immediate failure: the render is resized to the + reference's size before diffing, since the two are still expected to show + the same content at a slightly different trim/crop. + + On any breach, writes the (possibly resized) render and an amplified + difference image into ``debug_dir``, so the change can be inspected rather + than guessed at. With ``REGENERATE_REFERENCES`` set it overwrites the + reference instead and reports no mismatch. + """ + from PIL import Image + + ref_path = reference_dir / f"{name}{REFERENCE_SUFFIX}" + + if REGENERATE_REFERENCES: + reference_dir.mkdir(parents=True, exist_ok=True) + Image.open(actual_path).convert("RGBA").save( + ref_path, format="WEBP", lossless=True, method=6, quality=100, exact=True + ) + return None + + actual_im = Image.open(actual_path).convert("RGBA") + ref_im = Image.open(ref_path).convert("RGBA") + shape_note = "" + if actual_im.size != ref_im.size: + shape_note = f" (aligned: render was {actual_im.size}, reference is {ref_im.size})" + actual_im = actual_im.resize(ref_im.size, Image.BILINEAR) + + actual = np.asarray(actual_im).astype(np.int16) + ref = np.asarray(ref_im).astype(np.int16) + + diff = np.abs(actual - ref) + per_pixel = diff.max(axis=-1) + mean_abs = float(diff.mean()) + fraction = float((per_pixel > DIFF_THRESHOLD).mean()) + gross = float((per_pixel > GROSS_DIFF_THRESHOLD).mean()) + ssim_loss = 1.0 - _ssim(actual, ref) + + breaches = [] + if mean_abs > MAX_MEAN_ABS_DIFF: + breaches.append(f"mean|diff|={mean_abs:.3f} (limit {MAX_MEAN_ABS_DIFF})") + if fraction > MAX_FRACTION_DIFFERING: + breaches.append( + f"{fraction:.2%} of pixels differ by more than {DIFF_THRESHOLD} " + f"(limit {MAX_FRACTION_DIFFERING:.0%})" + ) + if gross > MAX_FRACTION_GROSSLY_DIFFERING: + breaches.append( + f"{gross:.3%} of pixels differ by more than {GROSS_DIFF_THRESHOLD} " + f"(limit {MAX_FRACTION_GROSSLY_DIFFERING:.1%})" + ) + if ssim_loss > MAX_SSIM_LOSS: + breaches.append(f"SSIM loss={ssim_loss:.4f} (limit {MAX_SSIM_LOSS})") + if not breaches: + return None + + actual_im.save(debug_dir / f"actual_{name}.png") + amplified = np.clip(diff[..., :3] * 8, 0, 255).astype("uint8") + Image.fromarray(amplified).save(debug_dir / f"diff_{name}.png") + return f"{name}: " + "; ".join(breaches) + shape_note + + +def _check_cross_renderer( + name: str, + quickflat_path: Path, + webgl_path: Path, + debug_dir: Path, +) -> Optional[str]: + """Compare a quickflat render directly against its webgl counterpart. + + Unlike ``_check_against_reference`` this has no stored fixture: it diffs + the two renders produced by *this* test run against each other. Both are + content-tight and transparent outside the flatmap, so webgl is resized to + quickflat's size and diffed as RGBA -- no coordinate correction is needed. + + RGB under fully transparent pixels is normalized first. It is undefined + there, and the two writers disagree: matplotlib leaves white, the browser + leaves black. Alpha itself stays in the comparison, so a render that + lost its transparency is still caught. + """ + from PIL import Image + + qf_im = Image.open(quickflat_path).convert("RGBA") + wg_im = Image.open(webgl_path).convert("RGBA") + shape_note = f" (resized: quickflat was {qf_im.size}, webgl was {wg_im.size})" + + qf = _normalize_transparent(np.asarray(qf_im)) + wg = _normalize_transparent(np.asarray(wg_im.resize(qf_im.size, Image.BILINEAR))) + + diff = np.abs(qf - wg) + mean_abs = float(diff.mean()) + fraction = float((diff.max(axis=-1) > CROSS_DIFF_THRESHOLD).mean()) + if mean_abs <= CROSS_MAX_MEAN_ABS_DIFF and fraction <= CROSS_MAX_FRACTION_DIFFERING: + return None + + amplified = np.clip(diff[..., :3] * 4, 0, 255).astype("uint8") + Image.fromarray(amplified).save(debug_dir / f"cross_diff_{name}.png") + return ( + f"cross_{name}: quickflat vs webgl mean|diff|={mean_abs:.3f} " + f"(limit {CROSS_MAX_MEAN_ABS_DIFF}), {fraction:.2%} of pixels differ by " + f"more than {CROSS_DIFF_THRESHOLD} (limit {CROSS_MAX_FRACTION_DIFFERING:.0%})" + f"{shape_note}" + ) + + +# Gaussian falloff from `seed`, used as the accuracy/alpha channel. +def _bump( + surf: cortex.polyutils.Surface, seed: int, sigma: float +) -> npt.NDArray[np.floating]: + d = np.linalg.norm(surf.pts - surf.pts[seed], axis=1) + return np.exp(-(d**2) / (2 * sigma**2)) + + +def _synth_arrays() -> dict: + """The clean volume and surface data all three builders start from. + + Shared so they cannot drift apart: what distinguishes the suites is which + elements they then NaN out, and that is the thing under test. + """ + zz, yy, xx = np.mgrid[0:31, 0:100, 0:100] + center = np.array([15, 50, 50]) + sigma_v = 25.0 + dist2 = (zz - center[0]) ** 2 + (yy - center[1]) ** 2 + (xx - center[2]) ** 2 + + # Vertex data is encoded by spatial coordinate, not by vertex index. + surfs = [ + cortex.polyutils.Surface(*d) for d in cortex.db.get_surf(subj, "fiducial") + ] + num_verts = [s.pts.shape[0] for s in surfs] + pts = np.vstack([surfs[0].pts, surfs[1].pts]) + y_centered = pts[:, 1] - pts[:, 1].mean() + + return dict( + xx=xx, yy=yy, zz=zz, + num_verts=num_verts, + data_vol=(xx - 50) / 50.0, # ~ [-1, 1] + accuracy_vol=np.exp(-dist2 / (2 * sigma_v**2)), # [0, 1] bump + red_vol=np.clip(xx / 99.0, 0, 1), + green_vol=np.clip(yy / 99.0, 0, 1), + blue_vol=np.clip(zz / 30.0, 0, 1), + data_vtx=y_centered / np.abs(y_centered).max(), # [-1, 1] + xyz_norm=(pts - pts.min(axis=0)) / (pts.max(axis=0) - pts.min(axis=0)), + accuracy_vtx=np.hstack([ + _bump(surfs[0], num_verts[0] // 2, sigma=40.0), + _bump(surfs[1], num_verts[1] // 2, sigma=40.0), + ]), + ) + + +def _dataview( + name: str, + *, + data_vol: npt.NDArray, + dim2_vol: npt.NDArray, + rgb_vol: tuple, + alpha_vol: npt.NDArray, + data_vtx: npt.NDArray, + dim2_vtx: npt.NDArray, + rgb_vtx: tuple, + alpha_vtx: npt.NDArray, +) -> Dataview: + """Construct one of the six dataview classes from prepared channels. + + One dispatch shared by all three suites, so adding a dataview class is a + single edit rather than three kept in lockstep. + """ + cmap_plain, cmap_2d = "viridis", "RdBu_r_alpha" + + if name == "Volume": + return cortex.Volume(data_vol, subj, xfmname, cmap=cmap_plain, vmin=-1, vmax=1) + elif name == "Vertex": + return cortex.Vertex(data_vtx, subj, cmap=cmap_plain, vmin=-1, vmax=1) + elif name == "Volume2D": + return cortex.Volume2D( + data_vol, dim2_vol, subj, xfmname, cmap=cmap_2d, + vmin=-1, vmax=1, vmin2=0, vmax2=1, + ) + elif name == "Vertex2D": + return cortex.Vertex2D( + data_vtx, dim2_vtx, subj, cmap=cmap_2d, + vmin=-1, vmax=1, vmin2=0, vmax2=1, + ) + elif name == "VolumeRGB": + red, green, blue = rgb_vol + return cortex.VolumeRGB( + cortex.Volume(red, subj, xfmname, vmin=0, vmax=1), + cortex.Volume(green, subj, xfmname, vmin=0, vmax=1), + cortex.Volume(blue, subj, xfmname, vmin=0, vmax=1), + subj, xfmname, + alpha=cortex.Volume(alpha_vol, subj, xfmname, vmin=0, vmax=1), + ) + elif name == "VertexRGB": + red, green, blue = rgb_vtx + return cortex.VertexRGB( + cortex.Vertex(red, subj, vmin=0, vmax=1), + cortex.Vertex(green, subj, vmin=0, vmax=1), + cortex.Vertex(blue, subj, vmin=0, vmax=1), + subj, + alpha=cortex.Vertex(alpha_vtx, subj, vmin=0, vmax=1), + ) + else: + raise ValueError(f"Unknown dataview: {name}") + + +def _build_alpha_dataview(name: str) -> Dataview: + """Build a single alpha-bearing dataview by name, with no NaNs anywhere.""" + a = _synth_arrays() + return _dataview( + name, + data_vol=a["data_vol"], + dim2_vol=a["accuracy_vol"], + rgb_vol=(a["red_vol"], a["green_vol"], a["blue_vol"]), + alpha_vol=a["accuracy_vol"], + data_vtx=a["data_vtx"], + dim2_vtx=a["accuracy_vtx"], + rgb_vtx=tuple(a["xyz_norm"][:, i] for i in range(3)), + alpha_vtx=a["accuracy_vtx"], + ) + + +def _build_nan_dataview(name: str) -> Dataview: + """Build a dataview with NaNs over roughly half of the primary data channel.""" + a = _synth_arrays() + xx, yy, zz = a["xx"], a["yy"], a["zz"] + + # The rule, as gh-695 states it, is that a NaN *anywhere* at a voxel -- the + # data, either 2D dimension, any RGB channel, or the alpha map -- renders + # fully transparent. These references pin the behavior as it is on main. + # Each of those gets NaN'd over its own region, so a single render exercises + # several branches of the rule at once and a failure still says which one + # moved. The vertex regions are disjoint; the volume ones (x>=50, y>=50, + # z>=15) overlap, which is harmless and additionally covers voxels carrying + # more than one NaN at once. Blue is deliberately left clean, as a control + # that not everything has simply gone transparent. + # + # The alpha map is NaN'd here too, on a third axis. That is not a duplicate + # of the nan_alpha suite: this covers alpha NaNs superposed on color NaNs, + # where nan_alpha isolates them with every color channel clean. + # + # Expect the alpha map's surviving NaN fraction to look smaller than its + # mask: where a color channel is already NaN the pipeline writes alpha's + # vmin over it. Both halves of that are worth rendering, which is why the + # regions are allowed to overlap. + def vol_nan(arr, mask): + out = arr.copy() + out[mask] = np.nan + return out + + primary = xx >= 50 # data, and red for RGB + secondary = yy >= 50 # 2D dimension 2, and green for RGB + tertiary = zz >= 15 # the alpha map, on a third independent axis + + # As above, in disjoint index ranges rather than spatial ones. + total = sum(a["num_verts"]) + idx = np.arange(total) + vtx_primary = idx >= total // 2 + vtx_secondary = idx < total // 4 + vtx_tertiary = (idx >= total // 4) & (idx < total // 2) + xyz = a["xyz_norm"] + + return _dataview( + name, + data_vol=vol_nan(a["data_vol"], primary), + dim2_vol=vol_nan(a["accuracy_vol"], secondary), + rgb_vol=( + vol_nan(a["red_vol"], primary), + vol_nan(a["green_vol"], secondary), + a["blue_vol"], + ), + alpha_vol=vol_nan(a["accuracy_vol"], tertiary), + data_vtx=vol_nan(a["data_vtx"], vtx_primary), + dim2_vtx=vol_nan(a["accuracy_vtx"], vtx_secondary), + rgb_vtx=( + vol_nan(xyz[:, 0], vtx_primary), + vol_nan(xyz[:, 1], vtx_secondary), + xyz[:, 2], + ), + alpha_vtx=vol_nan(a["accuracy_vtx"], vtx_tertiary), + ) + + +def _build_nan_alpha_dataview(name: str) -> Dataview: + """Build an RGB dataview whose *alpha map* carries NaNs, color channels clean. + + The other NaN suite puts NaNs in the data; this puts them in the alpha map, + which is a separate code path -- alpha is not color-mapped, it is used + directly as a blend weight, so a NaN reaches the compositing arithmetic + rather than a colormap lookup. + + Current behavior is that those elements render fully transparent, i.e. the + curvature underlay shows through, which is what the other NaN cases do too. + """ + a = _synth_arrays() + alpha_vol = a["accuracy_vol"].copy() + alpha_vol[a["xx"] >= 50] = np.nan + + total = sum(a["num_verts"]) + alpha_vtx = a["accuracy_vtx"].copy() + alpha_vtx[np.arange(total) >= total // 2] = np.nan + + return _dataview( + name, + data_vol=a["data_vol"], + dim2_vol=a["accuracy_vol"], + rgb_vol=(a["red_vol"], a["green_vol"], a["blue_vol"]), + alpha_vol=alpha_vol, + data_vtx=a["data_vtx"], + dim2_vtx=a["accuracy_vtx"], + rgb_vtx=tuple(a["xyz_norm"][:, i] for i in range(3)), + alpha_vtx=alpha_vtx, + ) + + +def _assert_no_failures(failures: list[str], tmp_path: Path) -> None: + """Fail with every mismatch at once, and say where to look at them.""" + assert not failures, ( + "Renders differ from expectations:\n " + + "\n ".join(failures) + + f"\n\nFor details, see {tmp_path} and {REFERENCE_ROOT}/README.md" + ) + + +def _render_and_check_dataview( + name: str, + view: Dataview, + reference_dir: Path, + tmp_path: Path, +) -> list[str]: + """Render a single dataview through both renderers and check it. + + Each render is checked three ways: quickflat vs its own reference, webgl vs + its own reference (both tight tolerances, see ``_check_against_reference``), + and quickflat vs webgl directly (loose tolerance, see + ``_check_cross_renderer``). + + Returns a list of failure messages (empty if no failures). Skips the test + if reference images are missing, and regenerates them if ``REGENERATE_REFERENCES`` + is set. + """ + from cortex.export.save_views import angle_view_params, save_3d_views + + flatmap_angle = ( + "flatmap", + { + **angle_view_params["flatmap"], + "surface.{subject}.curvature.smoothness": WEBGL_CURVATURE_SMOOTHNESS, + }, + ) + + # quickflat -> transparent PNG via make_png. + qf_path = tmp_path / f"quickflat_{name}.png" + cortex.quickflat.make_png( + str(qf_path), + view, + height=QUICKFLAT_HEIGHT, + with_curvature=True, + with_rois=False, + with_labels=False, + with_colorbar=False, + with_sulci=False, + with_borders=False, + curvature_threshold=QUICKFLAT_CURVATURE_THRESHOLD, + ) + + # webgl -> trimmed flatmap screenshot. + wg_path = Path( + save_3d_views( + view, + base_name=str(tmp_path / f"webgl_{name}"), + list_angles=[flatmap_angle], + list_surfaces=["flatmap"], + trim=True, + size=WEBGL_CANVAS, + sleep=10, + viewer_params=dict(labels_visible=[], overlays_visible=[]), + headless=True, + )[0] + ) + + # Fails rather than skips: wholesale absence -- an installed wheel, or a + # clone that has not fetched LFS -- is caught at import, so a single gap in + # a populated store means either a render that cannot succeed (Vertex2D) or + # an incomplete regeneration. A skip here would also be swallowed by the + # xfail on Vertex2D, hiding the day its render starts working. + if not REGENERATE_REFERENCES: + for prefix in ("quickflat", "webgl"): + reason = _unusable_reference( + reference_dir / f"{prefix}_{name}{REFERENCE_SUFFIX}" + ) + if reason is not None: + pytest.fail(reason) + + failures = [] + + # _check_against_reference rewrites the reference and returns None when + # regenerating, so this collects nothing on that path. + for prefix, path in [("quickflat", qf_path), ("webgl", wg_path)]: + msg = _check_against_reference(f"{prefix}_{name}", path, tmp_path, reference_dir) + if msg is not None: + failures.append(msg) + + if REGENERATE_REFERENCES: + pytest.skip(f"Regenerated {name} references in {reference_dir}") + + # Cross-renderer check (never regenerates, always compares) + msg = _check_cross_renderer(name, qf_path, wg_path, tmp_path) + if msg is not None: + failures.append(msg) + + return failures + + +def _render_and_check_webgl_only( + tag: str, + view: Dataview, + surface: str, + angle: str, + reference_dir: Path, + tmp_path: Path, +) -> list[str]: + """Render one non-flatmap view through webgl and check it against a reference. + + A cut-down ``_render_and_check_dataview``: no cross-renderer check because + we don't use quickflat. + + Curvature is left at pycortex's default (thresholded) here, unlike the + flatmap suites. Those un-threshold it to reduce cross-renderer + disagreement -- a reason that does not apply when there is no second + renderer -- so using the default recovers coverage of the default curvature + path, which the flatmap references explicitly do not provide. + """ + from cortex.export.save_views import save_3d_views + + wg_path = save_3d_views( + view, + base_name=str(tmp_path / f"webgl_{tag}"), + list_angles=[angle], + list_surfaces=[surface], + trim=True, + size=WEBGL_CANVAS, + sleep=10, + viewer_params=dict(labels_visible=[], overlays_visible=[]), + headless=True, + )[0] + + # Fails rather than skips, as in _render_and_check_dataview. + if not REGENERATE_REFERENCES: + reason = _unusable_reference( + reference_dir / f"webgl_{tag}{REFERENCE_SUFFIX}" + ) + if reason is not None: + pytest.fail(reason) + + msg = _check_against_reference( + f"webgl_{tag}", Path(wg_path), tmp_path, reference_dir + ) + if REGENERATE_REFERENCES: + pytest.skip(f"Regenerated webgl_{tag} in {reference_dir}") + return [msg] if msg is not None else [] + + +@pytest.mark.parametrize("name", DATAVIEW_NAMES) +def test_visual_comparison_alpha_dataviews(tmp_path, name): + """Render an alpha-bearing dataview through both renderers, and assert it matches. + + Plain Volume / Vertex have no native per-element alpha (pycortex's + bundled ``*_alpha`` colormaps are all 2D and only apply to the 2D + dataview types), so those two act as a no-alpha baseline. The other four + exercise alpha: Volume2D / Vertex2D via the 2D-alpha cmap ``RdBu_r_alpha``, + VolumeRGB / VertexRGB via the ``alpha=`` kwarg. + + Compared both within-renderer (against a stored reference) and + cross-renderer (quickflat vs webgl); see ``_render_and_check_dataview``. A + mismatch leaves ``actual_*.png`` and an amplified ``diff_*.png`` in the + test's ``tmp_path``. + """ + view = _build_alpha_dataview(name) + failures = _render_and_check_dataview(name, view, REFERENCE_DIR, tmp_path) + _assert_no_failures(failures, tmp_path) + + +@pytest.mark.parametrize("name", DATAVIEW_NAMES) +def test_visual_comparison_nan_dataviews(tmp_path, name): + """Render a NaN-bearing dataview through both renderers, and assert it matches. + + NaN is pycortex's convention for "no data at this voxel/vertex" -- both + renderers are expected to draw those elements as fully transparent (falling + through to the curvature underlay) rather than mapping NaN through the + colormap as if it were a real value. This test renders the six dataview + classes with the *primary* data channel (not alpha) containing NaNs over + roughly half of each volume/surface. + + Compared both within-renderer (against a stored reference) and + cross-renderer (quickflat vs webgl); see ``_render_and_check_dataview``. A + mismatch leaves ``actual_*.png`` and an amplified ``diff_*.png`` in the + test's ``tmp_path``. + """ + view = _build_nan_dataview(name) + failures = _render_and_check_dataview(name, view, NAN_REFERENCE_DIR, tmp_path) + _assert_no_failures(failures, tmp_path) + + +@pytest.mark.parametrize("name", NAN_ALPHA_DATAVIEW_NAMES) +def test_visual_comparison_nan_alpha_dataviews(tmp_path, name): + """Render an RGB dataview whose alpha map carries NaNs, and assert it matches. + + The other NaN suite puts NaNs in the data channels; this one puts them in the + alpha map. That is a distinct path -- alpha is not color-mapped, it is used + directly as a blend weight, so the NaN lands in the compositing arithmetic + rather than in a colormap lookup. Only ``VolumeRGB``/``VertexRGB`` take an + explicit ``alpha=``, so only those two are covered. + + Current behavior, which these references encode, is that NaN-alpha elements + render fully transparent and the curvature underlay shows through -- the same + outcome as a NaN in the data. + + Be aware that this behavior is not settled. gh-695, which unifies NaN and + alpha handling across quickflat, WebGL and the RGB dataviews, changes how the + surviving RGB is blended without changing the transparency itself. If that lands, expect these four references to need + regenerating; the transparency assertion should survive, the exact blend will + not. + """ + view = _build_nan_alpha_dataview(name) + failures = _render_and_check_dataview(name, view, NAN_ALPHA_REFERENCE_DIR, tmp_path) + _assert_no_failures(failures, tmp_path) + + +@pytest.mark.parametrize("surface,angle,name", NONFLAT_VIEWS) +def test_visual_comparison_nonflat_views(tmp_path, surface, angle, name): + """Render a non-flatmap view through webgl and assert it matches its reference. + + The other tests render flatmaps, which in webgl can have different behavior + from 3D views (for example, lighting). + + This is webgl only (quickflat only renders flatmaps), so no cross-renderer + check. Test both Volume and Vertex because they take different shader + paths. + """ + view = _build_alpha_dataview(name) + tag = f"{surface}_{angle}_{name}" + failures = _render_and_check_webgl_only( + tag, view, surface, angle, NONFLAT_REFERENCE_DIR, tmp_path + ) + _assert_no_failures(failures, tmp_path) diff --git a/cortex/tests/test_webgl_headless.py b/cortex/tests/test_webgl_headless.py index f97340ebc..c01ac3766 100644 --- a/cortex/tests/test_webgl_headless.py +++ b/cortex/tests/test_webgl_headless.py @@ -22,7 +22,7 @@ default_view_params, unfold_view_params, ) -from cortex.tests.testing_utils import has_playwright +from cortex.tests.testing_utils import has_playwright, wait_for_file pytestmark = pytest.mark.skipif( not has_playwright, reason="playwright and chromium are required" @@ -63,13 +63,50 @@ def make_dataview(dtype_name): raise ValueError(f"Unknown dtype_name: {dtype_name}") -def _wait_for_file(path, timeout=30): - """Poll until file exists and has nonzero size, raise after timeout.""" - for _ in range(int(timeout / 0.1)): - if os.path.exists(path) and os.path.getsize(path) > 0: - return - time.sleep(0.1) - raise RuntimeError(f"File {path!r} not written within {timeout}s") +def _assert_no_browser_failures(handle): + """Fail on uncaught JS exceptions, or on WebGL reporting its own failure. + + ``[pageerror]`` covers uncaught exceptions; ``filter_webgl_failures`` covers + a shader that compiled but failed to *link*, which three.js reports only on + console.error, so nothing raises and the render comes back blank. + + ``browser_errors`` is current to within ``EVENT_POLL_INTERVAL``, so it does + not matter whether this is called inside the ``with`` block. + + Not the only defense: a driver that silently links an over-allocating shader + reports nothing, which is what ``_assert_not_blank`` is for. Neither covers + a shader variant no test renders. + + Known limitation: ``browser_errors`` is cumulative and never cleared, so + with a handle shared across tests (``TestAddData``) one transient failure + fails every later test in the class too. Misattributed, not missed; a + watermark index from the previous call would isolate it. + """ + from cortex.export.headless import filter_webgl_failures + + errors = handle._pw_thread.browser_errors + pageerrors = [e for e in errors if "[pageerror]" in e] + assert not pageerrors, f"JS errors: {pageerrors}" + failures = filter_webgl_failures(errors) + assert not failures, f"WebGL reported a failure: {failures}" + + +def _assert_not_blank(path): + """Fail if the render came out as a single flat color. + + A shader that fails to link, or geometry that never reached the GPU, leaves + only the background -- otherwise indistinguishable from success, since the + png is written and nothing raises. + """ + from PIL import Image + + rgb = np.asarray(Image.open(path).convert("RGB")).reshape(-1, 3).astype(np.uint32) + # Packed to one int per pixel; np.unique on a structured view costs ~16x more. + ncolors = len(np.unique((rgb[:, 0] << 16) | (rgb[:, 1] << 8) | rgb[:, 2])) + assert ncolors > 10, ( + f"{path} has only {ncolors} distinct color(s); the brain was probably " + "never drawn." + ) # --------------------------------------------------------------------------- @@ -79,7 +116,22 @@ def _wait_for_file(path, timeout=30): @pytest.mark.parametrize( "dtype_name", - ["Volume", "Vertex", "VolumeRGB", "VertexRGB", "Volume2D", "Vertex2D"], + [ + "Volume", + "Vertex", + "VolumeRGB", + "VertexRGB", + "Volume2D", + # gh-714: the Vertex2D flatmap shader fails to link, so the render comes + # back blank and three.js reports it on console.error. Strict, so a + # render that starts succeeding reports an XPASS. + pytest.param( + "Vertex2D", + marks=pytest.mark.xfail( + strict=True, reason="gh-714: Vertex2D shader fails to link" + ), + ), + ], ) def test_datatype_renders(dtype_name, tmp_path): """Each data type should render in the headless viewer without errors.""" @@ -87,12 +139,13 @@ def test_datatype_renders(dtype_name, tmp_path): with cortex.export.headless_viewer(vol, viewer_params={}) as handle: outfile = str(tmp_path / "test.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) assert os.path.isfile(outfile) assert os.path.getsize(outfile) > 0 - # No uncaught JS errors - pageerrors = [e for e in handle._pw_thread.browser_errors if "[pageerror]" in e] - assert len(pageerrors) == 0, f"JS errors: {pageerrors}" + + # Browser errors first, since one may explain a blank file. + _assert_no_browser_failures(handle) + _assert_not_blank(outfile) # --------------------------------------------------------------------------- @@ -127,7 +180,7 @@ def test_angle(self, angle_name): time.sleep(1) outfile = str(type(self).tmp_dir / f"{angle_name}.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) assert os.path.isfile(outfile) assert os.path.getsize(outfile) > 1000, "Image too small — may be blank" @@ -164,7 +217,7 @@ def test_surface(self, surface_name): time.sleep(1) outfile = str(type(self).tmp_dir / f"{surface_name}.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) assert os.path.isfile(outfile) assert os.path.getsize(outfile) > 1000, "Image too small — may be blank" @@ -244,7 +297,7 @@ def test_overlay_visibility_changes_image(tmp_path): handle._set_view(**view) time.sleep(1) handle.getImage(f1, (512, 384)) - _wait_for_file(f1) + wait_for_file(f1) # Render WITHOUT overlays f2 = str(tmp_path / "without_overlay.png") @@ -254,7 +307,7 @@ def test_overlay_visibility_changes_image(tmp_path): handle._set_view(**view) time.sleep(1) handle.getImage(f2, (512, 384)) - _wait_for_file(f2) + wait_for_file(f2) img1 = np.array(Image.open(f1)) img2 = np.array(Image.open(f2)) @@ -299,7 +352,7 @@ def test_vertex_no_nan_renders_data(tmp_path): time.sleep(1) outfile = str(tmp_path / "vtx.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) n_red = _count_red_pixels(outfile) assert n_red > 1000, ( @@ -335,7 +388,7 @@ def render(data, name): time.sleep(1) outfile = str(tmp_path / f"{name}.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) return _count_red_pixels(outfile) n_full = render(full, "full") @@ -398,7 +451,7 @@ def test_vertexrgb_alpha_zero_renders_curvature_only(tmp_path): time.sleep(1) outfile = str(tmp_path / "alpha_zero.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) rgb = np.array(Image.open(outfile))[..., :3].astype(int) # Count strongly red-dominant pixels: with the bug, α=0 lets the @@ -459,7 +512,7 @@ def test_volumergb_alpha_half_renders_correct_blend(tmp_path): time.sleep(1) outfile = str(tmp_path / "volumergb_alpha_half.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) rgb = np.array(Image.open(outfile))[..., :3].astype(int) # Brain-region pixels are red-dominant under both correct and buggy @@ -549,7 +602,7 @@ def test_vertex2d_alpha_half_renders_correct_blend(tmp_path): # left over from a prior iteration (getImage writes async). outfile = str(tmp_path / f"vertex2d_alpha_half_{attempt}.png") handle.getImage(outfile, (512, 384)) - _wait_for_file(outfile) + wait_for_file(outfile) # Give the PNG writer a moment to finish flushing. time.sleep(1) try: @@ -641,7 +694,7 @@ def render(handle, name): outfile = str(tmp_path / f"{name}.png") time.sleep(1) handle.getImage(outfile, image_size) - _wait_for_file(outfile) + wait_for_file(outfile) return _count_red_pixels(outfile) # No ROI/sulci overlays or labels: their anti-aliased colored edges would @@ -687,6 +740,8 @@ def _addData_viewer(): vol1 = cortex.Volume(np.random.randn(*volshape), subj, xfmname) with cortex.export.headless_viewer(vol1, viewer_params={}) as handle: yield handle + # A final check after teardown, which flushes anything still queued. + _assert_no_browser_failures(handle) def _served_metadata(handle): @@ -726,7 +781,7 @@ def _image_array(handle, outfile, size=(512, 384)): from PIL import Image handle.getImage(outfile, size) - _wait_for_file(outfile) + wait_for_file(outfile) return np.asarray(Image.open(outfile).convert("RGB"), dtype=np.int16) @@ -744,8 +799,7 @@ def test_adds_dataview(self, _addData_viewer): handle.addData(second=vol2) time.sleep(2) - pageerrors = [e for e in handle._pw_thread.browser_errors if "[pageerror]" in e] - assert len(pageerrors) == 0, f"JS errors after addData: {pageerrors}" + _assert_no_browser_failures(handle) # "data" is the name webshow gives to a bare Dataview. assert set(handle.dataviews.attrs) == {"data", "second"} @@ -800,8 +854,7 @@ def test_replaces_existing_name(self, _addData_viewer): assert len(metadata["images"]) == 2 assert metadata["views"][1]["data"][0] not in previous_brains - pageerrors = [e for e in handle._pw_thread.browser_errors if "[pageerror]" in e] - assert len(pageerrors) == 0, f"JS errors after addData: {pageerrors}" + _assert_no_browser_failures(handle) def test_rejects_unknown_subject(self, _addData_viewer): """Surfaces cannot be added to a running viewer, so neither can subjects.""" @@ -847,210 +900,5 @@ def test_addData_vertex_data(tmp_path): assert "mosaic" not in metadata["data"][vertex_name] assert _fetch(handle, metadata["images"][vertex_name][0])[1:6] == b"NUMPY" - pageerrors = [e for e in handle._pw_thread.browser_errors if "[pageerror]" in e] - assert len(pageerrors) == 0, f"JS errors after addData: {pageerrors}" - - -# --------------------------------------------------------------------------- -# Group 10: Manual visual A/B comparison across all alpha-bearing dataviews -# --------------------------------------------------------------------------- - - -@pytest.mark.skipif( - not os.environ.get("RUN_VISUAL_COMPARISON"), - reason="Manual visual comparison; set RUN_VISUAL_COMPARISON=1 to run.", -) -def test_visual_comparison_alpha_dataviews(tmp_path): - """Render all 6 dataview types via quickshow + webgl, side-by-side. - - Skipped by default — set ``RUN_VISUAL_COMPARISON=1`` to run. Builds a - grid where each row is one dataview type (Volume, Vertex, Volume2D, - Vertex2D, VolumeRGB, VertexRGB) and the two columns are the matplotlib - (``cortex.quickshow``) reference vs the headless WebGL flatmap render. - Used as a manual smoke check that the alpha-blend fix - (``Package``-side premultiply for VertexRGB + cmap-LUT - ``premultiplyAlpha=true`` for the 2D-cmap path) keeps both viewers in - visual agreement across every alpha-encoding pattern. - - Plain Volume / Vertex have no native per-element alpha (pycortex's - bundled ``*_alpha`` colormaps are all 2D and only apply to the 2D - dataview types), so those two rows act as a no-alpha baseline. The - other four rows exercise alpha: Volume2D / Vertex2D via the 2D-alpha - cmap ``RdBu_r_alpha``, VolumeRGB / VertexRGB via the ``alpha=`` kwarg. - - Renders are intentionally low-resolution (quickshow ``height=256``, - webgl ``size=(512, 384)``) so the final composite PNG stays small. - Both viewers run with no labels, no ROIs, and curvature underlay on. - - The composite PNG is written under ``tmp_path`` and the absolute path - is printed at the end of the test so the file is easy to open. - """ - import matplotlib.pyplot as plt - - import cortex.polyutils - - # ------- Synthesize data and alpha maps (mirrors plot_data_with_alpha.py) - - - # Volumetric - zz, yy, xx = np.mgrid[0:31, 0:100, 0:100] - data_vol = (xx - 50) / 50.0 # ~ [-1, 1] - center = np.array([15, 50, 50]) - sigma_v = 25.0 - dist2 = ( - (zz - center[0]) ** 2 + (yy - center[1]) ** 2 + (xx - center[2]) ** 2 - ) - accuracy_vol = np.exp(-dist2 / (2 * sigma_v**2)) # [0, 1] bump - red_vol = np.clip(xx / 99.0, 0, 1) - green_vol = np.clip(yy / 99.0, 0, 1) - blue_vol = np.clip(zz / 30.0, 0, 1) - - # Surface (vertex) — encode by spatial coordinate, not vertex index - surfs = [ - cortex.polyutils.Surface(*d) - for d in cortex.db.get_surf(subj, "fiducial") - ] - num_verts = [s.pts.shape[0] for s in surfs] - pts = np.vstack([surfs[0].pts, surfs[1].pts]) - y_centered = pts[:, 1] - pts[:, 1].mean() - data_vtx = y_centered / np.abs(y_centered).max() # [-1, 1] - xyz_norm = (pts - pts.min(axis=0)) / (pts.max(axis=0) - pts.min(axis=0)) - - def _bump(surf, seed, sigma): - d = np.linalg.norm(surf.pts - surf.pts[seed], axis=1) - return np.exp(-(d**2) / (2 * sigma**2)) - - accuracy_vtx = np.hstack( - [ - _bump(surfs[0], num_verts[0] // 2, sigma=40.0), - _bump(surfs[1], num_verts[1] // 2, sigma=40.0), - ] - ) - - # ------- Build the six dataviews ---------------------------------------- - # Volume / Vertex have no native per-element alpha — pycortex's bundled - # `*_alpha` colormaps are all 2D LUTs and only apply to Volume2D / - # Vertex2D. So plain Volume / Vertex use a non-alpha cmap (`viridis`) - # and serve as the no-alpha baseline; Volume2D / Vertex2D pair data - # against accuracy via the 2D-alpha cmap `RdBu_r_alpha`; VolumeRGB / - # VertexRGB use the native `alpha=` kwarg. - - cmap_plain = "viridis" - cmap_2d = "RdBu_r_alpha" - - dataviews = [ - ( - "Volume", - cortex.Volume( - data_vol, subj, xfmname, - cmap=cmap_plain, vmin=-1, vmax=1, - ), - ), - ( - "Vertex", - cortex.Vertex( - data_vtx, subj, - cmap=cmap_plain, vmin=-1, vmax=1, - ), - ), - ( - "Volume2D", - cortex.Volume2D( - data_vol, accuracy_vol, subj, xfmname, - cmap=cmap_2d, - vmin=-1, vmax=1, vmin2=0, vmax2=1, - ), - ), - ( - "Vertex2D", - cortex.Vertex2D( - data_vtx, accuracy_vtx, subj, - cmap=cmap_2d, - vmin=-1, vmax=1, vmin2=0, vmax2=1, - ), - ), - ( - "VolumeRGB", - cortex.VolumeRGB( - cortex.Volume(red_vol, subj, xfmname, vmin=0, vmax=1), - cortex.Volume(green_vol, subj, xfmname, vmin=0, vmax=1), - cortex.Volume(blue_vol, subj, xfmname, vmin=0, vmax=1), - subj, xfmname, - alpha=cortex.Volume(accuracy_vol, subj, xfmname, vmin=0, vmax=1), - ), - ), - ( - "VertexRGB", - cortex.VertexRGB( - cortex.Vertex(xyz_norm[:, 0], subj, vmin=0, vmax=1), - cortex.Vertex(xyz_norm[:, 1], subj, vmin=0, vmax=1), - cortex.Vertex(xyz_norm[:, 2], subj, vmin=0, vmax=1), - subj, - alpha=cortex.Vertex(accuracy_vtx, subj, vmin=0, vmax=1), - ), - ), - ] - - # ------- Render each dataview through both paths ------------------------ - # Each WebGL render spins up its own headless browser via plot_panels; - # six sequential launches × ~15s sleep = ~90s+ end to end. That's fine - # for a manual A/B and avoids the broken `addData` path on headless. - - n = len(dataviews) - fig, axes = plt.subplots(n, 2, figsize=(7, 2.2 * n)) - - flatmap_panel = [ - { - "extent": [0.0, 0.0, 1.0, 1.0], - "view": {"angle": "flatmap", "surface": "flatmap"}, - } - ] - - for row, (name, view) in enumerate(dataviews): - # quickshow → low-res PNG - qs_path = tmp_path / f"qs_{name}.png" - qs_fig = cortex.quickshow( - view, - with_curvature=True, - with_rois=False, - with_labels=False, - with_colorbar=False, - with_sulci=False, - with_borders=False, - height=256, - ) - qs_fig.savefig(qs_path, bbox_inches="tight", pad_inches=0, dpi=80) - plt.close(qs_fig) - - # webgl → trimmed flatmap PNG via plot_panels (single flatmap panel) - wg_path = str(tmp_path / f"wg_{name}.png") - wg_fig = cortex.export.plot_panels( - view, - panels=flatmap_panel, - figsize=(6, 3), - windowsize=(512, 384), - save_name=wg_path, - sleep=10, - viewer_params=dict(labels_visible=[], overlays_visible=[]), - headless=True, - ) - plt.close(wg_fig) - - ax_qs, ax_wg = axes[row] - ax_qs.imshow(plt.imread(qs_path)) - ax_qs.set_title(f"{name} — quickshow", fontsize=9) - ax_qs.axis("off") - ax_wg.imshow(plt.imread(wg_path)) - ax_wg.set_title(f"{name} — webgl (flatmap)", fontsize=9) - ax_wg.axis("off") - - fig.suptitle( - "Alpha-bearing dataviews: quickshow vs WebGL", fontsize=11, - ) - fig.tight_layout() - out_path = tmp_path / "alpha_dataview_comparison.png" - fig.savefig(out_path, dpi=100, bbox_inches="tight") - plt.close(fig) + _assert_no_browser_failures(handle) - print(f"\nVisual comparison saved to:\n {out_path}\n") - assert out_path.exists() - assert out_path.stat().st_size > 0 diff --git a/cortex/tests/testing_utils.py b/cortex/tests/testing_utils.py index 6a72f6566..b0aac016c 100644 --- a/cortex/tests/testing_utils.py +++ b/cortex/tests/testing_utils.py @@ -11,3 +11,26 @@ has_playwright = True except Exception: has_playwright = False + + +def wait_for_file(path, timeout=30): + """Poll until `path` exists and has nonzero size; raise after `timeout`. + + TODO: cortex/export/save_views.py has a weaker inline copy of this loop -- + it checks existence only, so it accepts a file the browser has created but + not finished writing. Consolidating means promoting this into cortex/export/ + (the library cannot import from cortex/tests/), not deleting either copy. + + If that happens, keep the ``time.sleep(1)`` that follows that loop. It reads + as slack on the file wait but is not: it is the window in which event + polling delivers console messages, which the WebGL failure check on the next + line depends on. Re-label it rather than removing it. + """ + import os + import time + + for _ in range(int(timeout / 0.1)): + if os.path.exists(path) and os.path.getsize(path) > 0: + return + time.sleep(0.1) + raise RuntimeError(f"File {path!r} not written within {timeout}s") diff --git a/pyproject.toml b/pyproject.toml index ef7574e9e..64c6019ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,11 @@ dev = [ ] test = [ "pycortex[headless]", + # The visual-regression tests compare against renders from + # these builds. Both are unpinned for users. + # See cortex/tests/reference_images/README.md + "matplotlib==3.10.9", + "playwright==1.62.0", "ipykernel>=6.31.0", "nbclient>=0.10.2", "nbformat>=5.10.4", diff --git a/setup.py b/setup.py index 91f934e95..e41ba113f 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,10 @@ def run(self): # Don't use `extras_require` here. Put them in pyproject.toml . cmdclass=dict(install=my_install), include_package_data=True, + # Exclude reference renders for the visual-regression test from the wheel. + # They are still included in the source tarball by MANIFEST.in. + exclude_package_data={'cortex.tests': ['reference_images/*', + 'reference_images/*/*']}, classifiers=[ 'Development Status :: 6 - Mature', 'Intended Audience :: Science/Research',