test: fix what the run of #3282 showed, and use the 64-core runners (#3284)

Author: pavelfeldman

.github/workflows/ci.yml

@@ -43,59 +43,59 @@ jobs:
     env:
       # The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
       PYTHONPATH: scripts
+      # 16 workers on the 64-core runners, as upstream runs its tests there; two on
+      # the 3-core macOS runner, upstream's default of half the cores.
+      WORKERS: ${{ startsWith(matrix.os, 'macos') && '2' || '16' }}
     strategy:
       fail-fast: false
       matrix:
-        # ubuntu-22.04, not ubuntu-latest: the 24.04 image has no WebGL for headless
-        # Firefox (CA-008, CA-009).
-        os: [ubuntu-22.04, windows-latest, macos-latest]
-        python-version: ['3.14']
+        # Linux and Windows run on the 64-core runners of the organization, as
+        # upstream's tests_primary.yml; macOS on the hosted runner.
+        # 3.12 is our most used Python version (base)
+        os: [playwright-x64-ubuntu24-64-core, playwright-x64-server2025-64-core, macos-15-xlarge]
+        python-version: ['3.12']
         browser: [chromium, firefox, webkit]
-        exclude:
-        # WebKit on standard macOS-latest (currently macos-15-arm64) is unstable;
-        # upstream pins paid macos-15-xlarge for cross-browser webkit too.
-        - os: macos-latest
-          browser: webkit
         include:
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.10'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.10'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.10'
           browser: chromium
-        - os: macos-15-xlarge
-          python-version: '3.14'
-          browser: webkit
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.11'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.11'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.11'
           browser: chromium
-        - os: windows-latest
-          python-version: '3.12'
-          browser: chromium
-        - os: macos-latest
-          python-version: '3.12'
-          browser: chromium
-        - os: ubuntu-22.04
-          python-version: '3.12'
-          browser: chromium
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.13'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.13'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.13'
           browser: chromium
+        - os: playwright-x64-server2025-64-core
+          python-version: '3.14'
+          browser: chromium
+        - os: macos-15-xlarge
+          python-version: '3.14'
+          browser: chromium
+        - os: playwright-x64-ubuntu24-64-core
+          python-version: '3.14'
+          browser: chromium
+        # Intel macOS, once: *-large is Intel, *-xlarge is arm64, as upstream notes.
+        - os: macos-15-large
+          python-version: '3.12'
+          browser: chromium
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -119,34 +119,34 @@ jobs:
     - name: Test Wheel Installation
       run: pytest tests/test_installation.py --browser=${{ matrix.browser }}
     - name: Test Sync API
-      if: matrix.os != 'ubuntu-22.04'
-      run: pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os != 'Linux'
+      run: pytest tests/sync --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Sync API
-      if: matrix.os == 'ubuntu-22.04'
-      run: xvfb-run pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os == 'Linux'
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/sync --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
       # Also after a failed sync step: the two suites report on their own.
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
-      run: pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os != 'Linux' }}
+      run: pytest tests/async --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
-      run: xvfb-run pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os == 'Linux' }}
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/async --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
 
   test-stable:
     name: Stable
-    timeout-minutes: 90
+    # The branded browsers on the 3-core macOS runner take longer than the bundled ones.
+    timeout-minutes: 120
     env:
       # The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
       PYTHONPATH: scripts
+      WORKERS: ${{ startsWith(matrix.os, 'macos') && '2' || '16' }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-latest, macos-latest]
+        os: [playwright-x64-ubuntu24-64-core, playwright-x64-server2025-64-core, macos-15-xlarge]
         browser-channel: [chrome]
         include:
-        - os: windows-latest
-          browser-channel: msedge
-        - os: macos-latest
+        - os: playwright-x64-server2025-64-core
           browser-channel: msedge
     runs-on: ${{ matrix.os }}
     steps:
@@ -167,18 +167,18 @@ jobs:
     - name: Common Tests
       run: pytest tests/common --browser=chromium --browser-channel=${{ matrix.browser-channel }}
     - name: Test Sync API
-      if: matrix.os != 'ubuntu-22.04'
-      run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os != 'Linux'
+      run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Sync API
-      if: matrix.os == 'ubuntu-22.04'
-      run: xvfb-run pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os == 'Linux'
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
       # Also after a failed sync step: the two suites report on their own.
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
-      run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os != 'Linux' }}
+      run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
-      run: xvfb-run pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os == 'Linux' }}
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
 
   test_examples:
     name: Examples

local-requirements.txt

@@ -4,6 +4,7 @@ build==1.6.1
 mypy==2.4.0
 objgraph==3.6.2
 Pillow==12.3.0
+pixelmatch==0.3.0
 pre-commit==3.5.0
 pyOpenSSL==26.0.0
 pytest==9.0.3

tests/async/library/test_browsercontext_locale.py

@@ -130,6 +130,7 @@ async def test_LC_007_should_work_for_multiple_pages_sharing_same_process(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_LC_008_should_be_isolated_between_contexts(
     browser: Browser, server: Server
 ) -> None:

tests/async/library/test_browsercontext_pages.py

@@ -166,6 +166,7 @@ async def test_BC_058_should_return_bounding_box_with_page_scale(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_BC_059_should_not_leak_listeners_during_navigation_of_20_pages(
     context_factory: Callable[..., Awaitable[BrowserContext]], server: Server
 ) -> None:

tests/async/library/test_defaultbrowsercontext_2.py

@@ -210,6 +210,7 @@ async def test_DC_024_should_accept_relative_user_data_dir(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_DC_025_should_restore_state_from_user_data_dir(
     browser_type: BrowserType,
     server: Server,

tests/async/library/test_har.py

@@ -1326,6 +1326,7 @@ async def test_HA_057_should_close_the_context_when_saving_the_har_fails(
         await browser.close()
 
 
+@pytest.mark.slow
 async def test_HA_058_should_support_har_larger_than_512_mb(
     context_factory: Callable[..., Awaitable[BrowserContext]],
     server: Server,

tests/async/library/test_headful.py

@@ -121,6 +121,7 @@ async def test_HE_001_should_have_default_url_when_launching_browser_smoke(
     assert urls == ["about:blank"]
 
 
+@pytest.mark.slow
 async def test_HE_002_should_close_browser_with_beforeunload_page(
     launch_persistent: LaunchPersistent, server: Server
 ) -> None:

tests/async/library/test_tracing.py

@@ -1145,23 +1145,30 @@ async def test_TR_039_should_not_flush_console_events(
     page = await context.new_page()
     test_id = "TR-039"
     await context.tracing.start(name=test_id)
+    promise: "asyncio.Future[None]" = page._loop.create_future()
     counter = 0
 
-    def predicate(_: Any) -> bool:
+    def on_console(_: Any) -> None:
         nonlocal counter
         counter += 1
-        return counter == 100
+        if counter == 100:
+            promise.set_result(None)
 
-    async with page.expect_event("console", predicate=predicate):
-        await page.evaluate(
-            """() => {
+    # Python: a listener and a future, as upstream: expect_event() would be an
+    # action of its own in the trace, which flushes the console events.
+    page.on("console", on_console)
+
+    await page.evaluate(
+        """() => {
     window.builtins.setTimeout(() => {
       for (let i = 0; i < 100; ++i)
         console.log('hello ' + i);
     }, 10);
     return 31415926;
   }"""
-        )
+    )
+
+    await promise
 
     def read_trace() -> str:
         trace_name = next(

tests/async/library/test_video.py

@@ -819,6 +819,7 @@ async def test_VI_030_screencast_should_work_with_video_trace(
     assert is_almost_red(pixel) is True
 
 
+@pytest.mark.slow
 async def test_VI_031_should_save_as_video(browser: Browser, tmp_path: Path) -> None:
     videos_path = tmp_path
     size: ViewportSize = {"width": 320, "height": 240}

tests/async/page/test_page_screenshot.py

@@ -654,6 +654,7 @@ async def test_SS_038_page_screenshot_should_work_with_array_deleted(
 
 
 @pytest.mark.usefixtures("skip_firefox_headed")
+@pytest.mark.slow
 async def test_SS_039_page_screenshot_should_take_full_page_screenshots_during_navigation(
     page: Page, server: Server
 ) -> None: