Skip to content

Commit e25e4fc

Browse files
authored
test: fix the ports and the helpers that the full matrix showed (#3282)
1 parent 1818e98 commit e25e4fc

47 files changed

Lines changed: 886 additions & 134 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ jobs:
4040
build:
4141
name: Build
4242
timeout-minutes: 90
43+
env:
44+
# The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
45+
PYTHONPATH: scripts
4346
strategy:
4447
fail-fast: false
4548
matrix:
@@ -117,21 +120,24 @@ jobs:
117120
run: pytest tests/test_installation.py --browser=${{ matrix.browser }}
118121
- name: Test Sync API
119122
if: matrix.os != 'ubuntu-22.04'
120-
run: pytest tests/sync --browser=${{ matrix.browser }} -n auto
123+
run: pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
121124
- name: Test Sync API
122125
if: matrix.os == 'ubuntu-22.04'
123-
run: xvfb-run pytest tests/sync --browser=${{ matrix.browser }} -n auto
126+
run: xvfb-run pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
124127
- name: Test Async API
125128
# Also after a failed sync step: the two suites report on their own.
126129
if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
127-
run: pytest tests/async --browser=${{ matrix.browser }} -n auto
130+
run: pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
128131
- name: Test Async API
129132
if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
130-
run: xvfb-run pytest tests/async --browser=${{ matrix.browser }} -n auto
133+
run: xvfb-run pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
131134

132135
test-stable:
133136
name: Stable
134137
timeout-minutes: 90
138+
env:
139+
# The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
140+
PYTHONPATH: scripts
135141
strategy:
136142
fail-fast: false
137143
matrix:
@@ -162,17 +168,17 @@ jobs:
162168
run: pytest tests/common --browser=chromium --browser-channel=${{ matrix.browser-channel }}
163169
- name: Test Sync API
164170
if: matrix.os != 'ubuntu-22.04'
165-
run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto
171+
run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
166172
- name: Test Sync API
167173
if: matrix.os == 'ubuntu-22.04'
168-
run: xvfb-run pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto
174+
run: xvfb-run pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
169175
- name: Test Async API
170176
# Also after a failed sync step: the two suites report on their own.
171177
if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
172-
run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto
178+
run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
173179
- name: Test Async API
174180
if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
175-
run: xvfb-run pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto
181+
run: xvfb-run pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
176182

177183
test_examples:
178184
name: Examples

‎local-requirements.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ build==1.6.1
44
mypy==2.4.0
55
objgraph==3.6.2
66
Pillow==12.3.0
7-
pixelmatch==0.3.0
87
pre-commit==3.5.0
98
pyOpenSSL==26.0.0
109
pytest==9.0.3

‎playwright/_impl/_async_base.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ def __init__(self, future: "asyncio.Future[T]") -> None:
4646
self._event = AsyncEventInfo[T](future)
4747

4848
async def __aenter__(self) -> AsyncEventInfo[T]:
49+
# The wait takes its first step before the body runs. A task that is
50+
# cancelled before its first step, as __aexit__ cancels it when the body
51+
# raised before its first await, never runs its coroutine, and the
52+
# coroutines that the wait holds are never awaited: their listeners and
53+
# timers stay, and Python warns at their collection.
54+
await asyncio.sleep(0)
4955
return self._event
5056

5157
async def __aexit__(

‎playwright/_impl/_sync_base.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ def __init__(self, sync_base: "SyncBase", future: "asyncio.Future[T]") -> None:
6666
self._event = EventInfo[T](sync_base, future)
6767

6868
def __enter__(self) -> EventInfo[T]:
69+
# The wait takes its first step before the body runs; see
70+
# AsyncEventContextManager.__aenter__(). The body of a sync block may make
71+
# no call at all, so the event loop gets a turn here.
72+
sync_base = self._event._sync_base
73+
wait_for_future(
74+
sync_base._loop,
75+
sync_base._dispatcher_fiber,
76+
asyncio.ensure_future(asyncio.sleep(0)),
77+
)
6978
return self._event
7079

7180
def __exit__(

‎pyproject.toml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,9 @@ version_file = "playwright/_repo_version.py"
6666
addopts = "-Wall -rsx -vv -s"
6767
# The timeout of a test is 30 seconds, as upstream; a test marked slow gets three
6868
# times that, as test.slow() gives upstream (tests/conftest.py). The fixtures count,
69-
# as upstream counts them. The thread method ends the process of a test that hangs;
70-
# xdist replaces the worker. The signal method can not be used: it raises inside the
69+
# as upstream counts them. The thread method ends the process of a test that hangs,
70+
# after tests/conftest.py has written its stacks to the standard error; xdist
71+
# replaces the worker. The signal method can not be used: it raises inside the
7172
# running code, and asyncio's Handle._run() catches and logs every BaseException but
7273
# SystemExit and KeyboardInterrupt, so a timeout that fires while the event loop runs
7374
# a callback is lost and the test hangs on.
@@ -81,6 +82,14 @@ markers = [
8182
"skip_protocol_check",
8283
"slow: the test gets three times the timeout, as test.slow() upstream",
8384
]
85+
filterwarnings = [
86+
# Twisted 26 passes pyOpenSSL objects to pyOpenSSL 26, which wants cryptography
87+
# objects: Twisted's to fix, not visible to this code.
88+
"ignore:Passing pyOpenSSL .* objects is deprecated:DeprecationWarning:OpenSSL.SSL",
89+
# txaio (the web socket server of the tests) calls asyncio.iscoroutinefunction(),
90+
# deprecated in Python 3.14, in its latest version too.
91+
"ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning:txaio.tx",
92+
]
8493
junit_family = "xunit2"
8594
asyncio_mode = "auto"
8695
asyncio_default_fixture_loop_scope = "session"

‎scripts/pytest_dot_reporter.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Copyright (c) Microsoft Corporation.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""A pytest reporter that prints one character per test, like Playwright's dot reporter.
16+
17+
Load it with `-p pytest_dot_reporter` (scripts/ on PYTHONPATH); CI runs the suites
18+
with it. Each test is one character: `·` passed, `F` failed, `E` failed in a fixture,
19+
`°` skipped, `x` failed as expected, `X` passed unexpectedly, `×` failed and to be run
20+
again. Each failure prints in full as soon as it happens, between the dots, so that
21+
the log of a run that is cut short still has it; the end summary lists the failures
22+
again, one line each. Unlike the line reporter, it does not need a terminal.
23+
"""
24+
25+
from typing import Dict, Optional, Tuple
26+
27+
import pytest
28+
from _pytest.terminal import TerminalReporter
29+
30+
Location = Tuple[str, Optional[int], str]
31+
32+
_LINE_WIDTH = 80
33+
34+
# The character of each test outcome, and its colors.
35+
_MARKS: Dict[str, Tuple[str, Dict[str, bool]]] = {
36+
"passed": ("·", {"green": True}),
37+
"failed": ("F", {"red": True, "bold": True}),
38+
"error": ("E", {"red": True, "bold": True}),
39+
"skipped": ("°", {"yellow": True}),
40+
"xfailed": ("x", {"yellow": True}),
41+
"xpassed": ("X", {"yellow": True}),
42+
"rerun": ("×", {"light": True}),
43+
}
44+
45+
46+
class DotStatus:
47+
"""Takes over the per-test output of the terminal reporter."""
48+
49+
def __init__(self, reporter: TerminalReporter) -> None:
50+
self._reporter = reporter
51+
self._column = 0
52+
53+
def pytest_runtest_logstart(self, nodeid: str, location: Location) -> None:
54+
pass
55+
56+
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
57+
reporter = self._reporter
58+
reporter._tests_ran = True
59+
category, letter, _ = reporter.config.hook.pytest_report_teststatus(
60+
report=report, config=reporter.config
61+
)
62+
reporter._add_stats(category, [report])
63+
if report.failed or category == "rerun":
64+
self._print_failure(report, category)
65+
if not letter:
66+
# A fixture that ran: the test gets its character from its call.
67+
return
68+
if self._column == _LINE_WIDTH:
69+
reporter._tw.line()
70+
self._column = 0
71+
mark, markup = _MARKS.get(category, (letter[:1], {}))
72+
reporter._tw.write(mark, **markup)
73+
reporter._tw.flush()
74+
self._column += 1
75+
# Start the next output of the reporter on a new line.
76+
reporter.currentfspath = -2
77+
78+
def pytest_runtest_logfinish(self, nodeid: str, location: Location) -> None:
79+
pass
80+
81+
def summary_failures(self) -> None:
82+
# The failures were printed as they happened.
83+
pass
84+
85+
def summary_errors(self) -> None:
86+
pass
87+
88+
def _print_failure(self, report: pytest.TestReport, category: str) -> None:
89+
# As summary_errors() and summary_failures() print it, on a line of its own.
90+
reporter = self._reporter
91+
headline = reporter._getfailureheadline(report)
92+
if category == "rerun":
93+
headline = f"RERUN {headline}"
94+
elif report.when != "call":
95+
headline = f"ERROR at {report.when} of {headline}"
96+
reporter.write_sep("_", headline, red=True, bold=True)
97+
reporter._outrep_summary(report)
98+
reporter._tw.flush()
99+
self._column = 0
100+
101+
102+
@pytest.hookimpl(trylast=True)
103+
def pytest_configure(config: pytest.Config) -> None:
104+
reporter = config.pluginmanager.getplugin("terminalreporter")
105+
if not isinstance(reporter, TerminalReporter) or hasattr(config, "workerinput"):
106+
return
107+
# Keep the reporter object, which xdist already holds. Replace its per-test hooks
108+
# and its failure sections with DotStatus, then register it again so that pluggy
109+
# calls the replacements.
110+
status = DotStatus(reporter)
111+
config.pluginmanager.unregister(reporter)
112+
for name in (
113+
"pytest_runtest_logstart",
114+
"pytest_runtest_logreport",
115+
"pytest_runtest_logfinish",
116+
"summary_failures",
117+
"summary_errors",
118+
):
119+
setattr(reporter, name, getattr(status, name))
120+
config.pluginmanager.register(reporter, "terminalreporter")

‎test.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#!/usr/bin/env bash
22
# Runs tests/sync and then tests/async under xvfb, with one worker for every two
3-
# cores. On a terminal, the progress is one status line (scripts/pytest_line_reporter.py),
4-
# and the summary lists only failures and errors, without the warnings.
3+
# cores. On a terminal, the progress is one status line (scripts/pytest_line_reporter.py);
4+
# elsewhere it is one character per test, as on CI (scripts/pytest_dot_reporter.py).
5+
# The summary lists only failures and errors, without the warnings.
56
# Usage: ./test.sh [chromium|firefox|webkit] [extra pytest args]
67
set -u
78
cd "$(dirname "$0")"
@@ -14,7 +15,9 @@ python=python
1415
export PYTHONPATH="scripts${PYTHONPATH:+:$PYTHONPATH}"
1516
# --maxschedchunk=1: a worker gets one test at a time, as soon as it is free, so
1617
# that the slow tests at the end of the collection spread over all workers.
17-
args=(--browser "$browser" -n "$workers" --maxschedchunk=1 -p pytest_line_reporter -rfE --disable-warnings)
18+
reporter=pytest_dot_reporter
19+
[ -t 1 ] && reporter=pytest_line_reporter
20+
args=(--browser "$browser" -n "$workers" --maxschedchunk=1 -p "$reporter" -rfE --disable-warnings)
1821
status=0
1922
xvfb-run -a "$python" -m pytest tests/sync "${args[@]}" "$@" || status=$?
2023
xvfb-run -a "$python" -m pytest tests/async "${args[@]}" "$@" || status=$?

‎tests/async/library/test_global_fetch.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,12 @@ async def test_FE_027_should_set_playwright_as_user_agent(
546546
# on ubuntu: distro is 'ubuntu' and version is 'X.X'
547547
# on manjaro: distro is 'Manjaro' and version is 'unknown'
548548
assert (
549-
re.sub(r"<ARCH>; \w+ [^)]+", "<ARCH>; distro version", user_agent_masked, 1)
549+
re.sub(
550+
r"<ARCH>; \w+ [^)]+",
551+
"<ARCH>; distro version",
552+
user_agent_masked,
553+
count=1,
554+
)
550555
== "Playwright/X.X.X (<ARCH>; distro version) python/X.X" + suffix
551556
)
552557
elif is_mac:

‎tests/async/library/test_headful.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ async def test_HE_015_headless_and_headful_should_use_same_default_fonts(
477477
page.screenshot(), headless_page.screenshot()
478478
)
479479
# Port of utils.compare(image1, image2), which counts the different pixels.
480-
count = count_different_pixels(image1, image2, threshold=0.1)
480+
count = count_different_pixels(image1, image2)
481481
assert count == 0
482482
await headless_browser.close()
483483

‎tests/async/library/test_ignorehttpserrors.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ async def test_NE_140_should_work_with_web_socket(
124124
context = await browser.new_context(ignore_https_errors=True)
125125
page = await context.new_page()
126126
value = await page.evaluate(
127-
WEBSOCKET_VALUE, re.sub(r"https", "wss", https_server.PREFIX, 1) + "/ws"
127+
WEBSOCKET_VALUE, re.sub(r"https", "wss", https_server.PREFIX, count=1) + "/ws"
128128
)
129129
assert value == "incoming"
130130
await context.close()
@@ -136,7 +136,7 @@ async def test_NE_141_should_fail_with_web_socket_if_not_ignored(
136136
context = await browser.new_context()
137137
page = await context.new_page()
138138
value = await page.evaluate(
139-
WEBSOCKET_VALUE, re.sub(r"https", "wss", https_server.PREFIX, 1) + "/ws"
139+
WEBSOCKET_VALUE, re.sub(r"https", "wss", https_server.PREFIX, count=1) + "/ws"
140140
)
141141
assert value == "Error"
142142
await context.close()

0 commit comments

Comments
 (0)