test: migrate flaky plugin to rerunfailures (#2772)

Author: mxschmitt

.github/workflows/test_docker.yml

@@ -49,5 +49,5 @@ jobs:
         docker exec "${CONTAINER_ID}" pip install -r requirements.txt
         docker exec "${CONTAINER_ID}" pip install -e .
         docker exec "${CONTAINER_ID}" python -m build --wheel
-        docker exec "${CONTAINER_ID}" xvfb-run pytest -vv tests/sync/
-        docker exec "${CONTAINER_ID}" xvfb-run pytest -vv tests/async/
+        docker exec "${CONTAINER_ID}" xvfb-run pytest tests/sync/
+        docker exec "${CONTAINER_ID}" xvfb-run pytest tests/async/

local-requirements.txt

@@ -2,7 +2,6 @@ autobahn==23.1.2
 black==25.1.0
 build==1.2.2.post1
 flake8==7.1.2
-flaky==3.8.1
 mypy==1.15.0
 objgraph==3.6.2
 Pillow==11.1.0
@@ -13,6 +12,7 @@ pytest==8.3.5
 pytest-asyncio==0.25.3
 pytest-cov==6.0.0
 pytest-repeat==0.9.3
+pytest-rerunfailures==15.0
 pytest-timeout==2.3.1
 pytest-xdist==3.6.1
 requests==2.32.3

tests/async/test_browsercontext_proxy.py

@@ -17,7 +17,6 @@
 from typing import AsyncGenerator, Awaitable, Callable
 
 import pytest
-from flaky import flaky
 
 from playwright.async_api import Browser, BrowserContext
 from tests.server import Server, TestServerRequest
@@ -108,7 +107,6 @@ async def test_should_work_with_ip_port_notion(
     assert await page.title() == "Served by the proxy"
 
 
-@flaky  # Upstream flaky
 async def test_should_authenticate(
     context_factory: "Callable[..., Awaitable[BrowserContext]]", server: Server
 ) -> None:
@@ -139,7 +137,6 @@ def handler(req: TestServerRequest) -> None:
     )
 
 
-@flaky  # Upstream flaky
 async def test_should_authenticate_with_empty_password(
     context_factory: "Callable[..., Awaitable[BrowserContext]]", server: Server
 ) -> None:

tests/async/test_browsertype_connect.py

@@ -19,7 +19,6 @@
 from typing import Callable
 
 import pytest
-from flaky import flaky
 
 from playwright.async_api import BrowserType, Error, Playwright, Route
 from tests.conftest import RemoteServer
@@ -266,7 +265,6 @@ async def handle_request(route: Route) -> None:
     remote.kill()
 
 
-@flaky
 async def test_should_upload_large_file(
     browser_type: BrowserType,
     launch_server: Callable[[], RemoteServer],

tests/async/test_input.py

@@ -21,7 +21,6 @@
 from typing import Any
 
 import pytest
-from flaky import flaky
 
 from playwright._impl._path_utils import get_file_dirname
 from playwright.async_api import Error, FilePayload, Page
@@ -316,7 +315,6 @@ async def _listen_for_wheel_events(page: Page, selector: str) -> None:
     )
 
 
-@flaky
 async def test_should_upload_large_file(
     page: Page, server: Server, tmp_path: Path
 ) -> None:
@@ -383,7 +381,6 @@ async def test_set_input_files_should_preserve_last_modified_timestamp(
         assert abs(timestamps[i] - expected_timestamps[i]) < 1000
 
 
-@flaky
 async def test_should_upload_multiple_large_file(
     page: Page, server: Server, tmp_path: Path
 ) -> None:

tests/async/test_network.py

@@ -19,7 +19,6 @@
 from typing import Dict, List, Optional, Union
 
 import pytest
-from flaky import flaky
 from twisted.web import http
 
 from playwright.async_api import Browser, Error, Page, Request, Response, Route
@@ -819,7 +818,6 @@ async def test_set_extra_http_headers_should_work_with_extra_headers_from_browse
     assert request.getHeader("foo") == "bar"
 
 
-@flaky  # Flaky upstream https://devops.aslushnikov.com/flakiness2.html#filter_spec=should+override+extra+headers+from+browser+context&test_parameter_filters=%5B%5B%22browserName%22%2C%5B%5B%22webkit%22%2C%22include%22%5D%5D%5D%2C%5B%22video%22%2C%5B%5Btrue%2C%22exclude%22%5D%5D%5D%2C%5B%22platform%22%2C%5B%5B%22Windows%22%2C%22include%22%5D%5D%5D%5D
 async def test_set_extra_http_headers_should_override_extra_headers_from_browser_context(
     browser: Browser, server: Server
 ) -> None:

tests/async/test_resource_timing.py

@@ -15,7 +15,6 @@
 from typing import Dict
 
 import pytest
-from flaky import flaky
 
 from playwright.async_api import Browser, Page
 from tests.server import Server
@@ -33,7 +32,6 @@ async def test_should_work(page: Page, server: Server) -> None:
     assert timing["responseEnd"] < 10000
 
 
-@flaky
 async def test_should_work_for_subresource(
     page: Page, server: Server, is_win: bool, is_mac: bool, is_webkit: bool
 ) -> None:
@@ -51,7 +49,6 @@ async def test_should_work_for_subresource(
     assert timing["responseEnd"] < 10000
 
 
-@flaky  # Upstream flaky
 async def test_should_work_for_ssl(browser: Browser, https_server: Server) -> None:
     page = await browser.new_page(ignore_https_errors=True)
     async with page.expect_event("requestfinished") as request_info:

tests/async/test_websocket.py

@@ -16,7 +16,6 @@
 from typing import Union
 
 import pytest
-from flaky import flaky
 
 from playwright.async_api import Error, Page, WebSocket
 from tests.server import Server, WebSocketProtocol
@@ -151,7 +150,6 @@ def on_web_socket(ws: WebSocket) -> None:
     assert received == ["incoming", b"\x04\x02"]
 
 
-@flaky
 async def test_should_reject_wait_for_event_on_close_and_error(
     page: Page, server: Server
 ) -> None:

tests/async/test_worker.py

@@ -16,7 +16,6 @@
 from asyncio.futures import Future
 
 import pytest
-from flaky import flaky
 
 from playwright.async_api import Browser, ConsoleMessage, Error, Page, Worker
 from tests.server import Server
@@ -107,7 +106,6 @@ async def test_workers_should_report_errors(page: Page) -> None:
     assert "this is my error" in error_log.message
 
 
-@flaky  # Upstream flaky
 async def test_workers_should_clear_upon_navigation(server: Server, page: Page) -> None:
     await page.goto(server.EMPTY_PAGE)
     async with page.expect_event("worker") as event_info:
@@ -123,7 +121,6 @@ async def test_workers_should_clear_upon_navigation(server: Server, page: Page)
     assert len(page.workers) == 0
 
 
-@flaky  # Upstream flaky
 async def test_workers_should_clear_upon_cross_process_navigation(
     server: Server, page: Page
 ) -> None:

tests/conftest.py

@@ -34,6 +34,11 @@
 _dirname = get_file_dirname()
 
 
+def pytest_configure(config: pytest.Config) -> None:
+    if os.environ.get("CI"):
+        config.option.reruns = 3
+
+
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
     if "browser_name" in metafunc.fixturenames:
         browsers = metafunc.config.option.browser or ["chromium", "firefox", "webkit"]

tests/sync/test_resource_timing.py

@@ -15,7 +15,6 @@
 from typing import Dict
 
 import pytest
-from flaky import flaky
 
 from playwright.sync_api import Browser, Page
 from tests.server import Server
@@ -33,7 +32,6 @@ def test_should_work(page: Page, server: Server) -> None:
     assert timing["responseEnd"] < 10000
 
 
-@flaky
 def test_should_work_for_subresource(
     page: Page, server: Server, is_win: bool, is_mac: bool, is_webkit: bool
 ) -> None:
@@ -51,7 +49,6 @@ def test_should_work_for_subresource(
     assert timing["responseEnd"] < 10000
 
 
-@flaky  # Upstream flaky
 def test_should_work_for_ssl(
     browser: Browser, https_server: Server, is_mac: bool, is_webkit: bool
 ) -> None: