From 862631296ca3c47ea3dd1d521c77abce84d3a2fb Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sat, 14 Nov 2026 22:07:03 +0000 Subject: [PATCH] clean up `BrowserPool` and its plugins after a failed entry or exit --- src/crawlee/browsers/_browser_pool.py | 62 ++++- .../browsers/_playwright_browser_plugin.py | 24 +- .../browsers/_stagehand_browser_plugin.py | 57 ++-- tests/unit/browsers/test_browser_pool.py | 257 ++++++++++++++++++ .../test_playwright_browser_plugin.py | 94 +++++++ .../browsers/test_stagehand_browser_plugin.py | 140 +++++++++- 6 files changed, 597 insertions(+), 37 deletions(-) create mode 100644 tests/unit/browsers/test_browser_pool.py diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index b2a949efdc..b4c0a0e22a 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -214,12 +214,29 @@ async def __aenter__(self) -> BrowserPool: self._close_inactive_browsers_task.start() timeout = self._operation_timeout.total_seconds() + entered_plugins: list[BrowserPlugin] = [] try: for plugin in self._plugins: await asyncio.wait_for(plugin.__aenter__(), timeout) + entered_plugins.append(plugin) except TimeoutError: logger.warning('Initializing of the browser plugin %s timed out, will be skipped.', plugin) + except BaseException: + try: + await self._stop_recurring_tasks() + finally: + try: + plugin_exits = (entered_plugin.__aexit__(None, None, None) for entered_plugin in entered_plugins) + plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True) + finally: + self._active = False + for result in plugin_results: + if isinstance(result, BaseException): + logger.warning( + 'Exiting a browser plugin failed while rolling back the pool entry.', exc_info=result + ) + raise return self @@ -237,18 +254,39 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self._identify_inactive_browsers_task.stop() - await self._close_inactive_browsers_task.stop() + # Every browser and plugin is closed even when another one fails, since an inactive pool can't be exited again. + failures: list[BaseException] = [] + try: + try: + await self._stop_recurring_tasks() - for browser in self._active_browsers + self._inactive_browsers: - await browser.close(force=True) - self._active_browsers.clear() - self._inactive_browsers.clear() + browser_closes = ( + browser.close(force=True) for browser in self._active_browsers + self._inactive_browsers + ) + browser_results = await asyncio.gather(*browser_closes, return_exceptions=True) + failures.extend(result for result in browser_results if isinstance(result, BaseException)) + finally: + self._active_browsers.clear() + self._inactive_browsers.clear() - for plugin in self._plugins: - await plugin.__aexit__(exc_type, exc_value, exc_traceback) + try: + # Plugins left inactive by a timed-out entry have nothing to exit. + active_plugins = [plugin for plugin in self._plugins if plugin.active] + plugin_exits = (plugin.__aexit__(exc_type, exc_value, exc_traceback) for plugin in active_plugins) + plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True) + failures.extend(result for result in plugin_results if isinstance(result, BaseException)) + finally: + self._active = False + except BaseException: + # A cancellation above is raised instead, so the failures are only logged. + for failure in failures: + logger.warning('Closing the browser pool failed.', exc_info=failure) + raise - self._active = False + for failure in failures[1:]: + logger.warning('Closing the browser pool failed.', exc_info=failure) + if failures: + raise failures[0] @ensure_context async def new_page( @@ -392,6 +430,12 @@ async def _close_inactive_browsers(self) -> None: await browser.close() self._inactive_browsers.remove(browser) + async def _stop_recurring_tasks(self) -> None: + try: + await self._identify_inactive_browsers_task.stop() + finally: + await self._close_inactive_browsers_task.stop() + async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *args: Any) -> None: """Execute the provided hooks with the given arguments.""" for hook in hooks: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index 5cc53e7be8..475505220f 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -155,7 +155,21 @@ async def __aenter__(self) -> PlaywrightBrowserPlugin: raise RuntimeError(f'The {self.__class__.__name__} is already active.') self._active = True - self._playwright = await self._playwright_context_manager.__aenter__() + + try: + self._playwright = await self._playwright_context_manager.__aenter__() + except BaseException: + # The Playwright driver may already be running, so it's stopped too. + half_started_context_manager = self._playwright_context_manager + self._playwright_context_manager = async_playwright() + try: + await half_started_context_manager.__aexit__(None, None, None) + except Exception: + logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True) + finally: + self._active = False + raise + return self @override @@ -168,9 +182,11 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback) - self._playwright_context_manager = async_playwright() - self._active = False + try: + await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback) + finally: + self._playwright_context_manager = async_playwright() + self._active = False @override @ensure_context diff --git a/src/crawlee/browsers/_stagehand_browser_plugin.py b/src/crawlee/browsers/_stagehand_browser_plugin.py index a94a973bee..1d431acd15 100644 --- a/src/crawlee/browsers/_stagehand_browser_plugin.py +++ b/src/crawlee/browsers/_stagehand_browser_plugin.py @@ -163,19 +163,33 @@ async def __aenter__(self) -> StagehandBrowserPlugin: raise RuntimeError(f'The {self.__class__.__name__} is already active.') self._active = True - self._playwright = await self._playwright_context_manager.__aenter__() - if self._stagehand_options.env == 'LOCAL': - if 'executable_path' not in self._browser_launch_options: - chrome_path = self._playwright.chromium.executable_path - self._browser_launch_options['executable_path'] = chrome_path - logger.debug('Resolved Chromium path from Playwright: %s', chrome_path) - - self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path'] - - client = AsyncStagehand(**self._stagehand_init_params) - await client.__aenter__() - self._stagehand_client = client + try: + self._playwright = await self._playwright_context_manager.__aenter__() + + if self._stagehand_options.env == 'LOCAL': + if 'executable_path' not in self._browser_launch_options: + chrome_path = self._playwright.chromium.executable_path + self._browser_launch_options['executable_path'] = chrome_path + logger.debug('Resolved Chromium path from Playwright: %s', chrome_path) + + self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path'] + + client = AsyncStagehand(**self._stagehand_init_params) + await client.__aenter__() + self._stagehand_client = client + except BaseException: + # The Playwright driver may already be running, so it's stopped too. + self._playwright = None + playwright_context_manager = self._playwright_context_manager + self._playwright_context_manager = async_playwright() + try: + await playwright_context_manager.__aexit__(None, None, None) + except Exception: + logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True) + finally: + self._active = False + raise return self @@ -189,14 +203,17 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - if self._stagehand_client is not None: - await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback) - self._stagehand_client = None - - await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback) - self._playwright_context_manager = async_playwright() - self._playwright = None - self._active = False + try: + if self._stagehand_client is not None: + await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback) + finally: + try: + await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback) + finally: + self._stagehand_client = None + self._playwright = None + self._playwright_context_manager = async_playwright() + self._active = False @override @ensure_context diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py new file mode 100644 index 0000000000..b1b72f95be --- /dev/null +++ b/tests/unit/browsers/test_browser_pool.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import asyncio +from datetime import timedelta +from unittest.mock import AsyncMock + +import pytest + +from crawlee.browsers import BrowserPool + + +async def test_timed_out_plugin_is_not_exited() -> None: + """A pool whose plugin entry timed out exits cleanly without exiting that plugin.""" + entered_plugin = AsyncMock() + slow_plugin = AsyncMock() + slow_plugin.__aenter__.side_effect = asyncio.Event().wait + slow_plugin.active = False + browser_pool = BrowserPool(plugins=[entered_plugin, slow_plugin], operation_timeout=timedelta(milliseconds=10)) + + async with browser_pool: + pass + + assert not browser_pool.active + entered_plugin.__aexit__.assert_awaited_once() + slow_plugin.__aexit__.assert_not_awaited() + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('plugin failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_pool(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A pool whose entry fails or is cancelled stops its tasks and entered plugins and can be entered again.""" + entered_plugin = AsyncMock() + failing_plugin = AsyncMock() + failing_plugin.__aenter__.side_effect = error + browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin]) + identify_stop = AsyncMock(wraps=browser_pool._identify_inactive_browsers_task.stop) + close_stop = AsyncMock(wraps=browser_pool._close_inactive_browsers_task.stop) + monkeypatch.setattr(browser_pool._identify_inactive_browsers_task, 'stop', identify_stop) + monkeypatch.setattr(browser_pool._close_inactive_browsers_task, 'stop', close_stop) + + with pytest.raises(type(error)): + async with browser_pool: + pass + + assert not browser_pool.active + identify_stop.assert_awaited_once() + close_stop.assert_awaited_once() + entered_plugin.__aexit__.assert_awaited_once() + failing_plugin.__aexit__.assert_not_awaited() + + failing_plugin.__aenter__.side_effect = None + async with browser_pool: + assert browser_pool.active + + +async def test_failed_rollback_keeps_enter_error(caplog: pytest.LogCaptureFixture) -> None: + """A plugin that fails to exit while the pool rolls back a failed entry is logged and keeps the entry error.""" + entered_plugin = AsyncMock() + entered_plugin.__aexit__.side_effect = RuntimeError('exit failed') + failing_plugin = AsyncMock() + failing_plugin.__aenter__.side_effect = RuntimeError('enter failed') + browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin]) + + with pytest.raises(RuntimeError, match='enter failed'): + async with browser_pool: + pass + + assert not browser_pool.active + entered_plugin.__aexit__.assert_awaited_once() + assert 'Exiting a browser plugin failed while rolling back the pool entry.' in caplog.text + + +async def test_cancelled_rollback_exits_plugins(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool whose entry rollback is cancelled while stopping its tasks still exits the plugins it entered.""" + entered_plugin = AsyncMock() + failing_plugin = AsyncMock() + failing_plugin.__aenter__.side_effect = RuntimeError('enter failed') + browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin]) + monkeypatch.setattr( + browser_pool._identify_inactive_browsers_task, 'stop', AsyncMock(side_effect=asyncio.CancelledError()) + ) + + with pytest.raises(asyncio.CancelledError): + async with browser_pool: + pass + + assert not browser_pool.active + entered_plugin.__aexit__.assert_awaited_once() + monkeypatch.undo() + await browser_pool._identify_inactive_browsers_task.stop() + + +async def test_failed_exit_frees_pool() -> None: + """A pool whose plugin fails to exit is no longer active and can be entered again.""" + plugin = AsyncMock() + plugin.__aexit__.side_effect = RuntimeError('plugin failed') + browser_pool = BrowserPool(plugins=[plugin]) + + with pytest.raises(RuntimeError, match='plugin failed'): + async with browser_pool: + pass + + assert not browser_pool.active + + plugin.__aexit__.side_effect = None + async with browser_pool: + assert browser_pool.active + + +async def test_enter_during_exit_raises() -> None: + """A pool can't be entered again until its exit has exited its plugins.""" + exit_started = asyncio.Event() + release_exit = asyncio.Event() + + async def wait_for_release(*_: object) -> None: + exit_started.set() + await release_exit.wait() + + plugin = AsyncMock() + plugin.__aexit__.side_effect = wait_for_release + browser_pool = BrowserPool(plugins=[plugin]) + await browser_pool.__aenter__() + exit_task = asyncio.create_task(browser_pool.__aexit__(None, None, None)) + await exit_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await browser_pool.__aenter__() + + release_exit.set() + await exit_task + + +async def test_failed_browser_close_exits_plugins() -> None: + """A pool whose browser fails to close still closes its other browsers and exits its plugins.""" + plugin = AsyncMock() + failing_browser = AsyncMock() + failing_browser.close.side_effect = RuntimeError('close failed') + other_browser = AsyncMock() + browser_pool = BrowserPool(plugins=[plugin]) + + with pytest.raises(RuntimeError, match='close failed'): + async with browser_pool: + browser_pool._active_browsers.extend([failing_browser, other_browser]) + + assert not browser_pool.active + other_browser.close.assert_awaited_once() + plugin.__aexit__.assert_awaited_once() + + +async def test_plugin_exit_error_keeps_cancellation(caplog: pytest.LogCaptureFixture) -> None: + """A plugin that fails to exit is logged and doesn't replace a cancellation of the pool's exit.""" + failing_plugin = AsyncMock() + failing_plugin.__aexit__.side_effect = RuntimeError('exit failed') + close_started = asyncio.Event() + + async def close_until_cancelled(**_: object) -> None: + close_started.set() + await asyncio.Event().wait() + + browser = AsyncMock() + browser.close.side_effect = close_until_cancelled + browser_pool = BrowserPool(plugins=[failing_plugin]) + await browser_pool.__aenter__() + browser_pool._active_browsers.append(browser) + + exit_task = asyncio.create_task(browser_pool.__aexit__(None, None, None)) + await close_started.wait() + exit_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await exit_task + + failing_plugin.__aexit__.assert_awaited_once() + assert 'exit failed' in caplog.text + + +async def test_cancelled_plugin_exit_logs_close_error(caplog: pytest.LogCaptureFixture) -> None: + """A browser that fails to close is logged when the pool's exit is cancelled while exiting its plugins.""" + exit_started = asyncio.Event() + + async def exit_until_cancelled(*_: object) -> None: + exit_started.set() + await asyncio.Event().wait() + + plugin = AsyncMock() + plugin.__aexit__.side_effect = exit_until_cancelled + browser = AsyncMock() + browser.close.side_effect = RuntimeError('close failed') + browser_pool = BrowserPool(plugins=[plugin]) + await browser_pool.__aenter__() + browser_pool._active_browsers.append(browser) + + exit_task = asyncio.create_task(browser_pool.__aexit__(None, None, None)) + await exit_started.wait() + exit_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await exit_task + + assert not browser_pool.active + assert 'close failed' in caplog.text + + +async def test_cancelled_task_stop_stops_the_other(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool whose first task stop is cancelled still stops the other task and exits its plugins.""" + plugin = AsyncMock() + browser_pool = BrowserPool(plugins=[plugin]) + close_stop = AsyncMock(wraps=browser_pool._close_inactive_browsers_task.stop) + await browser_pool.__aenter__() + monkeypatch.setattr( + browser_pool._identify_inactive_browsers_task, 'stop', AsyncMock(side_effect=asyncio.CancelledError()) + ) + monkeypatch.setattr(browser_pool._close_inactive_browsers_task, 'stop', close_stop) + + with pytest.raises(asyncio.CancelledError): + await browser_pool.__aexit__(None, None, None) + + close_stop.assert_awaited_once() + plugin.__aexit__.assert_awaited_once() + monkeypatch.undo() + await browser_pool._identify_inactive_browsers_task.stop() + + +async def test_failed_plugin_exit_exits_others() -> None: + """A pool whose plugin fails to exit still exits its other plugins and raises the failure.""" + failing_plugin = AsyncMock() + failing_plugin.__aexit__.side_effect = RuntimeError('exit failed') + other_plugin = AsyncMock() + browser_pool = BrowserPool(plugins=[failing_plugin, other_plugin]) + + with pytest.raises(RuntimeError, match='exit failed'): + async with browser_pool: + pass + + assert not browser_pool.active + other_plugin.__aexit__.assert_awaited_once() + + +async def test_extra_exit_failures_are_logged(caplog: pytest.LogCaptureFixture) -> None: + """A pool whose plugins fail to exit raises the first failure and logs the others.""" + first_plugin = AsyncMock() + first_plugin.__aexit__.side_effect = RuntimeError('first failed') + second_plugin = AsyncMock() + second_plugin.__aexit__.side_effect = RuntimeError('second failed') + browser_pool = BrowserPool(plugins=[first_plugin, second_plugin]) + + with pytest.raises(RuntimeError, match='first failed'): + async with browser_pool: + pass + + assert 'Closing the browser pool failed.' in caplog.text + assert 'second failed' in caplog.text diff --git a/tests/unit/browsers/test_playwright_browser_plugin.py b/tests/unit/browsers/test_playwright_browser_plugin.py index f45b4449a3..fbe929ac92 100644 --- a/tests/unit/browsers/test_playwright_browser_plugin.py +++ b/tests/unit/browsers/test_playwright_browser_plugin.py @@ -1,5 +1,10 @@ from __future__ import annotations +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + from crawlee.browsers import PlaywrightBrowserPlugin @@ -29,3 +34,92 @@ async def test_executable_path_with_chrome_browser_type() -> None: assert plugin.browser_type == 'chromium' assert plugin.browser_launch_options['executable_path'] == '/path/to/chrome' assert plugin.browser_launch_options['channel'] == 'chrome' + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('playwright failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_plugin(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A plugin whose entry fails or is cancelled stops Playwright, stays inactive and can be entered again.""" + plugin = PlaywrightBrowserPlugin() + failing_context_manager = AsyncMock() + failing_context_manager.__aenter__.side_effect = error + monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager) + monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + + with pytest.raises(type(error)): + async with plugin: + pass + + assert not plugin.active + failing_context_manager.__aexit__.assert_awaited_once() + + async with plugin: + assert plugin.active + + +async def test_failed_rollback_keeps_enter_error( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A Playwright that fails to stop during an entry rollback is logged and the entry error is kept.""" + plugin = PlaywrightBrowserPlugin() + failing_context_manager = AsyncMock() + failing_context_manager.__aenter__.side_effect = RuntimeError('start failed') + failing_context_manager.__aexit__.side_effect = RuntimeError('stop failed') + monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager) + monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + + with pytest.raises(RuntimeError, match='start failed'): + async with plugin: + pass + + assert not plugin.active + assert 'Stopping Playwright failed while rolling back the plugin entry.' in caplog.text + + +async def test_failed_exit_frees_plugin(monkeypatch: pytest.MonkeyPatch) -> None: + """A plugin whose exit fails is no longer active and can be entered again.""" + plugin = PlaywrightBrowserPlugin() + failing_context_manager = AsyncMock() + failing_context_manager.__aexit__.side_effect = RuntimeError('stop failed') + monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager) + monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + + with pytest.raises(RuntimeError, match='stop failed'): + async with plugin: + pass + + assert not plugin.active + + async with plugin: + assert plugin.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A plugin can't be entered again until its exit has stopped Playwright.""" + plugin = PlaywrightBrowserPlugin() + stop_started = asyncio.Event() + release_stop = asyncio.Event() + + async def wait_for_release(*_: object) -> None: + stop_started.set() + await release_stop.wait() + + playwright_context_manager = AsyncMock() + playwright_context_manager.__aexit__.side_effect = wait_for_release + monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager) + monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + + await plugin.__aenter__() + exit_task = asyncio.create_task(plugin.__aexit__(None, None, None)) + await stop_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await plugin.__aenter__() + + release_stop.set() + await exit_task diff --git a/tests/unit/browsers/test_stagehand_browser_plugin.py b/tests/unit/browsers/test_stagehand_browser_plugin.py index eedcd4d0a5..e2eb23dc17 100644 --- a/tests/unit/browsers/test_stagehand_browser_plugin.py +++ b/tests/unit/browsers/test_stagehand_browser_plugin.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import asyncio +from unittest.mock import AsyncMock, Mock -from crawlee.browsers import StagehandBrowserPlugin, StagehandOptions +import pytest -if TYPE_CHECKING: - import pytest +from crawlee.browsers import StagehandBrowserPlugin, StagehandOptions def test_initial_state() -> None: @@ -71,3 +71,135 @@ def test_stagehand_options_defaults_when_not_provided() -> None: assert isinstance(plugin.stagehand_options, StagehandOptions) assert plugin.stagehand_options == StagehandOptions() + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('stagehand failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_stops_playwright(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A plugin whose Stagehand client fails to start stops Playwright and can be entered again.""" + plugin = StagehandBrowserPlugin() + playwright_context_manager = AsyncMock() + monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + stagehand_client = AsyncMock() + stagehand_client.__aenter__.side_effect = error + monkeypatch.setattr( + 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client) + ) + + with pytest.raises(type(error)): + async with plugin: + pass + + assert not plugin.active + playwright_context_manager.__aexit__.assert_awaited_once() + + stagehand_client.__aenter__.side_effect = None + async with plugin: + assert plugin.active + + +async def test_failed_exit_stops_playwright(monkeypatch: pytest.MonkeyPatch) -> None: + """A plugin whose Stagehand client fails to close still stops Playwright and can be entered again.""" + plugin = StagehandBrowserPlugin() + playwright_context_manager = AsyncMock() + monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + stagehand_client = AsyncMock() + stagehand_client.__aexit__.side_effect = RuntimeError('close failed') + monkeypatch.setattr( + 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client) + ) + + with pytest.raises(RuntimeError, match='close failed'): + async with plugin: + pass + + assert not plugin.active + playwright_context_manager.__aexit__.assert_awaited_once() + + stagehand_client.__aexit__.side_effect = None + async with plugin: + assert plugin.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A plugin can't be entered again until its exit has stopped Playwright.""" + plugin = StagehandBrowserPlugin() + stop_started = asyncio.Event() + release_stop = asyncio.Event() + + async def wait_for_release(*_: object) -> None: + stop_started.set() + await release_stop.wait() + + playwright_context_manager = AsyncMock() + playwright_context_manager.__aexit__.side_effect = wait_for_release + monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=AsyncMock())) + + await plugin.__aenter__() + exit_task = asyncio.create_task(plugin.__aexit__(None, None, None)) + await stop_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await plugin.__aenter__() + + release_stop.set() + await exit_task + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('playwright failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_playwright_start_frees_plugin(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A plugin whose Playwright fails to start stops it, stays inactive and can be entered again.""" + plugin = StagehandBrowserPlugin() + failing_context_manager = AsyncMock() + failing_context_manager.__aenter__.side_effect = error + monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=AsyncMock())) + + with pytest.raises(type(error)): + async with plugin: + pass + + assert not plugin.active + failing_context_manager.__aexit__.assert_awaited_once() + + async with plugin: + assert plugin.active + + +async def test_failed_rollback_keeps_enter_error( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A Playwright that fails to stop during an entry rollback is logged and the entry error is kept.""" + plugin = StagehandBrowserPlugin() + playwright_context_manager = AsyncMock() + playwright_context_manager.__aexit__.side_effect = RuntimeError('stop failed') + monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager) + monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock())) + stagehand_client = AsyncMock() + stagehand_client.__aenter__.side_effect = RuntimeError('stagehand failed') + monkeypatch.setattr( + 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client) + ) + + with pytest.raises(RuntimeError, match='stagehand failed'): + async with plugin: + pass + + assert not plugin.active + assert 'Stopping Playwright failed while rolling back the plugin entry.' in caplog.text