Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 53 additions & 9 deletions src/crawlee/browsers/_browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,12 +214,29 @@ async def __aenter__(self) -> BrowserPool:
self._close_inactive_browsers_task.start()

timeout = self._operation_timeout.total_seconds()
entered_plugins: list[BrowserPlugin] = []

try:
for plugin in self._plugins:
await asyncio.wait_for(plugin.__aenter__(), timeout)
entered_plugins.append(plugin)
except TimeoutError:
logger.warning('Initializing of the browser plugin %s timed out, will be skipped.', plugin)
except BaseException:
try:
await self._stop_recurring_tasks()
finally:
try:
plugin_exits = (entered_plugin.__aexit__(None, None, None) for entered_plugin in entered_plugins)
plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True)
finally:
self._active = False
for result in plugin_results:
if isinstance(result, BaseException):
logger.warning(
'Exiting a browser plugin failed while rolling back the pool entry.', exc_info=result
)
raise

return self

Expand All @@ -237,18 +254,39 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')

await self._identify_inactive_browsers_task.stop()
await self._close_inactive_browsers_task.stop()
# Every browser and plugin is closed even when another one fails, since an inactive pool can't be exited again.
failures: list[BaseException] = []
try:
try:
await self._stop_recurring_tasks()

for browser in self._active_browsers + self._inactive_browsers:
await browser.close(force=True)
self._active_browsers.clear()
self._inactive_browsers.clear()
browser_closes = (
browser.close(force=True) for browser in self._active_browsers + self._inactive_browsers
)
browser_results = await asyncio.gather(*browser_closes, return_exceptions=True)
failures.extend(result for result in browser_results if isinstance(result, BaseException))
finally:
self._active_browsers.clear()
self._inactive_browsers.clear()

for plugin in self._plugins:
await plugin.__aexit__(exc_type, exc_value, exc_traceback)
try:
# Plugins left inactive by a timed-out entry have nothing to exit.
active_plugins = [plugin for plugin in self._plugins if plugin.active]
plugin_exits = (plugin.__aexit__(exc_type, exc_value, exc_traceback) for plugin in active_plugins)
plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True)
failures.extend(result for result in plugin_results if isinstance(result, BaseException))
finally:
self._active = False
except BaseException:
# A cancellation above is raised instead, so the failures are only logged.
for failure in failures:
logger.warning('Closing the browser pool failed.', exc_info=failure)
raise

self._active = False
for failure in failures[1:]:
logger.warning('Closing the browser pool failed.', exc_info=failure)
if failures:
raise failures[0]

@ensure_context
async def new_page(
Expand Down Expand Up @@ -392,6 +430,12 @@ async def _close_inactive_browsers(self) -> None:
await browser.close()
self._inactive_browsers.remove(browser)

async def _stop_recurring_tasks(self) -> None:
try:
await self._identify_inactive_browsers_task.stop()
finally:
await self._close_inactive_browsers_task.stop()

async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *args: Any) -> None:
"""Execute the provided hooks with the given arguments."""
for hook in hooks:
Expand Down
24 changes: 20 additions & 4 deletions src/crawlee/browsers/_playwright_browser_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,21 @@ async def __aenter__(self) -> PlaywrightBrowserPlugin:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')

self._active = True
self._playwright = await self._playwright_context_manager.__aenter__()

try:
self._playwright = await self._playwright_context_manager.__aenter__()
except BaseException:
# The Playwright driver may already be running, so it's stopped too.
half_started_context_manager = self._playwright_context_manager
self._playwright_context_manager = async_playwright()
try:
await half_started_context_manager.__aexit__(None, None, None)
except Exception:
logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True)
finally:
self._active = False
raise

return self

@override
Expand All @@ -168,9 +182,11 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')

await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
self._playwright_context_manager = async_playwright()
self._active = False
try:
await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
finally:
self._playwright_context_manager = async_playwright()
self._active = False

@override
@ensure_context
Expand Down
57 changes: 37 additions & 20 deletions src/crawlee/browsers/_stagehand_browser_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,19 +163,33 @@ async def __aenter__(self) -> StagehandBrowserPlugin:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')

self._active = True
self._playwright = await self._playwright_context_manager.__aenter__()

if self._stagehand_options.env == 'LOCAL':
if 'executable_path' not in self._browser_launch_options:
chrome_path = self._playwright.chromium.executable_path
self._browser_launch_options['executable_path'] = chrome_path
logger.debug('Resolved Chromium path from Playwright: %s', chrome_path)

self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path']

client = AsyncStagehand(**self._stagehand_init_params)
await client.__aenter__()
self._stagehand_client = client
try:
self._playwright = await self._playwright_context_manager.__aenter__()

if self._stagehand_options.env == 'LOCAL':
if 'executable_path' not in self._browser_launch_options:
chrome_path = self._playwright.chromium.executable_path
self._browser_launch_options['executable_path'] = chrome_path
logger.debug('Resolved Chromium path from Playwright: %s', chrome_path)

self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path']

client = AsyncStagehand(**self._stagehand_init_params)
await client.__aenter__()
self._stagehand_client = client
except BaseException:
# The Playwright driver may already be running, so it's stopped too.
self._playwright = None
playwright_context_manager = self._playwright_context_manager
self._playwright_context_manager = async_playwright()
try:
await playwright_context_manager.__aexit__(None, None, None)
except Exception:
logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True)
finally:
self._active = False
raise

return self

Expand All @@ -189,14 +203,17 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')

if self._stagehand_client is not None:
await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback)
self._stagehand_client = None

await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
self._playwright_context_manager = async_playwright()
self._playwright = None
self._active = False
try:
if self._stagehand_client is not None:
await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback)
finally:
try:
await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
finally:
self._stagehand_client = None
self._playwright = None
self._playwright_context_manager = async_playwright()
self._active = False

@override
@ensure_context
Expand Down
Loading
Loading