Repository navigation
fix(browsers): Prevent a hang when closing a page never finishes #2441
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
9448688
52a1acf
87e07af
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -45,6 +45,9 @@ class BrowserPool: | |
| _GENERATED_PAGE_ID_LENGTH = 8 | ||
| """The length of the newly generated page ID.""" | ||
|
|
||
| _PAGE_CLOSE_TIMEOUT = timedelta(seconds=5) | ||
| """The time limit for closing a page, after which the pool releases the page anyway.""" | ||
|
|
||
| def __init__( | ||
| self, | ||
| plugins: Sequence[BrowserPlugin] | None = None, | ||
|
|
@@ -398,21 +401,38 @@ async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *arg | |
| await hook(*args) | ||
|
|
||
| def _override_page_close(self, crawlee_page: CrawleePage, browser_controller: BrowserController) -> None: | ||
| """Override the page's close method to execute pre and post close hooks.""" | ||
| if self._pre_page_close_hooks or self._post_page_close_hooks: | ||
| original_close = crawlee_page.page.close | ||
|
|
||
| async def close_with_hooks(*args: Any, **kwargs: Any) -> None: | ||
| try: | ||
| await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) | ||
| finally: | ||
| await original_close(*args, **kwargs) | ||
| await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) | ||
|
|
||
| # `Page.close` is a regular method, so type checkers reject reassigning it directly. | ||
| # Install the hook-wrapped version through an untyped reference to the page. | ||
| page: Any = crawlee_page.page | ||
| page.close = close_with_hooks | ||
| """Override the page's close method to execute pre and post close hooks and bound the close itself. | ||
|
|
||
| A browser can acknowledge the close and then never destroy the page, so neither the close call nor the page's | ||
| `close` event ever finishes. When the close does not finish within `_PAGE_CLOSE_TIMEOUT`, the page is released | ||
| from its browser anyway, and the browser is retired, so it gets no new pages and is closed once its remaining | ||
| pages are closed. The hooks themselves run without a time limit. | ||
| """ | ||
| original_close = crawlee_page.page.close | ||
|
|
||
| async def close_page(*args: Any, **kwargs: Any) -> None: | ||
| timeout = self._PAGE_CLOSE_TIMEOUT.total_seconds() | ||
| try: | ||
| await asyncio.wait_for(original_close(*args, **kwargs), timeout) | ||
| except asyncio.TimeoutError: | ||
| logger.warning( | ||
| f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it from ' | ||
| 'the pool anyway.' | ||
| ) | ||
| browser_controller.register_page_closed(crawlee_page.page) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Question: If the stuck page also blocks ✍️ Drafted by Claude Code
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I saw this once in a batch of ~600 sites: after an "Execution context was destroyed" retry, the deferred cleanup's I haven't been able to confirm your scenario. The hang wasn't reproducible on demand, the run never got as far as
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok, thank you, let's keep this PR to the page close hang only then, and let's see |
||
| self._retire_browser(browser_controller) | ||
|
|
||
| async def close_with_hooks(*args: Any, **kwargs: Any) -> None: | ||
| try: | ||
| await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) | ||
| finally: | ||
| await close_page(*args, **kwargs) | ||
| await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) | ||
|
|
||
| # `Page.close` is a regular method, so type checkers reject reassigning it directly. | ||
| # Install the wrapped version through an untyped reference to the page. | ||
| page: Any = crawlee_page.page | ||
| page.close = close_with_hooks | ||
|
|
||
| def pre_launch_hook( | ||
| self, hook: Callable[[str, BrowserPlugin], Awaitable[None]] | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.