From 5880cff785d7faee86281685fd6a87b279bd2338 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sun, 15 Nov 2026 17:14:01 +0000 Subject: [PATCH] use the HTTP client and session pool of `AdaptivePlaywrightCrawler` in its sub crawlers --- .../_adaptive_playwright_crawler.py | 5 +++ .../test_adaptive_playwright_crawler.py | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index b2145101e0..bc3fe41380 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -169,6 +169,11 @@ def __init__( # Sub crawlers related. playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions() + # The sub crawlers never run, so they would never enter or close defaults of their own. The client of the + # adaptive crawler already serves `send_request` in their contexts. + kwargs['session_pool'] = self._session_pool + kwargs['http_client'] = self._http_client + # The sub crawlers log under the adaptive crawler's logger, and only errors, since the adaptive crawler reports # the outcome of each request itself. static_logger = self._logger.getChild('static') diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 2db1cf4574..efbd2e34d8 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -31,7 +31,7 @@ from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import AdaptiveContextError from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor from crawlee.errors import RequestThrottledError -from crawlee.http_clients import HttpClient, HttpResponse +from crawlee.http_clients import HttpClient, HttpResponse, ImpitHttpClient from crawlee.sessions import SessionPool from crawlee.statistics import Statistics from crawlee.storage_backends import SqlStorageBackend @@ -736,6 +736,41 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_browser_handler.assert_called_once_with() +@pytest.mark.parametrize( + 'rendering_type', + [ + pytest.param('static', id='static'), + pytest.param('client only', id='browser'), + ], +) +async def test_closes_shared_default_client(test_urls: list[str], rendering_type: RenderingType) -> None: + """Both sub crawlers send through the default HTTP client of the crawler, which closes only that client.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor( + rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0]) + ), + respect_robots_txt_file=True, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + # The sub crawler fetches robots.txt with its HTTP client to filter the links. + await context.enqueue_links() + + with ( + patch.object( + ImpitHttpClient, 'send_request', autospec=True, side_effect=ImpitHttpClient.send_request + ) as send_request, + patch.object(ImpitHttpClient, 'cleanup', autospec=True, side_effect=ImpitHttpClient.cleanup) as cleanup, + ): + await crawler.run(test_urls[:1]) + + sending_clients = {send_call.args[0] for send_call in send_request.await_args_list} + closed_clients = {cleanup_call.args[0] for cleanup_call in cleanup.await_args_list} + assert sending_clients == {crawler._http_client} + assert closed_clients == {crawler._http_client} + + async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None: """Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.