Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@ def __init__(
# Sub crawlers related.
playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions()

# The sub crawlers never run, so they would never enter or close defaults of their own. The client of the
# adaptive crawler already serves `send_request` in their contexts.
kwargs['session_pool'] = self._session_pool
kwargs['http_client'] = self._http_client

# The sub crawlers log under the adaptive crawler's logger, and only errors, since the adaptive crawler reports
# the outcome of each request itself.
static_logger = self._logger.getChild('static')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import AdaptiveContextError
from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor
from crawlee.errors import RequestThrottledError
from crawlee.http_clients import HttpClient, HttpResponse
from crawlee.http_clients import HttpClient, HttpResponse, ImpitHttpClient
from crawlee.sessions import SessionPool
from crawlee.statistics import Statistics
from crawlee.storage_backends import SqlStorageBackend
Expand Down Expand Up @@ -736,6 +736,41 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
mocked_browser_handler.assert_called_once_with()


@pytest.mark.parametrize(
'rendering_type',
[
pytest.param('static', id='static'),
pytest.param('client only', id='browser'),
],
)
async def test_closes_shared_default_client(test_urls: list[str], rendering_type: RenderingType) -> None:
"""Both sub crawlers send through the default HTTP client of the crawler, which closes only that client."""
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(
rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0])
),
respect_robots_txt_file=True,
)

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
# The sub crawler fetches robots.txt with its HTTP client to filter the links.
await context.enqueue_links()

with (
patch.object(
ImpitHttpClient, 'send_request', autospec=True, side_effect=ImpitHttpClient.send_request
) as send_request,
patch.object(ImpitHttpClient, 'cleanup', autospec=True, side_effect=ImpitHttpClient.cleanup) as cleanup,
):
await crawler.run(test_urls[:1])

sending_clients = {send_call.args[0] for send_call in send_request.await_args_list}
closed_clients = {cleanup_call.args[0] for cleanup_call in cleanup.await_args_list}
assert sending_clients == {crawler._http_client}
assert closed_clients == {crawler._http_client}


async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None:
"""Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.

Expand Down
Loading