diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 9f498f0b13..d7cdfeeff9 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -2,8 +2,9 @@ import logging from collections.abc import Awaitable, Callable, Coroutine +from contextvars import ContextVar from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, field from logging import getLogger from random import random from typing import TYPE_CHECKING, Any, Generic, get_args @@ -30,7 +31,12 @@ from crawlee.crawlers._beautifulsoup._beautifulsoup_parser import BeautifulSoupParser from crawlee.crawlers._parsel._parsel_parser import ParselParser from crawlee.crawlers._playwright._playwright_crawler import _PlaywrightCrawlerAdditionalOptions -from crawlee.errors import ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError +from crawlee.errors import ( + ContextPipelineInitializationError, + RequestHandlerError, + RequestThrottledError, + UserDefinedErrorHandlerError, +) from crawlee.statistics import Statistics, StatisticsState from ._adaptive_playwright_crawler_statistics import AdaptivePlaywrightCrawlerStatisticState @@ -48,7 +54,7 @@ from typing_extensions import Unpack - from crawlee._types import UseStateFunction + from crawlee._types import SkippedReason, UseStateFunction from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions @@ -56,6 +62,13 @@ TStaticSelectResult = TypeVar('TStaticSelectResult') TStaticCrawlingContext = TypeVar('TStaticCrawlingContext', bound=ParsedHttpCrawlingContext) +_run_skipped_requests: ContextVar[list[tuple[str, SkippedReason]]] = ContextVar('_run_skipped_requests') +"""Requests skipped in the current sub crawler run. They are reported only if the result of the run is committed.""" + + +async def _collect_skipped_request(url: str, reason: SkippedReason) -> None: + _run_skipped_requests.get().append((url, reason)) + class _NonPersistentStatistics(Statistics): """Statistics compliant object that is not supposed to do anything when entering/exiting context. @@ -221,6 +234,13 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static) playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw) + # The sub crawlers check links against robots.txt in `enqueue_links`. They share the robots.txt cache of this + # crawler, so that each file is fetched once, and collect the links they skip in the current sub crawler run. + for sub_crawler in (static_crawler, playwright_crawler): + sub_crawler._robots_txt_file_cache = self._robots_txt_file_cache # noqa: SLF001 + sub_crawler._robots_txt_lock = self._robots_txt_lock # noqa: SLF001 + sub_crawler.on_skipped_request(_collect_skipped_request) + self._additional_context_managers = [ *self._additional_context_managers, self.rendering_type_predictor, @@ -285,6 +305,17 @@ def with_parsel_static_parser( **kwargs, ) + async def _report_skipped_requests( + self, context: BasicCrawlingContext, skipped_requests: list[tuple[str, SkippedReason]] + ) -> None: + """Report the requests skipped by the sub crawler run whose result is committed.""" + try: + for url, reason in skipped_requests: + await self._handle_skipped_request(url, reason) + except UserDefinedErrorHandlerError as e: + # Other crawlers call the callback inside the request handler, where its error fails the handler. + raise RequestHandlerError(e, context) from e + async def _crawl_one( self, rendering_type: RenderingType, @@ -296,7 +327,8 @@ async def _crawl_one( `SubCrawlerRun` contains either result of the crawl or the exception that was thrown during the crawl. Sub crawler pipeline call is dynamically created based on the `rendering_type`. A new context is created from passed `context` and `use_state` and is passed to sub crawler pipeline. The - original `request` is shared, so its mutations persist. Only `result` is isolated per sub crawler. + original `request` is shared, so its mutations persist. Only `result` and the skipped requests are isolated per + sub crawler. """ # A fresh result is injected into the new context to isolate `add_requests`/`push_data`/`kvs` calls between # sub crawlers. The `request` itself is shared, so mutations to it persist. @@ -315,6 +347,8 @@ async def _crawl_one( log=context.log, register_deferred_cleanup=context.register_deferred_cleanup, ) + skipped_requests: list[tuple[str, SkippedReason]] = [] + token = _run_skipped_requests.set(skipped_requests) try: await wait_for( @@ -327,9 +361,11 @@ async def _crawl_one( ), logger=self._logger, ) - return SubCrawlerRun(result=result) + return SubCrawlerRun(result=result, skipped_requests=skipped_requests) except Exception as e: return SubCrawlerRun(exception=e) + finally: + _run_skipped_requests.reset(token) def _pipeline_call_factory( self, rendering_type: RenderingType, context_linked_to_result: BasicCrawlingContext @@ -385,6 +421,7 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None: static_run = await self._crawl_one(rendering_type='static', context=context) if static_run.result and self.result_checker(static_run.result): self._context_result_map[context] = static_run.result + await self._report_skipped_requests(context, static_run.skipped_requests) return # The browser would hit the same rate-limited domain, so let the crawler defer the request. if ( @@ -446,6 +483,7 @@ async def use_initial_state_copy( self.rendering_type_predictor.store_result(context.request, detection_result) self._context_result_map[context] = pw_run.result + await self._report_skipped_requests(context, pw_run.skipped_requests) def pre_navigation_hook( self, @@ -512,3 +550,4 @@ def track_rendering_type_mispredictions(self) -> None: class SubCrawlerRun: result: RequestHandlerRunResult | None = None exception: Exception | None = None + skipped_requests: list[tuple[str, SkippedReason]] = field(default_factory=list) diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 76a2f40369..dc3e2f1fb7 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -15,6 +15,7 @@ from typing_extensions import override from crawlee import ConcurrencySettings, Request +from crawlee._utils.robots import RobotsTxtFile from crawlee.crawlers import ( AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext, @@ -43,6 +44,8 @@ from yarl import URL + from crawlee._types import SkippedReason + _H1_TEXT = 'Static' _H2_TEXT = 'Only in browser' @@ -695,6 +698,82 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_browser_handler.assert_called_once_with() +@pytest.mark.parametrize( + ('rendering_type', 'detection_probability', 'static_result_valid'), + [ + pytest.param('static', 0, True, id='static'), + pytest.param('client only', 0, True, id='browser'), + pytest.param('static', 0, False, id='static result rejected'), + pytest.param('static', 1, True, id='detection'), + ], +) +async def test_on_skipped_request( + server_url: URL, rendering_type: RenderingType, detection_probability: float, *, static_result_valid: bool +) -> None: + """Links skipped because of robots.txt reach the callback once, from the sub crawler run that is committed.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor( + rendering_types=cycle([rendering_type]), + detection_probability_recommendation=cycle([detection_probability]), + ), + result_checker=lambda _: static_result_valid, + respect_robots_txt_file=True, + ) + skip = Mock() + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + await context.enqueue_links() + + @crawler.on_skipped_request + async def skipped_hook(url: str, _reason: SkippedReason) -> None: + skip(url) + + await crawler.run([str(server_url / 'start_enqueue')]) + + assert skip.call_count == 4 + skip.assert_has_calls([call(str(server_url / f'page_{index}')) for index in range(1, 5)], any_order=True) + + +async def test_on_skipped_request_error_fails_request(server_url: URL) -> None: + """An error in the skipped request callback fails the request and the crawl goes on.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])), + respect_robots_txt_file=True, + max_request_retries=0, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + await context.enqueue_links() + + @crawler.on_skipped_request + async def skipped_hook(_url: str, _reason: SkippedReason) -> None: + raise RuntimeError('Skipped request callback failed') + + stats = await crawler.run([str(server_url / 'start_enqueue')]) + + assert stats.requests_failed == 1 + + +async def test_fetches_robots_txt_once(server_url: URL, monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler and its sub crawlers fetch the robots.txt file of a domain once, detection runs included.""" + find = AsyncMock(wraps=RobotsTxtFile.find) + monkeypatch.setattr(RobotsTxtFile, 'find', find) + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([1])), + respect_robots_txt_file=True, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + await context.enqueue_links() + + await crawler.run([str(server_url / 'start_enqueue')]) + + find.assert_awaited_once() + + async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None: """Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.