Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

import logging
from collections.abc import Awaitable, Callable, Coroutine
from contextvars import ContextVar
from copy import deepcopy
from dataclasses import dataclass
from dataclasses import dataclass, field
from logging import getLogger
from random import random
from typing import TYPE_CHECKING, Any, Generic, get_args
Expand All @@ -30,7 +31,12 @@
from crawlee.crawlers._beautifulsoup._beautifulsoup_parser import BeautifulSoupParser
from crawlee.crawlers._parsel._parsel_parser import ParselParser
from crawlee.crawlers._playwright._playwright_crawler import _PlaywrightCrawlerAdditionalOptions
from crawlee.errors import ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError
from crawlee.errors import (
ContextPipelineInitializationError,
RequestHandlerError,
RequestThrottledError,
UserDefinedErrorHandlerError,
)
from crawlee.statistics import Statistics, StatisticsState

from ._adaptive_playwright_crawler_statistics import AdaptivePlaywrightCrawlerStatisticState
Expand All @@ -48,14 +54,21 @@

from typing_extensions import Unpack

from crawlee._types import UseStateFunction
from crawlee._types import SkippedReason, UseStateFunction
from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions


TStaticParseResult = TypeVar('TStaticParseResult')
TStaticSelectResult = TypeVar('TStaticSelectResult')
TStaticCrawlingContext = TypeVar('TStaticCrawlingContext', bound=ParsedHttpCrawlingContext)

_run_skipped_requests: ContextVar[list[tuple[str, SkippedReason]]] = ContextVar('_run_skipped_requests')
"""Requests skipped in the current sub crawler run. They are reported only if the result of the run is committed."""


async def _collect_skipped_request(url: str, reason: SkippedReason) -> None:
_run_skipped_requests.get().append((url, reason))


class _NonPersistentStatistics(Statistics):
"""Statistics compliant object that is not supposed to do anything when entering/exiting context.
Expand Down Expand Up @@ -221,6 +234,13 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon
static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static)
playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw)

# The sub crawlers check links against robots.txt in `enqueue_links`. They share the robots.txt cache of this
# crawler, so that each file is fetched once, and collect the links they skip in the current sub crawler run.
for sub_crawler in (static_crawler, playwright_crawler):
sub_crawler._robots_txt_file_cache = self._robots_txt_file_cache # noqa: SLF001
Comment thread
Mantisus marked this conversation as resolved.
Resolved
sub_crawler._robots_txt_lock = self._robots_txt_lock # noqa: SLF001
sub_crawler.on_skipped_request(_collect_skipped_request)

self._additional_context_managers = [
*self._additional_context_managers,
self.rendering_type_predictor,
Expand Down Expand Up @@ -285,6 +305,17 @@ def with_parsel_static_parser(
**kwargs,
)

async def _report_skipped_requests(
self, context: BasicCrawlingContext, skipped_requests: list[tuple[str, SkippedReason]]
) -> None:
"""Report the requests skipped by the sub crawler run whose result is committed."""
try:
for url, reason in skipped_requests:
await self._handle_skipped_request(url, reason)
except UserDefinedErrorHandlerError as e:
# Other crawlers call the callback inside the request handler, where its error fails the handler.
raise RequestHandlerError(e, context) from e

async def _crawl_one(
self,
rendering_type: RenderingType,
Expand All @@ -296,7 +327,8 @@ async def _crawl_one(
`SubCrawlerRun` contains either result of the crawl or the exception that was thrown during the crawl.
Sub crawler pipeline call is dynamically created based on the `rendering_type`.
A new context is created from passed `context` and `use_state` and is passed to sub crawler pipeline. The
original `request` is shared, so its mutations persist. Only `result` is isolated per sub crawler.
original `request` is shared, so its mutations persist. Only `result` and the skipped requests are isolated per
sub crawler.
"""
# A fresh result is injected into the new context to isolate `add_requests`/`push_data`/`kvs` calls between
# sub crawlers. The `request` itself is shared, so mutations to it persist.
Expand All @@ -315,6 +347,8 @@ async def _crawl_one(
log=context.log,
register_deferred_cleanup=context.register_deferred_cleanup,
)
skipped_requests: list[tuple[str, SkippedReason]] = []
token = _run_skipped_requests.set(skipped_requests)

try:
await wait_for(
Expand All @@ -327,9 +361,11 @@ async def _crawl_one(
),
logger=self._logger,
)
return SubCrawlerRun(result=result)
return SubCrawlerRun(result=result, skipped_requests=skipped_requests)
except Exception as e:
return SubCrawlerRun(exception=e)
finally:
_run_skipped_requests.reset(token)

def _pipeline_call_factory(
self, rendering_type: RenderingType, context_linked_to_result: BasicCrawlingContext
Expand Down Expand Up @@ -385,6 +421,7 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None:
static_run = await self._crawl_one(rendering_type='static', context=context)
if static_run.result and self.result_checker(static_run.result):
self._context_result_map[context] = static_run.result
await self._report_skipped_requests(context, static_run.skipped_requests)
return
# The browser would hit the same rate-limited domain, so let the crawler defer the request.
if (
Expand Down Expand Up @@ -446,6 +483,7 @@ async def use_initial_state_copy(
self.rendering_type_predictor.store_result(context.request, detection_result)

self._context_result_map[context] = pw_run.result
await self._report_skipped_requests(context, pw_run.skipped_requests)

def pre_navigation_hook(
self,
Expand Down Expand Up @@ -512,3 +550,4 @@ def track_rendering_type_mispredictions(self) -> None:
class SubCrawlerRun:
result: RequestHandlerRunResult | None = None
exception: Exception | None = None
skipped_requests: list[tuple[str, SkippedReason]] = field(default_factory=list)
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from typing_extensions import override

from crawlee import ConcurrencySettings, Request
from crawlee._utils.robots import RobotsTxtFile
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
AdaptivePlaywrightCrawlingContext,
Expand Down Expand Up @@ -43,6 +44,8 @@

from yarl import URL

from crawlee._types import SkippedReason


_H1_TEXT = 'Static'
_H2_TEXT = 'Only in browser'
Expand Down Expand Up @@ -695,6 +698,82 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
mocked_browser_handler.assert_called_once_with()


@pytest.mark.parametrize(
('rendering_type', 'detection_probability', 'static_result_valid'),
[
pytest.param('static', 0, True, id='static'),
pytest.param('client only', 0, True, id='browser'),
pytest.param('static', 0, False, id='static result rejected'),
pytest.param('static', 1, True, id='detection'),
],
)
async def test_on_skipped_request(
server_url: URL, rendering_type: RenderingType, detection_probability: float, *, static_result_valid: bool
) -> None:
"""Links skipped because of robots.txt reach the callback once, from the sub crawler run that is committed."""
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(
rendering_types=cycle([rendering_type]),
detection_probability_recommendation=cycle([detection_probability]),
),
result_checker=lambda _: static_result_valid,
respect_robots_txt_file=True,
)
skip = Mock()

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
await context.enqueue_links()

@crawler.on_skipped_request
async def skipped_hook(url: str, _reason: SkippedReason) -> None:
skip(url)

await crawler.run([str(server_url / 'start_enqueue')])

assert skip.call_count == 4
skip.assert_has_calls([call(str(server_url / f'page_{index}')) for index in range(1, 5)], any_order=True)
Comment thread
Mantisus marked this conversation as resolved.
Resolved


async def test_on_skipped_request_error_fails_request(server_url: URL) -> None:
"""An error in the skipped request callback fails the request and the crawl goes on."""
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])),
respect_robots_txt_file=True,
max_request_retries=0,
)

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
await context.enqueue_links()

@crawler.on_skipped_request
async def skipped_hook(_url: str, _reason: SkippedReason) -> None:
raise RuntimeError('Skipped request callback failed')

stats = await crawler.run([str(server_url / 'start_enqueue')])

assert stats.requests_failed == 1


async def test_fetches_robots_txt_once(server_url: URL, monkeypatch: pytest.MonkeyPatch) -> None:
"""The crawler and its sub crawlers fetch the robots.txt file of a domain once, detection runs included."""
find = AsyncMock(wraps=RobotsTxtFile.find)
monkeypatch.setattr(RobotsTxtFile, 'find', find)
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([1])),
respect_robots_txt_file=True,
)

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
await context.enqueue_links()

await crawler.run([str(server_url / 'start_enqueue')])

find.assert_awaited_once()


async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None:
"""Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.

Expand Down
Loading