diff --git a/docs/03_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx
index b58f6dd3d0..b33e1b6037 100644
--- a/docs/03_concepts/03_crawling_context.mdx
+++ b/docs/03_concepts/03_crawling_context.mdx
@@ -46,6 +46,7 @@ The context provides helpers that operate on the crawler's storages and request
- `get_key_value_store` opens a key-value store, the default one unless you pass `id`, `name` or `alias`.
- `use_state` returns a dictionary shared by all handler calls of the crawler. The crawler keeps it in the default key-value store and saves it automatically. This helper is experimental and its interface may change.
- `send_request` sends an extra HTTP request through the crawler's HTTP client, with the session and proxy of the current request. Its response isn't parsed, and the request doesn't go through the request queue.
+- `extend_timeout` gives the running request handler, error handler or navigation hook more time and keeps the request locked in the request queue for longer. For details, see [Handler timeouts](./request-router#handler-timeouts).
The following example uses most of these helpers in a `ParselCrawler` handler and reads the results back from the crawler once the run finishes.
diff --git a/docs/03_concepts/04_request_router.mdx b/docs/03_concepts/04_request_router.mdx
index 01921c2309..a287bd7d22 100644
--- a/docs/03_concepts/04_request_router.mdx
+++ b/docs/03_concepts/04_request_router.mdx
@@ -9,6 +9,7 @@ import CodeBlock from '@theme/CodeBlock';
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
import BasicRequestHandlers from '!!raw-loader!roa-loader!./code_examples/request_router/basic_request_handlers.py';
+import HandlerTimeouts from '!!raw-loader!roa-loader!./code_examples/request_router/handler_timeouts.py';
import SimpleDefaultHandler from '!!raw-loader!roa-loader!./code_examples/request_router/simple_default_handler.py';
import CustomRouterDefaultOnly from '!!raw-loader!roa-loader!./code_examples/request_router/custom_router_default_only.py';
import HttpPreNavigation from '!!raw-loader!roa-loader!./code_examples/request_router/http_pre_navigation.py';
@@ -58,6 +59,16 @@ More complex crawling projects often require different processing logic for vari
{BasicRequestHandlers}
+### Handler timeouts
+
+The `request_handler_timeout` option of the crawler applies to every request handler. When one type of page needs more or less time than the others, pass `request_handler_timeout` to `router.handler()` or `router.default_handler()`. Handlers that don't set it keep the crawler's timeout. The timeout follows the label the request has before the [middlewares](#middleware) run. If a middleware changes the label, the new handler runs with the timeout of the old one.
+
+When the time a request needs only becomes clear while the handler runs, await `context.extend_timeout()` to give the handler more time. It also keeps the request locked in the request queue for longer. Called from a pre-navigation or post-navigation hook, it extends the navigation timeout instead. It raises an error when the delta isn't positive or when it's awaited on another event loop, such as one started in a separate thread. Callbacks that run outside the request processing, such as Playwright event handlers, can't extend the timeout.
+
+
+ {HandlerTimeouts}
+
+
## Middleware
Middlewares are functions registered with `router.use()` that execute before the matched request handler on every request, regardless of the request label. Multiple middlewares can be registered and are executed sequentially in the order they were registered. If a middleware raises an exception, the execution chain is interrupted and the handler is not called.
diff --git a/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py b/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py
index 8f74254be7..af273d5d49 100644
--- a/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py
+++ b/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py
@@ -14,9 +14,7 @@ async def main() -> None:
# been read yet.
crawler = FileDownloadCrawler(
stream=True,
- # Bounds the whole download: the connection, the response headers and the body.
- navigation_timeout=timedelta(minutes=5),
- # The body is downloaded inside the handler, so this bounds the transfer as well.
+ # The body is downloaded inside the handler, so this bounds the transfer.
request_handler_timeout=timedelta(minutes=5),
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
diff --git a/docs/03_concepts/code_examples/request_router/handler_timeouts.py b/docs/03_concepts/code_examples/request_router/handler_timeouts.py
new file mode 100644
index 0000000000..59338a66f8
--- /dev/null
+++ b/docs/03_concepts/code_examples/request_router/handler_timeouts.py
@@ -0,0 +1,55 @@
+import asyncio
+from datetime import timedelta
+from urllib.parse import urljoin
+
+from crawlee import Request
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.router import Router
+
+
+async def main() -> None:
+ router = Router[ParselCrawlingContext]()
+
+ # Category pages list many products, so give them more time than the rest
+ @router.handler('CATEGORY', request_handler_timeout=timedelta(minutes=2))
+ async def category_handler(context: ParselCrawlingContext) -> None:
+ context.log.info('Processing category page: %s', context.request.url)
+ await context.enqueue_links(selector='a[href*="/products/"]', label='PRODUCT')
+
+ # Product pages keep the timeout of the crawler
+ @router.handler('PRODUCT')
+ async def product_handler(context: ParselCrawlingContext) -> None:
+ context.log.info('Processing product page: %s', context.request.url)
+ image_urls = set(context.selector.css('.product-gallery img::attr(src)').getall())
+
+ # The handler downloads every image below, and their number is known only now,
+ # so ask for 5 more seconds per image
+ if image_urls:
+ await context.extend_timeout(timedelta(seconds=5 * len(image_urls)))
+
+ image_sizes = []
+ for image_url in image_urls:
+ response = await context.send_request(urljoin(context.request.url, image_url))
+ image = await response.read()
+ image_sizes.append(len(image))
+
+ await context.push_data({'url': context.request.url, 'image_sizes': image_sizes})
+
+ crawler = ParselCrawler(
+ request_handler=router,
+ request_handler_timeout=timedelta(seconds=30),
+ max_requests_per_crawl=10, # Limit the max requests per crawl.
+ )
+
+ await crawler.run(
+ [
+ Request.from_url(
+ 'https://warehouse-theme-metal.myshopify.com/collections/all',
+ label='CATEGORY',
+ ),
+ ]
+ )
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/docs/04_guides/11_secure_scraping.mdx b/docs/04_guides/11_secure_scraping.mdx
index fe5bbca877..253bdd5123 100644
--- a/docs/04_guides/11_secure_scraping.mdx
+++ b/docs/04_guides/11_secure_scraping.mdx
@@ -63,7 +63,7 @@ If only your crawler's storage sees the response, the same request is a *blind*
A hostile site can try to use up your CPU, memory, or queue rather than read anything:
- A *crawler trap* is a maze of randomly generated URLs linking to more generated pages. It's built to keep a crawler running forever and grow its queue without bound. Bound how far link-following goes with `max_crawl_depth`, and cap a whole run with `max_requests_per_crawl`. Both are unlimited by default, so set them yourself.
-- A single *oversized or slow response* can tie up a worker or blow up memory. Both the fetch and the handler are capped at 60 seconds by default. `navigation_timeout` covers the fetch, including reading the body, and is passed straight to the HTTP client. `request_handler_timeout` covers the handler. Raise them if a legitimate target needs longer, but keep the cap as tight as your workload allows.
+- A single *oversized or slow response* can tie up a worker or blow up memory. Both the fetch and the handler are capped at 60 seconds by default. `navigation_timeout` covers the fetch, including reading the body. `request_handler_timeout` covers the handler. With `FileDownloadCrawler` in stream mode, reading the body counts towards the handler timeout instead. Raise them if a legitimate target needs longer, but keep the cap as tight as your workload allows.
- A *decompression bomb* is an archive that expands to gigabytes when unpacked. If your crawler downloads and extracts archives, cap the output stream during extraction. The declared size is part of the attack and easy to forge.
## Untrusted content
diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md
index ee57b83421..ecbb8d0e93 100644
--- a/docs/06_upgrading/upgrading_to_v2.md
+++ b/docs/06_upgrading/upgrading_to_v2.md
@@ -180,7 +180,11 @@ The memory backend now handles values the same way as the file-system backend. I
`SqlStorageBackend` and `RedisStorageBackend` now lock a fetched request for 3 minutes by default, the same as Crawlee JS. In v1, the lock lasted 5 minutes. When a lock expires before the request is handled or reclaimed, another client sharing the queue can fetch the request and process it again.
-If processing a request can take longer than 3 minutes, call `RequestQueue.set_expected_request_processing_time()` with the longest time you expect. Every request fetched after that call stays locked at least that long. To give a single request that's still being processed more time, call `RequestQueue.extend_request_processing_time()`.
+A crawler keeps the lock at least as long as its longest request handler timeout plus 1 minute for the error handling, counting the timeouts of its router handlers. HTTP and Playwright crawlers add their navigation timeout, and `AdaptivePlaywrightCrawler` counts the request handler timeout twice, since it can run both sub crawlers on one request. The crawler caps this lock time at a day and logs a warning when its timeouts add up to more. If processing a request can take longer, call `RequestQueue.set_expected_request_processing_time()` with the longest time you expect. Every request fetched after that call stays locked at least that long. To give a single request that's still being processed more time, await `context.extend_timeout()` from the request handler. Outside a crawler, call `RequestQueue.extend_request_processing_time()`.
+
+## `BasicCrawlingContext` requires `extend_timeout`
+
+`BasicCrawlingContext` has a new required field, `extend_timeout`. If you build a crawling context yourself, such as in a test, pass it as well.
## Crawler `id` is a string
@@ -385,7 +389,7 @@ Responses returned by `context.send_request` are closed after the request is pro
The `timeout` of `send_request` is one deadline for the whole redirect chain and for reading the body, the same as Crawlee JS. A body read past it raises `TimeoutError`. By default, `timeout` is `'default'` and the default of the underlying library applies to each request of the chain. `ImpitHttpClient` allows 30 seconds, including reading the body. `CurlImpersonateHttpClient` allows 30 seconds to connect and 30 seconds without receiving data. `HttpxHttpClient` allows 5 seconds per operation. To change the default, pass `timeout` when you create the client.
-The crawlers pass the remaining `navigation_timeout` to the HTTP client, so it bounds reading the body too. With `FileDownloadCrawler(stream=True)`, the deadline also covers the body streamed in the request handler, with every HTTP client. In v1, only `ImpitHttpClient` bounded the streamed body, while `HttpxHttpClient` and `CurlImpersonateHttpClient` bounded only the connection. For long downloads, raise `navigation_timeout` together with `request_handler_timeout`.
+The crawlers bound the navigation request and reading its body with `navigation_timeout` themselves and send the request with `timeout=None`, so `context.extend_timeout()` can extend it. With `FileDownloadCrawler(stream=True)`, the body streamed in the request handler counts towards `request_handler_timeout`, so a body that stalls fails as a request handler timeout. In v1, `ImpitHttpClient` bounded the streamed body by the remaining `navigation_timeout`, while `HttpxHttpClient` and `CurlImpersonateHttpClient` bounded only the connection. For long downloads, raise `request_handler_timeout` or extend it from the handler.
### `timeout=None` disables the timeout
@@ -578,6 +582,8 @@ The warnings about an HTTP 429 that the request manager doesn't pace are reworde
The `ThrottlingRequestManager` warning about capping a delay now calls it a "requested wait" delay instead of a "Retry-After header" delay. Its DEBUG messages about a 429 ignored during an active backoff and about resetting a domain's rate limit state are removed, and its DEBUG message about a domain's crawl-delay shows the delay with one decimal place.
+The error of an `AdaptivePlaywrightCrawler` sub crawler that timed out now reads `rendering_type=static timed out after 60.0 seconds`, with a space before `seconds`.
+
Log records hold the message template and its arguments separately. `record.msg` is the template, such as `Retrying request to %s due to: %s. %s`, and `record.args` holds the values. To read the final text, use `record.getMessage()`, which formatters and handlers already do.
## Session retirement is permanent
diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py
index 85f90a3cfc..95e2ca546a 100644
--- a/src/crawlee/_types.py
+++ b/src/crawlee/_types.py
@@ -454,6 +454,29 @@ def __call__(
"""
+@docs_group('Functions')
+class ExtendTimeoutFunction(Protocol):
+ """A function for giving the running part of the request processing more time.
+
+ Called from a navigation hook, it extends the navigation timeout. Called from the request handler or an error
+ handler, it extends the timeout of that handler. It also keeps the request locked in the request manager for longer.
+ Use it when the time a request needs only becomes apparent while it runs. Otherwise, set `request_handler_timeout`
+ on the crawler or on the router handler. It doesn't extend a timeout that has already expired, or any timeout when
+ called from a callback that runs outside the request processing, such as a Playwright event callback.
+ """
+
+ def __call__(self, delta: timedelta) -> Coroutine[None, None, None]:
+ """Call dunder method.
+
+ Args:
+ delta: How much more time to give.
+
+ Raises:
+ ValueError: If `delta` isn't positive.
+ RuntimeError: If awaited on another event loop, such as one started in a separate thread.
+ """
+
+
@docs_group('Functions')
class ExtractLinksFunction(Protocol):
"""A function for extracting URLs to crawl based on elements selected by a given selector.
@@ -683,6 +706,9 @@ class BasicCrawlingContext:
register_deferred_cleanup: Callable[[DeferredCleanupCallback], None]
"""Register an async callback to be called after request processing completes (including error handlers)."""
+ extend_timeout: ExtendTimeoutFunction
+ """Give the running part of the request processing more time."""
+
async def get_snapshot(self) -> PageSnapshot:
"""Get snapshot of crawled page."""
return PageSnapshot()
diff --git a/src/crawlee/_utils/time.py b/src/crawlee/_utils/time.py
index 719886a45d..dc2a5228e6 100644
--- a/src/crawlee/_utils/time.py
+++ b/src/crawlee/_utils/time.py
@@ -3,17 +3,25 @@
import asyncio
import time
from contextlib import contextmanager
+from contextvars import ContextVar, copy_context
from dataclasses import dataclass
from datetime import timedelta
+from functools import partial
from typing import TYPE_CHECKING
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Callable, Generator
+ from contextvars import Context, Token
from types import TracebackType
+ _TimeoutExtender = Callable[[timedelta], bool]
+
_SECONDS_PER_MINUTE = 60
_SECONDS_PER_HOUR = 3600
+_active_timeout_extenders: ContextVar[tuple[_TimeoutExtender, ...]] = ContextVar('active_timeout_extenders', default=())
+"""Extenders of the `SharedTimeout`s entered in the current context, from the outermost to the innermost."""
+
@dataclass
class TimerResult:
@@ -40,13 +48,20 @@ def measure_time() -> Generator[TimerResult]:
class SharedTimeout:
"""Keeps track of a time budget shared by multiple independent async operations.
- Provides a reusable, non-reentrant context manager interface.
+ Provides a reusable, non-reentrant context manager interface. While entered, `extend_active_timeouts` can extend it.
"""
def __init__(self, timeout: timedelta) -> None:
+ self._total_timeout = timeout
self._remaining_timeout = timeout
self._active_timeout: asyncio.Timeout | None = None
self._activation_timestamp: float | None = None
+ self._extenders_token: Token[tuple[_TimeoutExtender, ...]] | None = None
+
+ @property
+ def total_timeout(self) -> timedelta:
+ """The whole time budget, including extensions."""
+ return self._total_timeout
async def __aenter__(self) -> timedelta:
if self._active_timeout is not None or self._activation_timestamp is not None:
@@ -55,6 +70,10 @@ async def __aenter__(self) -> timedelta:
self._activation_timestamp = time.monotonic()
self._active_timeout = new_timeout = asyncio.timeout(self._remaining_timeout.total_seconds())
await new_timeout.__aenter__()
+
+ # Entering and exiting must run in the same context. Otherwise, resetting the token fails.
+ extender = partial(self._extend, new_timeout)
+ self._extenders_token = _active_timeout_extenders.set((*_active_timeout_extenders.get(), extender))
return self._remaining_timeout
async def __aexit__(
@@ -63,15 +82,62 @@ async def __aexit__(
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
- if self._active_timeout is None or self._activation_timestamp is None:
+ if self._active_timeout is None or self._activation_timestamp is None or self._extenders_token is None:
raise RuntimeError('Logic error')
- await self._active_timeout.__aexit__(exc_type, exc_value, exc_traceback)
- elapsed = time.monotonic() - self._activation_timestamp
- self._remaining_timeout = self._remaining_timeout - timedelta(seconds=elapsed)
+ try:
+ await self._active_timeout.__aexit__(exc_type, exc_value, exc_traceback)
+ finally:
+ elapsed = time.monotonic() - self._activation_timestamp
+ self._remaining_timeout = self._remaining_timeout - timedelta(seconds=elapsed)
+
+ extenders_token = self._extenders_token
+ self._active_timeout = None
+ self._activation_timestamp = None
+ self._extenders_token = None
+ _active_timeout_extenders.reset(extenders_token)
+
+ def _extend(self, entered_timeout: asyncio.Timeout, delta: timedelta) -> bool:
+ """Extend the budget by `delta` and return whether the enclosing timeouts should be extended as well."""
+ # A timeout that has been exited since only gives its later uses the extra time.
+ is_running = entered_timeout is self._active_timeout
+ if is_running and entered_timeout.expired():
+ return False
+
+ if is_running:
+ deadline = entered_timeout.when()
+ if deadline is not None:
+ entered_timeout.reschedule(deadline + delta.total_seconds())
+
+ self._total_timeout = _add_saturating(self._total_timeout, delta)
+ self._remaining_timeout = _add_saturating(self._remaining_timeout, delta)
+ return is_running
+
+
+def _add_saturating(duration: timedelta, delta: timedelta) -> timedelta:
+ """Add `delta` to `duration`, staying at `timedelta.max` when the sum doesn't fit."""
+ try:
+ return duration + delta
+ except OverflowError:
+ return timedelta.max
+
+
+def extend_active_timeouts(delta: timedelta) -> None:
+ """Extend the `SharedTimeout`s entered in the current context by `delta`, from the innermost outwards.
+
+ Stops at a timeout that is no longer running, so the ones that still bound the caller stay as they are. Must be
+ called on the event loop the timeouts run on.
+ """
+ for extend in reversed(_active_timeout_extenders.get()):
+ if not extend(delta):
+ return
+
- self._active_timeout = None
- self._activation_timestamp = None
+def context_without_active_timeouts() -> Context:
+ """Copy the current context without the entered `SharedTimeout`s, so the code running in it can't extend them."""
+ context = copy_context()
+ context.run(_active_timeout_extenders.set, ())
+ return context
def format_duration(duration: timedelta | None) -> str:
diff --git a/src/crawlee/_utils/wait.py b/src/crawlee/_utils/wait.py
index ec77ac8ae7..9c1e9aa0d9 100644
--- a/src/crawlee/_utils/wait.py
+++ b/src/crawlee/_utils/wait.py
@@ -4,6 +4,8 @@
from contextlib import suppress
from typing import TYPE_CHECKING, TypeVar
+from crawlee._utils.time import SharedTimeout
+
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Sequence
from datetime import timedelta
@@ -46,6 +48,24 @@ async def wait_for(
raise RuntimeError('Unreachable code')
+async def wait_for_extendable(
+ operation: Callable[[], Awaitable[T]],
+ *,
+ timeout: timedelta,
+ timeout_message: Callable[[timedelta], str],
+) -> T:
+ """Wait for an async operation like `wait_for`, but under a timeout that `extend_active_timeouts` can extend.
+
+ The `timeout_message` function gets the whole timeout, including extensions.
+ """
+ shared_timeout = SharedTimeout(timeout)
+ try:
+ async with shared_timeout:
+ return await operation()
+ except TimeoutError as ex:
+ raise TimeoutError(timeout_message(shared_timeout.total_timeout)) from ex
+
+
async def wait_for_all_tasks_to_finish(
tasks: Sequence[asyncio.Task],
*,
diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
index 2559eff3a2..9b5ad9fdfd 100644
--- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
+++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -7,7 +7,7 @@
from more_itertools import partition
from pydantic import ValidationError
-from typing_extensions import TypeVar
+from typing_extensions import TypeVar, override
from crawlee._request import Request, RequestOptions, RequestState
from crawlee._utils.docs import docs_group
@@ -147,6 +147,11 @@ async def _manage_shared_navigation_timeout(
finally:
self._shared_navigation_timeouts.pop(request_id, None)
+ @override
+ def _get_max_processing_time(self) -> timedelta:
+ # The navigation phase runs before the request handler, under its own timeout.
+ return self._navigation_timeout + super()._get_max_processing_time()
+
async def _execute_pre_navigation_hooks(
self, context: BasicCrawlingContext
) -> AsyncGenerator[BasicCrawlingContext, None]:
@@ -312,7 +317,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
if context.session is not None and not self._save_response_cookies:
cookies = SessionCookies(context.session.cookies)
- async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
+ async with self._shared_navigation_timeouts[id(context.request)]:
response = await self._http_client.send_request(
context.request.url,
method=context.request.method,
@@ -321,7 +326,8 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
session=context.session,
cookies=cookies,
proxy_info=context.proxy_info,
- timeout=remaining_timeout,
+ # The crawler's timeouts, which `extend_timeout` can extend, bound the request and reading its body.
+ timeout=None,
)
try:
diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
index 8f2e74ab2f..922500495d 100644
--- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
+++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
@@ -13,7 +13,7 @@
from crawlee._types import BasicCrawlingContext, ConcurrencySettings, JsonSerializable, RequestHandlerRunResult
from crawlee._utils.docs import docs_group
-from crawlee._utils.wait import wait_for
+from crawlee._utils.wait import wait_for_extendable
from crawlee.crawlers import (
AbstractHttpCrawler,
AbstractHttpParser,
@@ -43,6 +43,7 @@
if TYPE_CHECKING:
from collections.abc import MutableMapping
+ from datetime import timedelta
from types import TracebackType
from typing import Unpack
@@ -296,6 +297,12 @@ def with_parsel_static_parser(
**kwargs,
)
+ @override
+ def _get_max_processing_time(self) -> timedelta:
+ # A request can run both sub crawlers one after another, each under the request handler timeout, which also
+ # covers its navigation.
+ return 2 * super()._get_max_processing_time()
+
async def _crawl_one(
self,
rendering_type: RenderingType,
@@ -336,18 +343,19 @@ async def get_input_state(
use_state=use_state_function,
log=context.log,
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
)
+ request_handler_timeout = self._resolve_request_handler_timeout(context.request.label)
try:
- await wait_for(
+ await wait_for_extendable(
lambda: self._pipeline_call_factory(
rendering_type=rendering_type, context_linked_to_result=context_linked_to_result
),
- timeout=self._request_handler_timeout,
- timeout_message=(
- f'{rendering_type=!s} timed out after {self._request_handler_timeout.total_seconds()}seconds'
+ timeout=request_handler_timeout,
+ timeout_message=lambda timeout: (
+ f'{rendering_type=!s} timed out after {timeout.total_seconds()} seconds'
),
- logger=self._logger,
)
return SubCrawlerRun(result=result)
except Exception as e:
diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py
index 9296285425..2bba136b26 100644
--- a/src/crawlee/crawlers/_basic/_basic_crawler.py
+++ b/src/crawlee/crawlers/_basic/_basic_crawler.py
@@ -49,8 +49,9 @@
from crawlee._utils.log import LoggerOnce, request_log_extra
from crawlee._utils.recurring_task import RecurringTask
from crawlee._utils.robots import RobotsTxtFile
+from crawlee._utils.time import context_without_active_timeouts, extend_active_timeouts, format_duration
from crawlee._utils.urls import UNSUPPORTED_SCHEME_MESSAGE, convert_to_absolute_url, filter_url, is_url_absolute
-from crawlee._utils.wait import wait_for
+from crawlee._utils.wait import wait_for, wait_for_extendable
from crawlee._utils.web import is_status_code_client_error, is_status_code_server_error
from crawlee.autoscaling import ConcurrencyConsumer, ConcurrencySystem
from crawlee.errors import (
@@ -89,6 +90,7 @@
ConcurrencySettings,
DeferredCleanupCallback,
EnqueueLinksFunction,
+ ExtendTimeoutFunction,
ExtractLinksFunction,
GetDataKwargs,
HttpMethod,
@@ -181,7 +183,7 @@ class _BasicCrawlerOptions(TypedDict):
concurrency. Cannot be combined with `concurrency_settings`."""
request_handler_timeout: NotRequired[timedelta]
- """Maximum duration allowed for a single request handler to run."""
+ """Maximum duration allowed for a single request handler to run. A router handler can override it."""
abort_on_error: NotRequired[bool]
"""If True, the crawler stops immediately when any request handler error occurs."""
@@ -287,6 +289,12 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
_SAFE_MIGRATION_WAIT = timedelta(seconds=20)
"""How long a migration or an abort waits for the requests in progress."""
+ _MAX_EXPECTED_PROCESSING_TIME = timedelta(days=1)
+ """The longest time the crawler asks the request manager to keep a fetched request locked."""
+
+ _ERROR_HANDLING_LOCK_MARGIN = timedelta(minutes=1)
+ """How much longer than its timeouts the crawler keeps a request locked, so the error handling still holds it."""
+
def __init__(
self,
*,
@@ -361,7 +369,8 @@ def __init__(
concurrency_system: A concurrency system to use instead of the default one, e.g. one shared between
crawlers to cap their combined concurrency. Keep it open with `async with` until every crawler using it
has finished. Cannot be combined with `concurrency_settings`.
- request_handler_timeout: Maximum duration allowed for a single request handler to run.
+ request_handler_timeout: Maximum duration allowed for a single request handler to run. A router handler
+ can override it.
statistics: A custom `Statistics` instance, allowing the use of non-default configuration.
abort_on_error: If True, the crawler stops immediately when any request handler error occurs.
keep_alive: If True, it will keep crawler alive even if there are no requests in queue.
@@ -711,6 +720,42 @@ async def get_request_manager(self) -> RequestManager:
return self._request_manager
+ def _get_expected_processing_time(self) -> timedelta:
+ """Get how long the request manager should keep a fetched request locked."""
+ try:
+ processing_time = self._get_max_processing_time() + self._ERROR_HANDLING_LOCK_MARGIN
+ except OverflowError:
+ # A timeout such as `timedelta.max`, given to mean no timeout at all.
+ processing_time = timedelta.max
+
+ if processing_time > self._MAX_EXPECTED_PROCESSING_TIME:
+ self._logger_once.log(
+ 'Processing a request may take longer than %s, the longest time the crawler asks to keep a fetched '
+ 'request locked. With a storage backend that several clients share, another client can then fetch the '
+ 'request and process it too.',
+ format_duration(self._MAX_EXPECTED_PROCESSING_TIME),
+ key='expected_processing_time_capped',
+ level=logging.WARNING,
+ )
+ return self._MAX_EXPECTED_PROCESSING_TIME
+
+ return processing_time
+
+ def _get_max_processing_time(self) -> timedelta:
+ """Get the longest time the crawler may spend on a request, not counting `context.extend_timeout()` calls.
+
+ A route may hold a request longer than the crawler's own timeout, and which routes a run hits isn't known up
+ front, so the longest timeout of any route counts. Subclasses add the phases they run outside the request
+ handler timeout.
+ """
+ handler_timeout = self._request_handler_timeout
+ # Not `self.router`, which would create an empty router, and the `router` setter would then refuse the user's.
+ max_route_timeout = self._router.get_max_request_handler_timeout() if self._router is not None else None
+ if max_route_timeout is not None:
+ handler_timeout = max(handler_timeout, max_route_timeout)
+
+ return handler_timeout
+
async def get_dataset(
self,
*,
@@ -805,6 +850,10 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta
if requests is not None:
await self.add_requests(requests)
+ # The hint only ever raises the lock time, so applying it on every run also covers routes added since.
+ request_manager = await self.get_request_manager()
+ await request_manager.set_expected_request_processing_time(self._get_expected_processing_time())
+
interrupted = False
def sigint_handler() -> None:
@@ -816,7 +865,10 @@ def sigint_handler() -> None:
run_task.cancel()
- run_task = asyncio.create_task(self._run_crawler(), name='run_crawler_task')
+ # A crawler run from a request handler of another crawler must not extend the timeout of that handler.
+ run_task = asyncio.create_task(
+ self._run_crawler(), name='run_crawler_task', context=context_without_active_timeouts()
+ )
loop = asyncio.get_running_loop()
remove_sigint_handler: Callable[[], object] | None = None
@@ -868,7 +920,6 @@ def sigint_handler() -> None:
# Starting against handled requests is not, since that is what resuming a crawl looks like. The statistics
# can't tell, as they don't count skipped requests and may be restored from a previous process.
if not self._run.handled_request:
- request_manager = await self.get_request_manager()
# The diagnostic must never fail the run.
try:
is_finished = (await request_manager.check_readiness()).status == 'finished'
@@ -1367,12 +1418,12 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC
try:
context.request.state = RequestState.ERROR_HANDLER
- await wait_for(
+ await wait_for_extendable(
partial(self._handle_request_retries, context, error),
timeout=self._internal_timeout,
- timeout_message='Handling request failure timed out after '
- f'{self._internal_timeout.total_seconds()} seconds',
- logger=self._logger,
+ timeout_message=lambda timeout: (
+ f'Handling request failure timed out after {timeout.total_seconds()} seconds'
+ ),
)
except UserDefinedErrorHandlerError:
context.request.state = RequestState.ERROR
@@ -1476,6 +1527,31 @@ async def send_request(
return send_request
+ def _prepare_extend_timeout_function(self, request: Request) -> ExtendTimeoutFunction:
+ loop = asyncio.get_running_loop()
+
+ async def extend_timeout(delta: timedelta) -> None:
+ if delta <= timedelta(0):
+ raise ValueError(f'The timeout extension must be positive, got {delta.total_seconds()} seconds.')
+
+ # The timeouts can only be extended on their own event loop. A thread started without the context of the
+ # request would also find no timeout to extend.
+ if asyncio.get_running_loop() is not loop:
+ raise RuntimeError('The timeout can only be extended from the event loop the crawler runs on.')
+
+ extend_active_timeouts(delta)
+
+ # A failed extension must not fail the request that asked for more time.
+ try:
+ request_manager = await self.get_request_manager()
+ await request_manager.extend_request_processing_time(request, delta)
+ except Exception:
+ self._logger.debug(
+ 'Extending the request processing time failed', exc_info=True, extra=request_log_extra(request)
+ )
+
+ return extend_timeout
+
def _convert_url_to_request_iterator(self, urls: Sequence[str | Request], base_url: str) -> Iterator[Request]:
"""Convert a sequence of URLs or Request objects to an iterator of Request objects."""
for url in urls:
@@ -1650,6 +1726,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
use_state=self.use_state,
log=self._logger,
register_deferred_cleanup=deferred_cleanup.append,
+ extend_timeout=self._prepare_extend_timeout_function(request),
)
self._context_result_map[context] = result
@@ -1826,17 +1903,21 @@ async def _defer_throttled_request(self, request: Request, error: RequestThrottl
async def _run_request_handler(self, context: BasicCrawlingContext) -> None:
context.request.state = RequestState.BEFORE_NAV
- await self._context_pipeline(
- context,
- lambda final_context: wait_for(
- lambda: self.router(final_context),
- timeout=self._request_handler_timeout,
- timeout_message=f'{self._request_handler_timeout_text}'
- f' {self._request_handler_timeout.total_seconds()} seconds',
- logger=self._logger,
- ),
+ await self._context_pipeline(context, self._run_router)
+
+ async def _run_router(self, context: TCrawlingContext) -> None:
+ request_handler_timeout = self._resolve_request_handler_timeout(context.request.label)
+ await wait_for_extendable(
+ lambda: self.router(context),
+ timeout=request_handler_timeout,
+ timeout_message=lambda timeout: f'{self._request_handler_timeout_text} {timeout.total_seconds()} seconds',
)
+ def _resolve_request_handler_timeout(self, label: str | None) -> timedelta:
+ """Get the timeout of the router handler for the label, or the crawler's own if the handler doesn't set one."""
+ route_timeout = self.router.get_request_handler_timeout(label)
+ return self._request_handler_timeout if route_timeout is None else route_timeout
+
def _raise_for_error_status_code(self, status_code: int) -> None:
"""Raise an exception if the given status code is considered an error.
diff --git a/src/crawlee/crawlers/_basic/_logging_utils.py b/src/crawlee/crawlers/_basic/_logging_utils.py
index 528d213cd6..17a39ad28a 100644
--- a/src/crawlee/crawlers/_basic/_logging_utils.py
+++ b/src/crawlee/crawlers/_basic/_logging_utils.py
@@ -29,6 +29,7 @@ def _get_filtered_traceback_parts_for_asyncio_timeout_error(traceback_parts: lis
"""Extract only the most relevant traceback parts from stack trace."""
ignore_pattern = (
r'([\\/]{1}asyncio[\\/]{1})|' # internal asyncio parts
+ r'([\\/]{1}crawlee[\\/]{1}_utils[\\/]{1}wait\.py)|' # crawlee wrapper that applies the timeout
r'(Traceback \(most recent call last\))|' # common part of the stack trace formatting
r'(asyncio\.exceptions\.CancelledError)' # internal asyncio exception
)
diff --git a/src/crawlee/crawlers/_file_download/_file_download_crawler.py b/src/crawlee/crawlers/_file_download/_file_download_crawler.py
index 1b7e87f8c1..8627455b0e 100644
--- a/src/crawlee/crawlers/_file_download/_file_download_crawler.py
+++ b/src/crawlee/crawlers/_file_download/_file_download_crawler.py
@@ -29,8 +29,9 @@ class FileDownloadCrawler(AbstractHttpCrawler[FileDownloadCrawlingContext, bytes
`await context.http_response.read()`. For large files, construct the crawler with `stream=True`. The request
handler then receives a response whose body has not been read yet and can consume it in chunks via
`context.http_response.read_stream()`. The connection stays open while the handler runs and is closed
- afterwards. A handler that returns without consuming the body just skips the download. In both modes, reading
- the body counts towards `navigation_timeout`, so raise it for long downloads.
+ afterwards. A handler that returns without consuming the body just skips the download. Reading the body counts
+ towards `navigation_timeout` by default and towards `request_handler_timeout` in stream mode, so raise the one that
+ applies for long downloads.
### Usage
diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
index 2803ee3c18..e14e8f8844 100644
--- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py
+++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
@@ -9,7 +9,7 @@
import playwright.async_api
from more_itertools import partition
from pydantic import ValidationError
-from typing_extensions import TypeVar
+from typing_extensions import TypeVar, override
from crawlee._request import Request, RequestOptions, RequestState
from crawlee._types import BasicCrawlingContext, ConcurrencySettings
@@ -243,6 +243,11 @@ async def _manage_shared_navigation_timeout(
finally:
self._shared_navigation_timeouts.pop(request_id, None)
+ @override
+ def _get_max_processing_time(self) -> timedelta:
+ # The navigation phase runs before the request handler, under its own timeout.
+ return self._navigation_timeout + super()._get_max_processing_time()
+
def _build_pre_nav_context(
self,
context: BasicCrawlingContext,
@@ -262,6 +267,7 @@ def _build_pre_nav_context(
get_key_value_store=context.get_key_value_store,
log=context.log,
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
page=page,
block_requests=cast('BlockRequestsFunction', partial(block_requests, page=page)),
goto_options=GotoOptions(**self._goto_options),
@@ -287,6 +293,7 @@ def _build_post_nav_context(
get_key_value_store=context.get_key_value_store,
log=context.log,
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
page=context.page,
block_requests=context.block_requests,
goto_options=context.goto_options,
@@ -314,6 +321,7 @@ def _build_crawling_context(
get_key_value_store=context.get_key_value_store,
log=context.log,
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
page=context.page,
goto_options=context.goto_options,
response=context.response,
diff --git a/src/crawlee/router.py b/src/crawlee/router.py
index 8ae22e6ff9..6a734c7c8d 100644
--- a/src/crawlee/router.py
+++ b/src/crawlee/router.py
@@ -1,12 +1,16 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
-from typing import Generic, TypeVar
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Generic, TypeVar, overload
from crawlee._request import RequestState
from crawlee._types import BasicCrawlingContext
from crawlee._utils.docs import docs_group
+if TYPE_CHECKING:
+ from datetime import timedelta
+
__all__ = ['Router']
from crawlee.errors import UserHandlerTimeoutError
@@ -15,17 +19,26 @@
RequestHandler = Callable[[TCrawlingContext], Awaitable[None]]
+@dataclass(frozen=True)
+class _Route(Generic[TCrawlingContext]):
+ handler: RequestHandler[TCrawlingContext]
+ request_handler_timeout: timedelta | None
+
+
@docs_group('Other')
class Router(Generic[TCrawlingContext]):
"""A request dispatching system that routes requests to registered handlers based on their labels.
The `Router` allows you to define and register request handlers for specific labels. When a request is received,
the router invokes the corresponding `request_handler` based on the request's `label`. If no matching handler
- is found, the default handler is used.
+ is found, the default handler is used. A handler can get its own `request_handler_timeout`, overriding the
+ crawler's.
### Usage
```python
+ from datetime import timedelta
+
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
from crawlee.router import Router
@@ -42,8 +55,8 @@ async def default_handler(context: HttpCrawlingContext) -> None:
context.log.info('Request without label %s ...', context.request.url)
- # Handler for category requests
- @router.handler(label='category')
+ # Handler for category requests, which take longer than the rest
+ @router.handler(label='category', request_handler_timeout=timedelta(minutes=2))
async def category_handler(context: HttpCrawlingContext) -> None:
context.log.info('Category request %s ...', context.request.url)
@@ -60,41 +73,93 @@ async def main() -> None:
"""
def __init__(self) -> None:
- self._default_handler: RequestHandler[TCrawlingContext] | None = None
- self._handlers_by_label = dict[str, RequestHandler[TCrawlingContext]]()
+ self._default_route: _Route[TCrawlingContext] | None = None
+ self._routes_by_label = dict[str, _Route[TCrawlingContext]]()
self._middlewares = list[RequestHandler[TCrawlingContext]]()
- def default_handler(self: Router, handler: RequestHandler[TCrawlingContext]) -> RequestHandler[TCrawlingContext]:
+ @overload
+ def default_handler(
+ self, handler: RequestHandler[TCrawlingContext], *, request_handler_timeout: timedelta | None = None
+ ) -> RequestHandler[TCrawlingContext]: ...
+
+ @overload
+ def default_handler(
+ self, *, request_handler_timeout: timedelta | None = None
+ ) -> Callable[[RequestHandler[TCrawlingContext]], RequestHandler[TCrawlingContext]]: ...
+
+ def default_handler(
+ self,
+ handler: RequestHandler[TCrawlingContext] | None = None,
+ *,
+ request_handler_timeout: timedelta | None = None,
+ ) -> (
+ RequestHandler[TCrawlingContext]
+ | Callable[[RequestHandler[TCrawlingContext]], RequestHandler[TCrawlingContext]]
+ ):
"""Register a default request handler.
The default request handler is invoked for requests that have either no label or a label for which we have
- no matching handler.
+ no matching handler. Use it as a decorator, either directly or called with `request_handler_timeout`.
+
+ Args:
+ handler: The request handler to register.
+ request_handler_timeout: Overrides the `request_handler_timeout` of the crawler for this handler.
"""
- if self._default_handler is not None:
- raise RuntimeError('A default handler is already configured')
- self._default_handler = handler
+ def wrapper(handler: RequestHandler[TCrawlingContext]) -> RequestHandler[TCrawlingContext]:
+ if self._default_route is not None:
+ raise RuntimeError('A default handler is already configured')
+
+ self._default_route = _Route(handler, request_handler_timeout)
+ return handler
+
+ if handler is None:
+ return wrapper
- return handler
+ return wrapper(handler)
def handler(
self,
label: str,
+ *,
+ request_handler_timeout: timedelta | None = None,
) -> Callable[[RequestHandler[TCrawlingContext]], Callable[[TCrawlingContext], Awaitable]]:
"""Register a request handler based on a label.
This decorator registers a request handler for a specific label. The handler will be invoked only for requests
that have the exact same label.
+
+ Args:
+ label: The label of the requests to handle.
+ request_handler_timeout: Overrides the `request_handler_timeout` of the crawler for this handler.
"""
- if label in self._handlers_by_label:
- raise RuntimeError(f'A handler for label `{label}` is already registered')
def wrapper(handler: Callable[[TCrawlingContext], Awaitable]) -> Callable[[TCrawlingContext], Awaitable]:
- self._handlers_by_label[label] = handler
+ if label in self._routes_by_label:
+ raise RuntimeError(f'A handler for label `{label}` is already registered')
+
+ self._routes_by_label[label] = _Route(handler, request_handler_timeout)
return handler
return wrapper
+ def get_request_handler_timeout(self, label: str | None) -> timedelta | None:
+ """Get the `request_handler_timeout` of the handler for requests with the given label.
+
+ Returns `None` when the handler doesn't override the timeout of the crawler.
+ """
+ route = self._get_route(label)
+ return None if route is None else route.request_handler_timeout
+
+ def get_max_request_handler_timeout(self) -> timedelta | None:
+ """Get the longest `request_handler_timeout` of all handlers, or `None` when no handler overrides it."""
+ routes = list(self._routes_by_label.values())
+ if self._default_route is not None:
+ routes.append(self._default_route)
+
+ timeouts = [route.request_handler_timeout for route in routes if route.request_handler_timeout is not None]
+ return max(timeouts, default=None)
+
def use(self, middleware: RequestHandler[TCrawlingContext]) -> RequestHandler[TCrawlingContext]:
"""Register a middleware.
@@ -109,18 +174,21 @@ async def __call__(self, context: TCrawlingContext) -> None:
await middleware(context)
context.request.state = RequestState.REQUEST_HANDLER
- if context.request.label is None or context.request.label not in self._handlers_by_label:
- if self._default_handler is None:
- raise RuntimeError(
- f'No handler matches label `{context.request.label}` and no default handler is configured'
- )
-
- user_defined_handler = self._default_handler
- else:
- user_defined_handler = self._handlers_by_label[context.request.label]
+ route = self._get_route(context.request.label)
+ if route is None:
+ raise RuntimeError(
+ f'No handler matches label `{context.request.label}` and no default handler is configured'
+ )
try:
- return await user_defined_handler(context)
+ return await route.handler(context)
except TimeoutError as e:
# Timeout in handler, but not timeout of handler.
raise UserHandlerTimeoutError('Timeout raised by user defined handler') from e
+
+ def _get_route(self, label: str | None) -> _Route[TCrawlingContext] | None:
+ """Get the route for requests with the given label, falling back to the default one."""
+ if label is not None and label in self._routes_by_label:
+ return self._routes_by_label[label]
+
+ return self._default_route
diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
index 813fe6e50e..dfef93118a 100644
--- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
+++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -643,6 +643,58 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
mocked_browser_handler.assert_called_once_with()
+async def test_route_timeout_in_sub_crawler(test_urls: list[str]) -> None:
+ """Test that the timeout of a router handler replaces the one of the crawler in the sub crawler runs."""
+ static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0]))
+ crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
+ max_request_retries=0,
+ rendering_type_predictor=static_only_predictor_no_detection,
+ request_handler_timeout=timedelta(seconds=0.5),
+ )
+ mocked_static_handler = Mock(name='static_handler')
+
+ @crawler.router.handler('SLOW', request_handler_timeout=timedelta(seconds=10))
+ async def slow_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
+ await asyncio.sleep(1)
+ with suppress(AdaptiveContextError):
+ context.page # noqa:B018 Intentionally "useless expression". Can trigger exception.
+ return
+ mocked_static_handler()
+
+ stats = await crawler.run([Request.from_url(test_urls[0], label='SLOW')])
+
+ assert stats.requests_finished == 1
+ mocked_static_handler.assert_called_once_with()
+
+
+async def test_pre_nav_hook_extends_timeout(test_urls: list[str]) -> None:
+ """Test that `extend_timeout` from a pre-navigation hook lets the static sub crawler run past its timeout."""
+ static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0]))
+ crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
+ max_request_retries=0,
+ rendering_type_predictor=static_only_predictor_no_detection,
+ request_handler_timeout=timedelta(seconds=0.5),
+ )
+ mocked_static_handler = Mock(name='static_handler')
+
+ @crawler.pre_navigation_hook
+ async def slow_hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ await asyncio.sleep(1)
+
+ @crawler.router.default_handler
+ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
+ with suppress(AdaptiveContextError):
+ context.page # noqa:B018 Intentionally "useless expression". Can trigger exception.
+ return
+ mocked_static_handler()
+
+ stats = await crawler.run(test_urls[:1])
+
+ assert stats.requests_finished == 1
+ mocked_static_handler.assert_called_once_with()
+
+
async def test_adaptive_playwright_crawler_default_predictor(test_urls: list[str]) -> None:
"""Test default rendering type predictor integration into crawler."""
diff --git a/tests/integration/crawlers/_file_download/test_file_download.py b/tests/integration/crawlers/_file_download/test_file_download.py
index d505f7b433..ea0e2d47ed 100644
--- a/tests/integration/crawlers/_file_download/test_file_download.py
+++ b/tests/integration/crawlers/_file_download/test_file_download.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from datetime import timedelta
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, Mock, call
@@ -117,6 +118,53 @@ async def request_handler(context: FileDownloadCrawlingContext) -> None:
handler_call.assert_called_once()
+async def test_stream_extension_reaches_body(server_url: URL, http_client: HttpClient) -> None:
+ """A handler that extends its timeout can stream a body that arrives after both timeouts."""
+ crawler = FileDownloadCrawler(
+ stream=True,
+ http_client=http_client,
+ navigation_timeout=timedelta(seconds=1),
+ request_handler_timeout=timedelta(seconds=0.5),
+ max_request_retries=0,
+ )
+ handler_call = Mock()
+
+ @crawler.router.default_handler
+ async def request_handler(context: FileDownloadCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ handler_call(body=b''.join([chunk async for chunk in context.http_response.read_stream()]))
+
+ # `curl_cffi` returns the response only with the first chunk of the body, so it arrives within the timeout.
+ stats = await crawler.run([str((server_url / 'slow_body').with_query(chunks=3, delay=0.5))])
+
+ assert stats.requests_finished == 1
+ handler_call.assert_called_once_with(body=b'xxx')
+
+
+async def test_stream_body_bounded_by_handler(server_url: URL, http_client: HttpClient) -> None:
+ """A streamed body counts towards the request handler timeout, not the navigation timeout."""
+ crawler = FileDownloadCrawler(
+ stream=True,
+ http_client=http_client,
+ navigation_timeout=timedelta(seconds=1),
+ request_handler_timeout=timedelta(seconds=2),
+ max_request_retries=0,
+ )
+ handler_call = Mock()
+ failed_request_handler = AsyncMock()
+ crawler.failed_request_handler(failed_request_handler)
+
+ @crawler.router.default_handler
+ async def request_handler(context: FileDownloadCrawlingContext) -> None:
+ handler_call(body=b''.join([chunk async for chunk in context.http_response.read_stream()]))
+
+ await crawler.run([str((server_url / 'slow_body').with_query(chunks=8, delay=0.5))])
+
+ handler_call.assert_not_called()
+ failed_request_handler.assert_called_once()
+ assert 'Request handler timed out' in str(failed_request_handler.call_args.args[1])
+
+
async def test_retry_failed_download(server_url: URL, http_client: HttpClient) -> None:
"""A mid-stream failure retries and gets a fresh, complete response."""
crawler = FileDownloadCrawler(stream=True, http_client=http_client, max_request_retries=1)
diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py
index 79b310b6f4..7713bdc6fa 100644
--- a/tests/integration/crawlers/_http/test_http_crawler.py
+++ b/tests/integration/crawlers/_http/test_http_crawler.py
@@ -1,10 +1,11 @@
from __future__ import annotations
+import asyncio
import json
import logging
import time
from datetime import UTC, datetime, timedelta
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
from unittest.mock import AsyncMock, Mock
from urllib.parse import parse_qs, urlencode
@@ -379,6 +380,69 @@ async def post_nav_hook(context: HttpCrawlingContext) -> None:
call_mock.assert_called_once_with(str(server_url))
+@pytest.mark.parametrize(
+ 'hook_type',
+ [
+ pytest.param('pre', id='pre-navigation-hook'),
+ pytest.param('post', id='post-navigation-hook'),
+ ],
+)
+async def test_extend_timeout_from_nav_hook(server_url: URL, hook_type: Literal['pre', 'post']) -> None:
+ """Test that `extend_timeout` from a navigation hook lets the navigation phase run past its timeout."""
+ crawler = HttpCrawler(request_handler=AsyncMock(), navigation_timeout=timedelta(seconds=0.5), max_request_retries=0)
+
+ async def slow_hook(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ await asyncio.sleep(1)
+
+ if hook_type == 'pre':
+ crawler.pre_navigation_hook(slow_hook)
+ else:
+ crawler.post_navigation_hook(slow_hook)
+
+ stats = await crawler.run([str(server_url)])
+
+ assert stats.requests_finished == 1
+
+
+async def test_extension_reaches_slow_body(http_client: HttpClient, server_url: URL) -> None:
+ """Test that `extend_timeout` from a post-navigation hook gives a body that arrives late more time."""
+ crawler = HttpCrawler(
+ http_client=http_client,
+ request_handler=AsyncMock(),
+ navigation_timeout=timedelta(seconds=1),
+ max_request_retries=0,
+ )
+
+ @crawler.post_navigation_hook
+ async def extend(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+
+ # `curl_cffi` returns the response only with the first chunk of the body, so it arrives within the timeout.
+ stats = await crawler.run([str((server_url / 'slow_body').with_query(chunks=3, delay=0.5))])
+
+ assert stats.requests_finished == 1
+
+
+async def test_nav_timeout_bounds_slow_body(http_client: HttpClient, server_url: URL) -> None:
+ """Test that the navigation timeout bounds reading a body that arrives late."""
+ request_handler = AsyncMock()
+ failed_request_handler = AsyncMock()
+ crawler = HttpCrawler(
+ http_client=http_client,
+ request_handler=request_handler,
+ navigation_timeout=timedelta(seconds=1),
+ max_request_retries=0,
+ )
+ crawler.failed_request_handler(failed_request_handler)
+
+ await crawler.run([str((server_url / 'slow_body').with_query(chunks=3, delay=0.5))])
+
+ request_handler.assert_not_called()
+ failed_request_handler.assert_called_once()
+ assert isinstance(failed_request_handler.call_args.args[1], TimeoutError)
+
+
async def test_http_crawler_navigation_hooks_order(server_url: URL) -> None:
"""Test that post-navigation hooks are executed in correct order."""
execution_order = []
diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
index 25750d60f8..58ff078752 100644
--- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
@@ -1261,6 +1261,35 @@ async def test_navigation_timeout_applies_to_hooks(server_url: URL) -> None:
assert request_handler.call_count == 0
+@pytest.mark.parametrize(
+ 'hook_type',
+ [
+ pytest.param('pre', id='pre-navigation-hook'),
+ pytest.param('post', id='post-navigation-hook'),
+ ],
+)
+async def test_extend_timeout_from_nav_hook(server_url: URL, hook_type: Literal['pre', 'post']) -> None:
+ """Test that `extend_timeout` from a navigation hook lets the navigation phase run past its timeout."""
+ crawler = PlaywrightCrawler(navigation_timeout=timedelta(seconds=1), max_request_retries=0)
+
+ request_handler = AsyncMock()
+ crawler.router.default_handler(request_handler)
+
+ async def slow_hook(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ await asyncio.sleep(1.5)
+
+ if hook_type == 'pre':
+ crawler.pre_navigation_hook(slow_hook)
+ else:
+ crawler.post_navigation_hook(slow_hook)
+
+ result = await crawler.run([str(server_url)])
+
+ assert result.requests_finished == 1
+ assert request_handler.call_count == 1
+
+
async def test_slow_navigation_does_not_count_toward_handler_timeout(server_url: URL) -> None:
crawler = PlaywrightCrawler(request_handler_timeout=timedelta(seconds=0.5))
diff --git a/tests/unit/_utils/test_shared_timeout.py b/tests/unit/_utils/test_shared_timeout.py
index 69c893837a..839c90ca0c 100644
--- a/tests/unit/_utils/test_shared_timeout.py
+++ b/tests/unit/_utils/test_shared_timeout.py
@@ -1,9 +1,10 @@
import asyncio
+from contextvars import copy_context
from datetime import timedelta
import pytest
-from crawlee._utils.time import SharedTimeout
+from crawlee._utils.time import SharedTimeout, _active_timeout_extenders, extend_active_timeouts
async def test_shared_timeout_tracks_elapsed_time() -> None:
@@ -54,3 +55,94 @@ async def test_shared_timeout_multiple_sequential_uses() -> None:
async with shared_timeout as remaining:
assert remaining < timedelta(seconds=0.8)
assert remaining > timedelta(seconds=0)
+
+
+async def test_extend_adds_to_budget() -> None:
+ """Test that the extension also counts towards the later uses of the timeout."""
+ shared_timeout = SharedTimeout(timedelta(seconds=1))
+
+ async with shared_timeout:
+ extend_active_timeouts(timedelta(seconds=5))
+
+ async with shared_timeout as remaining:
+ assert remaining > timedelta(seconds=5)
+
+
+async def test_extend_saturates_on_overflow() -> None:
+ """Test that extending a timeout of `timedelta.max` keeps it at `timedelta.max` instead of raising."""
+ shared_timeout = SharedTimeout(timedelta.max)
+
+ async with shared_timeout:
+ extend_active_timeouts(timedelta(seconds=5))
+
+ assert shared_timeout.total_timeout == timedelta.max
+
+
+async def test_extend_outside_is_noop() -> None:
+ """Test that extending outside an entered timeout leaves the timeout as it is."""
+ shared_timeout = SharedTimeout(timedelta(seconds=1))
+
+ extend_active_timeouts(timedelta(seconds=5))
+
+ async with shared_timeout as remaining:
+ assert remaining == timedelta(seconds=1)
+
+
+async def test_extend_reaches_enclosing() -> None:
+ """Test that the timeouts enclosing the innermost one are extended as well."""
+ async with SharedTimeout(timedelta(seconds=0.2)), SharedTimeout(timedelta(seconds=0.2)):
+ extend_active_timeouts(timedelta(seconds=5))
+ await asyncio.sleep(0.4)
+
+
+async def test_extend_stops_at_expired() -> None:
+ """Test that extending an expired timeout does nothing and leaves the enclosing timeouts as they are."""
+
+ async def extend_after_expiry() -> None:
+ async with SharedTimeout(timedelta(seconds=0.1)):
+ try:
+ await asyncio.sleep(1)
+ except asyncio.CancelledError:
+ extend_active_timeouts(timedelta(seconds=5))
+ raise
+
+ async def outlive_enclosing_timeout() -> None:
+ async with SharedTimeout(timedelta(seconds=0.3)):
+ with pytest.raises(TimeoutError):
+ await extend_after_expiry()
+
+ await asyncio.sleep(1)
+
+ with pytest.raises(TimeoutError):
+ await outlive_enclosing_timeout()
+
+
+async def test_extend_after_exit_budget_only() -> None:
+ """Test that extending through a copied context of an exited timeout only adds to its budget."""
+ outer_timeout = SharedTimeout(timedelta(seconds=1))
+ inner_timeout = SharedTimeout(timedelta(seconds=1))
+
+ async with outer_timeout:
+ async with inner_timeout:
+ context = copy_context()
+
+ context.run(extend_active_timeouts, timedelta(seconds=5))
+
+ async with inner_timeout as inner_remaining:
+ assert inner_remaining > timedelta(seconds=5)
+
+ async with outer_timeout as outer_remaining:
+ assert outer_remaining <= timedelta(seconds=1)
+
+
+async def test_expired_timeout_is_removed() -> None:
+ """Test that a timeout that expired is removed from the active timeouts of the task."""
+
+ async def expire() -> None:
+ async with SharedTimeout(timedelta(seconds=0.1)):
+ await asyncio.sleep(1)
+
+ with pytest.raises(TimeoutError):
+ await expire()
+
+ assert _active_timeout_extenders.get() == ()
diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
index 5295f15712..61cb89bedb 100644
--- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
+++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -2,6 +2,8 @@
import logging
import sys
+from datetime import timedelta
+from typing import TYPE_CHECKING
from unittest.mock import patch
import pytest
@@ -9,6 +11,9 @@
from crawlee.autoscaling import ConcurrencySystem
from crawlee.crawlers import AdaptivePlaywrightCrawler
+if TYPE_CHECKING:
+ from crawlee.crawlers import AdaptivePlaywrightCrawlingContext
+
async def test_accepts_concurrency_system() -> None:
"""A supplied concurrency system is used as is, since the lower default concurrency applies only without one."""
@@ -66,3 +71,14 @@ def test_default_statistics_log_to_default_crawler_logger() -> None:
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
assert crawler.statistics._periodic_message_logger is crawler.log
+
+
+def test_lock_time_counts_both_runs() -> None:
+ """Test that the expected processing time counts the request handler timeout once for each sub crawler."""
+ crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
+
+ @crawler.router.handler('SLOW', request_handler_timeout=timedelta(minutes=3))
+ async def slow_handler(_context: AdaptivePlaywrightCrawlingContext) -> None:
+ pass
+
+ assert crawler._get_expected_processing_time() == timedelta(minutes=7)
diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py
index b788bc1ddf..4bd8a4341d 100644
--- a/tests/unit/crawlers/_basic/test_basic_crawler.py
+++ b/tests/unit/crawlers/_basic/test_basic_crawler.py
@@ -26,6 +26,7 @@
from crawlee._request import Request, RequestState
from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs
from crawlee._utils.robots import RobotsTxtFile
+from crawlee._utils.time import SharedTimeout
from crawlee.autoscaling import AutoscaledPool, ConcurrencyConsumer, ConcurrencySystem
from crawlee.configuration import Configuration
from crawlee.crawlers import BasicCrawler
@@ -57,6 +58,7 @@
ThrottlingRequestManager,
WaitingStatus,
)
+from crawlee.router import Router
from crawlee.sessions import Session, SessionCookies, SessionPool
from crawlee.statistics import FinalStatistics, Statistics, StatisticsState
from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend
@@ -2559,6 +2561,332 @@ async def handler(context: BasicCrawlingContext) -> None:
assert mocked_handler_after_sleep.call_count == 1
+async def test_route_overrides_timeout() -> None:
+ """Test that the timeout of a router handler replaces the one of the crawler for the requests it handles."""
+ crawler = BasicCrawler(request_handler_timeout=timedelta(seconds=1), max_request_retries=0)
+ failed_request_handler = Mock()
+
+ @crawler.router.handler('LONG', request_handler_timeout=timedelta(seconds=10))
+ async def long_handler(_context: BasicCrawlingContext) -> None:
+ await asyncio.sleep(1.5)
+
+ @crawler.router.handler('SHORT', request_handler_timeout=timedelta(seconds=0.2))
+ async def short_handler(_context: BasicCrawlingContext) -> None:
+ await asyncio.sleep(0.5)
+
+ @crawler.failed_request_handler
+ async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
+ failed_request_handler(context.request.label, str(error))
+
+ stats = await crawler.run(
+ [
+ Request.from_url('https://a.placeholder.com', label='LONG'),
+ Request.from_url('https://b.placeholder.com', label='SHORT'),
+ ]
+ )
+
+ assert stats.requests_finished == 1
+ failed_request_handler.assert_called_once_with('SHORT', 'Request handler timed out after 0.2 seconds')
+
+
+async def test_extend_timeout_from_handler() -> None:
+ """Test that `extend_timeout` lets the request handler run past its timeout."""
+ crawler = BasicCrawler(request_handler_timeout=timedelta(seconds=0.5), max_request_retries=0)
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ await asyncio.sleep(1)
+
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+async def test_message_counts_extension() -> None:
+ """Test that the timeout message of an extended request handler states the extended timeout."""
+ crawler = BasicCrawler(request_handler_timeout=timedelta(seconds=0.2), max_request_retries=0)
+ failed_request_handler = Mock()
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=0.2))
+ await asyncio.sleep(1)
+
+ @crawler.failed_request_handler
+ async def failed_handler(_context: BasicCrawlingContext, error: Exception) -> None:
+ failed_request_handler(str(error))
+
+ await crawler.run(['https://a.placeholder.com'])
+
+ failed_request_handler.assert_called_once_with('Request handler timed out after 0.4 seconds')
+
+
+async def test_extend_timeout_from_middleware() -> None:
+ """Test that `extend_timeout` from a router middleware extends the timeout of the request handler."""
+ crawler = BasicCrawler(request_handler_timeout=timedelta(seconds=0.5), max_request_retries=0)
+
+ @crawler.router.use
+ async def middleware(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+
+ @crawler.router.default_handler
+ async def handler(_context: BasicCrawlingContext) -> None:
+ await asyncio.sleep(1)
+
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+async def test_extend_timeout_from_child_task() -> None:
+ """Test that a task spawned by the request handler can extend the timeout of the handler."""
+ crawler = BasicCrawler(request_handler_timeout=timedelta(seconds=0.5), max_request_retries=0)
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ async def extend() -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+
+ await asyncio.create_task(extend())
+ await asyncio.sleep(1)
+
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+async def test_error_handler_extends_timeout() -> None:
+ """Test that `extend_timeout` lets the error handler run past the timeout for handling a failure."""
+ crawler = BasicCrawler(
+ configuration=Configuration(internal_timeout=timedelta(seconds=1)),
+ max_request_retries=1,
+ )
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ if context.request.retry_count == 0:
+ raise RuntimeError('First attempt fails')
+
+ @crawler.error_handler
+ async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+ await asyncio.sleep(1.5)
+
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+async def test_extend_timeout_extends_lock() -> None:
+ """Test that `extend_timeout` also extends the lock of the request in the request manager."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue)
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+
+ with patch.object(queue, 'extend_request_processing_time', AsyncMock(return_value=True)) as extend_mock:
+ await crawler.run(['https://a.placeholder.com'])
+
+ extend_mock.assert_awaited_once_with(ANY, timedelta(seconds=10))
+ [extend_call] = extend_mock.await_args_list
+ assert extend_call.args[0].url == 'https://a.placeholder.com'
+
+
+@pytest.mark.parametrize(
+ 'copy_context',
+ [
+ pytest.param(True, id='thread-with-context'),
+ pytest.param(False, id='thread-without-context'),
+ ],
+)
+async def test_extend_timeout_off_loop_raises(*, copy_context: bool) -> None:
+ """Test that awaiting `extend_timeout` on another event loop raises, with or without the context of the request."""
+ crawler = BasicCrawler(max_request_retries=0)
+ extend_error = Mock()
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ def extend_in_thread() -> None:
+ try:
+ asyncio.run(context.extend_timeout(timedelta(seconds=10)))
+ except RuntimeError as error:
+ extend_error(error)
+
+ if copy_context:
+ await asyncio.to_thread(extend_in_thread)
+ else:
+ # Unlike `asyncio.to_thread`, `run_in_executor` doesn't copy the context to the thread.
+ await asyncio.get_running_loop().run_in_executor(None, extend_in_thread)
+
+ await crawler.run(['https://a.placeholder.com'])
+
+ extend_error.assert_called_once()
+
+
+@pytest.mark.parametrize(
+ 'delta',
+ [
+ pytest.param(timedelta(0), id='zero'),
+ pytest.param(timedelta(seconds=-1), id='negative'),
+ ],
+)
+async def test_extend_rejects_non_positive(delta: timedelta) -> None:
+ """Test that `extend_timeout` raises for a delta that isn't positive and leaves the request lock as it is."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue, max_request_retries=0)
+ extend_error = Mock()
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ try:
+ await context.extend_timeout(delta)
+ except ValueError as error:
+ extend_error(error)
+
+ with patch.object(queue, 'extend_request_processing_time', AsyncMock()) as extend_lock:
+ await crawler.run(['https://a.placeholder.com'])
+
+ extend_error.assert_called_once()
+ extend_lock.assert_not_awaited()
+
+
+async def test_lock_extension_failure_ignored() -> None:
+ """Test that a failure to extend the lock of the request doesn't fail the request."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue, max_request_retries=0)
+
+ @crawler.router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=10))
+
+ with patch.object(queue, 'extend_request_processing_time', AsyncMock(side_effect=RuntimeError('Lock lost'))):
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+@pytest.mark.parametrize(
+ ('crawler_timeout', 'route_timeout', 'expected_processing_time'),
+ [
+ pytest.param(timedelta(seconds=10), None, timedelta(minutes=1, seconds=10), id='crawler-timeout'),
+ pytest.param(timedelta(minutes=2), timedelta(minutes=1), timedelta(minutes=3), id='shorter-route'),
+ pytest.param(timedelta(seconds=10), timedelta(minutes=3), timedelta(minutes=4), id='longer-route'),
+ ],
+)
+async def test_expected_processing_time(
+ crawler_timeout: timedelta, route_timeout: timedelta | None, expected_processing_time: timedelta
+) -> None:
+ """Test that the lock time is the longest request handler timeout plus a minute for the error handling."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue, request_handler_timeout=crawler_timeout)
+
+ @crawler.router.handler('SLOW', request_handler_timeout=route_timeout)
+ async def slow_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ @crawler.router.default_handler
+ async def default_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ with patch.object(queue, 'set_expected_request_processing_time', AsyncMock()) as set_mock:
+ await crawler.run(['https://a.placeholder.com'])
+
+ set_mock.assert_awaited_once_with(expected_processing_time)
+
+
+@pytest.mark.parametrize(
+ 'route_timeout',
+ [
+ pytest.param(timedelta(days=2), id='over-a-day'),
+ pytest.param(timedelta.max, id='timedelta-max'),
+ ],
+)
+async def test_lock_time_capped_at_a_day(route_timeout: timedelta, caplog: pytest.LogCaptureFixture) -> None:
+ """Test that the expected processing time is capped at a day, with one warning per crawler, for a longer route."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue)
+
+ @crawler.router.handler('SLOW', request_handler_timeout=route_timeout)
+ async def slow_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ @crawler.router.default_handler
+ async def default_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ with (
+ patch.object(queue, 'set_expected_request_processing_time', AsyncMock()) as set_mock,
+ caplog.at_level(logging.WARNING),
+ ):
+ await crawler.run(['https://a.placeholder.com'])
+ await crawler.run(['https://b.placeholder.com'])
+
+ assert set_mock.await_args_list == [call(timedelta(days=1)), call(timedelta(days=1))]
+ cap_warnings = [record for record in caplog.records if 'the longest time the crawler asks' in record.message]
+ assert len(cap_warnings) == 1
+
+
+async def test_router_set_after_empty_run() -> None:
+ """Test that a router can still be set after a run without requests, which reads the router timeouts."""
+ crawler = BasicCrawler()
+ await crawler.run()
+
+ router = Router[BasicCrawlingContext]()
+ handler_call = Mock()
+
+ @router.default_handler
+ async def handler(context: BasicCrawlingContext) -> None:
+ handler_call(context.request.url)
+
+ crawler.router = router
+ await crawler.run(['https://a.placeholder.com'])
+
+ handler_call.assert_called_once_with('https://a.placeholder.com')
+
+
+async def test_late_route_extends_lock_time() -> None:
+ """Test that a route registered after the request manager was opened counts towards the expected processing time."""
+ queue = await RequestQueue.open()
+ crawler = BasicCrawler(request_manager=queue)
+ await crawler.add_requests(['https://a.placeholder.com'])
+
+ @crawler.router.handler('SLOW', request_handler_timeout=timedelta(minutes=10))
+ async def slow_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ @crawler.router.default_handler
+ async def default_handler(_context: BasicCrawlingContext) -> None:
+ pass
+
+ with patch.object(queue, 'set_expected_request_processing_time', AsyncMock()) as set_mock:
+ await crawler.run()
+
+ set_mock.assert_awaited_once_with(timedelta(minutes=11))
+
+
+async def test_nested_run_cannot_extend_outer() -> None:
+ """Test that a crawler run from a request handler doesn't extend the timeout of that handler."""
+ outer_crawler = BasicCrawler()
+ inner_crawler = BasicCrawler()
+
+ @inner_crawler.router.default_handler
+ async def inner_handler(context: BasicCrawlingContext) -> None:
+ await context.extend_timeout(timedelta(seconds=60))
+
+ @outer_crawler.router.default_handler
+ async def outer_handler(_context: BasicCrawlingContext) -> None:
+ await inner_crawler.run(['https://b.placeholder.com'])
+
+ with patch.object(SharedTimeout, '_extend', autospec=True, side_effect=SharedTimeout._extend) as extend:
+ await outer_crawler.run(['https://a.placeholder.com'])
+
+ # Only the timeout of the inner handler is extended, the outer one would be the second call.
+ extend.assert_called_once()
+
+
@pytest.mark.parametrize(
('keep_alive', 'max_requests_per_crawl', 'expected_handled_requests_count'),
[
@@ -2788,6 +3116,28 @@ async def handler(context: BasicCrawlingContext) -> None:
assert found_timeout_message, 'Expected log message about request handler error was not found.'
+async def test_timeout_log_hides_wrapper(caplog: pytest.LogCaptureFixture) -> None:
+ """Test that the error log of a timed out request handler leaves out the frame of the timeout wrapper."""
+ crawler = BasicCrawler(
+ configure_logging=False,
+ max_request_retries=0,
+ request_handler_timeout=timedelta(seconds=0.5),
+ )
+
+ @crawler.router.default_handler
+ async def handler(_context: BasicCrawlingContext) -> None:
+ await asyncio.sleep(10)
+
+ with caplog.at_level(logging.ERROR, logger='crawlee'):
+ await crawler.run(['https://a.placeholder.com'])
+
+ failed_messages = [record.message for record in caplog.records if 'reached maximum retries' in record.message]
+
+ assert len(failed_messages) == 1
+ assert 'Request handler timed out after 0.5 seconds' in failed_messages[0]
+ assert 'wait.py' not in failed_messages[0]
+
+
async def test_reduced_logs_from_time_out_in_request_handler(caplog: pytest.LogCaptureFixture) -> None:
crawler = BasicCrawler(configure_logging=False, max_request_retries=1)
diff --git a/tests/unit/crawlers/_basic/test_context_pipeline.py b/tests/unit/crawlers/_basic/test_context_pipeline.py
index 23f3686d16..ba13d21762 100644
--- a/tests/unit/crawlers/_basic/test_context_pipeline.py
+++ b/tests/unit/crawlers/_basic/test_context_pipeline.py
@@ -42,6 +42,7 @@ async def test_calls_consumer_without_middleware() -> None:
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
await pipeline(context, consumer)
@@ -70,6 +71,7 @@ async def middleware_a(context: BasicCrawlingContext) -> AsyncGenerator[Enhanced
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
)
events.append('middleware_a_out')
@@ -88,6 +90,7 @@ async def middleware_b(context: EnhancedCrawlingContext) -> AsyncGenerator[MoreE
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=context.register_deferred_cleanup,
+ extend_timeout=context.extend_timeout,
)
events.append('middleware_b_out')
@@ -104,6 +107,7 @@ async def middleware_b(context: EnhancedCrawlingContext) -> AsyncGenerator[MoreE
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
await pipeline(context, consumer)
@@ -131,6 +135,7 @@ async def test_wraps_consumer_errors() -> None:
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
with pytest.raises(RequestHandlerError):
@@ -161,6 +166,7 @@ async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingC
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
with pytest.raises(ContextPipelineInitializationError):
@@ -194,6 +200,7 @@ async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingC
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
with pytest.raises(ContextPipelineFinalizationError):
diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py
index b9076a0b88..f6e310daae 100644
--- a/tests/unit/crawlers/_http/test_http_crawler.py
+++ b/tests/unit/crawlers/_http/test_http_crawler.py
@@ -173,3 +173,14 @@ async def test_429_reported_once(monkeypatch: pytest.MonkeyPatch) -> None:
await crawler.run([url])
record_pacing_signal.assert_called_once_with(RateLimitedSignal(url))
+
+
+def test_lock_time_counts_navigation() -> None:
+ """Test that the expected processing time adds the navigation timeout to the longest request handler timeout."""
+ crawler = HttpCrawler(navigation_timeout=timedelta(minutes=1))
+
+ @crawler.router.handler('SLOW', request_handler_timeout=timedelta(minutes=3))
+ async def slow_handler(_context: HttpCrawlingContext) -> None:
+ pass
+
+ assert crawler._get_expected_processing_time() == timedelta(minutes=5)
diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
index 8a4b500788..253e3adcb7 100644
--- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import sys
+from datetime import timedelta
+from typing import TYPE_CHECKING
from unittest import mock
import pytest
@@ -14,6 +16,9 @@
PlaywrightCrawler,
)
+if TYPE_CHECKING:
+ from crawlee.crawlers import PlaywrightCrawlingContext
+
async def test_passing_configuration() -> None:
"""Check that the configuration is allowed to be passed to the Playwrightcrawler."""
@@ -34,6 +39,17 @@ async def test_accepts_concurrency_system() -> None:
assert crawler.concurrency_system is concurrency_system
+def test_lock_time_counts_navigation() -> None:
+ """Test that the expected processing time adds the navigation timeout to the longest request handler timeout."""
+ crawler = PlaywrightCrawler(navigation_timeout=timedelta(minutes=1))
+
+ @crawler.router.handler('SLOW', request_handler_timeout=timedelta(minutes=3))
+ async def slow_handler(_context: PlaywrightCrawlingContext) -> None:
+ pass
+
+ assert crawler._get_expected_processing_time() == timedelta(minutes=5)
+
+
def test_import_error_handled() -> None:
blocked = {
mod_name: None for mod_name in sys.modules if mod_name == 'playwright' or mod_name.startswith('playwright.')
diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py
index 1b36f9a013..b47b18e39b 100644
--- a/tests/unit/test_router.py
+++ b/tests/unit/test_router.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
+from datetime import timedelta
from unittest.mock import AsyncMock, Mock
import pytest
@@ -24,6 +25,7 @@ def __init__(self, *, label: str | None) -> None:
get_key_value_store=AsyncMock(),
log=logging.getLogger(),
register_deferred_cleanup=lambda _: None,
+ extend_timeout=AsyncMock(),
)
@@ -227,3 +229,100 @@ async def default_handler(_context: MockContext) -> None:
assert mock_execution_order.call_count == 1
assert mock_execution_order.call_args_list[0][1] == {'call': 'middleware_1'}
+
+
+async def test_default_handler_with_timeout() -> None:
+ """Test that the default handler registered with a timeout is returned and invoked."""
+ router = Router[MockContext]()
+ mock_default_handler = Mock()
+
+ async def default_handler(_context: MockContext) -> None:
+ mock_default_handler()
+
+ registered_handler = router.default_handler(request_handler_timeout=timedelta(seconds=30))(default_handler)
+
+ assert registered_handler is default_handler
+
+ await router(MockContext(label=None))
+
+ mock_default_handler.assert_called_once()
+
+
+@pytest.mark.parametrize(
+ 'label',
+ [
+ pytest.param(None, id='default-handler'),
+ pytest.param('LABEL', id='label-handler'),
+ ],
+)
+async def test_reused_decorator_raises(label: str | None) -> None:
+ """Test that applying a registering decorator a second time raises and keeps the first handler."""
+ router = Router[MockContext]()
+ if label is None:
+ decorator = router.default_handler(request_handler_timeout=timedelta(seconds=30))
+ else:
+ decorator = router.handler(label, request_handler_timeout=timedelta(seconds=30))
+
+ first_handler = AsyncMock()
+ decorator(first_handler)
+
+ with pytest.raises(RuntimeError, match='already'):
+ decorator(AsyncMock())
+
+ await router(MockContext(label=label))
+
+ first_handler.assert_awaited_once()
+
+
+@pytest.mark.parametrize(
+ ('label', 'expected_timeout'),
+ [
+ pytest.param('SLOW', timedelta(seconds=10), id='handler-with-timeout'),
+ pytest.param('PLAIN', None, id='handler-without-timeout'),
+ pytest.param('UNKNOWN', timedelta(seconds=30), id='unknown-label-falls-back-to-default-handler'),
+ pytest.param(None, timedelta(seconds=30), id='no-label-falls-back-to-default-handler'),
+ ],
+)
+def test_router_timeout_by_label(label: str | None, expected_timeout: timedelta | None) -> None:
+ """Test that the timeout is taken from the handler that the router would invoke for the label."""
+ router = Router[MockContext]()
+
+ @router.handler('SLOW', request_handler_timeout=timedelta(seconds=10))
+ async def slow_handler(_context: MockContext) -> None:
+ pass
+
+ @router.handler('PLAIN')
+ async def plain_handler(_context: MockContext) -> None:
+ pass
+
+ @router.default_handler(request_handler_timeout=timedelta(seconds=30))
+ async def default_handler(_context: MockContext) -> None:
+ pass
+
+ assert router.get_request_handler_timeout(label) == expected_timeout
+
+
+@pytest.mark.parametrize(
+ ('handler_timeout', 'default_handler_timeout', 'expected_timeout'),
+ [
+ pytest.param(None, None, None, id='no-overrides'),
+ pytest.param(timedelta(seconds=10), None, timedelta(seconds=10), id='handler-only'),
+ pytest.param(timedelta(seconds=10), timedelta(seconds=30), timedelta(seconds=30), id='default-is-longest'),
+ pytest.param(timedelta(seconds=60), timedelta(seconds=30), timedelta(seconds=60), id='handler-is-longest'),
+ ],
+)
+def test_router_max_handler_timeout(
+ handler_timeout: timedelta | None, default_handler_timeout: timedelta | None, expected_timeout: timedelta | None
+) -> None:
+ """Test that the longest timeout of all handlers is returned, including the default handler."""
+ router = Router[MockContext]()
+
+ @router.handler('LABEL', request_handler_timeout=handler_timeout)
+ async def handler(_context: MockContext) -> None:
+ pass
+
+ @router.default_handler(request_handler_timeout=default_handler_timeout)
+ async def default_handler(_context: MockContext) -> None:
+ pass
+
+ assert router.get_max_request_handler_timeout() == expected_timeout