From 0e691975981dd30cb2c096be781f46e23a7ece4d Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 17 Nov 2026 22:41:00 +0000 Subject: [PATCH 1/2] cancel right away on a zero timeout for event listeners and added requests --- docs/06_upgrading/upgrading_to_v2.md | 4 ++++ src/crawlee/_utils/wait.py | 2 +- tests/unit/_utils/test_wait.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/unit/_utils/test_wait.py diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 1174379fe4..763d139b7f 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -54,6 +54,10 @@ A custom storage backend has more names to update. The `StorageBackend` methods `RequestQueueBackend.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue backend, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in backends already implement it. +## A zero timeout cancels right away + +A zero `close_timeout` of an event manager, or a zero `timeout` passed to `EventManager.wait_for_all_listeners_to_complete()`, cancels the running listeners right away. A zero `wait_for_all_requests_to_be_added_timeout` in `add_requests()` does the same for the requests still being added. In v1, a zero timeout meant no limit. To wait without a limit, pass `None`. + ## Keyword-only arguments Secondary arguments of the public API are now keyword-only. The main argument of each call, such as a key, a value, a selector or a path, stays positional. A call that passes any of the following arguments positionally raises `TypeError`, so pass them by name: diff --git a/src/crawlee/_utils/wait.py b/src/crawlee/_utils/wait.py index 9c1e9aa0d9..3a39d4853d 100644 --- a/src/crawlee/_utils/wait.py +++ b/src/crawlee/_utils/wait.py @@ -82,7 +82,7 @@ async def wait_for_all_tasks_to_finish( if not tasks: return - timeout_secs = timeout.total_seconds() if timeout else None + timeout_secs = timeout.total_seconds() if timeout is not None else None try: _, pending = await asyncio.wait(tasks, timeout=timeout_secs) if pending: diff --git a/tests/unit/_utils/test_wait.py b/tests/unit/_utils/test_wait.py new file mode 100644 index 0000000000..65c6af6ca4 --- /dev/null +++ b/tests/unit/_utils/test_wait.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import asyncio +import logging +from datetime import timedelta + +from crawlee._utils.wait import wait_for_all_tasks_to_finish + + +async def test_zero_timeout_cancels_right_away() -> None: + """A zero timeout cancels the unfinished tasks without waiting for them.""" + task = asyncio.create_task(asyncio.Event().wait()) + + # Capped, as a zero timeout taken for no limit would wait for the task forever. + await asyncio.wait_for( + wait_for_all_tasks_to_finish([task], logger=logging.getLogger(__name__), timeout=timedelta(0)), + timeout=5, + ) + + assert task.cancelled() From 3aefe4017411dd643f57c77cbd3e33b96d17de05 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 18 Nov 2026 13:23:46 +0000 Subject: [PATCH 2/2] address review --- src/crawlee/crawlers/_basic/_basic_crawler.py | 3 ++- src/crawlee/events/_event_manager.py | 8 ++++++-- src/crawlee/request_loaders/_request_manager.py | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 11a6a7581c..adc4ef7c56 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1038,7 +1038,8 @@ async def add_requests( batch_size: The number of requests to add in one batch. wait_time_between_batches: Time to wait between adding batches. wait_for_all_requests_to_be_added: If True, wait for all requests to be added before returning. - wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. + wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. `None` waits + without a limit, zero cancels the requests still being added right away. """ allowed_requests = [] skipped = list[Request]() diff --git a/src/crawlee/events/_event_manager.py b/src/crawlee/events/_event_manager.py index 1b26cd5476..dcc74f9b5d 100644 --- a/src/crawlee/events/_event_manager.py +++ b/src/crawlee/events/_event_manager.py @@ -42,7 +42,9 @@ class EventManagerOptions(TypedDict): """Interval between emitted `PersistState` events to maintain state persistence.""" close_timeout: NotRequired[timedelta | None] - """Optional timeout for canceling pending event listeners if they exceed this duration.""" + """Optional timeout for canceling pending event listeners if they exceed this duration. `None` waits without + a limit, zero cancels them right away. + """ @docs_group('Event managers') @@ -65,6 +67,7 @@ def __init__( Args: persist_state_interval: Interval between emitted `PersistState` events to maintain state persistence. close_timeout: Optional timeout for canceling pending event listeners if they exceed this duration. + `None` waits without a limit, zero cancels them right away. """ self._persist_state_interval = persist_state_interval """Interval between the emitted `PersistState` events.""" @@ -226,7 +229,8 @@ async def wait_for_all_listeners_to_complete(self, *, timeout: timedelta | None Args: timeout: The maximum time to wait for the event listeners to finish. If they do not complete within - the specified timeout, they will be canceled. + the specified timeout, they will be canceled. `None` waits without a limit, zero cancels them right + away. """ # A waiter can't finish until the listeners it awaits do, so waiters must never await each other or # themselves - this is what happens when a listener waits or closes from within itself. Only a waiter diff --git a/src/crawlee/request_loaders/_request_manager.py b/src/crawlee/request_loaders/_request_manager.py index 671e5866f0..c24faf43f5 100644 --- a/src/crawlee/request_loaders/_request_manager.py +++ b/src/crawlee/request_loaders/_request_manager.py @@ -91,7 +91,8 @@ async def add_requests( batch_size: The number of requests to add in one batch. wait_time_between_batches: Time to wait between adding batches. wait_for_all_requests_to_be_added: If True, wait for all requests to be added before returning. - wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. + wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. `None` waits + without a limit, zero cancels the requests still being added right away. """ # Default and dumb implementation. processed_requests = list[ProcessedRequest]()