Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/06_upgrading/upgrading_to_v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ A custom storage backend has more names to update. The `StorageBackend` methods

`RequestQueueBackend.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue backend, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in backends already implement it.

## A zero timeout cancels right away

A zero `close_timeout` of an event manager, or a zero `timeout` passed to `EventManager.wait_for_all_listeners_to_complete()`, cancels the running listeners right away. A zero `wait_for_all_requests_to_be_added_timeout` in `add_requests()` does the same for the requests still being added. In v1, a zero timeout meant no limit. To wait without a limit, pass `None`.
Comment thread
Mantisus marked this conversation as resolved.
Resolved

## Keyword-only arguments

Secondary arguments of the public API are now keyword-only. The main argument of each call, such as a key, a value, a selector or a path, stays positional. A call that passes any of the following arguments positionally raises `TypeError`, so pass them by name:
Expand Down
2 changes: 1 addition & 1 deletion src/crawlee/_utils/wait.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ async def wait_for_all_tasks_to_finish(
if not tasks:
return

timeout_secs = timeout.total_seconds() if timeout else None
timeout_secs = timeout.total_seconds() if timeout is not None else None

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: Now that zero and None mean different things, the public docstrings should say it: None waits without a limit, zero cancels right away. Only the upgrading guide says it now. The docstrings are close_timeout and wait_for_all_listeners_to_complete(timeout=...) in events/_event_manager.py, and wait_for_all_requests_to_be_added_timeout in request_loaders/_request_manager.py and BasicCrawler.add_requests.

✍️ Drafted by Claude Code

try:
_, pending = await asyncio.wait(tasks, timeout=timeout_secs)
if pending:
Expand Down
3 changes: 2 additions & 1 deletion src/crawlee/crawlers/_basic/_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1038,7 +1038,8 @@ async def add_requests(
batch_size: The number of requests to add in one batch.
wait_time_between_batches: Time to wait between adding batches.
wait_for_all_requests_to_be_added: If True, wait for all requests to be added before returning.
wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added.
wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. `None` waits
without a limit, zero cancels the requests still being added right away.
"""
allowed_requests = []
skipped = list[Request]()
Expand Down
8 changes: 6 additions & 2 deletions src/crawlee/events/_event_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ class EventManagerOptions(TypedDict):
"""Interval between emitted `PersistState` events to maintain state persistence."""

close_timeout: NotRequired[timedelta | None]
"""Optional timeout for canceling pending event listeners if they exceed this duration."""
"""Optional timeout for canceling pending event listeners if they exceed this duration. `None` waits without
a limit, zero cancels them right away.
"""


@docs_group('Event managers')
Expand All @@ -65,6 +67,7 @@ def __init__(
Args:
persist_state_interval: Interval between emitted `PersistState` events to maintain state persistence.
close_timeout: Optional timeout for canceling pending event listeners if they exceed this duration.
`None` waits without a limit, zero cancels them right away.
"""
self._persist_state_interval = persist_state_interval
"""Interval between the emitted `PersistState` events."""
Expand Down Expand Up @@ -226,7 +229,8 @@ async def wait_for_all_listeners_to_complete(self, *, timeout: timedelta | None

Args:
timeout: The maximum time to wait for the event listeners to finish. If they do not complete within
the specified timeout, they will be canceled.
the specified timeout, they will be canceled. `None` waits without a limit, zero cancels them right
away.
"""
# A waiter can't finish until the listeners it awaits do, so waiters must never await each other or
# themselves - this is what happens when a listener waits or closes from within itself. Only a waiter
Expand Down
3 changes: 2 additions & 1 deletion src/crawlee/request_loaders/_request_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ async def add_requests(
batch_size: The number of requests to add in one batch.
wait_time_between_batches: Time to wait between adding batches.
wait_for_all_requests_to_be_added: If True, wait for all requests to be added before returning.
wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added.
wait_for_all_requests_to_be_added_timeout: Timeout for waiting for all requests to be added. `None` waits
without a limit, zero cancels the requests still being added right away.
"""
# Default and dumb implementation.
processed_requests = list[ProcessedRequest]()
Expand Down
20 changes: 20 additions & 0 deletions tests/unit/_utils/test_wait.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from __future__ import annotations

import asyncio
import logging
from datetime import timedelta

from crawlee._utils.wait import wait_for_all_tasks_to_finish


async def test_zero_timeout_cancels_right_away() -> None:
"""A zero timeout cancels the unfinished tasks without waiting for them."""
task = asyncio.create_task(asyncio.Event().wait())

# Capped, as a zero timeout taken for no limit would wait for the task forever.
await asyncio.wait_for(
wait_for_all_tasks_to_finish([task], logger=logging.getLogger(__name__), timeout=timedelta(0)),
timeout=5,
)

assert task.cancelled()
Loading