diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 1174379fe4..82609c9d68 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -493,6 +493,39 @@ class MyHttpClient(HttpClient): `SitemapRequestLoader.fetch_next_request` skips a URL while its request is still in progress, for example when another sitemap lists it too. In v1, it returned a request for the URL again, and `get_total_count` counted the URL twice. +## Crawlers manage only what they create + +A crawler now enters and exits only the objects it creates, the same as Crawlee JS. An object you pass is yours to enter and exit. The `Statistics` and the event managers are the exceptions, since the crawler enters them on every run. You can enter an event manager yourself as well. In v1, the crawler entered a passed object that wasn't active yet and exited it when its run finished, even while another crawler still used it. + +What you do with an object you pass depends on its type: + +- `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished. +- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`. +- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time. + +For example: + +```python +# v1 +crawler = ParselCrawler(session_pool=SessionPool(max_pool_size=10)) +await crawler.run(urls) + +# v2 +session_pool = SessionPool(max_pool_size=10) +crawler = ParselCrawler(session_pool=session_pool) + +async with session_pool: + await crawler.run(urls) +``` + +When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too. + +`RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`. + +`AdaptivePlaywrightCrawler.rendering_type_predictor` is now read-only. Pass your predictor to the constructor instead. + +The private `_additional_context_managers` argument of `BasicCrawler` is removed. To have the crawler enter and exit an object on every run, a subclass builds the object with `self._own_or_borrow()`. A subclass that gives its parent class a default object, such as an HTTP client, builds it this way before it calls `super().__init__()`. + ## Resetting statistics and the session pool `Statistics.reset()` is now synchronous and resets only the in-memory statistics, the same as Crawlee JS. In v1, it also cleared the persisted record. To clear the record too, call the new `reset_store()` once the statistics are exited: @@ -599,16 +632,21 @@ To keep the response cookies out of the sessions, pass `save_response_cookies=Fa ```python # v1 crawler = ParselCrawler(use_session_pool=False) +await crawler.run(urls) # v2 from crawlee.sessions import SessionCookies, SessionPool +session_pool = SessionPool(create_session_settings={'max_usage_count': 1}) crawler = ParselCrawler( retry_on_blocked=False, save_response_cookies=False, - session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), + session_pool=session_pool, ) +async with session_pool: + await crawler.run(urls) + # In a request handler response = await context.send_request(url, cookies=SessionCookies()) ``` diff --git a/src/crawlee/_utils/context.py b/src/crawlee/_utils/context.py index 9faa994376..ce5e519c47 100644 --- a/src/crawlee/_utils/context.py +++ b/src/crawlee/_utils/context.py @@ -3,11 +3,42 @@ import inspect from collections.abc import Callable from functools import wraps -from typing import Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast + +if TYPE_CHECKING: + from types import TracebackType T = TypeVar('T', bound=Callable[..., Any]) +class Collaborator(Protocol): + """An async context manager that reports through `active` whether it's entered.""" + + @property + def active(self) -> bool: + """Whether the object is entered and not exited yet.""" + + async def __aenter__(self) -> Any: + """Prepare the object for use. + + Raises: + RuntimeError: If the object is already active. + """ + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + exc_traceback: TracebackType | None, + /, + ) -> Any: + """Release what the object holds. + + Raises: + RuntimeError: If the object is not active. + """ + + def ensure_context(method: T) -> T: """Ensure the (async) context manager is initialized before executing the method. diff --git a/src/crawlee/autoscaling/_types.py b/src/crawlee/autoscaling/_types.py index 678a4f3590..ac872acf3c 100644 --- a/src/crawlee/autoscaling/_types.py +++ b/src/crawlee/autoscaling/_types.py @@ -7,6 +7,7 @@ from pydantic import Field from pydantic.dataclasses import dataclass as pydantic_dataclass +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group @@ -104,12 +105,13 @@ class ConcurrencyConsumer: @docs_group('Autoscaling') -class ConcurrencySystemInterface(Protocol): +class ConcurrencySystemInterface(Collaborator, Protocol): """The contract between an `AutoscaledPool` and the component that governs its concurrency. The system answers whether there is free compute for one more task and tracks the budget that tasks are booked against. `ConcurrencySystem` is the default implementation. Anything matching this contract can take its place, - without depending on its internals. + without depending on its internals. An implementation is an async context manager, and `AutoscaledPool.run` refuses + to run on one that isn't entered. Every allocation method is told which `ConcurrencyConsumer` is asking, so an implementation may allocate per consumer. `ConcurrencySystem` does not, and serves whoever asks first, which can starve a consumer that joins @@ -127,10 +129,6 @@ def desired_concurrency(self) -> int: def current_concurrency(self) -> int: """The number of parallel tasks currently booked against this system.""" - @property - def active(self) -> bool: - """Whether the system is ready to be booked against. `AutoscaledPool.run` refuses to run when it is not.""" - def has_capacity_for_task(self, consumer: ConcurrencyConsumer) -> bool: """Report whether one more task may start right now on behalf of `consumer`. diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 922500495d..b2145101e0 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -135,7 +135,8 @@ def __init__( Args: rendering_type_predictor: Object that implements RenderingTypePredictor and is capable of predicting which - rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used. + rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used. Keep a + predictor passed in open with `async with` until every crawler using it has finished. result_checker: Function that evaluates whether crawling result is valid or not. result_comparator: Function that compares two crawling results and decides whether they are equivalent. static_parser: Implementation of `AbstractHttpParser`. Parser that will be used for static crawling. @@ -144,11 +145,10 @@ def __init__( save_response_cookies: Whether to save the response cookies to the session, in both the static and the browser sub crawler. statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of - non-default configuration. + non-default configuration. The crawler enters it on every run, so don't enter it yourself. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. """ # Adaptive crawling related. - self.rendering_type_predictor = rendering_type_predictor or DefaultRenderingTypePredictor() self.result_checker = result_checker or (lambda _: True) self.result_comparator = result_comparator or create_default_comparator(result_checker) @@ -164,6 +164,8 @@ def __init__( super().__init__(statistics=adaptive_statistics, **kwargs) + self._rendering_type_predictor = self._own_or_borrow(rendering_type_predictor, DefaultRenderingTypePredictor) + # Sub crawlers related. playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions() @@ -180,15 +182,16 @@ def __init__( # Initialize sub crawlers to create their pipelines. static_crawler_class = AbstractHttpCrawler.create_parsed_http_crawler_class(static_parser=static_parser) + # The sub crawlers never run themselves, so this crawler enters and exits their statistics. static_crawler = static_crawler_class( parser=static_parser, save_response_cookies=save_response_cookies, - statistics=_NonPersistentStatistics(), + statistics=self._own_or_borrow(None, _NonPersistentStatistics), **basic_crawler_kwargs_for_static_crawler, ) playwright_crawler = PlaywrightCrawler( save_response_cookies=save_response_cookies, - statistics=_NonPersistentStatistics(), + statistics=self._own_or_borrow(None, _NonPersistentStatistics), **playwright_crawler_specific_kwargs, **basic_crawler_kwargs_for_pw_crawler, ) @@ -229,19 +232,22 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static) playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw) - self._additional_context_managers = [ - *self._additional_context_managers, - self.rendering_type_predictor, - static_crawler.statistics, - playwright_crawler.statistics, - playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member. - ] + # The default pool is the one the browser sub crawler built. + self._own_or_borrow( + playwright_crawler_specific_kwargs.get('browser_pool'), + lambda: playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member. + ) # Sub crawler pipeline related self._pw_context_pipeline = playwright_crawler._context_pipeline # noqa:SLF001 # Intentional access to private member. self._static_context_pipeline = static_crawler._context_pipeline # noqa:SLF001 # Intentional access to private member. self._static_parser = static_parser + @property + def rendering_type_predictor(self) -> RenderingTypePredictor: + """The predictor that decides which sub crawler processes a request.""" + return self._rendering_type_predictor + @classmethod def with_beautifulsoup_static_parser( cls, diff --git a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py index de80baf56f..dd3e979ba1 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py @@ -68,6 +68,11 @@ def __init__(self) -> None: # Flag to indicate the state. self._active = False + @property + def active(self) -> bool: + """Whether the predictor is initialized and not cleared since.""" + return self._active + @abstractmethod def predict(self, request: Request) -> RenderingTypePrediction: """Get `RenderingTypePrediction` based on the input request. @@ -164,15 +169,20 @@ async def initialize(self) -> None: await super().initialize() if not self._state.is_initialized: - await self._state.initialize() + try: + await self._state.initialize() + except BaseException: + await super().clear() + raise @override async def clear(self) -> None: """Clear the predictor state.""" - await super().clear() - - if self._state.is_initialized: - await self._state.teardown() + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + await super().clear() @override def predict(self, request: Request) -> RenderingTypePrediction: diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 11a6a7581c..494d42d0ba 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -13,7 +13,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence from contextlib import AsyncExitStack from datetime import timedelta -from functools import partial +from functools import cached_property, partial from http import HTTPStatus from io import StringIO from pathlib import Path @@ -42,6 +42,7 @@ SendRequestFunction, SkippedReason, ) +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group from crawlee._utils.file import atomic_write, export_csv_to_stream, export_json_to_stream from crawlee._utils.globs import check_url_patterns @@ -84,7 +85,6 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping, MutableMapping - from contextlib import AbstractAsyncContextManager from crawlee._types import ( ConcurrencySettings, @@ -113,6 +113,7 @@ TRequestIterator = TypeVar('TRequestIterator', str, Request) TParams = ParamSpec('TParams') T = TypeVar('T') +TCollaborator = TypeVar('TCollaborator', bound=Collaborator) ErrorHandler = Callable[[TCrawlingContext, Exception], Awaitable[Request | None]] FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]] @@ -135,13 +136,15 @@ class _BasicCrawlerOptions(TypedDict): """Manager of requests that should be processed by the crawler.""" session_pool: NotRequired[SessionPool] - """A custom `SessionPool` instance, allowing the use of non-default configuration.""" + """A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open with `async with` + until every crawler using it has finished.""" proxy_configuration: NotRequired[ProxyConfiguration] """HTTP proxy configuration used when making requests.""" http_client: NotRequired[HttpClient] - """HTTP client used by `BasicCrawlingContext.send_request` method.""" + """HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with` until every + crawler using it has finished.""" max_request_retries: NotRequired[int] """Specifies the maximum number of retries allowed for a request if its processing fails. @@ -177,7 +180,8 @@ class _BasicCrawlerOptions(TypedDict): concurrency_system: NotRequired[ConcurrencySystemInterface] """A concurrency system to use instead of the default one, e.g. one shared between crawlers to cap their combined - concurrency. Cannot be combined with `concurrency_settings`.""" + concurrency. Keep it open with `async with` until every crawler using it has finished. Cannot be combined with + `concurrency_settings`.""" request_handler_timeout: NotRequired[timedelta] """Maximum duration allowed for a single request handler to run. A router handler can override it.""" @@ -206,10 +210,6 @@ class _BasicCrawlerOptions(TypedDict): ignore_http_error_status_codes: NotRequired[Iterable[int]] """HTTP status codes that are typically considered errors but should be treated as successful responses.""" - _additional_context_managers: NotRequired[Sequence[AbstractAsyncContextManager]] - """Additional context managers used throughout the crawler lifecycle. Intended for use by - subclasses rather than direct instantiation of `BasicCrawler`.""" - respect_robots_txt_file: NotRequired[bool] """If set to `True`, the crawler will automatically try to fetch the robots.txt file for each domain, and skip those that are not allowed. This also prevents disallowed URLs to be added via `EnqueueLinksFunction`.""" @@ -240,7 +240,8 @@ class _BasicCrawlerOptionsGeneric(TypedDict, Generic[TCrawlingContext, TStatisti subclasses rather than direct instantiation of `BasicCrawler`.""" statistics: NotRequired[Statistics[TStatisticsState]] - """A custom `Statistics` instance, allowing the use of non-default configuration.""" + """A custom `Statistics` instance, allowing the use of non-default configuration. The crawler enters it on every + run, so don't enter it yourself. Crawlers can share it only when they run one after another.""" class BasicCrawlerOptions( @@ -325,7 +326,6 @@ def __init__( | None = None, id: str | None = None, _context_pipeline: ContextPipeline[TCrawlingContext] | None = None, - _additional_context_managers: Sequence[AbstractAsyncContextManager] | None = None, ) -> None: """Initialize a new instance. @@ -334,9 +334,11 @@ def __init__( event_manager: The event manager for managing events for the crawler and all its components. storage_backend: The storage backend for managing storages for the crawler and all its components. request_manager: Manager of requests that should be processed by the crawler. - session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. + session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open + with `async with` until every crawler using it has finished. proxy_configuration: HTTP proxy configuration used when making requests. - http_client: HTTP client used by `BasicCrawlingContext.send_request` method. + http_client: HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with` + until every crawler using it has finished. request_handler: A callable responsible for handling requests. max_request_retries: Specifies the maximum number of retries allowed for a request if its processing fails. This includes retries due to navigation errors or errors thrown from user-supplied functions @@ -366,7 +368,9 @@ def __init__( has finished. Cannot be combined with `concurrency_settings`. request_handler_timeout: Maximum duration allowed for a single request handler to run. A router handler can override it. - statistics: A custom `Statistics` instance, allowing the use of non-default configuration. + statistics: A custom `Statistics` instance, allowing the use of non-default configuration. The crawler + enters it on every run, so don't enter it yourself. Crawlers can share it only when they run one after + another. abort_on_error: If True, the crawler stops immediately when any request handler error occurs. keep_alive: If True, it will keep crawler alive even if there are no requests in queue. Use `crawler.stop()` to exit the crawler. @@ -387,8 +391,6 @@ def __init__( it, the ID follows the order in which crawlers are created. _context_pipeline: Enables extending the request lifecycle and modifying the crawling context. Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. - _additional_context_managers: Additional context managers used throughout the crawler lifecycle. - Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. """ self._id = str(BasicCrawler.__next_id) if id is None else id self._default_rq_alias = None if BasicCrawler.__next_id == 0 else f'__default_{self._id}__' @@ -415,7 +417,7 @@ def __init__( # Core components self._request_manager = request_manager - self._session_pool = session_pool or SessionPool() + self._session_pool = self._own_or_borrow(session_pool, SessionPool) self._proxy_configuration = proxy_configuration self._additional_http_error_status_codes = ( @@ -425,7 +427,7 @@ def __init__( set(ignore_http_error_status_codes) if ignore_http_error_status_codes else set() ) - self._http_client = http_client or ImpitHttpClient() + self._http_client = self._own_or_borrow(http_client, ImpitHttpClient) # Request router setup self._router: Router[TCrawlingContext] | None = None @@ -506,9 +508,6 @@ async def persist_state_factory() -> KeyValueStore: ), ) - # Additional context managers to enter and exit - self._additional_context_managers = _additional_context_managers or [] - # Internal, not explicitly configurable components self._robots_txt_file_cache: LRUCache[str, RobotsTxtFile] = LRUCache(maxsize=1000) self._robots_txt_lock = asyncio.Lock() @@ -522,6 +521,8 @@ async def persist_state_factory() -> KeyValueStore: self._concurrency_settings = concurrency_settings self._borrowed_concurrency_system = concurrency_system + if concurrency_system is not None: + self._borrowed_collaborators.append(concurrency_system) # The default system is built per run, so that a run is not scaled on the load and concurrency of the previous # one. A supplied system is known up front. @@ -540,6 +541,32 @@ async def persist_state_factory() -> KeyValueStore: # The last run started on this instance, which stays readable after it finishes. self._last_run: CrawlerRun[TStatisticsState] | None = None + @cached_property + def _owned_collaborators(self) -> list[Collaborator]: + """The collaborators the crawler owns, which it enters and exits on every run.""" + return [] + + @cached_property + def _borrowed_collaborators(self) -> list[Collaborator]: + """The collaborators passed in, which the caller enters and exits, so `run()` only checks they're active.""" + return [] + + def _own_or_borrow(self, passed: TCollaborator | None, build_default: Callable[[], TCollaborator]) -> TCollaborator: + """Return the collaborator passed in, or build the default one, and record which of them the crawler owns. + + It needs nothing that `BasicCrawler.__init__` sets, so a subclass can call it before `super().__init__()` and + pass the result down. A parent class that gets an already recorded collaborator leaves its record as is. + """ + if passed is None: + default = build_default() + self._owned_collaborators.append(default) + return default + + if passed not in self._owned_collaborators and passed not in self._borrowed_collaborators: + self._borrowed_collaborators.append(passed) + + return passed + @property def log(self) -> logging.Logger: """The logger used by the crawler.""" @@ -816,6 +843,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta requests: The requests to be enqueued before the crawler starts. Raises: + RuntimeError: If the crawler is already running, an object passed to it, such as a `SessionPool`, isn't + entered, or its statistics are already active. PersistentRateLimitError: If a domain throttled by a `ThrottlingRequestManager` has rate-limited every request for longer than the manager's `max_domain_stall` and no other requests are left. """ @@ -824,6 +853,19 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta 'This crawler instance is already running, you can add more requests to it via `crawler.add_requests()`' ) + for collaborator in self._borrowed_collaborators: + if not collaborator.active: + raise RuntimeError( + f'The {type(collaborator).__name__} passed to the crawler is not active. Enter it with ' + '`async with` before running the crawler.' + ) + + if self._statistics.active: + raise RuntimeError( + "The statistics of the crawler are already active. The crawler enters them itself, so don't enter " + 'them yourself. Crawlers can share statistics only when they run one after another.' + ) + self._last_run = self._create_run() try: @@ -831,8 +873,10 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta self._statistics.reset() await self._statistics.reset_store() - self._session_pool.reset() - await self._session_pool.reset_store() + # A session pool passed in belongs to the caller, who may still use it in another crawler. + if self._session_pool in self._owned_collaborators: + self._session_pool.reset() + await self._session_pool.reset_store() if requests is not None: await self.add_requests(requests) @@ -973,24 +1017,16 @@ async def _run_crawler(self) -> None: # A system supplied by the caller is only borrowed, and its lifecycle belongs to whoever created it. The one # the crawler builds for this run is entered and exited below. - owned_concurrency_system = self._concurrency_system if self._borrowed_concurrency_system is None else None - - # Collect the context managers to be entered. Context managers that are already active are excluded, - # as they were likely entered by the caller, who will also be responsible for exiting them. - contexts_to_enter.extend( - [ - cm - for cm in ( - owned_concurrency_system, - self._statistics, - self._session_pool, - self._http_client, - self._crawler_state_rec_task, - *self._additional_context_managers, - ) - if cm and getattr(cm, 'active', False) is False - ] - ) + if self._borrowed_concurrency_system is None: + contexts_to_enter.append(self._concurrency_system) + + # The statistics count only while they're entered, so the crawler enters them on every run, even when they + # were passed in. + contexts_to_enter.append(self._statistics) + contexts_to_enter.extend(self._owned_collaborators) + + # The status messages of the recurring task read the statistics, so it starts after them and stops before them. + contexts_to_enter.append(self._crawler_state_rec_task) async with AsyncExitStack() as exit_stack: for context in contexts_to_enter: diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 39888d9e95..be323f006b 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -129,7 +129,8 @@ def __init__( """Initialize a new instance. Args: - browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. + browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it + open with `async with` until every crawler using it has finished. user_data_dir: Path to a user data directory, which stores browser session data like cookies and local storage. browser_type: The type of browser to launch: @@ -162,49 +163,48 @@ def __init__( self._shared_navigation_timeouts: dict[int, SharedTimeout] = {} self._save_response_cookies = save_response_cookies - if browser_pool: - # Raise an exception if browser_pool is provided together with other browser-related arguments. - if any( - param not in [None, 'default'] - for param in ( - user_data_dir, - use_incognito_pages, - headless, - browser_type, - browser_launch_options, - browser_new_context_options, - fingerprint_generator, - ) - ): - raise ValueError( - 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, ' - '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or ' - '`fingerprint_generator` arguments when `browser_pool` is provided.' - ) + if browser_pool and any( + param not in [None, 'default'] + for param in ( + user_data_dir, + use_incognito_pages, + headless, + browser_type, + browser_launch_options, + browser_new_context_options, + fingerprint_generator, + ) + ): + raise ValueError( + 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, ' + '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or ' + '`fingerprint_generator` arguments when `browser_pool` is provided.' + ) - # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments. - else: - if fingerprint_generator == 'default': - generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = ( - [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None - ) + # Before `super().__init__()`, so that a failure here doesn't take up a crawler id. + generator: FingerprintGenerator | None = None + if fingerprint_generator != 'default': + generator = fingerprint_generator + elif browser_pool is None: + generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = ( + [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None + ) - fingerprint_generator = DefaultFingerprintGenerator( - header_options=HeaderGeneratorOptions(browsers=generator_browser_type) - ) + generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=generator_browser_type) + ) - browser_pool = BrowserPool.with_default_plugin( + def build_browser_pool() -> BrowserPool: + return BrowserPool.with_default_plugin( headless=headless, browser_type=browser_type, user_data_dir=user_data_dir, browser_launch_options=browser_launch_options, browser_new_context_options=browser_new_context_options, use_incognito_pages=use_incognito_pages, - fingerprint_generator=fingerprint_generator, + fingerprint_generator=generator, ) - self._browser_pool = browser_pool - # Compose the context pipeline with the Playwright-specific context enhancer. kwargs['_context_pipeline'] = ( ContextPipeline() @@ -216,11 +216,11 @@ def __init__( .compose(self._handle_blocked_request_by_content) .compose(self._create_crawling_context) ) - kwargs['_additional_context_managers'] = [self._browser_pool] self._pre_navigation_hooks: list[Callable[[TPreNavContext], Awaitable[None]]] = [] self._post_navigation_hooks: list[Callable[[TPostNavContext], Awaitable[None]]] = [] - kwargs['http_client'] = PlaywrightHttpClient() if not kwargs.get('http_client') else kwargs['http_client'] + # Before `super().__init__()`, which would otherwise build the default `ImpitHttpClient`. + kwargs['http_client'] = self._own_or_borrow(kwargs.get('http_client'), PlaywrightHttpClient) # Lower the default concurrency for browser crawlers, unless the caller configured it themselves. if kwargs.get('concurrency_settings') is None and kwargs.get('concurrency_system') is None: @@ -231,6 +231,8 @@ def __init__( super().__init__(**kwargs) + self._browser_pool = self._own_or_borrow(browser_pool, build_browser_pool) + async def _manage_shared_navigation_timeout( self, context: BasicCrawlingContext ) -> AsyncGenerator[BasicCrawlingContext, None]: @@ -647,7 +649,8 @@ class _PlaywrightCrawlerAdditionalOptions(TypedDict): """ browser_pool: NotRequired[BrowserPool] - """A `BrowserPool` instance to be used for launching the browsers and getting pages.""" + """A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it open with `async with` + until every crawler using it has finished.""" browser_type: NotRequired[BrowserType] """The type of browser to launch: diff --git a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py index cc24f91d6b..82dcc03c6c 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py +++ b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py @@ -12,6 +12,9 @@ from ._types import PydanticAiUsageStats if TYPE_CHECKING: + from types import TracebackType + from typing import Self + from pydantic_ai.models import Model from pydantic_ai.usage import UsageLimits @@ -27,7 +30,7 @@ class BasePydanticAiHtmlExtractor(ABC): An HTML extractor turns a page into a validated Pydantic model with the help of an LLM. This abstract base implements the parts the built-in extractors share: resolving the model, composing the task instructions with - the distiller's prompt notes, and accumulating token usage. + the distiller's prompt notes, and accumulating token usage. It also provides a default `async with` lifecycle. The public interface is the `PydanticAiHtmlExtractor` protocol. The concrete extractors are `PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`. @@ -56,6 +59,40 @@ def __init__( self._base_instructions = self._compose_instructions(instructions, distiller) self._usage_limits = usage_limits self._ai_usage = PydanticAiUsageStats() + self._active = False + + @property + def active(self) -> bool: + """Whether the extractor is entered and not exited yet.""" + return self._active + + async def __aenter__(self) -> Self: + """Prepare the extractor for extraction. + + Raises: + RuntimeError: If the extractor is already active. + """ + if self._active: + raise RuntimeError(f'The {type(self).__name__} is already active.') + + self._active = True + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + exc_traceback: TracebackType | None, + ) -> None: + """Release what the extractor holds. + + Raises: + RuntimeError: If the extractor is not active. + """ + if not self._active: + raise RuntimeError(f'The {type(self).__name__} is not active.') + + self._active = False @property def ai_usage(self) -> PydanticAiUsageStats: diff --git a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py index 55b61e3e65..1428811635 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py +++ b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py @@ -1,7 +1,6 @@ from __future__ import annotations import warnings -from contextlib import AbstractAsyncContextManager from typing import TYPE_CHECKING from parsel import Selector @@ -90,7 +89,8 @@ def __init__( `model` nor `extractor` is given. Provide at most one of `model` or `extractor`. extractor: A pre-configured `PydanticAiHtmlExtractor`, for full control over the distiller, instructions, caching, usage limits, and model fallback. Pass a `PydanticAiSelectorExtractor` here for - cached-selector extraction. Provide at most one of `model` or `extractor`. + cached-selector extraction. Keep it open with `async with` until every crawler using it has finished. + Provide at most one of `model` or `extractor`. kwargs: Additional keyword arguments to pass to the underlying `AbstractHttpCrawler`. """ # Call the notification only once. @@ -103,10 +103,9 @@ def __init__( if model is not None and extractor is not None: raise ValueError('Provide at most one of `model` or `extractor`.') - if extractor is None: - extractor = PydanticAiDirectExtractor(model if model is not None else _DEFAULT_AI_MODEL) - - self._extractor = extractor + # Before `super().__init__()`, so that a failure here doesn't take up a crawler id. + default_model = model if model is not None else _DEFAULT_AI_MODEL + self._extractor = self._own_or_borrow(extractor, lambda: PydanticAiDirectExtractor(default_model)) async def final_step( context: ParsedHttpCrawlingContext[Selector], @@ -121,13 +120,6 @@ async def final_step( kwargs['_context_pipeline'] = self._create_static_content_crawler_pipeline().compose(final_step) - # If the extractor is an async context manager, add it to the crawler's additional context managers so it's - # properly entered and exited around the crawl. - if isinstance(extractor, AbstractAsyncContextManager): - kwargs['_additional_context_managers'] = [ - *kwargs.get('_additional_context_managers', []), - extractor, - ] super().__init__( parser=ParselParser(), **kwargs, diff --git a/src/crawlee/crawlers/_pydantic_ai/_types.py b/src/crawlee/crawlers/_pydantic_ai/_types.py index 4ddad30eda..5dccf5d6a1 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_types.py +++ b/src/crawlee/crawlers/_pydantic_ai/_types.py @@ -5,6 +5,7 @@ from pydantic import BaseModel +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group if TYPE_CHECKING: @@ -52,12 +53,13 @@ def add(self, usage: RunUsage) -> None: @docs_group('Other') -class PydanticAiHtmlExtractor(Protocol): +class PydanticAiHtmlExtractor(Collaborator, Protocol): """Interface for HTML extractors. An extractor turns an HTML page into a validated Pydantic model using an LLM. The input format (cleaned HTML, skeleton, Markdown, ...) is decided by the `PydanticAiHtmlDistiller` an implementation composes. The model and base - instructions are set at construction. Each `extract` call runs one extraction. The built-in extractors are + instructions are set at construction. Each `extract` call runs one extraction. An extractor is an async context + manager, and `PydanticAiCrawler` refuses to run with one that isn't entered. The built-in extractors are `PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`. """ diff --git a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py index df71060797..226ce4db0a 100644 --- a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py +++ b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py @@ -109,7 +109,8 @@ def __init__( stagehand_options: Stagehand-specific configuration (model, API key, env, etc.). Cannot be specified if `browser_pool` is provided. browser_pool: A pre-configured `BrowserPool`. All plugins must be instances of `StagehandBrowserPlugin`. - If omitted, a pool is created automatically from the other browser arguments. + If omitted, a pool is created automatically from the other browser arguments. Keep a pool passed in + open with `async with` until every crawler using it has finished. user_data_dir: Path to a user data directory, which stores browser session data like cookies and local storage. Cannot be specified if `browser_pool` is provided. headless: Whether to run the browser in headless mode. Defaults to the value from Crawlee's @@ -143,12 +144,13 @@ def __init__( 'Cannot specify `stagehand_options`, `user_data_dir`, `headless`, ' '`browser_launch_options`, `browser_new_context_options` when `browser_pool` is provided.' ) - else: + + def build_browser_pool() -> BrowserPool: launch_options = dict(browser_launch_options or {}) if headless is not None: launch_options['headless'] = headless - browser_pool = BrowserPool( + return BrowserPool( plugins=[ StagehandBrowserPlugin( stagehand_options=stagehand_options, @@ -159,6 +161,8 @@ def __init__( ] ) + browser_pool = self._own_or_borrow(browser_pool, build_browser_pool) + super().__init__( browser_pool=browser_pool, goto_options=goto_options, diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index e69322a003..65498c2418 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -389,15 +389,17 @@ async def __aenter__(self) -> HttpClient: return self async def __aexit__( - self, exc_type: BaseException | None, exc_value: BaseException | None, traceback: TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_traceback: TracebackType | None ) -> None: """Deinitialize the client and clean up resources when exiting the context manager. Raises: - RuntimeError: If the context manager is already active. + RuntimeError: If the context manager is not active. """ if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self.cleanup() - self._active = False + try: + await self.cleanup() + finally: + self._active = False diff --git a/src/crawlee/sessions/_session_pool.py b/src/crawlee/sessions/_session_pool.py index f206c64f58..a855224d6a 100644 --- a/src/crawlee/sessions/_session_pool.py +++ b/src/crawlee/sessions/_session_pool.py @@ -118,12 +118,20 @@ async def __aenter__(self) -> SessionPool: self._active = True - state = await self._state.initialize() - state.max_pool_size = self._max_pool_size - self._remove_retired_sessions() - - if not state.sessions: - await self._fill_sessions_to_max() + try: + state = await self._state.initialize() + state.max_pool_size = self._max_pool_size + self._remove_retired_sessions() + + if not state.sessions: + await self._fill_sessions_to_max() + except BaseException: + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + self._active = False + raise return self @@ -141,9 +149,10 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self._state.teardown() - - self._active = False + try: + await self._state.teardown() + finally: + self._active = False @overload def get_state(self, *, as_dict: Literal[True]) -> dict: ... diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py index 74ff13e5ba..3c684a4da1 100644 --- a/src/crawlee/statistics/_statistics.py +++ b/src/crawlee/statistics/_statistics.py @@ -172,22 +172,27 @@ async def __aenter__(self) -> Self: if self._active: raise RuntimeError(f'The {self.__class__.__name__} is already active.') - await self._state.initialize() - - # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated - # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count - # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat the - # new run as already checkpointed. - self.state.accumulated_runtime = self.state.crawler_runtime - self.state.crawler_last_started_at = None - self.state.crawler_finished_at = None - self.state.stats_persisted_at = None - - # Start periodic logging and let it print the restored state before the new run starts being measured. - self._periodic_logger.start() - await asyncio.sleep(0.01) self._active = True + try: + await self._state.initialize() + + # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated + # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count + # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat + # the new run as already checkpointed. + self.state.accumulated_runtime = self.state.crawler_runtime + self.state.crawler_last_started_at = None + self.state.crawler_finished_at = None + self.state.stats_persisted_at = None + + # Start periodic logging and let it print the restored state before the new run starts being measured. + self._periodic_logger.start() + await asyncio.sleep(0.01) + except BaseException: + await self._stop_capturing() + raise + self.state.crawler_last_started_at = datetime.now(UTC) self.state.crawler_started_at = self.state.crawler_started_at or self.state.crawler_last_started_at return self @@ -206,11 +211,10 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - # Stop logging; the finish timestamp freezes the measured run before the final persist. - await self._periodic_logger.stop() + # Before the first await, so that a cancelled exit doesn't count the time until the next run as runtime. self.state.crawler_finished_at = datetime.now(UTC) - self._active = False - await self._state.teardown() + + await self._stop_capturing() @property def state(self) -> TStatisticsState: @@ -316,6 +320,16 @@ async def reset_store(self) -> None: """ await self._state.reset_store() + async def _stop_capturing(self) -> None: + try: + await self._periodic_logger.stop() + finally: + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + self._active = False + def _log(self) -> None: stats = self.calculate() if self._statistics_log_format == 'table': diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index c6931a9571..2db1cf4574 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -29,6 +29,7 @@ AdaptivePlaywrightCrawlerStatisticState, ) from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import AdaptiveContextError +from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor from crawlee.errors import RequestThrottledError from crawlee.http_clients import HttpClient, HttpResponse from crawlee.sessions import SessionPool @@ -123,6 +124,12 @@ def predict(self, request: Request) -> RenderingTypePrediction: def store_result(self, request: Request, rendering_type: RenderingType) -> None: pass + @property + @override + def active(self) -> bool: + # Lets the tests pass the predictor without entering it. + return True + @dataclass(frozen=True) class TestInput: @@ -573,6 +580,17 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: assert stored_results == [saved_data] +async def test_passed_predictor_left_open(test_urls: list[str]) -> None: + """A rendering type predictor passed to the crawler stays active after the run, for the caller to exit.""" + async with DefaultRenderingTypePredictor() as predictor: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(rendering_type_predictor=predictor) + crawler.router.default_handler(AsyncMock()) + + await crawler.run(test_urls[:1]) + + assert predictor.active + + async def test_adaptive_playwright_crawler_statistics_in_init() -> None: """Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics.""" persistence_enabled = True @@ -1040,7 +1058,7 @@ async def test_throttled_static_429_is_deferred(monkeypatch: pytest.MonkeyPatch) throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.side_effect = [ Mock( spec=HttpResponse, @@ -1090,7 +1108,7 @@ async def test_static_handler_raised_throttled_error_is_deferred(monkeypatch: py throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = Mock( spec=HttpResponse, status_code=200, diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py index ed4d5108bf..95be64034f 100644 --- a/tests/integration/crawlers/_http/test_http_crawler.py +++ b/tests/integration/crawlers/_http/test_http_crawler.py @@ -485,13 +485,14 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No sessions_cookies: dict[str, dict[str, str]] = {} response_cookies: dict[str, dict[str, str]] = {} + session_pool = SessionPool( + max_pool_size=1, + create_session_settings={ + 'max_error_score': 50, + }, + ) crawler = HttpCrawler( - session_pool=SessionPool( - max_pool_size=1, - create_session_settings={ - 'max_error_score': 50, - }, - ), + session_pool=session_pool, http_client=http_client, max_request_retries=10, concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), @@ -513,16 +514,19 @@ async def handler(context: HttpCrawlingContext) -> None: if context.request.user_data.get('retire_session'): context.session.retire() - await crawler.run( - [ - # The first request sets the cookie in the session - str(server_url.with_path('set_cookies').extend_query(a=1)), - # With the second request, we check the cookies in the session and set retire - Request.from_url(str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True}), - # The third request is made with a new session to make sure it does not use another session's cookies - Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'), - ] - ) + async with session_pool: + await crawler.run( + [ + # The first request sets the cookie in the session + str(server_url.with_path('set_cookies').extend_query(a=1)), + # With the second request, we check the cookies in the session and set retire + Request.from_url( + str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True} + ), + # The third request is made with a new session to make sure it does not use another session's cookies + Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'), + ] + ) assert len(sessions_cookies) == 2 assert len(response_cookies) == 2 @@ -644,10 +648,8 @@ async def handler(context: HttpCrawlingContext) -> None: async def test_save_response_cookies_disabled(server_url: URL) -> None: """Test that `save_response_cookies=False` keeps the response cookies out of the session, but sends its cookies.""" preset_cookie = CookieParam(name='preset', value='1', domain=server_url.host or '') - crawler = HttpCrawler( - session_pool=SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}), - save_response_cookies=False, - ) + session_pool = SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}) + crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False) handler_call = Mock() @crawler.router.default_handler @@ -659,14 +661,16 @@ async def handler(context: HttpCrawlingContext) -> None: ) # The cookie set by the redirecting response is still sent to the redirect target. - await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + async with session_pool: + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) handler_call.assert_called_once_with(sent_cookies={'preset': '1', 'a': '1'}, session_cookies={'preset'}) async def test_context_send_request_saves_cookies(server_url: URL) -> None: """Test that `context.send_request` stores response cookies in the session with `save_response_cookies=False`.""" - crawler = HttpCrawler(session_pool=SessionPool(max_pool_size=1), save_response_cookies=False) + session_pool = SessionPool(max_pool_size=1) + crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False) handler_call = Mock() @crawler.router.default_handler @@ -675,7 +679,8 @@ async def handler(context: HttpCrawlingContext) -> None: session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call(session_cookies={cookie['name'] for cookie in session_cookies}) - await crawler.run([str(server_url / 'cookies')]) + async with session_pool: + await crawler.run([str(server_url / 'cookies')]) handler_call.assert_called_once_with(session_cookies={'b'}) diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py index 888ffa0535..15ab38c78b 100644 --- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py @@ -434,12 +434,15 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) sessions: dict[str, Session] = {} response_cookies: dict[str, dict[str, str]] = {} - async with BrowserPool( - plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)], - browser_inactive_threshold=timedelta(minutes=10), - ) as browser_pool: + async with ( + BrowserPool( + plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)], + browser_inactive_threshold=timedelta(minutes=10), + ) as browser_pool, + SessionPool(max_pool_size=1) as session_pool, + ): crawler = PlaywrightCrawler( - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), browser_pool=browser_pool, ) @@ -604,7 +607,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_send_request_skips_cookies_deleted_by_page(server_url: URL) -> None: """Test that `context.send_request` doesn't send a session cookie that the page deleted in the handler.""" - crawler = PlaywrightCrawler(session_pool=SessionPool(max_pool_size=1)) + session_pool = SessionPool(max_pool_size=1) + crawler = PlaywrightCrawler(session_pool=session_pool) sent_cookies: list[dict[str, str]] = [] @crawler.router.default_handler @@ -617,7 +621,8 @@ async def second_handler(context: PlaywrightCrawlingContext) -> None: response = await context.send_request(str(server_url / 'cookies')) sent_cookies.append(json.loads(await response.read())['cookies']) - await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + async with session_pool: + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) assert sent_cookies == [{}] @@ -983,7 +988,8 @@ async def test_send_request_with_client(server_url: URL) -> None: """Check that the persist context works with fingerprints.""" check_data: dict[str, Any] = {} - crawler = PlaywrightCrawler(http_client=ImpitHttpClient(headers={'user-agent': 'My User-Agent'})) + http_client = ImpitHttpClient(headers={'user-agent': 'My User-Agent'}) + crawler = PlaywrightCrawler(http_client=http_client) @crawler.router.default_handler async def request_handler(context: PlaywrightCrawlingContext) -> None: @@ -992,7 +998,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: send_request_response = await context.send_request(str(server_url / 'user-agent')) check_data['send_request'] = dict(json.loads(await send_request_response.read())) - await crawler.run([str(server_url / 'user-agent')]) + async with http_client: + await crawler.run([str(server_url / 'user-agent')]) assert check_data['default'].get('user-agent') is not None assert check_data['send_request']['user-agent'] == 'My User-Agent' @@ -1518,9 +1525,10 @@ async def test_throttled_429_is_deferred(monkeypatch: pytest.MonkeyPatch) -> Non record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = PlaywrightCrawler( request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, max_session_rotations=0, ) @@ -1537,7 +1545,8 @@ async def fulfill_with_next_status(context: PlaywrightPreNavCrawlingContext) -> async def handler(context: PlaywrightCrawlingContext) -> None: handled.append(context) - await crawler.run([url]) + async with session_pool: + await crawler.run([url]) assert [context.response.status for context in handled] == [200] failed_request_handler.assert_not_called() diff --git a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py index 905878c17f..37ade376b3 100644 --- a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py +++ b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py @@ -71,7 +71,8 @@ async def request_handler(context: PydanticAiCrawlingContext) -> None: await context.extract(_Article, scope='article', additional_instructions='hint') with patch.object(extractor, 'extract', extract_mock): - await crawler.run([Request.from_url(str(server_url), label='detail')]) + async with extractor: + await crawler.run([Request.from_url(str(server_url), label='detail')]) first_call, second_call, third_call = extract_mock.call_args_list diff --git a/tests/unit/_statistics/test_enter_exit.py b/tests/unit/_statistics/test_enter_exit.py new file mode 100644 index 0000000000..2324af6941 --- /dev/null +++ b/tests/unit/_statistics/test_enter_exit.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from crawlee.statistics import Statistics + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('initialize failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_stats(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """Statistics whose entry fails or is cancelled stay inactive and can be entered again.""" + statistics = Statistics.with_default_state() + monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with statistics: + pass + + assert not statistics.active + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_failed_exit_frees_stats(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose exit fails are no longer active, so they can be entered again.""" + statistics = Statistics.with_default_state() + + with pytest.raises(RuntimeError, match='teardown failed'): + async with statistics: + monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not statistics.active + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics can't be entered again until their exit has torn the state down.""" + statistics = Statistics.with_default_state() + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await statistics.__aenter__() + monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(statistics.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await statistics.__aenter__() + + release_teardown.set() + await exit_task + + +async def test_failed_start_tears_down_stats(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose entry fails after loading the state tear the state down and can be entered again.""" + statistics = Statistics.with_default_state() + teardown = AsyncMock(wraps=statistics._state.teardown) + monkeypatch.setattr(statistics._state, 'teardown', teardown) + monkeypatch.setattr(statistics._periodic_logger, 'start', Mock(side_effect=RuntimeError('start failed'))) + + with pytest.raises(RuntimeError, match='start failed'): + async with statistics: + pass + + assert not statistics.active + teardown.assert_awaited_once() + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_cancelled_exit_finishes_run(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose exit is cancelled still record when the run finished and tear their state down.""" + statistics = Statistics.with_default_state() + + with pytest.raises(asyncio.CancelledError): + async with statistics: + monkeypatch.setattr(statistics._periodic_logger, 'stop', AsyncMock(side_effect=asyncio.CancelledError())) + + assert statistics.state.crawler_finished_at is not None + assert statistics._state.is_initialized is False + monkeypatch.undo() + await statistics._periodic_logger.stop() diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 61cb89bedb..94d84a87b3 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -9,7 +9,9 @@ import pytest from crawlee.autoscaling import ConcurrencySystem +from crawlee.browsers import BrowserPool from crawlee.crawlers import AdaptivePlaywrightCrawler +from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor if TYPE_CHECKING: from crawlee.crawlers import AdaptivePlaywrightCrawlingContext @@ -23,6 +25,26 @@ async def test_accepts_concurrency_system() -> None: assert crawler.concurrency_system is concurrency_system +async def test_run_needs_entered_predictor() -> None: + """A rendering type predictor passed to the crawler has to be entered before the crawler runs.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=DefaultRenderingTypePredictor() + ) + + with pytest.raises(RuntimeError, match='DefaultRenderingTypePredictor passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_run_needs_entered_browser_pool() -> None: + """A browser pool passed to the Playwright sub crawler has to be entered before the crawler runs.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + playwright_crawler_specific_kwargs={'browser_pool': BrowserPool()} + ) + + with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + @pytest.mark.parametrize( 'optional_module_name', [ diff --git a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py index 681307cc9e..72d01f7515 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py @@ -1,5 +1,8 @@ from __future__ import annotations +import asyncio +from unittest.mock import AsyncMock + import pytest from crawlee import Request @@ -217,3 +220,63 @@ def test_url_similarity(url_1: str, url_2: str, expected_rounded_similarity: flo round(calculate_url_similarity(url_1=get_url_components(url_1), url_2=get_url_components(url_2)), 2) == expected_rounded_similarity ) + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('initialize failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_predictor(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A predictor whose entry fails or is cancelled stays inactive and can be entered again.""" + predictor = DefaultRenderingTypePredictor() + monkeypatch.setattr(predictor._state, 'initialize', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with predictor: + pass + + assert not predictor.active + monkeypatch.undo() + + async with predictor: + assert predictor.active + + +async def test_failed_exit_frees_predictor(monkeypatch: pytest.MonkeyPatch) -> None: + """A predictor whose exit fails is no longer active, so it can be entered again.""" + predictor = DefaultRenderingTypePredictor() + + with pytest.raises(RuntimeError, match='teardown failed'): + async with predictor: + monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not predictor.active + monkeypatch.undo() + + async with predictor: + assert predictor.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A predictor can't be entered again until its exit has torn the state down.""" + predictor = DefaultRenderingTypePredictor() + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await predictor.__aenter__() + monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(predictor.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await predictor.__aenter__() + + release_teardown.set() + await exit_task diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 1577561704..3e7f93330f 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -45,7 +45,7 @@ EventMigratingData, LocalEventManager, ) -from crawlee.http_clients import HttpClient, HttpResponse +from crawlee.http_clients import HttpClient, HttpResponse, ImpitHttpClient from crawlee.proxy_configuration import ProxyConfiguration from crawlee.request_loaders import ( FinishedStatus, @@ -625,7 +625,7 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception async def test_send_request_forwards_arguments_to_http_client() -> None: """Test that `context.send_request` passes all its arguments to the HTTP client.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @@ -667,7 +667,7 @@ async def handler(context: BasicCrawlingContext) -> None: async def test_send_request_default_timeout() -> None: """Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = Mock(spec=HttpResponse) crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @@ -689,7 +689,7 @@ async def handler(context: BasicCrawlingContext) -> None: ) async def test_send_request_response_closed_after_request(*, handler_fails: bool) -> None: """Test that a response of `context.send_request` stays open in the handler and is closed after the request.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response crawler = BasicCrawler(http_client=http_client, max_request_retries=0, max_requests_per_crawl=1) @@ -1478,6 +1478,164 @@ async def test_repeated_run_starts_with_fresh_statistics() -> None: assert crawler.statistics.state.requests_finished == 1 +async def test_crawlers_share_stats_in_turn() -> None: + """Statistics passed to two crawlers that run one after the other add up the requests and the runtime of both.""" + statistics = Statistics.with_default_state() + first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + await first_crawler.run(['https://a.placeholder.com']) + started_at = statistics.state.crawler_started_at + first_runtime = statistics.state.crawler_runtime + stats = await second_crawler.run(['https://b.placeholder.com']) + + assert stats.requests_finished == 2 + assert statistics.state.crawler_started_at == started_at + assert statistics.state.accumulated_runtime == first_runtime + + +async def test_shared_stats_concurrent_raises() -> None: + """Two crawlers can't run with the same statistics at the same time, since the first one has them entered.""" + statistics = Statistics.with_default_state() + handler_started = asyncio.Event() + release_handler = asyncio.Event() + + async def blocking_handler(_context: BasicCrawlingContext) -> None: + handler_started.set() + await release_handler.wait() + + first_crawler = BasicCrawler(request_handler=blocking_handler, statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com'])) + await handler_started.wait() + + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await second_crawler.run(['https://b.placeholder.com']) + + release_handler.set() + stats = await first_run + + assert stats.requests_finished == 1 + + +async def test_shared_stats_started_together(monkeypatch: pytest.MonkeyPatch) -> None: + """A crawler started while another crawler is still entering the shared statistics raises.""" + statistics = Statistics.with_default_state() + entry_started = asyncio.Event() + release_entry = asyncio.Event() + initialize = statistics._state.initialize + + async def hold_first_initialize() -> StatisticsState: + # Only the first entry waits, so that an unblocked second entry finishes instead of hanging the test. + if not entry_started.is_set(): + entry_started.set() + await release_entry.wait() + return await initialize() + + monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=hold_first_initialize)) + first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com'])) + await entry_started.wait() + + # The first entry makes the statistics active before its first await, so the second crawler stops in `run()`. + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await second_crawler.run(['https://b.placeholder.com']) + + release_entry.set() + stats = await first_run + + assert stats.requests_finished == 1 + assert not statistics.active + + +async def test_run_after_failed_exit(monkeypatch: pytest.MonkeyPatch) -> None: + """A failed exit of an object the crawler built doesn't keep it active, so the next run works.""" + monkeypatch.setattr(ImpitHttpClient, 'cleanup', AsyncMock(side_effect=[RuntimeError('cleanup failed'), None])) + handler = AsyncMock() + crawler = BasicCrawler(request_handler=handler) + + with pytest.raises(RuntimeError, match='cleanup failed'): + await crawler.run(['https://a.placeholder.com']) + + await crawler.run(['https://b.placeholder.com']) + + assert handler.await_count == 2 + + +async def test_run_rejects_entered_stats() -> None: + """Statistics the caller entered make `run()` raise, since the crawler enters them itself.""" + async with Statistics.with_default_state() as statistics: + crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_repeated_run_keeps_sessions() -> None: + """A repeated `run()` keeps the sessions of a persisted session pool passed to the crawler, and leaves it open.""" + async with SessionPool(max_pool_size=1, persistence_enabled=True) as session_pool: + session_ids = set(session_pool.get_state(as_dict=False).sessions) + crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + assert set(session_pool.get_state(as_dict=False).sessions) == session_ids + assert session_pool.active + + +async def test_repeated_run_resets_own_sessions() -> None: + """A repeated `run()` replaces the sessions of the session pool the crawler built.""" + crawler = BasicCrawler() + session_ids = list[set[str]]() + + @crawler.router.default_handler + async def handler(_context: BasicCrawlingContext) -> None: + session_ids.append(set(crawler._session_pool.get_state(as_dict=False).sessions)) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + first_ids, second_ids = session_ids + assert first_ids.isdisjoint(second_ids) + + +async def test_run_needs_entered_session_pool() -> None: + """A session pool passed to the crawler has to be entered before the crawler runs.""" + session_pool = SessionPool() + crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool) + + with pytest.raises(RuntimeError, match='SessionPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + async with session_pool: + stats = await crawler.run(['https://a.placeholder.com']) + + assert stats.requests_finished == 1 + + +async def test_run_needs_entered_http_client() -> None: + """An HTTP client passed to the crawler has to be entered before the crawler runs.""" + http_client = ImpitHttpClient() + crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client) + + with pytest.raises(RuntimeError, match='ImpitHttpClient passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_passed_http_client_left_open() -> None: + """The crawler doesn't close an HTTP client passed to it.""" + http_client = ImpitHttpClient() + crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client) + + async with http_client: + await crawler.run(['https://a.placeholder.com']) + assert http_client.active + + def test_crawler_log() -> None: crawler = BasicCrawler() assert isinstance(crawler.log, logging.Logger) @@ -2968,7 +3126,8 @@ async def signal_idle_reached(*args: object, **kwargs: object) -> bool: ], ) async def test_session_retire_in_user_handler(*, retire: bool) -> None: - crawler = BasicCrawler(session_pool=SessionPool(max_pool_size=1)) + session_pool = SessionPool(max_pool_size=1) + crawler = BasicCrawler(session_pool=session_pool) sessions = list[str]() @crawler.router.default_handler @@ -2980,7 +3139,8 @@ async def handler(context: BasicCrawlingContext) -> None: await context.add_requests(['https://b.placeholder.com']) - await crawler.run(['https://a.placeholder.com']) + async with session_pool: + await crawler.run(['https://a.placeholder.com']) # The session should differ if `retire` was called and match otherwise since pool size == 1 if retire: @@ -3024,7 +3184,8 @@ def create_session() -> Session: check_sessions = [str(session_id) for session_id in range(10)] used_sessions = list[str]() - crawler = BasicCrawler(session_pool=SessionPool(create_session_function=create_session_function())) + session_pool = SessionPool(create_session_function=create_session_function()) + crawler = BasicCrawler(session_pool=session_pool) @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: @@ -3035,7 +3196,8 @@ async def handler(context: BasicCrawlingContext) -> None: for session_id in range(10) ] - await crawler.run(requests) + async with session_pool: + await crawler.run(requests) assert len(used_sessions) == 10 assert set(used_sessions) == set(check_sessions) @@ -3524,20 +3686,23 @@ async def test_crawler_intermediate_statistics() -> None: crawler = BasicCrawler() check_time = timedelta(seconds=0.1) + handler_started = asyncio.Event() + @crawler.router.default_handler async def handler(_: BasicCrawlingContext) -> None: + handler_started.set() await asyncio.sleep(check_time.total_seconds() * 5) - # Start crawler and wait until statistics are initialized. Use a generous timeout so that crawler startup - # has enough time to reach the active state on slow CI runners under xdist load. + # The handler starts only once the statistics are entered and measure the run. Use a generous timeout so that + # crawler startup has enough time on slow CI runners under xdist load. crawler_task = asyncio.create_task(crawler.run(['https://a.placeholder.com'])) - assert await poll_until_condition(lambda: crawler.statistics.active, timeout=30) - active_seen_at = datetime.now(UTC) + await asyncio.wait_for(handler_started.wait(), timeout=30) + started_seen_at = datetime.now(UTC) # Wait some time and check that runtime is updated. The runtime is measured on the wall clock, which on Windows # ticks at ~15.6 ms independently of the `asyncio.sleep` clock, so bound it by a wall-clock interval inside the run. await asyncio.sleep(check_time.total_seconds()) - min_runtime = datetime.now(UTC) - active_seen_at + min_runtime = datetime.now(UTC) - started_seen_at crawler.statistics.calculate() assert min_runtime > timedelta(0) assert crawler.statistics.state.crawler_runtime >= min_runtime @@ -3929,16 +4094,18 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo """A `RequestThrottledError` from the handler starts a backoff without costing a retry or session reputation.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = BasicCrawler( request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, ) crawler.failed_request_handler(failed_request_handler) handler = AsyncMock(side_effect=[RequestThrottledError, None]) crawler.router.default_handler(handler) - stats = await crawler.run([THROTTLED_URL]) + async with session_pool: + stats = await crawler.run([THROTTLED_URL]) assert handler.await_count == 2 failed_request_handler.assert_not_called() diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py index d80425ad8c..b15ccbb10f 100644 --- a/tests/unit/crawlers/_http/test_http_crawler.py +++ b/tests/unit/crawlers/_http/test_http_crawler.py @@ -51,7 +51,7 @@ async def test_navigation_response_closed_after_request(*, status_code: int, exp """Test that the response of the navigation request stays open in the handler and is closed after the request.""" url = 'https://a.placeholder.com' response = mock_response(status_code, url) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = response crawler = HttpCrawler(http_client=http_client, max_request_retries=0) handler_call = Mock() @@ -71,7 +71,7 @@ async def test_body_read_bound_by_navigation_timeout() -> None: url = 'https://a.placeholder.com' response = mock_response(200, url) response.read.side_effect = asyncio.Event().wait - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = response crawler = HttpCrawler( http_client=http_client, @@ -100,14 +100,15 @@ async def test_body_read_bound_by_navigation_timeout() -> None: async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None: """A throttled 429 is retried later without spending a retry, a session rotation or session reputation.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = HttpCrawler( http_client=http_client, request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, max_session_rotations=0, **crawler_kwargs, @@ -119,7 +120,8 @@ async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeyp async def handler(context: HttpCrawlingContext) -> None: handled.append(context) - await crawler.run([url]) + async with session_pool: + await crawler.run([url]) assert [context.http_response.status_code for context in handled] == [200] failed_request_handler.assert_not_called() @@ -134,7 +136,7 @@ async def handler(context: HttpCrawlingContext) -> None: async def test_tandem_429_deferred(monkeypatch: pytest.MonkeyPatch) -> None: """A 429 reaches a throttler wrapped in a `RequestManagerTandem`, so the request is deferred without a retry.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] failed_request_handler = AsyncMock() @@ -163,7 +165,7 @@ async def test_429_reported_once(monkeypatch: pytest.MonkeyPatch) -> None: throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] crawler = HttpCrawler(http_client=http_client, request_manager=throttler) diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 253e3adcb7..d0a0c815c0 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -11,8 +11,10 @@ service_locator, ) from crawlee.autoscaling import ConcurrencySystem +from crawlee.browsers import BrowserPool from crawlee.configuration import Configuration from crawlee.crawlers import ( + BasicCrawler, PlaywrightCrawler, ) @@ -50,6 +52,25 @@ async def slow_handler(_context: PlaywrightCrawlingContext) -> None: assert crawler._get_expected_processing_time() == timedelta(minutes=5) +async def test_run_needs_entered_browser_pool() -> None: + """A browser pool passed to the crawler has to be entered before the crawler runs.""" + crawler = PlaywrightCrawler(browser_pool=BrowserPool()) + + with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +def test_unsupported_browser_type_takes_no_id() -> None: + """A crawler with a browser type its default fingerprint generator doesn't support doesn't use up a crawler id.""" + first_crawler = BasicCrawler() + + with pytest.raises(ValueError, match='Unsupported browser type'): + PlaywrightCrawler(browser_type='edge') # ty: ignore[invalid-argument-type] + + second_crawler = BasicCrawler() + assert int(second_crawler._id) == int(first_crawler._id) + 1 + + def test_import_error_handled() -> None: blocked = { mod_name: None for mod_name in sys.modules if mod_name == 'playwright' or mod_name.startswith('playwright.') diff --git a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py index 759a277b18..74ffa04260 100644 --- a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py +++ b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py @@ -159,3 +159,14 @@ async def test_raise_for_invalid_output() -> None: # The extractor's usage stats reflect the 3 failed attempts (1 initial + 2 retries). assert extractor.ai_usage.requests == 3 + + +async def test_enter_sets_active() -> None: + """Entering the extractor makes it active until it's exited.""" + extractor = PydanticAiDirectExtractor(TestModel()) + assert extractor.active is False + + async with extractor: + assert extractor.active is True + + assert extractor.active is False diff --git a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py index 7cd2aaad4d..356dd3e79f 100644 --- a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py +++ b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py @@ -1,14 +1,17 @@ from __future__ import annotations import sys -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest +from pydantic_ai.exceptions import UserError from pydantic_ai.models.test import TestModel from crawlee.crawlers import ( + BasicCrawler, PydanticAiCrawler, PydanticAiDirectExtractor, + PydanticAiSelectorExtractor, ) @@ -42,6 +45,37 @@ def test_exposes_extractor_and_usage() -> None: assert crawler.ai_usage is extractor.ai_usage +@pytest.mark.parametrize( + 'extractor_class', + [ + pytest.param(PydanticAiDirectExtractor, id='direct'), + pytest.param(PydanticAiSelectorExtractor, id='selector'), + ], +) +async def test_run_needs_entered_extractor( + extractor_class: type[PydanticAiDirectExtractor | PydanticAiSelectorExtractor], +) -> None: + """An extractor passed to the crawler has to be entered before the crawler runs.""" + crawler = PydanticAiCrawler(extractor=extractor_class(TestModel())) + + with pytest.raises(RuntimeError, match=f'{extractor_class.__name__} passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_owns_default_extractor(monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler enters and exits the default extractor it builds.""" + extractor_enter = AsyncMock() + extractor_exit = AsyncMock() + monkeypatch.setattr(PydanticAiDirectExtractor, '__aenter__', extractor_enter) + monkeypatch.setattr(PydanticAiDirectExtractor, '__aexit__', extractor_exit) + crawler = PydanticAiCrawler(model=TestModel()) + + await crawler.run() + + extractor_enter.assert_awaited_once() + extractor_exit.assert_awaited_once() + + def test_import_error_handled() -> None: # The `ai` extra is optional, so accessing the crawler without `pydantic_ai` installed must raise a clear error. blocked = {name: None for name in sys.modules if name == 'pydantic_ai' or name.startswith('pydantic_ai.')} @@ -51,3 +85,14 @@ def test_import_error_handled() -> None: sys.modules.pop(name, None) with pytest.raises(ImportError): from crawlee.crawlers._pydantic_ai import PydanticAiCrawler # noqa: F401 PLC0415 + + +def test_unknown_model_takes_no_id() -> None: + """A crawler with an unknown model doesn't use up a crawler id.""" + first_crawler = BasicCrawler() + + with pytest.raises(UserError, match='Unknown model'): + PydanticAiCrawler(model='no-such-provider:model') + + second_crawler = BasicCrawler() + assert int(second_crawler._id) == int(first_crawler._id) + 1 diff --git a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py index 2604ca53da..3e052cdf5e 100644 --- a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py +++ b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -41,6 +41,28 @@ def test_init_with_valid_browser_pool() -> None: StagehandCrawler(browser_pool=pool) +async def test_run_needs_entered_browser_pool() -> None: + """A browser pool passed to the crawler has to be entered before the crawler runs.""" + crawler = StagehandCrawler(browser_pool=BrowserPool(plugins=[StagehandBrowserPlugin()])) + + with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_owns_default_browser_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler enters and exits the browser pool it builds itself.""" + pool_enter = AsyncMock() + pool_exit = AsyncMock() + monkeypatch.setattr(BrowserPool, '__aenter__', pool_enter) + monkeypatch.setattr(BrowserPool, '__aexit__', pool_exit) + crawler = StagehandCrawler() + + await crawler.run() + + pool_enter.assert_awaited_once() + pool_exit.assert_awaited_once() + + def test_forwards_save_response_cookies() -> None: """`save_response_cookies` reaches the `PlaywrightCrawler` that saves the cookies of the page.""" crawler = StagehandCrawler(save_response_cookies=False) diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index adde6d710d..cb365ba98f 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -312,3 +312,25 @@ async def test_ignore_tls_errors_reaches_fetch( await client.send_request(url, proxy_info=proxy_info, ignore_tls_errors=request_flag) assert fetch.call_args.kwargs['ignore_tls_errors'] is expected_flag + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A client can't be entered again until its exit has cleaned it up.""" + http_client = ImpitHttpClient() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + async def wait_for_release() -> None: + cleanup_started.set() + await release_cleanup.wait() + + await http_client.__aenter__() + monkeypatch.setattr(http_client, 'cleanup', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(http_client.__aexit__(None, None, None)) + await cleanup_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await http_client.__aenter__() + + release_cleanup.set() + await exit_task diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py index 404612af69..8c29723737 100644 --- a/tests/unit/sessions/test_session_pool.py +++ b/tests/unit/sessions/test_session_pool.py @@ -1,8 +1,10 @@ from __future__ import annotations +import asyncio import logging from datetime import UTC, datetime from typing import TYPE_CHECKING +from unittest.mock import AsyncMock import pytest @@ -275,3 +277,66 @@ async def test_methods_raise_error_when_not_active() -> None: async with session_pool: assert session_pool.active is True + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('fill failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_pool(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A pool whose entry fails or is cancelled tears its state down, stays inactive and can be entered again.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + teardown = AsyncMock(wraps=session_pool._state.teardown) + monkeypatch.setattr(session_pool._state, 'teardown', teardown) + monkeypatch.setattr(session_pool, '_fill_sessions_to_max', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with session_pool: + pass + + assert not session_pool.active + teardown.assert_awaited_once() + monkeypatch.undo() + + async with session_pool: + assert session_pool.active + + +async def test_failed_exit_frees_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool whose exit fails is no longer active, so it can be entered again.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + + with pytest.raises(RuntimeError, match='teardown failed'): + async with session_pool: + monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not session_pool.active + monkeypatch.undo() + + async with session_pool: + assert session_pool.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool can't be entered again until its exit has torn the state down.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await session_pool.__aenter__() + monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(session_pool.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await session_pool.__aenter__() + + release_teardown.set() + await exit_task