diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 82609c9d68..45b6c5130a 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -520,6 +520,24 @@ async with session_pool: When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too. +The `with_beautifulsoup_static_parser` and `with_parsel_static_parser` factory methods of `AdaptivePlaywrightCrawler` pass `statistics` to the crawler as they are, instead of a copy built with `Statistics.replace_state_model`, so the instance you pass is the one that counts. The instance must use `AdaptivePlaywrightCrawlerStatisticState` as its state model: + +```python +# v1 +crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + statistics=Statistics.with_default_state(), +) +await crawler.run(urls) + +# v2 +crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + statistics=Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState), +) +await crawler.run(urls) +``` + +`Statistics.replace_state_model` is removed, since it existed only for that copy. To get statistics with another state model, create a new `Statistics` instance with the `state_model` you need. + `RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`. `AdaptivePlaywrightCrawler.rendering_type_predictor` is now read-only. Pass your predictor to the constructor instead. diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index b2145101e0..f1085fa2ab 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -147,6 +147,9 @@ def __init__( statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of non-default configuration. The crawler enters it on every run, so don't enter it yourself. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. + + Raises: + ValueError: If the state model of `statistics` isn't `AdaptivePlaywrightCrawlerStatisticState`. """ # Adaptive crawling related. self.result_checker = result_checker or (lambda _: True) @@ -156,6 +159,13 @@ def __init__( if kwargs.get('concurrency_settings') is None and kwargs.get('concurrency_system') is None: kwargs['concurrency_settings'] = ConcurrencySettings(desired_concurrency=1) + # The crawler counts its own fields in the statistics, which would fail on every request with another model. + if statistics is not None and not issubclass(statistics._state_model, AdaptivePlaywrightCrawlerStatisticState): # noqa: SLF001 + raise ValueError( + 'The `statistics` of `AdaptivePlaywrightCrawler` must use `AdaptivePlaywrightCrawlerStatisticState` as ' + 'their state model, e.g. `Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState)`.' + ) + adaptive_statistics = statistics or Statistics( state_model=AdaptivePlaywrightCrawlerStatisticState, periodic_message_logger=kwargs.get('logger') @@ -258,13 +268,10 @@ def with_beautifulsoup_static_parser( parser_type: BeautifulSoupParserType = 'lxml', playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None, save_response_cookies: bool = True, - statistics: Statistics[StatisticsState] | None = None, + statistics: Statistics[AdaptivePlaywrightCrawlerStatisticState] | None = None, **kwargs: Unpack[_BasicCrawlerOptions], ) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[BeautifulSoup], BeautifulSoup, Tag]: """Create `AdaptivePlaywrightCrawler` that uses `BeautifulSoup` for parsing static content.""" - adaptive_statistics = ( - statistics.replace_state_model(AdaptivePlaywrightCrawlerStatisticState) if statistics is not None else None - ) return AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[BeautifulSoup], BeautifulSoup, Tag]( rendering_type_predictor=rendering_type_predictor, result_checker=result_checker, @@ -272,7 +279,7 @@ def with_beautifulsoup_static_parser( static_parser=BeautifulSoupParser(parser=parser_type), playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs, save_response_cookies=save_response_cookies, - statistics=adaptive_statistics, + statistics=statistics, **kwargs, ) @@ -285,13 +292,10 @@ def with_parsel_static_parser( result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None, save_response_cookies: bool = True, - statistics: Statistics[StatisticsState] | None = None, + statistics: Statistics[AdaptivePlaywrightCrawlerStatisticState] | None = None, **kwargs: Unpack[_BasicCrawlerOptions], ) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[Selector], Selector, Selector]: """Create `AdaptivePlaywrightCrawler` that uses `Parcel` for parsing static content.""" - adaptive_statistics = ( - statistics.replace_state_model(AdaptivePlaywrightCrawlerStatisticState) if statistics is not None else None - ) return AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[Selector], Selector, Selector]( rendering_type_predictor=rendering_type_predictor, result_checker=result_checker, @@ -299,7 +303,7 @@ def with_parsel_static_parser( static_parser=ParselParser(), playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs, save_response_cookies=save_response_cookies, - statistics=adaptive_statistics, + statistics=statistics, **kwargs, ) diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py index 3c684a4da1..bcee222414 100644 --- a/src/crawlee/statistics/_statistics.py +++ b/src/crawlee/statistics/_statistics.py @@ -24,7 +24,6 @@ from crawlee.storages import KeyValueStore TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState) -TNewStatisticsState = TypeVar('TNewStatisticsState', bound=StatisticsState, default=StatisticsState) logger = getLogger(__name__) @@ -97,6 +96,7 @@ def __init__( self.error_tracker_retry = ErrorTracker(save_error_snapshots=False) self._requests_in_progress = dict[str, RequestProcessingRecord]() + self._state_model = state_model self._state = RecoverableState( default_state=state_model(stats_id=self._id), @@ -115,20 +115,6 @@ def __init__( # Flag to indicate the context state. self._active = False - def replace_state_model(self, state_model: type[TNewStatisticsState]) -> Statistics[TNewStatisticsState]: - """Create near copy of the `Statistics` with replaced `state_model`.""" - new_statistics: Statistics[TNewStatisticsState] = Statistics( - persistence_enabled=self._state._persistence_enabled, # noqa: SLF001 - persist_state_key=self._state._persist_state_key, # noqa: SLF001 - persist_state_kvs_factory=self._state._persist_state_kvs_factory, # noqa: SLF001 - log_message=self._log_message, - periodic_message_logger=self._periodic_message_logger, - state_model=state_model, - id=self._id, - ) - new_statistics._periodic_logger = self._periodic_logger # Accessing private member to create copy like-object. - return new_statistics - @staticmethod def with_default_state( *, diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 2db1cf4574..0816dfc501 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import logging from contextlib import suppress from dataclasses import dataclass from datetime import timedelta @@ -591,33 +590,18 @@ async def test_passed_predictor_left_open(test_urls: list[str]) -> None: assert predictor.active -async def test_adaptive_playwright_crawler_statistics_in_init() -> None: - """Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics.""" - persistence_enabled = True - persist_state_kvs_name = 'some-name' - persist_state_key = 'come key' - log_message = 'some message' - periodic_message_logger = logging.getLogger('some logger') - log_interval = timedelta(minutes=2) - statistics = Statistics.with_default_state( - persistence_enabled=persistence_enabled, - persist_state_kvs_name=persist_state_kvs_name, - persist_state_key=persist_state_key, - log_message=log_message, - periodic_message_logger=periodic_message_logger, - log_interval=log_interval, +async def test_passed_stats_count_runs(test_urls: list[str]) -> None: + """Statistics passed to a factory method count the adaptive fields of the crawl.""" + statistics = Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState) + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])), + statistics=statistics, ) + crawler.router.default_handler(AsyncMock()) - crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(statistics=statistics) - await crawler.run([]) # ensure that statistics get initialized - - assert type(crawler._statistics.state) is AdaptivePlaywrightCrawlerStatisticState - - assert crawler._statistics._state._persistence_enabled == persistence_enabled - assert crawler._statistics._state._persist_state_key == persist_state_key + await crawler.run(test_urls[:1]) - assert crawler._statistics._log_message == log_message - assert crawler._statistics._periodic_message_logger == periodic_message_logger + assert statistics.state.http_only_request_handler_runs == 1 async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_urls: list[str]) -> None: diff --git a/tests/unit/_statistics/test_persistence.py b/tests/unit/_statistics/test_persistence.py index cc00ef41d1..8c7a54da0f 100644 --- a/tests/unit/_statistics/test_persistence.py +++ b/tests/unit/_statistics/test_persistence.py @@ -274,13 +274,6 @@ def test_id_defaults_to_counter() -> None: assert Statistics.with_default_state()._id == '2' -def test_replace_state_model_keeps_id() -> None: - """`replace_state_model` keeps the ID of the original statistics.""" - statistics = Statistics.with_default_state(id='custom') - - assert statistics.replace_state_model(StatisticsState)._id == 'custom' - - async def test_reentering_keeps_counters_and_runtime() -> None: """Entering the statistics again keeps the counters and the runtime of the previous run.""" statistics = Statistics.with_default_state() diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 94d84a87b3..19acc795cc 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -11,7 +11,11 @@ from crawlee.autoscaling import ConcurrencySystem from crawlee.browsers import BrowserPool from crawlee.crawlers import AdaptivePlaywrightCrawler +from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawler_statistics import ( + AdaptivePlaywrightCrawlerStatisticState, +) from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor +from crawlee.statistics import Statistics if TYPE_CHECKING: from crawlee.crawlers import AdaptivePlaywrightCrawlingContext @@ -45,6 +49,22 @@ async def test_run_needs_entered_browser_pool() -> None: await crawler.run(['https://a.placeholder.com']) +def test_factory_passes_stats_as_is() -> None: + """A factory method passes the given statistics to the crawler as they are.""" + statistics = Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState) + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(statistics=statistics) + + assert crawler.statistics is statistics + + +def test_rejects_plain_statistics() -> None: + """Statistics without the adaptive state model are rejected when the crawler is created.""" + with pytest.raises(ValueError, match='AdaptivePlaywrightCrawlerStatisticState'): + AdaptivePlaywrightCrawler.with_parsel_static_parser( + statistics=Statistics.with_default_state() # ty: ignore[invalid-argument-type] + ) + + @pytest.mark.parametrize( 'optional_module_name', [