Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/06_upgrading/upgrading_to_v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,24 @@ async with session_pool:

When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too.

The `with_beautifulsoup_static_parser` and `with_parsel_static_parser` factory methods of `AdaptivePlaywrightCrawler` pass `statistics` to the crawler as they are, instead of a copy built with `Statistics.replace_state_model`, so the instance you pass is the one that counts. The instance must use `AdaptivePlaywrightCrawlerStatisticState` as its state model:

```python
# v1
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
statistics=Statistics.with_default_state(),
)
await crawler.run(urls)

# v2
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
statistics=Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState),
)
await crawler.run(urls)
```

`Statistics.replace_state_model` is removed, since it existed only for that copy. To get statistics with another state model, create a new `Statistics` instance with the `state_model` you need.

`RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`.

`AdaptivePlaywrightCrawler.rendering_type_predictor` is now read-only. Pass your predictor to the constructor instead.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ def __init__(
statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of
non-default configuration. The crawler enters it on every run, so don't enter it yourself.
kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`.

Raises:
ValueError: If the state model of `statistics` isn't `AdaptivePlaywrightCrawlerStatisticState`.
"""
# Adaptive crawling related.
self.result_checker = result_checker or (lambda _: True)
Expand All @@ -156,6 +159,13 @@ def __init__(
if kwargs.get('concurrency_settings') is None and kwargs.get('concurrency_system') is None:
kwargs['concurrency_settings'] = ConcurrencySettings(desired_concurrency=1)

# The crawler counts its own fields in the statistics, which would fail on every request with another model.
if statistics is not None and not issubclass(statistics._state_model, AdaptivePlaywrightCrawlerStatisticState): # noqa: SLF001
raise ValueError(
'The `statistics` of `AdaptivePlaywrightCrawler` must use `AdaptivePlaywrightCrawlerStatisticState` as '
'their state model, e.g. `Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState)`.'
)

adaptive_statistics = statistics or Statistics(
state_model=AdaptivePlaywrightCrawlerStatisticState,
periodic_message_logger=kwargs.get('logger')
Expand Down Expand Up @@ -258,21 +268,18 @@ def with_beautifulsoup_static_parser(
parser_type: BeautifulSoupParserType = 'lxml',
playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None,
save_response_cookies: bool = True,
statistics: Statistics[StatisticsState] | None = None,
statistics: Statistics[AdaptivePlaywrightCrawlerStatisticState] | None = None,
**kwargs: Unpack[_BasicCrawlerOptions],
) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[BeautifulSoup], BeautifulSoup, Tag]:
"""Create `AdaptivePlaywrightCrawler` that uses `BeautifulSoup` for parsing static content."""
adaptive_statistics = (
statistics.replace_state_model(AdaptivePlaywrightCrawlerStatisticState) if statistics is not None else None
)
return AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[BeautifulSoup], BeautifulSoup, Tag](
rendering_type_predictor=rendering_type_predictor,
result_checker=result_checker,
result_comparator=result_comparator,
static_parser=BeautifulSoupParser(parser=parser_type),
playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs,
save_response_cookies=save_response_cookies,
statistics=adaptive_statistics,
statistics=statistics,
**kwargs,
)

Expand All @@ -285,21 +292,18 @@ def with_parsel_static_parser(
result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None,
playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None,
save_response_cookies: bool = True,
statistics: Statistics[StatisticsState] | None = None,
statistics: Statistics[AdaptivePlaywrightCrawlerStatisticState] | None = None,
**kwargs: Unpack[_BasicCrawlerOptions],
) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[Selector], Selector, Selector]:
"""Create `AdaptivePlaywrightCrawler` that uses `Parcel` for parsing static content."""
adaptive_statistics = (
statistics.replace_state_model(AdaptivePlaywrightCrawlerStatisticState) if statistics is not None else None
)
return AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[Selector], Selector, Selector](
rendering_type_predictor=rendering_type_predictor,
result_checker=result_checker,
result_comparator=result_comparator,
static_parser=ParselParser(),
playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs,
save_response_cookies=save_response_cookies,
statistics=adaptive_statistics,
statistics=statistics,
**kwargs,
)

Expand Down
16 changes: 1 addition & 15 deletions src/crawlee/statistics/_statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from crawlee.storages import KeyValueStore

TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
TNewStatisticsState = TypeVar('TNewStatisticsState', bound=StatisticsState, default=StatisticsState)
logger = getLogger(__name__)


Expand Down Expand Up @@ -97,6 +96,7 @@ def __init__(
self.error_tracker_retry = ErrorTracker(save_error_snapshots=False)

self._requests_in_progress = dict[str, RequestProcessingRecord]()
self._state_model = state_model

self._state = RecoverableState(
default_state=state_model(stats_id=self._id),
Expand All @@ -115,20 +115,6 @@ def __init__(
# Flag to indicate the context state.
self._active = False

def replace_state_model(self, state_model: type[TNewStatisticsState]) -> Statistics[TNewStatisticsState]:
"""Create near copy of the `Statistics` with replaced `state_model`."""
new_statistics: Statistics[TNewStatisticsState] = Statistics(
persistence_enabled=self._state._persistence_enabled, # noqa: SLF001
persist_state_key=self._state._persist_state_key, # noqa: SLF001
persist_state_kvs_factory=self._state._persist_state_kvs_factory, # noqa: SLF001
log_message=self._log_message,
periodic_message_logger=self._periodic_message_logger,
state_model=state_model,
id=self._id,
)
new_statistics._periodic_logger = self._periodic_logger # Accessing private member to create copy like-object.
return new_statistics

@staticmethod
def with_default_state(
*,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import asyncio
import logging
from contextlib import suppress
from dataclasses import dataclass
from datetime import timedelta
Expand Down Expand Up @@ -591,33 +590,18 @@ async def test_passed_predictor_left_open(test_urls: list[str]) -> None:
assert predictor.active


async def test_adaptive_playwright_crawler_statistics_in_init() -> None:
"""Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics."""
persistence_enabled = True
persist_state_kvs_name = 'some-name'
persist_state_key = 'come key'
log_message = 'some message'
periodic_message_logger = logging.getLogger('some logger')
log_interval = timedelta(minutes=2)
statistics = Statistics.with_default_state(
persistence_enabled=persistence_enabled,
persist_state_kvs_name=persist_state_kvs_name,
persist_state_key=persist_state_key,
log_message=log_message,
periodic_message_logger=periodic_message_logger,
log_interval=log_interval,
async def test_passed_stats_count_runs(test_urls: list[str]) -> None:
"""Statistics passed to a factory method count the adaptive fields of the crawl."""
statistics = Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState)
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])),
statistics=statistics,
)
crawler.router.default_handler(AsyncMock())

crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(statistics=statistics)
await crawler.run([]) # ensure that statistics get initialized

assert type(crawler._statistics.state) is AdaptivePlaywrightCrawlerStatisticState

assert crawler._statistics._state._persistence_enabled == persistence_enabled
assert crawler._statistics._state._persist_state_key == persist_state_key
await crawler.run(test_urls[:1])

assert crawler._statistics._log_message == log_message
assert crawler._statistics._periodic_message_logger == periodic_message_logger
assert statistics.state.http_only_request_handler_runs == 1


async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_urls: list[str]) -> None:
Expand Down
7 changes: 0 additions & 7 deletions tests/unit/_statistics/test_persistence.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,13 +274,6 @@ def test_id_defaults_to_counter() -> None:
assert Statistics.with_default_state()._id == '2'


def test_replace_state_model_keeps_id() -> None:
"""`replace_state_model` keeps the ID of the original statistics."""
statistics = Statistics.with_default_state(id='custom')

assert statistics.replace_state_model(StatisticsState)._id == 'custom'


async def test_reentering_keeps_counters_and_runtime() -> None:
"""Entering the statistics again keeps the counters and the runtime of the previous run."""
statistics = Statistics.with_default_state()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@
from crawlee.autoscaling import ConcurrencySystem
from crawlee.browsers import BrowserPool
from crawlee.crawlers import AdaptivePlaywrightCrawler
from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawler_statistics import (
AdaptivePlaywrightCrawlerStatisticState,
)
from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor
from crawlee.statistics import Statistics

if TYPE_CHECKING:
from crawlee.crawlers import AdaptivePlaywrightCrawlingContext
Expand Down Expand Up @@ -45,6 +49,22 @@ async def test_run_needs_entered_browser_pool() -> None:
await crawler.run(['https://a.placeholder.com'])


def test_factory_passes_stats_as_is() -> None:
"""A factory method passes the given statistics to the crawler as they are."""
statistics = Statistics(state_model=AdaptivePlaywrightCrawlerStatisticState)
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(statistics=statistics)

assert crawler.statistics is statistics


def test_rejects_plain_statistics() -> None:
"""Statistics without the adaptive state model are rejected when the crawler is created."""
with pytest.raises(ValueError, match='AdaptivePlaywrightCrawlerStatisticState'):
AdaptivePlaywrightCrawler.with_parsel_static_parser(
statistics=Statistics.with_default_state() # ty: ignore[invalid-argument-type]
)


@pytest.mark.parametrize(
'optional_module_name',
[
Expand Down
Loading