Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@ async def main() -> None:
'gpt-5.4-nano',
provider=OpenAIProvider(api_key='your-openai-api-key'),
)
extractor = PydanticAiSelectorExtractor(
model=model,
# Pages the cached selectors cannot handle fall back to direct extraction.
fallback=PydanticAiDirectExtractor(model=model),
)
crawler = PydanticAiCrawler(
extractor=PydanticAiSelectorExtractor(
model=model,
# Pages the cached selectors cannot handle fall back to direct extraction.
fallback=PydanticAiDirectExtractor(model=model),
),
extractor=extractor,
max_requests_per_crawl=10,
)

Expand All @@ -49,7 +50,8 @@ async def article_handler(context: PydanticAiCrawlingContext) -> None:

await context.push_data(article.model_dump())

await crawler.run(['https://crawlee.dev/blog'])
async with extractor:
await crawler.run(['https://crawlee.dev/blog'])


if __name__ == '__main__':
Expand Down
2 changes: 1 addition & 1 deletion docs/06_upgrading/upgrading_to_v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ A crawler now enters and exits only the objects it creates, the same as Crawlee
What you do with an object you pass depends on its type:

- `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished.
- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`.
- The `extractor` of `PydanticAiCrawler`: enter it the same way. Used on its own, `PydanticAiSelectorExtractor` has to be entered before `extract()` too, or it raises a `RuntimeError`. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`.
- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time.

For example:
Expand Down
4 changes: 3 additions & 1 deletion src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ class Product(BaseModel):

model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...'))
extractor = PydanticAiDirectExtractor(model=model)
product = await extractor.extract('<html>...</html>', Product)

async with extractor:
product = await extractor.extract('<html>...</html>', Product)
```
"""

Expand Down
72 changes: 36 additions & 36 deletions src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import re
import types
from collections import defaultdict
from contextlib import AbstractAsyncContextManager
from contextlib import AsyncExitStack
from enum import Enum
from functools import lru_cache
from logging import getLogger
Expand All @@ -18,6 +18,7 @@
from pydantic_ai.exceptions import UnexpectedModelBehavior
from pydantic_ai.usage import RunUsage

from crawlee._utils.context import ensure_context
from crawlee._utils.docs import docs_group
from crawlee._utils.recoverable_state import RecoverableState

Expand All @@ -27,7 +28,7 @@

if TYPE_CHECKING:
from types import TracebackType
from typing import Any
from typing import Any, Self

from parsel import Selector
from pydantic.fields import FieldInfo
Expand Down Expand Up @@ -254,8 +255,8 @@ class PydanticAiSelectorExtractor(BasePydanticAiHtmlExtractor):
asks the model for a new map, validates it against the live page, and caches it. A bucket keeps several maps,
so A/B-tested markup variants can coexist.

The cache is a `RecoverableState` persisted to a `KeyValueStore`. As an async context manager it loads at
startup and saves at shutdown. Used standalone, it initializes lazily.
The cache is a `RecoverableState` persisted to a `KeyValueStore`. It loads when the extractor is entered with
`async with` and is saved when it's exited, so `extract` needs an entered extractor.

With a `fallback` extractor, unsupported schemas and generation failures degrade to it. Infrastructure errors
such as credentials, HTTP, and usage limits propagate.
Expand All @@ -280,7 +281,9 @@ class Product(BaseModel):

model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...'))
extractor = PydanticAiSelectorExtractor(model=model, fallback=PydanticAiDirectExtractor(model=model))
product = await extractor.extract('<html>...</html>', Product, cache_tag='product')

async with extractor:
product = await extractor.extract('<html>...</html>', Product, cache_tag='product')
```
"""

Expand Down Expand Up @@ -310,7 +313,8 @@ def __init__(
automatically.
retries: How many times the model may fix failing selectors within one generation.
max_variants: Cap on cached selector maps per bucket.
fallback: Extractor to degrade to when generation fails or the schema shape is unsupported.
fallback: Extractor to degrade to when generation fails or the schema shape is unsupported. The selector
extractor enters and exits it together with itself, so don't enter it yourself.
usage_limits: Optional pydantic-ai `UsageLimits` applied to every generation run.
persistence: Whether the selector cache is persisted. Disable for ephemeral runs or tests.
"""
Expand All @@ -327,7 +331,7 @@ def __init__(
self._max_variants = max_variants
self._fallback = fallback
self._persistence = persistence
self._share_usage_with_fallback()
self._exit_stack = AsyncExitStack()

self._selector_cache: RecoverableState[SelectorCacheState] = RecoverableState(
default_state=SelectorCacheState(),
Expand All @@ -336,8 +340,6 @@ def __init__(
logger=logger,
)
self._locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
self._init_lock = asyncio.Lock()
self._active = False

def set_ai_usage(self, value: PydanticAiUsageStats) -> None:
"""Adopt `value` and re-share it with the fallback chain."""
Expand All @@ -349,23 +351,26 @@ def _share_usage_with_fallback(self) -> None:
if self._fallback is not None:
self._fallback.set_ai_usage(self._ai_usage)

@property
def active(self) -> bool:
"""Whether the extractor is in its async context-manager scope."""
return self._active
async def __aenter__(self) -> Self:
"""Enter the fallback chain and initialize the selector cache.

async def __aenter__(self) -> PydanticAiSelectorExtractor:
"""Initialize the selector cache eagerly and enter the fallback chain."""
if self._active:
raise RuntimeError(f'The {type(self).__name__} is already active.')
Raises:
RuntimeError: If the extractor or its fallback is already active.
"""
await super().__aenter__()

if not self._selector_cache.is_initialized:
await self._selector_cache.initialize()
async with AsyncExitStack() as stack:
stack.push_async_callback(super().__aexit__, None, None, None)

if self._fallback is not None:
await stack.enter_async_context(self._fallback)
# Shared after a successful entry, so the fallback counts into whichever extractor holds it.
self._share_usage_with_fallback()

if isinstance(self._fallback, AbstractAsyncContextManager):
await self._fallback.__aenter__()
await self._selector_cache.initialize()
stack.push_async_callback(self._selector_cache.teardown)
self._exit_stack = stack.pop_all()

self._active = True
return self

async def __aexit__(
Expand All @@ -374,17 +379,17 @@ async def __aexit__(
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
"""Persist the selector cache one final time, detach from events, and exit the fallback chain."""
if not self._active:
"""Persist the selector cache one final time, detach from events, and exit the fallback chain.

Raises:
RuntimeError: If the extractor is not active.
"""
if not self.active:
raise RuntimeError(f'The {type(self).__name__} is not active.')

try:
if isinstance(self._fallback, AbstractAsyncContextManager):
await self._fallback.__aexit__(exc_type, exc_value, exc_traceback)
finally:
await self._selector_cache.teardown()
self._active = False
await self._exit_stack.__aexit__(exc_type, exc_value, exc_traceback)

@ensure_context
async def extract(
self,
content: str | Selector,
Expand All @@ -405,6 +410,7 @@ async def extract(
additional_instructions: Extra instructions appended for this call only.

Raises:
RuntimeError: If the extractor isn't entered with `async with`.
ValueError: When the schema shape is unsupported and no fallback is configured, or when `scope` matches
nothing.
"""
Expand All @@ -429,12 +435,6 @@ async def extract(
'Configure a fallback extractor or use PydanticAiDirectExtractor for it.'
)

if not self._selector_cache.is_initialized:
async with self._init_lock:
if not self._selector_cache.is_initialized:
# Lazy init for standalone use. Under `PydanticAiCrawler` the context manager inits it at startup.
await self._selector_cache.initialize()

cache_digest = _build_cache_digest(schema, scope, cache_tag)
variants = self._selector_cache.current_value.selectors.setdefault(cache_digest, [])

Expand Down
Loading
Loading