diff --git a/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py
index 7acd3353bd..4d74f5fc34 100644
--- a/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py
+++ b/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py
@@ -25,12 +25,13 @@ async def main() -> None:
'gpt-5.4-nano',
provider=OpenAIProvider(api_key='your-openai-api-key'),
)
+ extractor = PydanticAiSelectorExtractor(
+ model=model,
+ # Pages the cached selectors cannot handle fall back to direct extraction.
+ fallback=PydanticAiDirectExtractor(model=model),
+ )
crawler = PydanticAiCrawler(
- extractor=PydanticAiSelectorExtractor(
- model=model,
- # Pages the cached selectors cannot handle fall back to direct extraction.
- fallback=PydanticAiDirectExtractor(model=model),
- ),
+ extractor=extractor,
max_requests_per_crawl=10,
)
@@ -49,7 +50,8 @@ async def article_handler(context: PydanticAiCrawlingContext) -> None:
await context.push_data(article.model_dump())
- await crawler.run(['https://crawlee.dev/blog'])
+ async with extractor:
+ await crawler.run(['https://crawlee.dev/blog'])
if __name__ == '__main__':
diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md
index 82609c9d68..141f3894df 100644
--- a/docs/06_upgrading/upgrading_to_v2.md
+++ b/docs/06_upgrading/upgrading_to_v2.md
@@ -500,7 +500,7 @@ A crawler now enters and exits only the objects it creates, the same as Crawlee
What you do with an object you pass depends on its type:
- `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished.
-- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`.
+- The `extractor` of `PydanticAiCrawler`: enter it the same way. Used on its own, `PydanticAiSelectorExtractor` has to be entered before `extract()` too, or it raises a `RuntimeError`. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`.
- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time.
For example:
diff --git a/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py
index a82241b6a9..9418a8fbbc 100644
--- a/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py
+++ b/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py
@@ -52,7 +52,9 @@ class Product(BaseModel):
model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...'))
extractor = PydanticAiDirectExtractor(model=model)
- product = await extractor.extract('...', Product)
+
+ async with extractor:
+ product = await extractor.extract('...', Product)
```
"""
diff --git a/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py
index 3103608097..ae5ee071d4 100644
--- a/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py
+++ b/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py
@@ -6,7 +6,7 @@
import re
import types
from collections import defaultdict
-from contextlib import AbstractAsyncContextManager
+from contextlib import AsyncExitStack
from enum import Enum
from functools import lru_cache
from logging import getLogger
@@ -18,6 +18,7 @@
from pydantic_ai.exceptions import UnexpectedModelBehavior
from pydantic_ai.usage import RunUsage
+from crawlee._utils.context import ensure_context
from crawlee._utils.docs import docs_group
from crawlee._utils.recoverable_state import RecoverableState
@@ -27,7 +28,7 @@
if TYPE_CHECKING:
from types import TracebackType
- from typing import Any
+ from typing import Any, Self
from parsel import Selector
from pydantic.fields import FieldInfo
@@ -254,8 +255,8 @@ class PydanticAiSelectorExtractor(BasePydanticAiHtmlExtractor):
asks the model for a new map, validates it against the live page, and caches it. A bucket keeps several maps,
so A/B-tested markup variants can coexist.
- The cache is a `RecoverableState` persisted to a `KeyValueStore`. As an async context manager it loads at
- startup and saves at shutdown. Used standalone, it initializes lazily.
+ The cache is a `RecoverableState` persisted to a `KeyValueStore`. It loads when the extractor is entered with
+ `async with` and is saved when it's exited, so `extract` needs an entered extractor.
With a `fallback` extractor, unsupported schemas and generation failures degrade to it. Infrastructure errors
such as credentials, HTTP, and usage limits propagate.
@@ -280,7 +281,9 @@ class Product(BaseModel):
model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...'))
extractor = PydanticAiSelectorExtractor(model=model, fallback=PydanticAiDirectExtractor(model=model))
- product = await extractor.extract('...', Product, cache_tag='product')
+
+ async with extractor:
+ product = await extractor.extract('...', Product, cache_tag='product')
```
"""
@@ -310,7 +313,8 @@ def __init__(
automatically.
retries: How many times the model may fix failing selectors within one generation.
max_variants: Cap on cached selector maps per bucket.
- fallback: Extractor to degrade to when generation fails or the schema shape is unsupported.
+ fallback: Extractor to degrade to when generation fails or the schema shape is unsupported. The selector
+ extractor enters and exits it together with itself, so don't enter it yourself.
usage_limits: Optional pydantic-ai `UsageLimits` applied to every generation run.
persistence: Whether the selector cache is persisted. Disable for ephemeral runs or tests.
"""
@@ -327,7 +331,7 @@ def __init__(
self._max_variants = max_variants
self._fallback = fallback
self._persistence = persistence
- self._share_usage_with_fallback()
+ self._exit_stack = AsyncExitStack()
self._selector_cache: RecoverableState[SelectorCacheState] = RecoverableState(
default_state=SelectorCacheState(),
@@ -336,8 +340,6 @@ def __init__(
logger=logger,
)
self._locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
- self._init_lock = asyncio.Lock()
- self._active = False
def set_ai_usage(self, value: PydanticAiUsageStats) -> None:
"""Adopt `value` and re-share it with the fallback chain."""
@@ -349,23 +351,26 @@ def _share_usage_with_fallback(self) -> None:
if self._fallback is not None:
self._fallback.set_ai_usage(self._ai_usage)
- @property
- def active(self) -> bool:
- """Whether the extractor is in its async context-manager scope."""
- return self._active
+ async def __aenter__(self) -> Self:
+ """Enter the fallback chain and initialize the selector cache.
- async def __aenter__(self) -> PydanticAiSelectorExtractor:
- """Initialize the selector cache eagerly and enter the fallback chain."""
- if self._active:
- raise RuntimeError(f'The {type(self).__name__} is already active.')
+ Raises:
+ RuntimeError: If the extractor or its fallback is already active.
+ """
+ await super().__aenter__()
- if not self._selector_cache.is_initialized:
- await self._selector_cache.initialize()
+ async with AsyncExitStack() as stack:
+ stack.push_async_callback(super().__aexit__, None, None, None)
+
+ if self._fallback is not None:
+ await stack.enter_async_context(self._fallback)
+ # Shared after a successful entry, so the fallback counts into whichever extractor holds it.
+ self._share_usage_with_fallback()
- if isinstance(self._fallback, AbstractAsyncContextManager):
- await self._fallback.__aenter__()
+ await self._selector_cache.initialize()
+ stack.push_async_callback(self._selector_cache.teardown)
+ self._exit_stack = stack.pop_all()
- self._active = True
return self
async def __aexit__(
@@ -374,17 +379,17 @@ async def __aexit__(
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
- """Persist the selector cache one final time, detach from events, and exit the fallback chain."""
- if not self._active:
+ """Persist the selector cache one final time, detach from events, and exit the fallback chain.
+
+ Raises:
+ RuntimeError: If the extractor is not active.
+ """
+ if not self.active:
raise RuntimeError(f'The {type(self).__name__} is not active.')
- try:
- if isinstance(self._fallback, AbstractAsyncContextManager):
- await self._fallback.__aexit__(exc_type, exc_value, exc_traceback)
- finally:
- await self._selector_cache.teardown()
- self._active = False
+ await self._exit_stack.__aexit__(exc_type, exc_value, exc_traceback)
+ @ensure_context
async def extract(
self,
content: str | Selector,
@@ -405,6 +410,7 @@ async def extract(
additional_instructions: Extra instructions appended for this call only.
Raises:
+ RuntimeError: If the extractor isn't entered with `async with`.
ValueError: When the schema shape is unsupported and no fallback is configured, or when `scope` matches
nothing.
"""
@@ -429,12 +435,6 @@ async def extract(
'Configure a fallback extractor or use PydanticAiDirectExtractor for it.'
)
- if not self._selector_cache.is_initialized:
- async with self._init_lock:
- if not self._selector_cache.is_initialized:
- # Lazy init for standalone use. Under `PydanticAiCrawler` the context manager inits it at startup.
- await self._selector_cache.initialize()
-
cache_digest = _build_cache_digest(schema, scope, cache_tag)
variants = self._selector_cache.current_value.selectors.setdefault(cache_digest, [])
diff --git a/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py b/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py
index 4d5692b37e..a82126172e 100644
--- a/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py
+++ b/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py
@@ -2,6 +2,7 @@
import asyncio
from typing import TYPE_CHECKING, Literal
+from unittest.mock import AsyncMock
import pytest
from pydantic import BaseModel, Field, create_model
@@ -13,6 +14,7 @@
from crawlee.crawlers import PydanticAiDirectExtractor, PydanticAiSelectorExtractor, PydanticAiUsageStats
if TYPE_CHECKING:
+ from collections.abc import Callable
from typing import Any
from pydantic_ai.messages import ModelMessage
@@ -113,12 +115,14 @@ async def test_rejects_unsupported_schema(annotation: Any, match: str) -> None:
# The schema shape is checked before any model call, so an unsupported one raises with the reason.
schema = create_model('_Unsupported', field=(annotation, ...))
- with pytest.raises(ValueError, match=match):
- await _extractor().extract('
', schema)
+ async with _extractor() as extractor:
+ with pytest.raises(ValueError, match=match):
+ await extractor.extract('', schema)
async def test_extracts_scalar() -> None:
- result = await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Item)
+ async with _extractor(_model(NAME_SELECTORS)) as extractor:
+ result = await extractor.extract(NAME_HTML, _Item)
assert result == _Item(name='X')
@@ -127,14 +131,16 @@ async def test_extracts_collection_of_scalars() -> None:
plan = {'selectors': {'items': {'selector': '.a::text'}, 'unique': {'selector': '.b::text'}}}
html = 'xypp
'
- result = await _extractor(_model(plan)).extract(html, _Collections)
+ async with _extractor(_model(plan)) as extractor:
+ result = await extractor.extract(html, _Collections)
# The list keeps order and duplicates; the set is deduplicated.
assert result == _Collections(items=['x', 'y'], unique={'p'})
async def test_extracts_list_of_items() -> None:
- result = await _extractor(_model(POSTS_SELECTORS)).extract(LIST_HTML, _Posts)
+ async with _extractor(_model(POSTS_SELECTORS)) as extractor:
+ result = await extractor.extract(LIST_HTML, _Posts)
assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')])
@@ -142,14 +148,16 @@ async def test_extracts_list_of_items() -> None:
async def test_extracts_literal_field() -> None:
plan = {'selectors': {'status': {'selector': '.s::text'}}}
- result = await _extractor(_model(plan)).extract('in_stock
', _Status)
+ async with _extractor(_model(plan)) as extractor:
+ result = await extractor.extract('in_stock
', _Status)
assert result == _Status(status='in_stock')
async def test_extracts_field_with_alias() -> None:
# The selector map is keyed by the field name, so by-name validation lets a schema with an alias accept it.
- result = await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Aliased)
+ async with _extractor(_model(NAME_SELECTORS)) as extractor:
+ result = await extractor.extract(NAME_HTML, _Aliased)
assert result.name == 'X'
@@ -157,8 +165,10 @@ async def test_extracts_field_with_alias() -> None:
async def test_reuses_cached_plan() -> None:
extractor = _extractor(_model(NAME_SELECTORS))
- assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X')
- assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X')
+ async with extractor:
+ assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X')
+ assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X')
+
# The second extract is served from the cache, so the model was consulted only once.
assert extractor.ai_usage.requests == 1
@@ -166,10 +176,11 @@ async def test_reuses_cached_plan() -> None:
async def test_concurrent_generate() -> None:
extractor = _extractor(_model(NAME_SELECTORS))
- results = await asyncio.gather(
- extractor.extract(NAME_HTML, _Item, cache_tag='test'),
- extractor.extract(NAME_HTML, _Item, cache_tag='test'),
- )
+ async with extractor:
+ results = await asyncio.gather(
+ extractor.extract(NAME_HTML, _Item, cache_tag='test'),
+ extractor.extract(NAME_HTML, _Item, cache_tag='test'),
+ )
assert results == [_Item(name='X'), _Item(name='X')]
assert extractor.ai_usage.requests == 1
@@ -184,11 +195,12 @@ async def test_cached_plan_for_optional_field() -> None:
}
extractor = _extractor(_model(plan))
- first_call = await extractor.extract(
- '', _Nested, cache_tag='test'
- )
- # The second page has no item, but the cached plan is still valid and returns None for the optional field.
- second_call = await extractor.extract('T2
', _Nested, cache_tag='test')
+ async with extractor:
+ first_call = await extractor.extract(
+ '', _Nested, cache_tag='test'
+ )
+ # The second page has no item, but the cached plan is still valid and returns None for the optional field.
+ second_call = await extractor.extract('T2
', _Nested, cache_tag='test')
assert first_call == _Nested(title='T', item=_Item(name='X'))
assert second_call == _Nested(title='T2', item=None)
@@ -206,7 +218,8 @@ async def test_omits_selector_for_optional_field(schema: type[BaseModel], expect
plan = {'selectors': {'title': {'selector': 'h1::text'}}}
extractor = _extractor(_model(plan))
- result = await extractor.extract('T
', schema, cache_tag='test')
+ async with extractor:
+ result = await extractor.extract('T
', schema, cache_tag='test')
assert result == expected
assert extractor.ai_usage.requests == 1
@@ -217,8 +230,9 @@ async def test_cache_key_ignores_field_description() -> None:
schema_b = create_model('_DescB', name=(str, Field(description='no description')))
extractor = _extractor(_model(NAME_SELECTORS))
- await extractor.extract(NAME_HTML, schema_a, cache_tag='test')
- await extractor.extract(NAME_HTML, schema_b, cache_tag='test')
+ async with extractor:
+ await extractor.extract(NAME_HTML, schema_a, cache_tag='test')
+ await extractor.extract(NAME_HTML, schema_b, cache_tag='test')
# Same shape and tag, so the second extract reuses the cached plan with no model call.
assert extractor.ai_usage.requests == 1
@@ -227,9 +241,10 @@ async def test_cache_key_ignores_field_description() -> None:
async def test_caches_different_tags() -> None:
extractor = _extractor(_model(NAME_SELECTORS))
- await extractor.extract(NAME_HTML, _Item, cache_tag='a')
- await extractor.extract(NAME_HTML, _Item, cache_tag='b') # different bucket, generated again
- await extractor.extract(NAME_HTML, _Item, cache_tag='a') # first bucket still cached
+ async with extractor:
+ await extractor.extract(NAME_HTML, _Item, cache_tag='a')
+ await extractor.extract(NAME_HTML, _Item, cache_tag='b') # different bucket, generated again
+ await extractor.extract(NAME_HTML, _Item, cache_tag='a') # first bucket still cached
# One generation per tag. A shared bucket would have served 'b' from cache, leaving the count at one.
assert extractor.ai_usage.requests == 2
@@ -251,11 +266,12 @@ async def test_eviction_of_oldest_variant() -> None:
max_variants=2,
)
- for key in ('a', 'b', 'c'):
- await extractor.extract(pages[key], _Item, cache_tag='test')
+ async with extractor:
+ for key in ('a', 'b', 'c'):
+ await extractor.extract(pages[key], _Item, cache_tag='test')
- # With max_variants=2 the 'a' plan was evicted, so extracting 'a' again generates a fourth time.
- await extractor.extract(pages['a'], _Item, cache_tag='test')
+ # With max_variants=2 the 'a' plan was evicted, so extracting 'a' again generates a fourth time.
+ await extractor.extract(pages['a'], _Item, cache_tag='test')
assert extractor.ai_usage.requests == 4
@@ -286,7 +302,8 @@ async def test_retries_with_invalid_plan(invalid_plan: dict[str, Any]) -> None:
# Each plan trips a different validation guard. The model is asked to fix it, then returns a valid plan.
extractor = _extractor(_model(invalid_plan, POSTS_SELECTORS))
- result = await extractor.extract(LIST_HTML, _Posts, cache_tag='test')
+ async with extractor:
+ result = await extractor.extract(LIST_HTML, _Posts, cache_tag='test')
assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')])
assert extractor.ai_usage.requests == 2 # the first plan failed validation, the second one succeeded
@@ -302,7 +319,8 @@ async def test_retries_with_invalid_data() -> None:
)
)
- result = await extractor.extract(html, _Status, cache_tag='test')
+ async with extractor:
+ result = await extractor.extract(html, _Status, cache_tag='test')
assert result == _Status(status='in_stock')
assert extractor.ai_usage.requests == 2
@@ -311,7 +329,8 @@ async def test_retries_with_invalid_data() -> None:
async def test_unsupported_schema_delegates_to_fallback() -> None:
fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'data': {'k': 'v'}}))
- result = await _extractor(fallback=fallback).extract('', _Mapping)
+ async with _extractor(fallback=fallback) as extractor:
+ result = await extractor.extract('', _Mapping)
assert result == _Mapping(data={'k': 'v'})
@@ -321,9 +340,8 @@ async def test_generation_failure_delegates_to_fallback() -> None:
bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}}
fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'name': 'from-fallback'}))
- result = await _extractor(_model(bad_plan), fallback=fallback, retries=0).extract(
- NAME_HTML, _Item, cache_tag='test'
- )
+ async with _extractor(_model(bad_plan), fallback=fallback, retries=0) as extractor:
+ result = await extractor.extract(NAME_HTML, _Item, cache_tag='test')
assert result == _Item(name='from-fallback')
@@ -331,20 +349,23 @@ async def test_generation_failure_delegates_to_fallback() -> None:
async def test_generation_failure_raises() -> None:
bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}}
- with pytest.raises(UnexpectedModelBehavior):
- await _extractor(_model(bad_plan), retries=0).extract(NAME_HTML, _Item, cache_tag='test')
+ async with _extractor(_model(bad_plan), retries=0) as extractor:
+ with pytest.raises(UnexpectedModelBehavior):
+ await extractor.extract(NAME_HTML, _Item, cache_tag='test')
async def test_scope_raises() -> None:
- with pytest.raises(ValueError, match='matched nothing'):
- await _extractor().extract('x
', _Item, scope='.missing')
+ async with _extractor() as extractor:
+ with pytest.raises(ValueError, match='matched nothing'):
+ await extractor.extract('x
', _Item, scope='.missing')
async def test_fallback_shares_usage_accumulator() -> None:
fallback = PydanticAiDirectExtractor(TestModel())
extractor = _extractor(fallback=fallback)
- assert fallback.ai_usage is extractor.ai_usage
+ async with extractor:
+ assert fallback.ai_usage is extractor.ai_usage
def test_set_ai_usage_reshares_with_fallback() -> None:
@@ -367,13 +388,87 @@ async def test_active_state() -> None:
assert extractor.active is False
-async def test_enter_propagates_to_fallback() -> None:
- inner = _extractor()
- outer = _extractor(fallback=inner)
+@pytest.mark.parametrize(
+ 'make_fallback',
+ [
+ pytest.param(lambda: PydanticAiDirectExtractor(TestModel()), id='direct'),
+ pytest.param(_extractor, id='selector'),
+ ],
+)
+async def test_enter_propagates_to_fallback(make_fallback: Callable[[], PydanticAiHtmlExtractor]) -> None:
+ """Entering the extractor enters its fallback, and exiting exits it."""
+ fallback = make_fallback()
- async with outer:
- assert inner.active is True
- assert inner.active is False
+ async with _extractor(fallback=fallback):
+ assert fallback.active is True
+ assert fallback.active is False
+
+
+async def test_fallback_enter_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A fallback that fails to enter leaves the extractor inactive and its selector cache unloaded."""
+ monkeypatch.setattr(PydanticAiDirectExtractor, '__aenter__', AsyncMock(side_effect=RuntimeError('fallback failed')))
+ extractor = _extractor(fallback=PydanticAiDirectExtractor(TestModel()))
+
+ with pytest.raises(RuntimeError, match='fallback failed'):
+ await extractor.__aenter__()
+
+ assert extractor.active is False
+ assert extractor._selector_cache.is_initialized is False
+
+
+async def test_failed_cache_load_exits_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A selector cache that fails to load leaves the extractor and its fallback inactive."""
+ fallback = PydanticAiDirectExtractor(TestModel())
+ extractor = _extractor(fallback=fallback)
+ monkeypatch.setattr(extractor._selector_cache, 'initialize', AsyncMock(side_effect=RuntimeError('load failed')))
+
+ with pytest.raises(RuntimeError, match='load failed'):
+ await extractor.__aenter__()
+
+ assert extractor.active is False
+ assert fallback.active is False
+
+
+async def test_fallback_exit_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A fallback that fails to exit still leaves the extractor inactive and its selector cache closed."""
+ monkeypatch.setattr(PydanticAiDirectExtractor, '__aexit__', AsyncMock(side_effect=RuntimeError('fallback failed')))
+ extractor = _extractor(fallback=PydanticAiDirectExtractor(TestModel()))
+
+ with pytest.raises(RuntimeError, match='fallback failed'):
+ async with extractor:
+ pass
+
+ assert extractor.active is False
+ assert extractor._selector_cache.is_initialized is False
+
+
+async def test_fallback_held_by_another_extractor() -> None:
+ """A fallback another extractor holds can't be entered again, and its usage still counts into the holder."""
+ fallback = PydanticAiDirectExtractor(TestModel())
+ first = _extractor(fallback=fallback)
+ second = _extractor(fallback=fallback)
+
+ async with first:
+ with pytest.raises(RuntimeError, match='already active'):
+ await second.__aenter__()
+
+ assert second.active is False
+ assert fallback.ai_usage is first.ai_usage
+
+
+async def test_fallback_used_in_turn() -> None:
+ """Extractors that use one fallback in turn each count its usage into their own `ai_usage`."""
+ fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'data': {'k': 'v'}}))
+ first = _extractor(fallback=fallback)
+ second = _extractor(fallback=fallback)
+
+ async with first:
+ await first.extract('', _Mapping)
+ async with second:
+ await second.extract('', _Mapping)
+
+ assert first.ai_usage.requests == 1
+ assert second.ai_usage.requests == 1
async def test_double_enter_raises() -> None:
@@ -400,3 +495,9 @@ async def test_cache_persists_across_instances() -> None:
async with PydanticAiSelectorExtractor(_model(NAME_SELECTORS), kvs_cache_key='shared-cache') as second:
assert await second.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X')
assert second.ai_usage.requests == 0
+
+
+async def test_extract_needs_entered_extractor() -> None:
+ """An extractor that isn't entered with `async with` refuses to extract."""
+ with pytest.raises(RuntimeError, match='not active'):
+ await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Item)