diff --git a/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py index 7acd3353bd..4d74f5fc34 100644 --- a/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py +++ b/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py @@ -25,12 +25,13 @@ async def main() -> None: 'gpt-5.4-nano', provider=OpenAIProvider(api_key='your-openai-api-key'), ) + extractor = PydanticAiSelectorExtractor( + model=model, + # Pages the cached selectors cannot handle fall back to direct extraction. + fallback=PydanticAiDirectExtractor(model=model), + ) crawler = PydanticAiCrawler( - extractor=PydanticAiSelectorExtractor( - model=model, - # Pages the cached selectors cannot handle fall back to direct extraction. - fallback=PydanticAiDirectExtractor(model=model), - ), + extractor=extractor, max_requests_per_crawl=10, ) @@ -49,7 +50,8 @@ async def article_handler(context: PydanticAiCrawlingContext) -> None: await context.push_data(article.model_dump()) - await crawler.run(['https://crawlee.dev/blog']) + async with extractor: + await crawler.run(['https://crawlee.dev/blog']) if __name__ == '__main__': diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 82609c9d68..141f3894df 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -500,7 +500,7 @@ A crawler now enters and exits only the objects it creates, the same as Crawlee What you do with an object you pass depends on its type: - `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished. -- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`. +- The `extractor` of `PydanticAiCrawler`: enter it the same way. Used on its own, `PydanticAiSelectorExtractor` has to be entered before `extract()` too, or it raises a `RuntimeError`. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`. - `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time. For example: diff --git a/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py index a82241b6a9..9418a8fbbc 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py +++ b/src/crawlee/crawlers/_pydantic_ai/_direct_extractor.py @@ -52,7 +52,9 @@ class Product(BaseModel): model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...')) extractor = PydanticAiDirectExtractor(model=model) - product = await extractor.extract('...', Product) + + async with extractor: + product = await extractor.extract('...', Product) ``` """ diff --git a/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py index 3103608097..ae5ee071d4 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py +++ b/src/crawlee/crawlers/_pydantic_ai/_selector_extractor.py @@ -6,7 +6,7 @@ import re import types from collections import defaultdict -from contextlib import AbstractAsyncContextManager +from contextlib import AsyncExitStack from enum import Enum from functools import lru_cache from logging import getLogger @@ -18,6 +18,7 @@ from pydantic_ai.exceptions import UnexpectedModelBehavior from pydantic_ai.usage import RunUsage +from crawlee._utils.context import ensure_context from crawlee._utils.docs import docs_group from crawlee._utils.recoverable_state import RecoverableState @@ -27,7 +28,7 @@ if TYPE_CHECKING: from types import TracebackType - from typing import Any + from typing import Any, Self from parsel import Selector from pydantic.fields import FieldInfo @@ -254,8 +255,8 @@ class PydanticAiSelectorExtractor(BasePydanticAiHtmlExtractor): asks the model for a new map, validates it against the live page, and caches it. A bucket keeps several maps, so A/B-tested markup variants can coexist. - The cache is a `RecoverableState` persisted to a `KeyValueStore`. As an async context manager it loads at - startup and saves at shutdown. Used standalone, it initializes lazily. + The cache is a `RecoverableState` persisted to a `KeyValueStore`. It loads when the extractor is entered with + `async with` and is saved when it's exited, so `extract` needs an entered extractor. With a `fallback` extractor, unsupported schemas and generation failures degrade to it. Infrastructure errors such as credentials, HTTP, and usage limits propagate. @@ -280,7 +281,9 @@ class Product(BaseModel): model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...')) extractor = PydanticAiSelectorExtractor(model=model, fallback=PydanticAiDirectExtractor(model=model)) - product = await extractor.extract('...', Product, cache_tag='product') + + async with extractor: + product = await extractor.extract('...', Product, cache_tag='product') ``` """ @@ -310,7 +313,8 @@ def __init__( automatically. retries: How many times the model may fix failing selectors within one generation. max_variants: Cap on cached selector maps per bucket. - fallback: Extractor to degrade to when generation fails or the schema shape is unsupported. + fallback: Extractor to degrade to when generation fails or the schema shape is unsupported. The selector + extractor enters and exits it together with itself, so don't enter it yourself. usage_limits: Optional pydantic-ai `UsageLimits` applied to every generation run. persistence: Whether the selector cache is persisted. Disable for ephemeral runs or tests. """ @@ -327,7 +331,7 @@ def __init__( self._max_variants = max_variants self._fallback = fallback self._persistence = persistence - self._share_usage_with_fallback() + self._exit_stack = AsyncExitStack() self._selector_cache: RecoverableState[SelectorCacheState] = RecoverableState( default_state=SelectorCacheState(), @@ -336,8 +340,6 @@ def __init__( logger=logger, ) self._locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock) - self._init_lock = asyncio.Lock() - self._active = False def set_ai_usage(self, value: PydanticAiUsageStats) -> None: """Adopt `value` and re-share it with the fallback chain.""" @@ -349,23 +351,26 @@ def _share_usage_with_fallback(self) -> None: if self._fallback is not None: self._fallback.set_ai_usage(self._ai_usage) - @property - def active(self) -> bool: - """Whether the extractor is in its async context-manager scope.""" - return self._active + async def __aenter__(self) -> Self: + """Enter the fallback chain and initialize the selector cache. - async def __aenter__(self) -> PydanticAiSelectorExtractor: - """Initialize the selector cache eagerly and enter the fallback chain.""" - if self._active: - raise RuntimeError(f'The {type(self).__name__} is already active.') + Raises: + RuntimeError: If the extractor or its fallback is already active. + """ + await super().__aenter__() - if not self._selector_cache.is_initialized: - await self._selector_cache.initialize() + async with AsyncExitStack() as stack: + stack.push_async_callback(super().__aexit__, None, None, None) + + if self._fallback is not None: + await stack.enter_async_context(self._fallback) + # Shared after a successful entry, so the fallback counts into whichever extractor holds it. + self._share_usage_with_fallback() - if isinstance(self._fallback, AbstractAsyncContextManager): - await self._fallback.__aenter__() + await self._selector_cache.initialize() + stack.push_async_callback(self._selector_cache.teardown) + self._exit_stack = stack.pop_all() - self._active = True return self async def __aexit__( @@ -374,17 +379,17 @@ async def __aexit__( exc_value: BaseException | None, exc_traceback: TracebackType | None, ) -> None: - """Persist the selector cache one final time, detach from events, and exit the fallback chain.""" - if not self._active: + """Persist the selector cache one final time, detach from events, and exit the fallback chain. + + Raises: + RuntimeError: If the extractor is not active. + """ + if not self.active: raise RuntimeError(f'The {type(self).__name__} is not active.') - try: - if isinstance(self._fallback, AbstractAsyncContextManager): - await self._fallback.__aexit__(exc_type, exc_value, exc_traceback) - finally: - await self._selector_cache.teardown() - self._active = False + await self._exit_stack.__aexit__(exc_type, exc_value, exc_traceback) + @ensure_context async def extract( self, content: str | Selector, @@ -405,6 +410,7 @@ async def extract( additional_instructions: Extra instructions appended for this call only. Raises: + RuntimeError: If the extractor isn't entered with `async with`. ValueError: When the schema shape is unsupported and no fallback is configured, or when `scope` matches nothing. """ @@ -429,12 +435,6 @@ async def extract( 'Configure a fallback extractor or use PydanticAiDirectExtractor for it.' ) - if not self._selector_cache.is_initialized: - async with self._init_lock: - if not self._selector_cache.is_initialized: - # Lazy init for standalone use. Under `PydanticAiCrawler` the context manager inits it at startup. - await self._selector_cache.initialize() - cache_digest = _build_cache_digest(schema, scope, cache_tag) variants = self._selector_cache.current_value.selectors.setdefault(cache_digest, []) diff --git a/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py b/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py index 4d5692b37e..a82126172e 100644 --- a/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py +++ b/tests/unit/crawlers/_pydantic_ai/test_selector_extractor.py @@ -2,6 +2,7 @@ import asyncio from typing import TYPE_CHECKING, Literal +from unittest.mock import AsyncMock import pytest from pydantic import BaseModel, Field, create_model @@ -13,6 +14,7 @@ from crawlee.crawlers import PydanticAiDirectExtractor, PydanticAiSelectorExtractor, PydanticAiUsageStats if TYPE_CHECKING: + from collections.abc import Callable from typing import Any from pydantic_ai.messages import ModelMessage @@ -113,12 +115,14 @@ async def test_rejects_unsupported_schema(annotation: Any, match: str) -> None: # The schema shape is checked before any model call, so an unsupported one raises with the reason. schema = create_model('_Unsupported', field=(annotation, ...)) - with pytest.raises(ValueError, match=match): - await _extractor().extract('
', schema) + async with _extractor() as extractor: + with pytest.raises(ValueError, match=match): + await extractor.extract('
', schema) async def test_extracts_scalar() -> None: - result = await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Item) + async with _extractor(_model(NAME_SELECTORS)) as extractor: + result = await extractor.extract(NAME_HTML, _Item) assert result == _Item(name='X') @@ -127,14 +131,16 @@ async def test_extracts_collection_of_scalars() -> None: plan = {'selectors': {'items': {'selector': '.a::text'}, 'unique': {'selector': '.b::text'}}} html = '
xypp
' - result = await _extractor(_model(plan)).extract(html, _Collections) + async with _extractor(_model(plan)) as extractor: + result = await extractor.extract(html, _Collections) # The list keeps order and duplicates; the set is deduplicated. assert result == _Collections(items=['x', 'y'], unique={'p'}) async def test_extracts_list_of_items() -> None: - result = await _extractor(_model(POSTS_SELECTORS)).extract(LIST_HTML, _Posts) + async with _extractor(_model(POSTS_SELECTORS)) as extractor: + result = await extractor.extract(LIST_HTML, _Posts) assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')]) @@ -142,14 +148,16 @@ async def test_extracts_list_of_items() -> None: async def test_extracts_literal_field() -> None: plan = {'selectors': {'status': {'selector': '.s::text'}}} - result = await _extractor(_model(plan)).extract('
in_stock
', _Status) + async with _extractor(_model(plan)) as extractor: + result = await extractor.extract('
in_stock
', _Status) assert result == _Status(status='in_stock') async def test_extracts_field_with_alias() -> None: # The selector map is keyed by the field name, so by-name validation lets a schema with an alias accept it. - result = await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Aliased) + async with _extractor(_model(NAME_SELECTORS)) as extractor: + result = await extractor.extract(NAME_HTML, _Aliased) assert result.name == 'X' @@ -157,8 +165,10 @@ async def test_extracts_field_with_alias() -> None: async def test_reuses_cached_plan() -> None: extractor = _extractor(_model(NAME_SELECTORS)) - assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') - assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + async with extractor: + assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + # The second extract is served from the cache, so the model was consulted only once. assert extractor.ai_usage.requests == 1 @@ -166,10 +176,11 @@ async def test_reuses_cached_plan() -> None: async def test_concurrent_generate() -> None: extractor = _extractor(_model(NAME_SELECTORS)) - results = await asyncio.gather( - extractor.extract(NAME_HTML, _Item, cache_tag='test'), - extractor.extract(NAME_HTML, _Item, cache_tag='test'), - ) + async with extractor: + results = await asyncio.gather( + extractor.extract(NAME_HTML, _Item, cache_tag='test'), + extractor.extract(NAME_HTML, _Item, cache_tag='test'), + ) assert results == [_Item(name='X'), _Item(name='X')] assert extractor.ai_usage.requests == 1 @@ -184,11 +195,12 @@ async def test_cached_plan_for_optional_field() -> None: } extractor = _extractor(_model(plan)) - first_call = await extractor.extract( - '

T

X
', _Nested, cache_tag='test' - ) - # The second page has no item, but the cached plan is still valid and returns None for the optional field. - second_call = await extractor.extract('

T2

', _Nested, cache_tag='test') + async with extractor: + first_call = await extractor.extract( + '

T

X
', _Nested, cache_tag='test' + ) + # The second page has no item, but the cached plan is still valid and returns None for the optional field. + second_call = await extractor.extract('

T2

', _Nested, cache_tag='test') assert first_call == _Nested(title='T', item=_Item(name='X')) assert second_call == _Nested(title='T2', item=None) @@ -206,7 +218,8 @@ async def test_omits_selector_for_optional_field(schema: type[BaseModel], expect plan = {'selectors': {'title': {'selector': 'h1::text'}}} extractor = _extractor(_model(plan)) - result = await extractor.extract('

T

', schema, cache_tag='test') + async with extractor: + result = await extractor.extract('

T

', schema, cache_tag='test') assert result == expected assert extractor.ai_usage.requests == 1 @@ -217,8 +230,9 @@ async def test_cache_key_ignores_field_description() -> None: schema_b = create_model('_DescB', name=(str, Field(description='no description'))) extractor = _extractor(_model(NAME_SELECTORS)) - await extractor.extract(NAME_HTML, schema_a, cache_tag='test') - await extractor.extract(NAME_HTML, schema_b, cache_tag='test') + async with extractor: + await extractor.extract(NAME_HTML, schema_a, cache_tag='test') + await extractor.extract(NAME_HTML, schema_b, cache_tag='test') # Same shape and tag, so the second extract reuses the cached plan with no model call. assert extractor.ai_usage.requests == 1 @@ -227,9 +241,10 @@ async def test_cache_key_ignores_field_description() -> None: async def test_caches_different_tags() -> None: extractor = _extractor(_model(NAME_SELECTORS)) - await extractor.extract(NAME_HTML, _Item, cache_tag='a') - await extractor.extract(NAME_HTML, _Item, cache_tag='b') # different bucket, generated again - await extractor.extract(NAME_HTML, _Item, cache_tag='a') # first bucket still cached + async with extractor: + await extractor.extract(NAME_HTML, _Item, cache_tag='a') + await extractor.extract(NAME_HTML, _Item, cache_tag='b') # different bucket, generated again + await extractor.extract(NAME_HTML, _Item, cache_tag='a') # first bucket still cached # One generation per tag. A shared bucket would have served 'b' from cache, leaving the count at one. assert extractor.ai_usage.requests == 2 @@ -251,11 +266,12 @@ async def test_eviction_of_oldest_variant() -> None: max_variants=2, ) - for key in ('a', 'b', 'c'): - await extractor.extract(pages[key], _Item, cache_tag='test') + async with extractor: + for key in ('a', 'b', 'c'): + await extractor.extract(pages[key], _Item, cache_tag='test') - # With max_variants=2 the 'a' plan was evicted, so extracting 'a' again generates a fourth time. - await extractor.extract(pages['a'], _Item, cache_tag='test') + # With max_variants=2 the 'a' plan was evicted, so extracting 'a' again generates a fourth time. + await extractor.extract(pages['a'], _Item, cache_tag='test') assert extractor.ai_usage.requests == 4 @@ -286,7 +302,8 @@ async def test_retries_with_invalid_plan(invalid_plan: dict[str, Any]) -> None: # Each plan trips a different validation guard. The model is asked to fix it, then returns a valid plan. extractor = _extractor(_model(invalid_plan, POSTS_SELECTORS)) - result = await extractor.extract(LIST_HTML, _Posts, cache_tag='test') + async with extractor: + result = await extractor.extract(LIST_HTML, _Posts, cache_tag='test') assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')]) assert extractor.ai_usage.requests == 2 # the first plan failed validation, the second one succeeded @@ -302,7 +319,8 @@ async def test_retries_with_invalid_data() -> None: ) ) - result = await extractor.extract(html, _Status, cache_tag='test') + async with extractor: + result = await extractor.extract(html, _Status, cache_tag='test') assert result == _Status(status='in_stock') assert extractor.ai_usage.requests == 2 @@ -311,7 +329,8 @@ async def test_retries_with_invalid_data() -> None: async def test_unsupported_schema_delegates_to_fallback() -> None: fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'data': {'k': 'v'}})) - result = await _extractor(fallback=fallback).extract('
', _Mapping) + async with _extractor(fallback=fallback) as extractor: + result = await extractor.extract('
', _Mapping) assert result == _Mapping(data={'k': 'v'}) @@ -321,9 +340,8 @@ async def test_generation_failure_delegates_to_fallback() -> None: bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}} fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'name': 'from-fallback'})) - result = await _extractor(_model(bad_plan), fallback=fallback, retries=0).extract( - NAME_HTML, _Item, cache_tag='test' - ) + async with _extractor(_model(bad_plan), fallback=fallback, retries=0) as extractor: + result = await extractor.extract(NAME_HTML, _Item, cache_tag='test') assert result == _Item(name='from-fallback') @@ -331,20 +349,23 @@ async def test_generation_failure_delegates_to_fallback() -> None: async def test_generation_failure_raises() -> None: bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}} - with pytest.raises(UnexpectedModelBehavior): - await _extractor(_model(bad_plan), retries=0).extract(NAME_HTML, _Item, cache_tag='test') + async with _extractor(_model(bad_plan), retries=0) as extractor: + with pytest.raises(UnexpectedModelBehavior): + await extractor.extract(NAME_HTML, _Item, cache_tag='test') async def test_scope_raises() -> None: - with pytest.raises(ValueError, match='matched nothing'): - await _extractor().extract('
x
', _Item, scope='.missing') + async with _extractor() as extractor: + with pytest.raises(ValueError, match='matched nothing'): + await extractor.extract('
x
', _Item, scope='.missing') async def test_fallback_shares_usage_accumulator() -> None: fallback = PydanticAiDirectExtractor(TestModel()) extractor = _extractor(fallback=fallback) - assert fallback.ai_usage is extractor.ai_usage + async with extractor: + assert fallback.ai_usage is extractor.ai_usage def test_set_ai_usage_reshares_with_fallback() -> None: @@ -367,13 +388,87 @@ async def test_active_state() -> None: assert extractor.active is False -async def test_enter_propagates_to_fallback() -> None: - inner = _extractor() - outer = _extractor(fallback=inner) +@pytest.mark.parametrize( + 'make_fallback', + [ + pytest.param(lambda: PydanticAiDirectExtractor(TestModel()), id='direct'), + pytest.param(_extractor, id='selector'), + ], +) +async def test_enter_propagates_to_fallback(make_fallback: Callable[[], PydanticAiHtmlExtractor]) -> None: + """Entering the extractor enters its fallback, and exiting exits it.""" + fallback = make_fallback() - async with outer: - assert inner.active is True - assert inner.active is False + async with _extractor(fallback=fallback): + assert fallback.active is True + assert fallback.active is False + + +async def test_fallback_enter_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """A fallback that fails to enter leaves the extractor inactive and its selector cache unloaded.""" + monkeypatch.setattr(PydanticAiDirectExtractor, '__aenter__', AsyncMock(side_effect=RuntimeError('fallback failed'))) + extractor = _extractor(fallback=PydanticAiDirectExtractor(TestModel())) + + with pytest.raises(RuntimeError, match='fallback failed'): + await extractor.__aenter__() + + assert extractor.active is False + assert extractor._selector_cache.is_initialized is False + + +async def test_failed_cache_load_exits_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + """A selector cache that fails to load leaves the extractor and its fallback inactive.""" + fallback = PydanticAiDirectExtractor(TestModel()) + extractor = _extractor(fallback=fallback) + monkeypatch.setattr(extractor._selector_cache, 'initialize', AsyncMock(side_effect=RuntimeError('load failed'))) + + with pytest.raises(RuntimeError, match='load failed'): + await extractor.__aenter__() + + assert extractor.active is False + assert fallback.active is False + + +async def test_fallback_exit_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """A fallback that fails to exit still leaves the extractor inactive and its selector cache closed.""" + monkeypatch.setattr(PydanticAiDirectExtractor, '__aexit__', AsyncMock(side_effect=RuntimeError('fallback failed'))) + extractor = _extractor(fallback=PydanticAiDirectExtractor(TestModel())) + + with pytest.raises(RuntimeError, match='fallback failed'): + async with extractor: + pass + + assert extractor.active is False + assert extractor._selector_cache.is_initialized is False + + +async def test_fallback_held_by_another_extractor() -> None: + """A fallback another extractor holds can't be entered again, and its usage still counts into the holder.""" + fallback = PydanticAiDirectExtractor(TestModel()) + first = _extractor(fallback=fallback) + second = _extractor(fallback=fallback) + + async with first: + with pytest.raises(RuntimeError, match='already active'): + await second.__aenter__() + + assert second.active is False + assert fallback.ai_usage is first.ai_usage + + +async def test_fallback_used_in_turn() -> None: + """Extractors that use one fallback in turn each count its usage into their own `ai_usage`.""" + fallback = PydanticAiDirectExtractor(TestModel(custom_output_args={'data': {'k': 'v'}})) + first = _extractor(fallback=fallback) + second = _extractor(fallback=fallback) + + async with first: + await first.extract('
', _Mapping) + async with second: + await second.extract('
', _Mapping) + + assert first.ai_usage.requests == 1 + assert second.ai_usage.requests == 1 async def test_double_enter_raises() -> None: @@ -400,3 +495,9 @@ async def test_cache_persists_across_instances() -> None: async with PydanticAiSelectorExtractor(_model(NAME_SELECTORS), kvs_cache_key='shared-cache') as second: assert await second.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') assert second.ai_usage.requests == 0 + + +async def test_extract_needs_entered_extractor() -> None: + """An extractor that isn't entered with `async with` refuses to extract.""" + with pytest.raises(RuntimeError, match='not active'): + await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Item)