Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/crawlee/_utils/recoverable_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ async def initialize(self) -> TStateModel:
ValidationError: If the persisted record is invalid. The record is kept, and every call raises again until
it's fixed or cleared by `reset_store`.
"""
# Checked without the lock first, so that a call on an initialized state doesn't wait for a save in progress.
if self._initialized:
return self.current_value

async with self._lock:
if self._initialized:
return self.current_value
Expand All @@ -124,7 +128,7 @@ async def initialize(self) -> TStateModel:

# Registered only after a successful load, so that a failed one leaves no listener behind.
if self._persistence_enabled:
self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_quietly)
self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_on_event)

self._initialized = True
return self.current_value
Expand All @@ -142,7 +146,7 @@ async def teardown(self) -> None:
if not self._persistence_enabled:
return

self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_quietly)
self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_on_event)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: KeyValueStore.drop() and purge() (#2467) rely on the new guarantee: teardown() waits for an event save in progress, and no event save runs after it returns. The teardown() docstring could say so; now only the comment on the private listener does.

✍️ Drafted by Claude Code

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd keep it in the comment. The teardown() docstring already says it persists the state one last time, so no save follows it. Waiting for a save in progress is how that's kept, and a caller doesn't act on it.

await self._persist_state_quietly()

@property
Expand Down Expand Up @@ -228,6 +232,15 @@ def _event_manager(self) -> EventManager:

return service_locator.get_event_manager()

async def _persist_state_on_event(self, event_data: EventPersistStateData) -> None:
# An event emitted before `teardown` can run this listener after it, when the record may already be cleared.
# The lock makes `teardown` wait for a save in progress, and the check skips a save that starts later.
async with self._lock:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Important: This listener now holds self._lock for the whole save, and initialize() takes the same lock before it checks _initialized. KeyValueStore.get_auto_saved_value() calls initialize() on every call, so every use_state() call now waits for a periodic save in progress. With a 1 s set_value, get_auto_saved_value() took 1.96 s here and 0 s on v2. A fast path at the top of initialize() fixes it:

if self._initialized:
    return self.current_value

async with self._lock:
    ...

✍️ Drafted by Claude Code

if not self._initialized:
return

await self._persist_state_quietly(event_data)

async def _persist_state_quietly(self, event_data: EventPersistStateData | None = None) -> None:
try:
await self.persist_state(event_data)
Expand Down
72 changes: 72 additions & 0 deletions tests/unit/_utils/test_recoverable_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,3 +516,75 @@ async def slow_get_value(key: str) -> object:
recoverable_state.current_value.counter = 1
await _emit_persist_state(event_manager)
assert await get_value(KEY) == {'counter': 42, 'message': 'hello'}


async def test_persist_state_event_before_teardown_saves_nothing_after_it(
event_manager: EventManager, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A PERSIST_STATE event emitted right before `teardown` adds no save to the one `teardown` makes."""
kvs = await KeyValueStore.open()
set_value = AsyncMock(side_effect=kvs.set_value)
monkeypatch.setattr(kvs, 'set_value', set_value)
recoverable_state = _build_state()
await recoverable_state.initialize()

event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False))
await recoverable_state.teardown()
await event_manager.wait_for_all_listeners_to_complete()

set_value.assert_awaited_once()


async def test_teardown_waits_for_save_in_progress(
event_manager: EventManager, monkeypatch: pytest.MonkeyPatch
) -> None:
"""`teardown` waits for a save that a PERSIST_STATE event started, so the save can't finish after it."""
kvs = await KeyValueStore.open()
set_value = kvs.set_value
save_started = asyncio.Event()
finish_save = asyncio.Event()

async def slow_set_value(key: str, value: object, *, content_type: str | None = None) -> None:
save_started.set()
await finish_save.wait()
await set_value(key, value, content_type=content_type)

monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value))
recoverable_state = _build_state()
await recoverable_state.initialize()

event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False))
await save_started.wait()
monkeypatch.setattr(kvs, 'set_value', set_value)
teardown_task = asyncio.create_task(recoverable_state.teardown())
done, _ = await asyncio.wait([teardown_task], timeout=0.1)
finish_save.set()
await teardown_task

assert not done


async def test_initialize_skips_wait_for_save_in_progress(
event_manager: EventManager, monkeypatch: pytest.MonkeyPatch
) -> None:
"""`initialize` on an initialized state doesn't wait for a save that a PERSIST_STATE event started."""
kvs = await KeyValueStore.open()
save_started = asyncio.Event()
finish_save = asyncio.Event()

async def slow_set_value(*_args: object, **_kwargs: object) -> None:
save_started.set()
await finish_save.wait()

monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value))
recoverable_state = _build_state()
await recoverable_state.initialize()

event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False))
await save_started.wait()
initialize_task = asyncio.create_task(recoverable_state.initialize())
done, _ = await asyncio.wait([initialize_task], timeout=1)
finish_save.set()
await recoverable_state.teardown()

assert initialize_task in done
Loading