diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 17fb1ae809..a67b7417d2 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -115,6 +115,10 @@ async def initialize(self) -> TStateModel: ValidationError: If the persisted record is invalid. The record is kept, and every call raises again until it's fixed or cleared by `reset_store`. """ + # Checked without the lock first, so that a call on an initialized state doesn't wait for a save in progress. + if self._initialized: + return self.current_value + async with self._lock: if self._initialized: return self.current_value @@ -124,7 +128,7 @@ async def initialize(self) -> TStateModel: # Registered only after a successful load, so that a failed one leaves no listener behind. if self._persistence_enabled: - self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_quietly) + self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_on_event) self._initialized = True return self.current_value @@ -142,7 +146,7 @@ async def teardown(self) -> None: if not self._persistence_enabled: return - self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_quietly) + self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_on_event) await self._persist_state_quietly() @property @@ -228,6 +232,15 @@ def _event_manager(self) -> EventManager: return service_locator.get_event_manager() + async def _persist_state_on_event(self, event_data: EventPersistStateData) -> None: + # An event emitted before `teardown` can run this listener after it, when the record may already be cleared. + # The lock makes `teardown` wait for a save in progress, and the check skips a save that starts later. + async with self._lock: + if not self._initialized: + return + + await self._persist_state_quietly(event_data) + async def _persist_state_quietly(self, event_data: EventPersistStateData | None = None) -> None: try: await self.persist_state(event_data) diff --git a/tests/unit/_utils/test_recoverable_state.py b/tests/unit/_utils/test_recoverable_state.py index 71cf2d4271..16fbca83fd 100644 --- a/tests/unit/_utils/test_recoverable_state.py +++ b/tests/unit/_utils/test_recoverable_state.py @@ -516,3 +516,75 @@ async def slow_get_value(key: str) -> object: recoverable_state.current_value.counter = 1 await _emit_persist_state(event_manager) assert await get_value(KEY) == {'counter': 42, 'message': 'hello'} + + +async def test_persist_state_event_before_teardown_saves_nothing_after_it( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """A PERSIST_STATE event emitted right before `teardown` adds no save to the one `teardown` makes.""" + kvs = await KeyValueStore.open() + set_value = AsyncMock(side_effect=kvs.set_value) + monkeypatch.setattr(kvs, 'set_value', set_value) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await recoverable_state.teardown() + await event_manager.wait_for_all_listeners_to_complete() + + set_value.assert_awaited_once() + + +async def test_teardown_waits_for_save_in_progress( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """`teardown` waits for a save that a PERSIST_STATE event started, so the save can't finish after it.""" + kvs = await KeyValueStore.open() + set_value = kvs.set_value + save_started = asyncio.Event() + finish_save = asyncio.Event() + + async def slow_set_value(key: str, value: object, *, content_type: str | None = None) -> None: + save_started.set() + await finish_save.wait() + await set_value(key, value, content_type=content_type) + + monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value)) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await save_started.wait() + monkeypatch.setattr(kvs, 'set_value', set_value) + teardown_task = asyncio.create_task(recoverable_state.teardown()) + done, _ = await asyncio.wait([teardown_task], timeout=0.1) + finish_save.set() + await teardown_task + + assert not done + + +async def test_initialize_skips_wait_for_save_in_progress( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """`initialize` on an initialized state doesn't wait for a save that a PERSIST_STATE event started.""" + kvs = await KeyValueStore.open() + save_started = asyncio.Event() + finish_save = asyncio.Event() + + async def slow_set_value(*_args: object, **_kwargs: object) -> None: + save_started.set() + await finish_save.wait() + + monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value)) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await save_started.wait() + initialize_task = asyncio.create_task(recoverable_state.initialize()) + done, _ = await asyncio.wait([initialize_task], timeout=1) + finish_save.set() + await recoverable_state.teardown() + + assert initialize_task in done