From ae8b374003e4c14ea976834a63a9546fc4c018b3 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 17 Nov 2026 22:57:50 +0000 Subject: [PATCH 1/2] keep event-triggered state saves from running after teardown --- src/crawlee/_utils/recoverable_state.py | 13 +++++- tests/unit/_utils/test_recoverable_state.py | 46 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 17fb1ae809..cb07d4d60e 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -124,7 +124,7 @@ async def initialize(self) -> TStateModel: # Registered only after a successful load, so that a failed one leaves no listener behind. if self._persistence_enabled: - self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_quietly) + self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state_on_event) self._initialized = True return self.current_value @@ -142,7 +142,7 @@ async def teardown(self) -> None: if not self._persistence_enabled: return - self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_quietly) + self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state_on_event) await self._persist_state_quietly() @property @@ -228,6 +228,15 @@ def _event_manager(self) -> EventManager: return service_locator.get_event_manager() + async def _persist_state_on_event(self, event_data: EventPersistStateData) -> None: + # An event emitted before `teardown` can run this listener after it, when the record may already be cleared. + # The lock makes `teardown` wait for a save in progress, and the check skips a save that starts later. + async with self._lock: + if not self._initialized: + return + + await self._persist_state_quietly(event_data) + async def _persist_state_quietly(self, event_data: EventPersistStateData | None = None) -> None: try: await self.persist_state(event_data) diff --git a/tests/unit/_utils/test_recoverable_state.py b/tests/unit/_utils/test_recoverable_state.py index 71cf2d4271..76f6f3cf94 100644 --- a/tests/unit/_utils/test_recoverable_state.py +++ b/tests/unit/_utils/test_recoverable_state.py @@ -516,3 +516,49 @@ async def slow_get_value(key: str) -> object: recoverable_state.current_value.counter = 1 await _emit_persist_state(event_manager) assert await get_value(KEY) == {'counter': 42, 'message': 'hello'} + + +async def test_persist_state_event_before_teardown_saves_nothing_after_it( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """A PERSIST_STATE event emitted right before `teardown` adds no save to the one `teardown` makes.""" + kvs = await KeyValueStore.open() + set_value = AsyncMock(side_effect=kvs.set_value) + monkeypatch.setattr(kvs, 'set_value', set_value) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await recoverable_state.teardown() + await event_manager.wait_for_all_listeners_to_complete() + + set_value.assert_awaited_once() + + +async def test_teardown_waits_for_save_in_progress( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """`teardown` waits for a save that a PERSIST_STATE event started, so the save can't finish after it.""" + kvs = await KeyValueStore.open() + set_value = kvs.set_value + save_started = asyncio.Event() + finish_save = asyncio.Event() + + async def slow_set_value(key: str, value: object, *, content_type: str | None = None) -> None: + save_started.set() + await finish_save.wait() + await set_value(key, value, content_type=content_type) + + monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value)) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await save_started.wait() + monkeypatch.setattr(kvs, 'set_value', set_value) + teardown_task = asyncio.create_task(recoverable_state.teardown()) + done, _ = await asyncio.wait([teardown_task], timeout=0.1) + finish_save.set() + await teardown_task + + assert not done From dc3d92a644e3af1bbee87c787676cf3862380649 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 18 Nov 2026 13:29:35 +0000 Subject: [PATCH 2/2] address review --- src/crawlee/_utils/recoverable_state.py | 4 ++++ tests/unit/_utils/test_recoverable_state.py | 26 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index cb07d4d60e..a67b7417d2 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -115,6 +115,10 @@ async def initialize(self) -> TStateModel: ValidationError: If the persisted record is invalid. The record is kept, and every call raises again until it's fixed or cleared by `reset_store`. """ + # Checked without the lock first, so that a call on an initialized state doesn't wait for a save in progress. + if self._initialized: + return self.current_value + async with self._lock: if self._initialized: return self.current_value diff --git a/tests/unit/_utils/test_recoverable_state.py b/tests/unit/_utils/test_recoverable_state.py index 76f6f3cf94..16fbca83fd 100644 --- a/tests/unit/_utils/test_recoverable_state.py +++ b/tests/unit/_utils/test_recoverable_state.py @@ -562,3 +562,29 @@ async def slow_set_value(key: str, value: object, *, content_type: str | None = await teardown_task assert not done + + +async def test_initialize_skips_wait_for_save_in_progress( + event_manager: EventManager, monkeypatch: pytest.MonkeyPatch +) -> None: + """`initialize` on an initialized state doesn't wait for a save that a PERSIST_STATE event started.""" + kvs = await KeyValueStore.open() + save_started = asyncio.Event() + finish_save = asyncio.Event() + + async def slow_set_value(*_args: object, **_kwargs: object) -> None: + save_started.set() + await finish_save.wait() + + monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=slow_set_value)) + recoverable_state = _build_state() + await recoverable_state.initialize() + + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await save_started.wait() + initialize_task = asyncio.create_task(recoverable_state.initialize()) + done, _ = await asyncio.wait([initialize_task], timeout=1) + finish_save.set() + await recoverable_state.teardown() + + assert initialize_task in done