From 542ba3df5cd98340a344d87285bcdc7d876e64c7 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 17 Nov 2026 23:34:13 +0000 Subject: [PATCH 1/2] drop autosaved values on purge and keep them when a purge or drop fails --- docs/06_upgrading/upgrading_to_v2.md | 4 + src/crawlee/storages/_key_value_store.py | 33 ++++-- tests/unit/storages/test_key_value_store.py | 125 ++++++++++++++++++-- 3 files changed, 143 insertions(+), 19 deletions(-) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 1174379fe4..4b7c7726c4 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -301,6 +301,10 @@ for urls in batches: In v1, `RequestQueue.purge()` worked around a storage backend that didn't support purging. It dropped an unnamed queue and opened the default queue in its place, and it skipped a named queue with a warning. In v2, it raises the backend's `NotImplementedError`, the same as Crawlee JS. The Apify platform's storage backend doesn't support purging, so on the platform, call `drop()` or open a new queue instead. +## `KeyValueStore.purge` drops autosaved values + +`KeyValueStore.purge()` now also drops the values that `get_auto_saved_value()` and `use_state` keep in memory for the store, the same as `drop()`. Their later changes aren't saved, and the next `get_auto_saved_value()` or `use_state` call starts from the default value. In v1, a later save could write them back into the purged store. + ## `check_readiness` replaces `is_empty` and `is_finished` `RequestLoader` and `RequestManager` no longer have `is_empty` and `is_finished`. A single `check_readiness` method replaces them and returns one of four statuses: diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py index 5a974dfd38..ec4e84627c 100644 --- a/src/crawlee/storages/_key_value_store.py +++ b/src/crawlee/storages/_key_value_store.py @@ -18,7 +18,7 @@ from ._utils import validate_storage_name if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Awaitable, Callable from crawlee.configuration import Configuration from crawlee.storage_backends import StorageBackend @@ -141,12 +141,11 @@ async def drop(self) -> None: storage_instance_manager = service_locator.storage_instance_manager storage_instance_manager.remove_from_cache(self) - await self._clear_cache() # Clear cache with persistent values. - await self._backend.drop() + await self._clear_store_and_cache(self._backend.drop) @override async def purge(self) -> None: - await self._backend.purge() + await self._clear_store_and_cache(self._backend.purge) @overload async def get_value(self, key: str) -> Any: ... @@ -370,11 +369,23 @@ async def persist_autosaved_values(self) -> None: for value in cache.values(): await value.persist_state() - async def _clear_cache(self) -> None: - """Clear cache with autosaved values.""" - if self.id in self._autosaved_values: - async with self._autosave_lock: - cache = self._autosaved_values[self.id] - for value in cache.values(): + async def _clear_store_and_cache(self, clear_method: Callable[[], Awaitable[None]]) -> None: + """Clear the store with the backend's `clear_method`, and the cache of autosaved values with it.""" + # The autosaved values go with the records, or their next save would write them back. The lock is held until + # the records are gone, so that a concurrent `get_auto_saved_value` can't load one of them. + async with self._autosave_lock: + cache = self._autosaved_values.get(self.id, {}) + initialized_values = [value for value in cache.values() if value.is_initialized] + + try: + for value in initialized_values: await value.teardown() - cache.clear() + + await clear_method() + except (Exception, asyncio.CancelledError): + # A failed call keeps the values, so they listen for `PERSIST_STATE` events again. + for value in initialized_values: + await value.initialize() + raise + + cache.clear() diff --git a/tests/unit/storages/test_key_value_store.py b/tests/unit/storages/test_key_value_store.py index d0a7c79d03..2b215f6aef 100644 --- a/tests/unit/storages/test_key_value_store.py +++ b/tests/unit/storages/test_key_value_store.py @@ -10,6 +10,7 @@ from crawlee import service_locator from crawlee.configuration import Configuration +from crawlee.events import Event, EventPersistStateData from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend, SqlStorageBackend, StorageBackend from crawlee.storage_backends._memory import MemoryKeyValueStoreBackend from crawlee.storage_backends.models import KeyValueStoreRecord @@ -1322,14 +1323,14 @@ async def insert_autosaved_value() -> None: await asyncio.wait_for(asyncio.gather(persist_task, insert_task), timeout=5) -async def test_clear_cache_with_concurrent_new_autosaved_key( +async def test_purge_with_concurrent_new_autosaved_key( kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test clearing the cache while adding a new autosaved key.""" + """An autosaved key added during a purge stays cached after it.""" await kvs.get_auto_saved_value('existing') - # Park the clear loop mid-iteration by blocking the underlying write, then insert a new key. + # Park the purge in the last save of the existing value by blocking the underlying write, then insert a new key. clear_entered = asyncio.Event() release_clear = asyncio.Event() original_set_value = kvs.set_value @@ -1341,21 +1342,22 @@ async def blocking_set_value(*args: Any, **kwargs: Any) -> None: monkeypatch.setattr(kvs, 'set_value', blocking_set_value) - clear_task = asyncio.create_task(kvs._clear_cache()) + clear_task = asyncio.create_task(kvs.purge()) await asyncio.wait_for(clear_entered.wait(), timeout=5) insert_started = asyncio.Event() - async def insert_autosaved_value() -> None: + async def insert_autosaved_value() -> Any: insert_started.set() - await kvs.get_auto_saved_value('added-during-clear') + return await kvs.get_auto_saved_value('added-during-clear') insert_task = asyncio.create_task(insert_autosaved_value()) await asyncio.wait_for(insert_started.wait(), timeout=5) release_clear.set() - # Should not raise `RuntimeError: dictionary changed size during iteration`. - await asyncio.wait_for(asyncio.gather(clear_task, insert_task), timeout=5) + _, added_value = await asyncio.wait_for(asyncio.gather(clear_task, insert_task), timeout=5) + + assert await kvs.get_auto_saved_value('added-during-clear') is added_value async def test_get_auto_saved_value_retries_cancelled_load(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: @@ -1413,3 +1415,110 @@ async def test_mutating_read_value_does_not_change_stored_value(kvs: KeyValueSto value['key'] = 'changed' assert await kvs.get_value('test-key') == {'key': 'value'} + + +async def _emit_persist_state() -> None: + event_manager = service_locator.get_event_manager() + async with event_manager: + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await event_manager.wait_for_all_listeners_to_complete() + + +async def test_purge_drops_autosaved_values(kvs: KeyValueStore) -> None: + """A purge drops the autosaved values, so neither the next save nor the next read brings them back.""" + autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) + autosaved_value['count'] = 1 + + await kvs.purge() + await kvs.persist_autosaved_values() + await _emit_persist_state() + + assert await kvs.get_value('state') is None + assert await kvs.get_auto_saved_value('state', {'count': 0}) == {'count': 0} + + +async def test_purge_with_concurrent_get_auto_saved_value(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: + """A `get_auto_saved_value` call made during a purge doesn't load the value that is being purged.""" + autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) + autosaved_value['count'] = 1 + + purge_started = asyncio.Event() + release_purge = asyncio.Event() + backend_purge = kvs._backend.purge + + async def blocking_purge() -> None: + purge_started.set() + await release_purge.wait() + await backend_purge() + + monkeypatch.setattr(kvs._backend, 'purge', AsyncMock(side_effect=blocking_purge)) + + purge_task = asyncio.create_task(kvs.purge()) + await purge_started.wait() + + # The read gets time to load the record while the purge is parked. + read_task = asyncio.create_task(kvs.get_auto_saved_value('state', {'count': 0})) + await asyncio.wait([read_task], timeout=0.1) + release_purge.set() + await purge_task + + assert await read_task == {'count': 0} + + +@pytest.mark.parametrize('operation', [pytest.param('purge', id='purge'), pytest.param('drop', id='drop')]) +async def test_failed_purge_or_drop_keeps_autosaved_values( + kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch, operation: str +) -> None: + """A purge or drop that the storage backend rejects leaves the autosaved values saved on `PERSIST_STATE`.""" + autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) + purge_or_drop = getattr(kvs, operation) + + with monkeypatch.context() as failing_backend: + failing_backend.setattr(kvs._backend, operation, AsyncMock(side_effect=NotImplementedError)) + with pytest.raises(NotImplementedError): + await purge_or_drop() + + autosaved_value['count'] = 1 + await _emit_persist_state() + + assert await kvs.get_value('state') == {'count': 1} + assert await kvs.get_auto_saved_value('state') is autosaved_value + + +async def test_failed_purge_with_invalid_autosaved_record(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed purge raises the storage backend's error when an autosaved value with an invalid record is cached.""" + await kvs.set_value('state', [1, 2, 3]) + with pytest.raises(ValidationError): + await kvs.get_auto_saved_value('state') + + monkeypatch.setattr(kvs._backend, 'purge', AsyncMock(side_effect=NotImplementedError)) + with pytest.raises(NotImplementedError): + await kvs.purge() + + +@pytest.mark.parametrize('operation', [pytest.param('purge', id='purge'), pytest.param('drop', id='drop')]) +async def test_cancelled_purge_or_drop_keeps_autosaved_values( + kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch, operation: str +) -> None: + """A purge or drop cancelled during the last save of an autosaved value leaves it saved on `PERSIST_STATE`.""" + autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) + set_value = kvs.set_value + save_started = asyncio.Event() + + async def hanging_set_value(*_args: Any, **_kwargs: Any) -> None: + save_started.set() + await asyncio.sleep(3600) + + monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=hanging_set_value)) + purge_or_drop = getattr(kvs, operation) + task = asyncio.create_task(purge_or_drop()) + await save_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + monkeypatch.setattr(kvs, 'set_value', set_value) + autosaved_value['count'] = 1 + await _emit_persist_state() + + assert await kvs.get_value('state') == {'count': 1} From d85a39211e5f13844151035ed48bf68952991d45 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 18 Nov 2026 13:58:16 +0000 Subject: [PATCH 2/2] address review --- docs/06_upgrading/upgrading_to_v2.md | 2 +- src/crawlee/storages/_key_value_store.py | 4 +- tests/unit/storages/test_key_value_store.py | 68 +++++++++++++++------ 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 4b7c7726c4..cbce040202 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -303,7 +303,7 @@ In v1, `RequestQueue.purge()` worked around a storage backend that didn't suppor ## `KeyValueStore.purge` drops autosaved values -`KeyValueStore.purge()` now also drops the values that `get_auto_saved_value()` and `use_state` keep in memory for the store, the same as `drop()`. Their later changes aren't saved, and the next `get_auto_saved_value()` or `use_state` call starts from the default value. In v1, a later save could write them back into the purged store. +`KeyValueStore.purge()` now also drops the values that `get_auto_saved_value()` and `use_state()` keep in memory for the store, the same as `drop()`. Their later changes aren't saved, and the next `get_auto_saved_value()` or `use_state()` call starts from the default value. In v1, a later save could write them back into the purged store. ## `check_readiness` replaces `is_empty` and `is_finished` diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py index ec4e84627c..15e180ecdf 100644 --- a/src/crawlee/storages/_key_value_store.py +++ b/src/crawlee/storages/_key_value_store.py @@ -138,11 +138,11 @@ async def open( @override async def drop(self) -> None: + await self._clear_store_and_cache(self._backend.drop) + storage_instance_manager = service_locator.storage_instance_manager storage_instance_manager.remove_from_cache(self) - await self._clear_store_and_cache(self._backend.drop) - @override async def purge(self) -> None: await self._clear_store_and_cache(self._backend.purge) diff --git a/tests/unit/storages/test_key_value_store.py b/tests/unit/storages/test_key_value_store.py index 2b215f6aef..b7e478f022 100644 --- a/tests/unit/storages/test_key_value_store.py +++ b/tests/unit/storages/test_key_value_store.py @@ -1331,33 +1331,33 @@ async def test_purge_with_concurrent_new_autosaved_key( await kvs.get_auto_saved_value('existing') # Park the purge in the last save of the existing value by blocking the underlying write, then insert a new key. - clear_entered = asyncio.Event() - release_clear = asyncio.Event() + purge_entered = asyncio.Event() + release_purge = asyncio.Event() original_set_value = kvs.set_value async def blocking_set_value(*args: Any, **kwargs: Any) -> None: - clear_entered.set() - await asyncio.wait_for(release_clear.wait(), timeout=5) + purge_entered.set() + await asyncio.wait_for(release_purge.wait(), timeout=5) await original_set_value(*args, **kwargs) monkeypatch.setattr(kvs, 'set_value', blocking_set_value) - clear_task = asyncio.create_task(kvs.purge()) - await asyncio.wait_for(clear_entered.wait(), timeout=5) + purge_task = asyncio.create_task(kvs.purge()) + await asyncio.wait_for(purge_entered.wait(), timeout=5) insert_started = asyncio.Event() async def insert_autosaved_value() -> Any: insert_started.set() - return await kvs.get_auto_saved_value('added-during-clear') + return await kvs.get_auto_saved_value('added-during-purge') insert_task = asyncio.create_task(insert_autosaved_value()) await asyncio.wait_for(insert_started.wait(), timeout=5) - release_clear.set() + release_purge.set() - _, added_value = await asyncio.wait_for(asyncio.gather(clear_task, insert_task), timeout=5) + _, added_value = await asyncio.wait_for(asyncio.gather(purge_task, insert_task), timeout=5) - assert await kvs.get_auto_saved_value('added-during-clear') is added_value + assert await kvs.get_auto_saved_value('added-during-purge') is added_value async def test_get_auto_saved_value_retries_cancelled_load(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: @@ -1437,6 +1437,21 @@ async def test_purge_drops_autosaved_values(kvs: KeyValueStore) -> None: assert await kvs.get_auto_saved_value('state', {'count': 0}) == {'count': 0} +async def test_drop_drops_autosaved_values(storage_backend: StorageBackend, monkeypatch: pytest.MonkeyPatch) -> None: + """A drop drops the autosaved values, so no later save writes them to the dropped store.""" + kvs = await KeyValueStore.open(name='drop-test', storage_backend=storage_backend) + autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) + await kvs.drop() + + set_value = AsyncMock() + monkeypatch.setattr(kvs, 'set_value', set_value) + autosaved_value['count'] = 1 + await kvs.persist_autosaved_values() + await _emit_persist_state() + + set_value.assert_not_called() + + async def test_purge_with_concurrent_get_auto_saved_value(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: """A `get_auto_saved_value` call made during a purge doesn't load the value that is being purged.""" autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) @@ -1448,17 +1463,18 @@ async def test_purge_with_concurrent_get_auto_saved_value(kvs: KeyValueStore, mo async def blocking_purge() -> None: purge_started.set() - await release_purge.wait() + await asyncio.wait_for(release_purge.wait(), timeout=5) await backend_purge() monkeypatch.setattr(kvs._backend, 'purge', AsyncMock(side_effect=blocking_purge)) purge_task = asyncio.create_task(kvs.purge()) - await purge_started.wait() + await asyncio.wait_for(purge_started.wait(), timeout=5) # The read gets time to load the record while the purge is parked. read_task = asyncio.create_task(kvs.get_auto_saved_value('state', {'count': 0})) await asyncio.wait([read_task], timeout=0.1) + assert not read_task.done() release_purge.set() await purge_task @@ -1485,6 +1501,18 @@ async def test_failed_purge_or_drop_keeps_autosaved_values( assert await kvs.get_auto_saved_value('state') is autosaved_value +async def test_failed_drop_keeps_store_cached( + kvs: KeyValueStore, storage_backend: StorageBackend, monkeypatch: pytest.MonkeyPatch +) -> None: + """A drop that the storage backend rejects leaves the store cached, so opening it again returns the same store.""" + with monkeypatch.context() as failing_backend: + failing_backend.setattr(kvs._backend, 'drop', AsyncMock(side_effect=NotImplementedError)) + with pytest.raises(NotImplementedError): + await kvs.drop() + + assert await KeyValueStore.open(storage_backend=storage_backend) is kvs + + async def test_failed_purge_with_invalid_autosaved_record(kvs: KeyValueStore, monkeypatch: pytest.MonkeyPatch) -> None: """A failed purge raises the storage backend's error when an autosaved value with an invalid record is cached.""" await kvs.set_value('state', [1, 2, 3]) @@ -1502,22 +1530,22 @@ async def test_cancelled_purge_or_drop_keeps_autosaved_values( ) -> None: """A purge or drop cancelled during the last save of an autosaved value leaves it saved on `PERSIST_STATE`.""" autosaved_value = await kvs.get_auto_saved_value('state', {'count': 0}) - set_value = kvs.set_value save_started = asyncio.Event() async def hanging_set_value(*_args: Any, **_kwargs: Any) -> None: save_started.set() await asyncio.sleep(3600) - monkeypatch.setattr(kvs, 'set_value', AsyncMock(side_effect=hanging_set_value)) purge_or_drop = getattr(kvs, operation) - task = asyncio.create_task(purge_or_drop()) - await save_started.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - monkeypatch.setattr(kvs, 'set_value', set_value) + with monkeypatch.context() as hanging_save: + hanging_save.setattr(kvs, 'set_value', AsyncMock(side_effect=hanging_set_value)) + task = asyncio.create_task(purge_or_drop()) + await asyncio.wait_for(save_started.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + autosaved_value['count'] = 1 await _emit_persist_state()