From 669990b3595cf8d75a3e1c6da6772f2e01a19648 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 17 Nov 2026 13:32:26 +0000 Subject: [PATCH 1/3] close Redis connections and keep each backend's storages separate --- docs/03_concepts/12_storage_backends.mdx | 4 ++ .../redis_storage_backend_basic_example.py | 16 +++-- ...s_storage_backend_configuration_example.py | 28 ++++---- docs/06_upgrading/upgrading_to_v2.md | 30 +++++++- .../storage_backends/_redis/_backend_mixin.py | 50 ++++++++++---- .../_redis/_dataset_backend.py | 19 +++--- .../_redis/_key_value_store_backend.py | 20 +++--- .../_redis/_request_queue_backend.py | 28 ++++---- .../_redis/_storage_backend.py | 61 +++++++++++++++-- .../_redis/test_redis_kvs_backend.py | 47 ++++++++++++- .../_redis/test_redis_rq_backend.py | 20 ++++++ .../_redis/test_redis_storage_backend.py | 68 ++++++++++++++++++- 12 files changed, 317 insertions(+), 74 deletions(-) diff --git a/docs/03_concepts/12_storage_backends.mdx b/docs/03_concepts/12_storage_backends.mdx index 3cc110157f..ccded3ca4f 100644 --- a/docs/03_concepts/12_storage_backends.mdx +++ b/docs/03_concepts/12_storage_backends.mdx @@ -383,6 +383,10 @@ Data persistence in Redis depends on your [database configuration](https://redis The backend requires either a Redis connection string or a pre-configured Redis client instance. Use a pre-configured client when you need custom Redis settings such as connection pooling, timeouts, or SSL/TLS encryption. +Use the backend as an async context manager, so that its connections are closed when you're done. It closes a client it created from the connection string, but not one you pass in. Storages opened through a backend stop working once it's closed. + +Each backend opens its own storages. With `purge_on_start` enabled, it purges the unnamed storages, such as the default dataset, when it opens them. As a result, a new backend connected to the same database wipes the data an earlier one wrote to these storages. + {RedisStorageBackendBasicExample} diff --git a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py index 3798cb8c70..a4a655c333 100644 --- a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py +++ b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py @@ -1,10 +1,14 @@ from crawlee.crawlers import ParselCrawler from crawlee.storage_backends import RedisStorageBackend -# Create a new instance of storage backend using connection string. -# 'redis://localhost:6379' is the just placeholder, replace it with your actual -# connection string. -storage_backend = RedisStorageBackend(connection_string='redis://localhost:6379') -# And pass it to the crawler. -crawler = ParselCrawler(storage_backend=storage_backend) +async def main() -> None: + # Create a new instance of the storage backend using a connection string. + # 'redis://localhost:6379' is just a placeholder. Replace it with your actual + # connection string. + # Use the context manager to ensure that connections are properly cleaned up. + async with RedisStorageBackend( + connection_string='redis://localhost:6379' + ) as storage_backend: + # And pass it to the crawler. + crawler = ParselCrawler(storage_backend=storage_backend) diff --git a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py index 0c35151763..8b21bbcc3f 100644 --- a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py +++ b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py @@ -4,24 +4,26 @@ from crawlee.crawlers import ParselCrawler from crawlee.storage_backends import RedisStorageBackend -# Create a new instance of storage backend using a Redis client with custom settings. -# Replace host and port with your actual Redis server configuration. -# Other Redis client settings can be adjusted as needed. -storage_backend = RedisStorageBackend( - redis=Redis( + +async def main() -> None: + # Create a Redis client with custom settings. + # Replace host and port with your actual Redis server configuration. + # Other Redis client settings can be adjusted as needed. + redis = Redis( host='localhost', port=6379, retry_on_timeout=True, socket_keepalive=True, socket_connect_timeout=10, ) -) -# Create a configuration with custom settings. -configuration = Configuration(purge_on_start=False) + # The storage backend doesn't close a client you pass in, so close it yourself. + async with redis, RedisStorageBackend(redis=redis) as storage_backend: + # Create a configuration with custom settings. + configuration = Configuration(purge_on_start=False) -# And pass them to the crawler. -crawler = ParselCrawler( - storage_backend=storage_backend, - configuration=configuration, -) + # And pass them to the crawler. + crawler = ParselCrawler( + storage_backend=storage_backend, + configuration=configuration, + ) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index ee57b83421..f638ac3b21 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -182,6 +182,34 @@ The memory backend now handles values the same way as the file-system backend. I If processing a request can take longer than 3 minutes, call `RequestQueue.set_expected_request_processing_time()` with the longest time you expect. Every request fetched after that call stays locked at least that long. To give a single request that's still being processed more time, call `RequestQueue.extend_request_processing_time()`. +## Each `RedisStorageBackend` opens its own storages + +In v1, all `RedisStorageClient` instances in a process shared the storages opened through any of them, even when they connected to different Redis servers. Now each `RedisStorageBackend` instance opens its own storages. With `purge_on_start` enabled, a new instance purges the unnamed storages, such as the default dataset, when it opens them. When it connects to the same database as an earlier instance, the data the earlier one wrote to these storages is lost. To read what a crawler stored in them, open them through the crawler's backend. + +Use `RedisStorageBackend` as an async context manager. On exit, it closes the Redis client it created from the connection string. In v1, nothing closed that client. A client you pass in stays open, so close it yourself. Storages opened through a closed backend stop working. + +For example: + +```python +# v1 +storage_client = RedisStorageClient(connection_string='redis://localhost:6379') +crawler = ParselCrawler(storage_client=storage_client) +await crawler.run(['https://crawlee.dev']) + +dataset = await Dataset.open( + storage_client=RedisStorageClient(connection_string='redis://localhost:6379') +) + +# v2 +async with RedisStorageBackend( + connection_string='redis://localhost:6379' +) as storage_backend: + crawler = ParselCrawler(storage_backend=storage_backend) + await crawler.run(['https://crawlee.dev']) + + dataset = await Dataset.open(storage_backend=storage_backend) +``` + ## Crawler `id` is a string The `id` argument of `BasicCrawler` and its subclasses is now a `str`, so pass a string wherever you passed an integer. `StatisticsState.stats_id` and `EventCrawlerStatusData.crawler_id` are strings too, and `crawler_id` holds the crawler's `id`. In v1, it held the Python object ID of the crawler. @@ -291,7 +319,7 @@ for urls in batches: await crawler.run(urls) ``` -`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged once per process, when they're first opened. +`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged when a storage backend first opens them. ## `RequestQueue.purge` raises when the storage backend can't purge diff --git a/src/crawlee/storage_backends/_redis/_backend_mixin.py b/src/crawlee/storage_backends/_redis/_backend_mixin.py index 4fe2c59434..bef01a3779 100644 --- a/src/crawlee/storage_backends/_redis/_backend_mixin.py +++ b/src/crawlee/storage_backends/_redis/_backend_mixin.py @@ -20,6 +20,8 @@ from crawlee.storage_backends.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata + from ._storage_backend import RedisStorageBackend + logger = getLogger(__name__) @@ -46,17 +48,17 @@ class RedisBackendMixin: _STORAGE_TYPE: ClassVar[str] """Human-readable storage type for error messages.""" - def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: + def __init__(self, storage_name: str, storage_id: str, storage_backend: RedisStorageBackend) -> None: self._storage_name = storage_name self._storage_id = storage_id - self._redis = redis + self._storage_backend = storage_backend self._scripts_loaded = False @property def redis(self) -> Redis: """Return the Redis client instance.""" - return self._redis + return self._storage_backend.redis @property def metadata_key(self) -> str: @@ -107,7 +109,7 @@ async def _open( name: str | None, alias: str | None, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata], - redis: Redis, + storage_backend: RedisStorageBackend, extra_metadata_fields: dict[str, Any], instance_kwargs: dict[str, Any], ) -> Self: @@ -117,7 +119,7 @@ async def _open( id: The ID of the storage. If not provided, a random ID will be generated. name: The name of the storage for named (global scope) storages. alias: The alias of the storage for unnamed (run scope) storages. - redis: Redis client instance. + storage_backend: The storage backend providing the Redis client. metadata_model: Pydantic model for metadata validation. extra_metadata_fields: Storage-specific metadata fields. instance_kwargs: Additional arguments for the backend constructor. @@ -129,17 +131,19 @@ async def _open( storage_id: str | None = None # Determine if storage exists by ID or name if id: - storage_name = await cls._get_metadata_name_by_id(id=id, redis=redis) + storage_name = await cls._get_metadata_name_by_id(id=id, redis=storage_backend.redis) storage_id = id if storage_name is None: raise ValueError(f'{cls._STORAGE_TYPE} with ID "{id}" does not exist.') else: - metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis) + metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=storage_backend.redis) storage_name = internal_name if metadata_data is not None else None storage_id = metadata_data['id'] if metadata_data is not None else None # If both storage_name and storage_id are found, open existing storage if storage_name and storage_id: - backend = cls(storage_name=storage_name, storage_id=storage_id, redis=redis, **instance_kwargs) + backend = cls( + storage_name=storage_name, storage_id=storage_id, storage_backend=storage_backend, **instance_kwargs + ) async with backend._get_pipeline() as pipe: await backend._update_metadata(pipe, update_accessed_at=True) # Otherwise, create a new storage @@ -153,12 +157,28 @@ async def _open( modified_at=now, **extra_metadata_fields, ) - backend = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) + backend = cls( + storage_name=internal_name, storage_id=metadata.id, storage_backend=storage_backend, **instance_kwargs + ) created = await backend._create_metadata_and_storage(internal_name, metadata.model_dump()) # The storage was probably not created due to a race condition. Let's try to open it using the name. if not created: - metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis, with_wait=True) - backend = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) + metadata_data = await cls._get_metadata_by_name( + name=internal_name, redis=storage_backend.redis, with_wait=True + ) + if metadata_data is None: + raise RuntimeError( + f'{cls._STORAGE_TYPE} "{internal_name}" is being created by another client, but its metadata ' + 'did not appear in time.' + ) + + # The storage that won the race has its own ID, which the backend must use to drop it. + backend = cls( + storage_name=internal_name, + storage_id=metadata_data['id'], + storage_backend=storage_backend, + **instance_kwargs, + ) # Ensure Lua scripts are loaded await backend._ensure_scripts_loaded() @@ -177,7 +197,7 @@ async def _ensure_scripts_loaded(self) -> None: @asynccontextmanager async def _get_pipeline(self, *, with_execute: bool = True) -> AsyncGenerator[Pipeline]: """Create a new Redis pipeline.""" - async with self._redis.pipeline() as pipe: + async with self.redis.pipeline() as pipe: try: pipe.multi() yield pipe @@ -192,7 +212,7 @@ async def _create_script(self, script_name: str) -> AsyncScript: """Load a Lua script from a file and return a Script object.""" script_content = await asyncio.to_thread(read_lua_script, script_name) - return self._redis.register_script(script_content) + return self.redis.register_script(script_content) async def _create_metadata_and_storage(self, storage_name: str, metadata: dict) -> bool: index_id_to_name = f'{self._MAIN_KEY}:id_to_name' @@ -202,7 +222,7 @@ async def _create_metadata_and_storage(self, storage_name: str, metadata: dict) metadata['modified_at'] = metadata['modified_at'].isoformat() # Try to create name_to_id index entry, if it already exists, return False. - name_to_id = await await_redis_response(self._redis.hsetnx(index_name_to_id, storage_name, metadata['id'])) + name_to_id = await await_redis_response(self.redis.hsetnx(index_name_to_id, storage_name, metadata['id'])) # If name already exists, return False. Probably an attempt at parallel creation. if not name_to_id: return False @@ -244,7 +264,7 @@ async def _get_metadata( self, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata] ) -> DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata: """Retrieve the storage metadata.""" - metadata_dict = await self._get_metadata_by_name(name=self._storage_name, redis=self._redis) + metadata_dict = await self._get_metadata_by_name(name=self._storage_name, redis=self.redis) if metadata_dict is None: raise ValueError(f'{self._STORAGE_TYPE} with name "{self._storage_name}" does not exist.') async with self._get_pipeline() as pipe: diff --git a/src/crawlee/storage_backends/_redis/_dataset_backend.py b/src/crawlee/storage_backends/_redis/_dataset_backend.py index 8f6b0e9874..e2a5624e6b 100644 --- a/src/crawlee/storage_backends/_redis/_dataset_backend.py +++ b/src/crawlee/storage_backends/_redis/_dataset_backend.py @@ -17,12 +17,13 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping - from redis.asyncio import Redis from redis.asyncio.client import Pipeline from redis.commands.json._util import JsonType from crawlee._types import JsonSerializable + from ._storage_backend import RedisStorageBackend + logger = getLogger(__name__) @@ -57,7 +58,7 @@ class RedisDatasetBackend(DatasetBackend, RedisBackendMixin): _STORAGE_TYPE = 'Dataset' """Human-readable storage type for error messages.""" - def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: + def __init__(self, storage_name: str, storage_id: str, storage_backend: RedisStorageBackend) -> None: """Initialize a new instance. Preferably use the `RedisDatasetBackend.open` class method to create a new instance. @@ -65,9 +66,9 @@ def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: Args: storage_name: Internal storage name used for Redis keys. storage_id: Unique identifier for the dataset. - redis: Redis client instance. + storage_backend: The storage backend providing the Redis client. """ - super().__init__(storage_name=storage_name, storage_id=storage_id, redis=redis) + super().__init__(storage_name=storage_name, storage_id=storage_id, storage_backend=storage_backend) @property def _items_key(self) -> str: @@ -81,7 +82,7 @@ async def open( id: str | None, name: str | None, alias: str | None, - redis: Redis, + storage_backend: RedisStorageBackend, ) -> RedisDatasetBackend: """Open or create a new Redis dataset backend. @@ -93,7 +94,7 @@ async def open( id: The ID of the dataset. If not provided, a random ID will be generated. name: The name of the dataset for named (global scope) storages. alias: The alias of the dataset for unnamed (run scope) storages. - redis: Redis client instance. + storage_backend: The storage backend providing the Redis client. Returns: An instance for the opened or created storage backend. @@ -102,7 +103,7 @@ async def open( id=id, name=name, alias=alias, - redis=redis, + storage_backend=storage_backend, metadata_model=DatasetMetadata, extra_metadata_fields={'item_count': 0}, instance_kwargs={}, @@ -207,7 +208,7 @@ async def get_data( if json_path == '$': json_path = '$[*]' - data = await await_redis_response(self._redis.json().get(self._items_key, json_path)) + data = await await_redis_response(self.redis.json().get(self._items_key, json_path)) if not isinstance(data, list): data = [] @@ -295,7 +296,7 @@ async def iterate_items( json_path = f'$[{batch_start}:{batch_end}]' # Get batch of items - batch_items = await await_redis_response(self._redis.json().get(self._items_key, json_path)) + batch_items = await await_redis_response(self.redis.json().get(self._items_key, json_path)) # Handle case where batch_items might be None or not a list if batch_items is None or not isinstance(batch_items, list): diff --git a/src/crawlee/storage_backends/_redis/_key_value_store_backend.py b/src/crawlee/storage_backends/_redis/_key_value_store_backend.py index 4d4d9a7055..db4434c403 100644 --- a/src/crawlee/storage_backends/_redis/_key_value_store_backend.py +++ b/src/crawlee/storage_backends/_redis/_key_value_store_backend.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from redis.asyncio import Redis + from ._storage_backend import RedisStorageBackend logger = getLogger(__name__) @@ -64,12 +64,12 @@ class RedisKeyValueStoreBackend(KeyValueStoreBackend, RedisBackendMixin): A single record larger than this is still fetched, but alone in its batch. """ - def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: + def __init__(self, storage_name: str, storage_id: str, storage_backend: RedisStorageBackend) -> None: """Initialize a new instance. Preferably use the `RedisKeyValueStoreBackend.open` class method to create a new instance. """ - super().__init__(storage_name=storage_name, storage_id=storage_id, redis=redis) + super().__init__(storage_name=storage_name, storage_id=storage_id, storage_backend=storage_backend) @property def _items_key(self) -> str: @@ -88,7 +88,7 @@ async def open( id: str | None, name: str | None, alias: str | None, - redis: Redis, + storage_backend: RedisStorageBackend, ) -> RedisKeyValueStoreBackend: """Open or create a new Redis key-value store backend. @@ -100,7 +100,7 @@ async def open( id: The ID of the key-value store. If not provided, a random ID will be generated. name: The name of the key-value store for named (global scope) storages. alias: The alias of the key-value store for unnamed (run scope) storages. - redis: Redis client instance. + storage_backend: The storage backend providing the Redis client. Returns: An instance for the opened or created storage backend. @@ -109,7 +109,7 @@ async def open( id=id, name=name, alias=alias, - redis=redis, + storage_backend=storage_backend, metadata_model=KeyValueStoreMetadata, extra_metadata_fields={}, instance_kwargs={}, @@ -175,7 +175,7 @@ async def set_value(self, *, key: str, value: Any, content_type: str | None = No @retry_on_error(RedisError) @override async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: - serialized_metadata_item = await await_redis_response(self._redis.hget(self._metadata_items_key, key)) + serialized_metadata_item = await await_redis_response(self.redis.hget(self._metadata_items_key, key)) async with self._get_pipeline() as pipe: await self._update_metadata(pipe, **MetadataUpdateParams(update_accessed_at=True)) @@ -191,7 +191,7 @@ async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: return KeyValueStoreRecord(value=None, **metadata_item.model_dump()) # Query the record by key - value_bytes = expect_bytes(await await_redis_response(self._redis.hget(self._items_key, key))) + value_bytes = expect_bytes(await await_redis_response(self.redis.hget(self._items_key, key))) return self._build_record(metadata_item, value_bytes) @@ -248,7 +248,7 @@ async def iterate_keys( exclusive_start_key: str | None = None, limit: int | None = None, ) -> AsyncIterator[KeyValueStoreRecordMetadata]: - items_data = await await_redis_response(self._redis.hgetall(self._metadata_items_key)) + items_data = await await_redis_response(self.redis.hgetall(self._metadata_items_key)) if not items_data: return # No items to iterate over @@ -309,7 +309,7 @@ async def iterate_entries( async def _fetch_records(self, batch: list[KeyValueStoreRecordMetadata]) -> list[KeyValueStoreRecord]: """Fetch the values of the given records with a single HMGET call and return the deserialized records.""" keys = [item.key for item in batch] - values = [expect_bytes(v) for v in await await_redis_response(self._redis.hmget(self._items_key, keys))] + values = [expect_bytes(v) for v in await await_redis_response(self.redis.hmget(self._items_key, keys))] # A missing value means the record was deleted after its metadata was listed, so it is skipped silently. records = ( diff --git a/src/crawlee/storage_backends/_redis/_request_queue_backend.py b/src/crawlee/storage_backends/_redis/_request_queue_backend.py index 284946570b..4e8c9cf2be 100644 --- a/src/crawlee/storage_backends/_redis/_request_queue_backend.py +++ b/src/crawlee/storage_backends/_redis/_request_queue_backend.py @@ -21,10 +21,11 @@ if TYPE_CHECKING: from collections.abc import Sequence - from redis.asyncio import Redis from redis.asyncio.client import Pipeline from redis.commands.core import AsyncScript + from ._storage_backend import RedisStorageBackend + logger = getLogger(__name__) @@ -90,7 +91,7 @@ def __init__( self, storage_name: str, storage_id: str, - redis: Redis, + storage_backend: RedisStorageBackend, dedup_strategy: Literal['default', 'bloom'] = 'default', bloom_error_rate: float = 1e-7, ) -> None: @@ -98,7 +99,7 @@ def __init__( Preferably use the `RedisRequestQueueBackend.open` class method to create a new instance. """ - super().__init__(storage_name=storage_name, storage_id=storage_id, redis=redis) + super().__init__(storage_name=storage_name, storage_id=storage_id, storage_backend=storage_backend) self._dedup_strategy = dedup_strategy """Deduplication strategy for the queue.""" @@ -173,7 +174,7 @@ async def open( id: str | None, name: str | None, alias: str | None, - redis: Redis, + storage_backend: RedisStorageBackend, dedup_strategy: Literal['default', 'bloom'] = 'default', bloom_error_rate: float = 1e-7, ) -> RedisRequestQueueBackend: @@ -187,7 +188,7 @@ async def open( id: The ID of the request queue. If not provided, a random ID will be generated. name: The name of the dataset for named (global scope) storages. alias: The alias of the dataset for unnamed (run scope) storages. - redis: Redis client instance. + storage_backend: The storage backend providing the Redis client. dedup_strategy: Strategy for request queue deduplication. Options are: - 'default': Uses Redis sets for exact deduplication. - 'bloom': Uses Redis Bloom filters for probabilistic deduplication with lower memory usage. When using @@ -202,7 +203,7 @@ async def open( id=id, name=name, alias=alias, - redis=redis, + storage_backend=storage_backend, metadata_model=RequestQueueMetadata, extra_metadata_fields={ 'had_multiple_clients': False, @@ -325,6 +326,7 @@ async def add_batch_of_requests( self._data_key, ], args=[int(forefront), json.dumps(new_unique_keys), json.dumps(new_request_data)], + client=self.redis, ) actually_added = set(json.loads(script_results)) @@ -373,6 +375,7 @@ async def fetch_next_request(self) -> Request | None: requests_json = await self._fetch_script( keys=[self._queue_key, self._in_progress_key, self._data_key], args=[self.client_key, blocked_until_timestamp, self._MAX_BATCH_FETCH_SIZE], + client=self.redis, ) async with self._get_pipeline() as pipe: @@ -390,7 +393,7 @@ async def fetch_next_request(self) -> Request | None: @retry_on_error(RedisError) @override async def get_request(self, unique_key: str) -> Request | None: - request_data = await await_redis_response(self._redis.hget(self._data_key, unique_key)) + request_data = await await_redis_response(self.redis.hget(self._data_key, unique_key)) if isinstance(request_data, (str, bytes, bytearray)): return Request.model_validate_json(request_data) @@ -402,7 +405,7 @@ async def get_request(self, unique_key: str) -> Request | None: async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None: # `hdel` is an atomic operation, so we can be sure that if it returns 1, the request was in progress and is now # removed from `in_progress`. - deleted = await await_redis_response(self._redis.hdel(self._in_progress_key, request.unique_key)) + deleted = await await_redis_response(self.redis.hdel(self._in_progress_key, request.unique_key)) # If not deleted, the request was not in progress. if not deleted: @@ -445,7 +448,7 @@ async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | # If we fail to mark the request as handled after removing it from in_progress, we restore request in # `in_progress` hash. await await_redis_response( - self._redis.hset( + self.redis.hset( self._in_progress_key, request.unique_key, json.dumps({'client_id': self.client_key, 'blocked_until_timestamp': blocked_until}), @@ -461,7 +464,7 @@ async def reclaim_request( *, forefront: bool = False, ) -> ProcessedRequest | None: - check_in_progress = await await_redis_response(self._redis.hexists(self._in_progress_key, request.unique_key)) + check_in_progress = await await_redis_response(self.redis.hexists(self._in_progress_key, request.unique_key)) if not check_in_progress: logger.info('Reclaiming request %s that is not in progress.', request.unique_key) return None @@ -514,7 +517,7 @@ async def is_empty(self) -> bool: self._next_reclaim_stale = datetime.now(tz=UTC) + self._RECLAIM_INTERVAL # Check if there are any requests in the queue. - requests_in_queue = await await_redis_response(self._redis.llen(self._queue_key)) + requests_in_queue = await await_redis_response(self.redis.llen(self._queue_key)) return requests_in_queue == 0 @retry_on_error(RedisError) @@ -545,6 +548,7 @@ async def extend_request_processing_time(self, request: Request, delta: timedelt extended = await self._extend_lock_script( keys=[self._in_progress_key], args=[request.unique_key, self.client_key, current_time, delta_ms], + client=self.redis, ) return extended == 1 @@ -581,7 +585,7 @@ async def _reclaim_stale_requests(self) -> None: current_time = int(datetime.now(tz=UTC).timestamp() * 1000) await self._reclaim_stale_script( - keys=[self._in_progress_key, self._queue_key, self._data_key], args=[current_time] + keys=[self._in_progress_key, self._queue_key, self._data_key], args=[current_time], client=self.redis ) @override diff --git a/src/crawlee/storage_backends/_redis/_storage_backend.py b/src/crawlee/storage_backends/_redis/_storage_backend.py index df0bfef2e7..e99de67950 100644 --- a/src/crawlee/storage_backends/_redis/_storage_backend.py +++ b/src/crawlee/storage_backends/_redis/_storage_backend.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Literal +from typing import TYPE_CHECKING, Literal from redis.asyncio import Redis from typing_extensions import override @@ -14,6 +14,11 @@ from ._key_value_store_backend import RedisKeyValueStoreBackend from ._request_queue_backend import RedisRequestQueueBackend +if TYPE_CHECKING: + from collections.abc import Hashable + from types import TracebackType + from typing import Self + @docs_group('Storage backends') class RedisStorageBackend(StorageBackend): @@ -51,7 +56,7 @@ def __init__( Args: connection_string: Redis connection string (e.g., "redis://localhost:6379"). Supports standard Redis URL format with optional database selection. - redis: Pre-configured Redis client instance. + redis: Pre-configured Redis client instance. The backend doesn't close it. queue_dedup_strategy: Strategy for request queue deduplication. Options are: - 'default': Uses Redis sets for exact deduplication. - 'bloom': Uses Redis Bloom filters for probabilistic deduplication with lower memory usage. When using @@ -72,6 +77,8 @@ def __init__( self._redis = Redis.from_url(connection_string) self._redis: Redis # to help type checker + self._owns_redis = redis is None + self._closed = False self._queue_dedup_strategy = queue_dedup_strategy self._queue_bloom_error_rate = queue_bloom_error_rate @@ -85,6 +92,50 @@ def __init__( stacklevel=2, ) + async def __aenter__(self) -> Self: + """Async context manager entry.""" + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + exc_traceback: TracebackType | None, + ) -> None: + """Async context manager exit.""" + await self.close() + + @property + def redis(self) -> Redis: + """The Redis client of the storage backend. + + Raises: + RuntimeError: If the storage backend is closed. + """ + # A closed client would silently open new connections that nobody closes. + if self._closed: + raise RuntimeError(f'The {self.__class__.__name__} is closed.') + + return self._redis + + async def close(self) -> None: + """Close the Redis client, unless it was passed in. + + The storage backend and the storages opened with it can't be used afterwards. + """ + if self._closed: + return + + self._closed = True + + if self._owns_redis: + await self._redis.aclose() + + @override + def get_storage_backend_cache_key(self, configuration: Configuration) -> Hashable: + # Storages stop working when their backend closes, so a different backend instance must not get them. + return super().get_storage_backend_cache_key(configuration), id(self) + @override async def create_dataset_backend( self, @@ -100,7 +151,7 @@ async def create_dataset_backend( id=id, name=name, alias=alias, - redis=self._redis, + storage_backend=self, ) await self._purge_if_needed(backend, configuration) @@ -121,7 +172,7 @@ async def create_kvs_backend( id=id, name=name, alias=alias, - redis=self._redis, + storage_backend=self, ) await self._purge_if_needed(backend, configuration) @@ -142,7 +193,7 @@ async def create_rq_backend( id=id, name=name, alias=alias, - redis=self._redis, + storage_backend=self, dedup_strategy=self._queue_dedup_strategy, bloom_error_rate=self._queue_bloom_error_rate, ) diff --git a/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py b/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py index 80d413f3e1..d84c5bf826 100644 --- a/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py +++ b/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py @@ -9,14 +9,14 @@ from fakeredis import FakeAsyncRedis from redis.exceptions import RedisError +from crawlee.configuration import Configuration from crawlee.storage_backends import RedisStorageBackend +from crawlee.storage_backends._redis import RedisKeyValueStoreBackend from crawlee.storage_backends._redis._utils import await_redis_response if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterator - from crawlee.storage_backends._redis import RedisKeyValueStoreBackend - @pytest.fixture async def kvs_backend( @@ -372,3 +372,46 @@ async def test_decoding_redis_client_is_rejected(suppress_user_warning: None) -> with pytest.raises(TypeError, match='decode_responses'): _ = [record async for record in kvs_backend.iterate_entries()] + + +async def test_lost_race_uses_existing_id( + redis_client: FakeAsyncRedis, + monkeypatch: pytest.MonkeyPatch, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A client that loses the race to create a store opens it under the ID of the store that won.""" + configuration = Configuration(purge_on_start=False) + storage_backend = RedisStorageBackend(redis=redis_client) + winner = await storage_backend.create_kvs_backend(name='raced', configuration=configuration) + winner_id = (await winner.get_metadata()).id + + original_lookup = RedisKeyValueStoreBackend._get_metadata_by_name + + async def lookup(name: str, redis: FakeAsyncRedis, *, with_wait: bool = False) -> dict | None: + # The first lookup comes before the other client created the store, so it finds nothing. + if lookup_mock.await_count == 1: + return None + return await original_lookup(name, redis, with_wait=with_wait) + + lookup_mock = AsyncMock(side_effect=lookup) + monkeypatch.setattr(RedisKeyValueStoreBackend, '_get_metadata_by_name', staticmethod(lookup_mock)) + loser = await storage_backend.create_kvs_backend(name='raced', configuration=configuration) + await loser.drop() + + assert await await_redis_response(redis_client.hget('key_value_stores:id_to_name', winner_id)) is None + + +async def test_lost_race_without_metadata_raises( + redis_client: FakeAsyncRedis, + monkeypatch: pytest.MonkeyPatch, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A client that loses the race to create a store raises when the metadata of the store that won never appears.""" + configuration = Configuration(purge_on_start=False) + storage_backend = RedisStorageBackend(redis=redis_client) + await storage_backend.create_kvs_backend(name='raced', configuration=configuration) + + monkeypatch.setattr(RedisKeyValueStoreBackend, '_get_metadata_by_name', staticmethod(AsyncMock(return_value=None))) + + with pytest.raises(RuntimeError, match=r'did not appear in time'): + await storage_backend.create_kvs_backend(name='raced', configuration=configuration) diff --git a/tests/unit/storage_backends/_redis/test_redis_rq_backend.py b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py index 2044003049..11e6e9b12b 100644 --- a/tests/unit/storage_backends/_redis/test_redis_rq_backend.py +++ b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py @@ -455,3 +455,23 @@ async def test_extend_rejects_non_positive_delta(rq_backend: RedisRequestQueueBa """Test that a lock extension must be positive.""" with pytest.raises(ValueError, match='must be positive'): await rq_backend.extend_request_processing_time(Request.from_url('https://example.com/1'), delta) + + +async def test_closed_backend_skips_scripts( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A queue of a closed backend raises before its scripts lock or extend any request.""" + request = Request.from_url('https://example.com/1') + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + rq_backend = await storage_backend.create_rq_backend(name='test_request_queue') + await rq_backend.add_batch_of_requests([request]) + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await rq_backend.fetch_next_request() + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await rq_backend.extend_request_processing_time(request, timedelta(minutes=10)) + + assert await await_redis_response(redis_client.hlen(rq_backend._in_progress_key)) == 0 diff --git a/tests/unit/storage_backends/_redis/test_redis_storage_backend.py b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py index 5d5d5180a8..023aa0c3ae 100644 --- a/tests/unit/storage_backends/_redis/test_redis_storage_backend.py +++ b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py @@ -1,9 +1,18 @@ from __future__ import annotations import sys -from unittest.mock import patch +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, patch import pytest +from redis.asyncio import Redis + +from crawlee.configuration import Configuration +from crawlee.storage_backends import RedisStorageBackend +from crawlee.storages import Dataset + +if TYPE_CHECKING: + from fakeredis import FakeAsyncRedis def test_import_error_handled() -> None: @@ -14,3 +23,60 @@ def test_import_error_handled() -> None: sys.modules.pop(mod_name, None) with pytest.raises(ImportError): from crawlee.storage_backends._redis import RedisStorageBackend # noqa: F401 PLC0415 + + +async def test_close_closes_own_client(suppress_user_warning: None) -> None: # noqa: ARG001 + """A client created from the connection string is closed with the backend, and only once.""" + with patch.object(Redis, 'aclose', AsyncMock()) as aclose: + async with RedisStorageBackend(connection_string='redis://localhost:6379') as storage_backend: + pass + + await storage_backend.close() + + aclose.assert_awaited_once() + + +async def test_close_keeps_passed_client( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A client passed in belongs to the caller, so the backend doesn't close it.""" + with patch.object(redis_client, 'aclose', AsyncMock()) as aclose: + async with RedisStorageBackend(redis=redis_client): + pass + + aclose.assert_not_awaited() + + +async def test_closed_backend_raises( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A closed backend refuses to open storages and to serve the ones opened before.""" + configuration = Configuration(purge_on_start=False) + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + kvs_backend = await storage_backend.create_kvs_backend(configuration=configuration) + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await kvs_backend.set_value(key='key', value='value') + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await storage_backend.create_kvs_backend(configuration=configuration) + + +async def test_new_backend_skips_storages_of_closed_one( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A new backend opens its own storage instead of the cached one bound to a closed backend.""" + configuration = Configuration(purge_on_start=False) + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + old_dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) + await dataset.push_data({'key': 'value'}) + + assert dataset is not old_dataset From 4b5668da1ac1fad855e68c3254cfcec12b6b1918 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 18 Nov 2026 13:05:28 +0000 Subject: [PATCH 2/3] address review --- .../redis_storage_backend_basic_example.py | 3 +++ ...s_storage_backend_configuration_example.py | 3 +++ docs/06_upgrading/upgrading_to_v2.md | 2 +- .../_redis/_request_queue_backend.py | 10 +++++++-- .../_redis/test_redis_rq_backend.py | 21 +++++++++++++++++++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py index a4a655c333..9aa4475d57 100644 --- a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py +++ b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_basic_example.py @@ -12,3 +12,6 @@ async def main() -> None: ) as storage_backend: # And pass it to the crawler. crawler = ParselCrawler(storage_backend=storage_backend) + + # Run the crawler inside the block, while the storage backend is open. + await crawler.run(['https://crawlee.dev']) diff --git a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py index 8b21bbcc3f..96ba48fc9c 100644 --- a/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py +++ b/docs/03_concepts/code_examples/storage_backends/redis_storage_backend_configuration_example.py @@ -27,3 +27,6 @@ async def main() -> None: storage_backend=storage_backend, configuration=configuration, ) + + # Run the crawler inside the block, while the storage backend is open. + await crawler.run(['https://crawlee.dev']) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index f638ac3b21..f66b7035ec 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -319,7 +319,7 @@ for urls in batches: await crawler.run(urls) ``` -`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged when a storage backend first opens them. +`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged once per process, when they're first opened. ## `RequestQueue.purge` raises when the storage backend can't purge diff --git a/src/crawlee/storage_backends/_redis/_request_queue_backend.py b/src/crawlee/storage_backends/_redis/_request_queue_backend.py index 4e8c9cf2be..c3f8ad3506 100644 --- a/src/crawlee/storage_backends/_redis/_request_queue_backend.py +++ b/src/crawlee/storage_backends/_redis/_request_queue_backend.py @@ -363,6 +363,9 @@ async def add_batch_of_requests( @retry_on_error(RedisError) @override async def fetch_next_request(self) -> Request | None: + # Cached requests are served only while the storage backend is open. + redis = self.redis + if self._pending_fetch_cache: return self._pending_fetch_cache.popleft() @@ -375,7 +378,7 @@ async def fetch_next_request(self) -> Request | None: requests_json = await self._fetch_script( keys=[self._queue_key, self._in_progress_key, self._data_key], args=[self.client_key, blocked_until_timestamp, self._MAX_BATCH_FETCH_SIZE], - client=self.redis, + client=redis, ) async with self._get_pipeline() as pipe: @@ -507,6 +510,9 @@ async def reclaim_request( @retry_on_error(RedisError) @override async def is_empty(self) -> bool: + # Cached requests are reported only while the storage backend is open. + redis = self.redis + # Requests buffered for fetching mean the queue is not empty. if self._pending_fetch_cache: return False @@ -517,7 +523,7 @@ async def is_empty(self) -> bool: self._next_reclaim_stale = datetime.now(tz=UTC) + self._RECLAIM_INTERVAL # Check if there are any requests in the queue. - requests_in_queue = await await_redis_response(self.redis.llen(self._queue_key)) + requests_in_queue = await await_redis_response(redis.llen(self._queue_key)) return requests_in_queue == 0 @retry_on_error(RedisError) diff --git a/tests/unit/storage_backends/_redis/test_redis_rq_backend.py b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py index 11e6e9b12b..ef4e892499 100644 --- a/tests/unit/storage_backends/_redis/test_redis_rq_backend.py +++ b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py @@ -475,3 +475,24 @@ async def test_closed_backend_skips_scripts( await rq_backend.extend_request_processing_time(request, timedelta(minutes=10)) assert await await_redis_response(redis_client.hlen(rq_backend._in_progress_key)) == 0 + + +async def test_closed_backend_skips_fetch_cache( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A queue of a closed backend raises instead of serving the requests it fetched earlier.""" + requests = [Request.from_url('https://example.com/1'), Request.from_url('https://example.com/2')] + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + rq_backend = await storage_backend.create_rq_backend(name='test_request_queue') + await rq_backend.add_batch_of_requests(requests) + await rq_backend.fetch_next_request() + + assert len(rq_backend._pending_fetch_cache) == 1 + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await rq_backend.fetch_next_request() + + with pytest.raises(RuntimeError, match=r'RedisStorageBackend is closed'): + await rq_backend.is_empty() From 9fef00e251698dcdba7151d11160ebf28251b57a Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 18 Nov 2026 13:16:43 +0000 Subject: [PATCH 3/3] add `test_new_backend_purges_default_dataset` --- .../_redis/test_redis_storage_backend.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/storage_backends/_redis/test_redis_storage_backend.py b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py index 023aa0c3ae..2dd489c396 100644 --- a/tests/unit/storage_backends/_redis/test_redis_storage_backend.py +++ b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py @@ -80,3 +80,21 @@ async def test_new_backend_skips_storages_of_closed_one( await dataset.push_data({'key': 'value'}) assert dataset is not old_dataset + + +async def test_new_backend_purges_default_dataset( + redis_client: FakeAsyncRedis, + suppress_user_warning: None, # noqa: ARG001 +) -> None: + """A new backend with `purge_on_start` enabled purges the default dataset an earlier backend wrote.""" + configuration = Configuration(purge_on_start=True) + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) + await dataset.push_data({'key': 'value'}) + + async with RedisStorageBackend(redis=redis_client) as storage_backend: + dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) + data = await dataset.get_data() + + assert data.items == []