diff --git a/docs/03_concepts/12_storage_backends.mdx b/docs/03_concepts/12_storage_backends.mdx index 3cc110157f..2dfe67601a 100644 --- a/docs/03_concepts/12_storage_backends.mdx +++ b/docs/03_concepts/12_storage_backends.mdx @@ -162,6 +162,10 @@ To use it, you need to install Crawlee with the appropriate extra dependency: By default, SqlStorageBackend uses SQLite. To use a different database, just provide the appropriate connection string via the `connection_string` parameter. No other code changes are needed—the same backend works for all supported databases. +Use the backend as an async context manager, so that its connections are closed when you're done. It disposes an engine it created, but not one you pass in. Storages opened through a backend stop working once it's closed. + +Each backend opens its own storages. With `purge_on_start` enabled, it purges the unnamed storages, such as the default dataset, when it opens them. As a result, a new backend connected to the same database wipes the data an earlier one wrote to these storages. + {SQLStorageBackendBasicExample} diff --git a/docs/03_concepts/code_examples/storage_backends/sql_storage_backend_configuration_example.py b/docs/03_concepts/code_examples/storage_backends/sql_storage_backend_configuration_example.py index 879054fc96..7118be3e30 100644 --- a/docs/03_concepts/code_examples/storage_backends/sql_storage_backend_configuration_example.py +++ b/docs/03_concepts/code_examples/storage_backends/sql_storage_backend_configuration_example.py @@ -6,28 +6,31 @@ async def main() -> None: - # Create a new instance of storage backend. - # On first run, also creates tables in your PostgreSQL database. - # Use the context manager to ensure that connections are properly cleaned up. - async with SqlStorageBackend( - # Create an `engine` with the desired configuration - engine=create_async_engine( - 'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres', - future=True, - pool_size=5, - max_overflow=10, - pool_recycle=3600, - pool_pre_ping=True, - echo=False, - ) - ) as storage_backend: - # Create a configuration with custom settings. - configuration = Configuration( - purge_on_start=False, - ) + # Create an `engine` with the desired configuration. + engine = create_async_engine( + 'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres', + future=True, + pool_size=5, + max_overflow=10, + pool_recycle=3600, + pool_pre_ping=True, + echo=False, + ) - # And pass them to the crawler. - crawler = ParselCrawler( - storage_backend=storage_backend, - configuration=configuration, - ) + try: + # Create a new instance of the storage backend. + # On first run, also creates tables in your PostgreSQL database. + async with SqlStorageBackend(engine=engine) as storage_backend: + # Create a configuration with custom settings. + configuration = Configuration( + purge_on_start=False, + ) + + # And pass them to the crawler. + crawler = ParselCrawler( + storage_backend=storage_backend, + configuration=configuration, + ) + finally: + # The backend doesn't dispose an engine you pass in, so dispose it yourself. + await engine.dispose() diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index ee57b83421..6a101b9ef2 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -182,6 +182,33 @@ The memory backend now handles values the same way as the file-system backend. I If processing a request can take longer than 3 minutes, call `RequestQueue.set_expected_request_processing_time()` with the longest time you expect. Every request fetched after that call stays locked at least that long. To give a single request that's still being processed more time, call `RequestQueue.extend_request_processing_time()`. +## Each `SqlStorageBackend` opens its own storages + +In v1, all `SqlStorageClient` instances in a process shared the storages opened through any of them, even when they connected to different databases. Now each `SqlStorageBackend` instance opens its own storages. With `purge_on_start` enabled, a new instance purges the unnamed storages, such as the default dataset, when it opens them. When it connects to the same database as an earlier instance, the data the earlier one wrote to these storages is lost. To read what a crawler stored in them, open them through the crawler's backend. + +Use `SqlStorageBackend` as an async context manager. On exit, it disposes the engine it created from the connection string or for the default SQLite database. In v1, it also disposed an engine you passed in. Now it leaves that engine open, so dispose it yourself. Storages opened through a closed backend stop working. In v1, they silently opened a new connection pool that nobody closed. + +For example: + +```python +# v1 +async with SqlStorageClient(engine=engine) as storage_client: + crawler = ParselCrawler(storage_client=storage_client) + await crawler.run(['https://crawlee.dev']) + + dataset = await Dataset.open(storage_client=SqlStorageClient(engine=engine)) + +# v2 +try: + async with SqlStorageBackend(engine=engine) as storage_backend: + crawler = ParselCrawler(storage_backend=storage_backend) + await crawler.run(['https://crawlee.dev']) + + dataset = await Dataset.open(storage_backend=storage_backend) +finally: + await engine.dispose() +``` + ## Crawler `id` is a string The `id` argument of `BasicCrawler` and its subclasses is now a `str`, so pass a string wherever you passed an integer. `StatisticsState.stats_id` and `EventCrawlerStatusData.crawler_id` are strings too, and `crawler_id` holds the crawler's `id`. In v1, it held the Python object ID of the crawler. @@ -291,7 +318,7 @@ for urls in batches: await crawler.run(urls) ``` -`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged once per process, when they're first opened. +`Configuration.purge_on_start` isn't affected. Unnamed storages are still purged once per process, when they're first opened. With `SqlStorageBackend` and `RedisStorageBackend`, each instance now opens its own storages, so it purges them once per instance. ## `RequestQueue.purge` raises when the storage backend can't purge diff --git a/src/crawlee/storage_backends/_sql/_storage_backend.py b/src/crawlee/storage_backends/_sql/_storage_backend.py index 2c6def7b8b..4c9e83c712 100644 --- a/src/crawlee/storage_backends/_sql/_storage_backend.py +++ b/src/crawlee/storage_backends/_sql/_storage_backend.py @@ -21,6 +21,7 @@ from ._request_queue_backend import SqlRequestQueueBackend if TYPE_CHECKING: + from collections.abc import Hashable from types import TracebackType from sqlalchemy.engine.interfaces import DBAPIConnection @@ -65,14 +66,16 @@ def __init__( Args: connection_string: Database connection string (e.g., "sqlite+aiosqlite:///crawlee.db"). If not provided, defaults to SQLite database in the storage directory. - engine: Pre-configured AsyncEngine instance. If provided, connection_string is ignored. + engine: Pre-configured AsyncEngine instance. The backend doesn't dispose it. """ if engine is not None and connection_string is not None: raise ValueError('Either connection_string or engine must be provided, not both.') self._connection_string = connection_string self._engine = engine + self._owns_engine = engine is None self._initialized = False + self._closed = False self.session_maker: async_sessionmaker[AsyncSession] | None = None self._listeners_registered = False @@ -100,7 +103,13 @@ async def __aexit__( @property def engine(self) -> AsyncEngine: - """Get the SQLAlchemy AsyncEngine instance.""" + """Get the SQLAlchemy AsyncEngine instance. + + Raises: + RuntimeError: If the storage backend is closed. + """ + self._raise_if_closed() + if self._engine is None: raise ValueError('Engine is not initialized. Call initialize() before accessing the engine.') return self._engine @@ -114,7 +123,12 @@ async def initialize(self, configuration: Configuration) -> None: This method creates all necessary tables if they don't exist. Should be called before using the storage backend. + + Raises: + RuntimeError: If the storage backend is closed. """ + self._raise_if_closed() + if not self._initialized: engine = self._get_or_create_engine(configuration) @@ -155,8 +169,17 @@ async def initialize(self, configuration: Configuration) -> None: self._initialized = True async def close(self) -> None: - """Close the database connection pool.""" - if self._engine is not None: + """Close the database connection pool, unless the engine was passed in. + + The storage backend and the storages opened with it can't be used afterwards. + """ + if self._closed: + return + + self._closed = True + self.session_maker = None + + if self._engine is not None and self._owns_engine: if self._listeners_registered: event.remove(self._engine.sync_engine, 'connect', self._on_connect) self._listeners_registered = False @@ -169,11 +192,21 @@ def create_session(self) -> AsyncSession: Returns: A new AsyncSession instance. + + Raises: + RuntimeError: If the storage backend is closed. """ + self._raise_if_closed() + if self.session_maker is None: self.session_maker = async_sessionmaker(self._engine, expire_on_commit=False, autoflush=False) return self.session_maker() + @override + def get_storage_backend_cache_key(self, configuration: Configuration) -> Hashable: + # Storages stop working when their backend closes, so a different backend instance must not get them. + return super().get_storage_backend_cache_key(configuration), id(self) + @override async def create_dataset_backend( self, @@ -240,6 +273,11 @@ async def create_rq_backend( await self._purge_if_needed(backend, configuration) return backend + def _raise_if_closed(self) -> None: + # Without this check, `initialize()` would create a new engine that nobody disposes. + if self._closed: + raise RuntimeError(f'The {self.__class__.__name__} is closed.') + def _get_or_create_engine(self, configuration: Configuration) -> AsyncEngine: """Get or create the database engine based on configuration.""" if self._engine is not None: diff --git a/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py b/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py index 6306c65fbb..b55920f05b 100644 --- a/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py +++ b/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py @@ -83,6 +83,8 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: assert 'dataset_records' in tables assert 'datasets' in tables + await engine.dispose() + async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL dataset creates proper tables and metadata records.""" diff --git a/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py b/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py index ad4230e962..cdc0e2145c 100644 --- a/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py +++ b/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py @@ -85,6 +85,8 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: assert 'key_value_stores' in tables assert 'key_value_store_records' in tables + await engine.dispose() + async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL key-value store creates proper tables and metadata records.""" diff --git a/tests/unit/storage_backends/_sql/test_sql_rq_backend.py b/tests/unit/storage_backends/_sql/test_sql_rq_backend.py index e77ef5e86a..939d9a3e55 100644 --- a/tests/unit/storage_backends/_sql/test_sql_rq_backend.py +++ b/tests/unit/storage_backends/_sql/test_sql_rq_backend.py @@ -100,6 +100,8 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: assert 'request_queue_records' in tables assert 'request_queue_state' in tables + await engine.dispose() + async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL request queue creates proper tables and metadata records.""" diff --git a/tests/unit/storage_backends/_sql/test_sql_storage_backend.py b/tests/unit/storage_backends/_sql/test_sql_storage_backend.py index a42c5001bc..db31241fab 100644 --- a/tests/unit/storage_backends/_sql/test_sql_storage_backend.py +++ b/tests/unit/storage_backends/_sql/test_sql_storage_backend.py @@ -2,14 +2,15 @@ import sys from typing import TYPE_CHECKING -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest from sqlalchemy import text -from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from crawlee.configuration import Configuration from crawlee.storage_backends import SqlStorageBackend +from crawlee.storages import Dataset if TYPE_CHECKING: from pathlib import Path @@ -53,6 +54,89 @@ async def test_sqlite_wal_mode_not_applied_with_custom_engine(tmp_path: Path) -> result = await conn.execute(text('PRAGMA journal_mode')) assert result.scalar() != 'wal' + await engine.dispose() + + +async def test_closed_backend_raises(tmp_path: Path) -> None: + """A closed backend refuses to open storages and to serve the ones opened before.""" + configuration = Configuration(storage_dir=str(tmp_path)) + + async with SqlStorageBackend() as storage_backend: + kvs_backend = await storage_backend.create_kvs_backend(configuration=configuration) + + with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): + await kvs_backend.set_value(key='key', value='value') + + with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): + await storage_backend.create_kvs_backend(configuration=configuration) + + with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): + _ = storage_backend.engine + + +async def test_backend_closed_before_first_use_refuses_initialization(tmp_path: Path) -> None: + """A backend closed before its first use doesn't create an engine afterwards.""" + async with SqlStorageBackend() as storage_backend: + pass + + with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): + await storage_backend.initialize(Configuration(storage_dir=str(tmp_path))) + + +async def test_close_keeps_passed_engine(tmp_path: Path) -> None: + """An engine passed in belongs to the caller, so the backend doesn't dispose it.""" + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "test.db"}') + + with patch.object(AsyncEngine, 'dispose', AsyncMock()) as dispose: + async with SqlStorageBackend(engine=engine): + pass + + dispose.assert_not_awaited() + await engine.dispose() + + +async def test_close_disposes_own_engine(tmp_path: Path) -> None: + """An engine created from the connection string is disposed with the backend, and only once.""" + storage_backend = SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{tmp_path / "test.db"}') + await storage_backend.initialize(Configuration(storage_dir=str(tmp_path))) + engine = storage_backend.engine + + with patch.object(AsyncEngine, 'dispose', AsyncMock()) as dispose: + await storage_backend.close() + await storage_backend.close() + + dispose.assert_awaited_once() + await engine.dispose() + + +async def test_new_backend_skips_storages_of_closed_one(tmp_path: Path) -> None: + """A new backend opens its own storage instead of the cached one bound to a closed backend.""" + configuration = Configuration(storage_dir=str(tmp_path), purge_on_start=False) + + async with SqlStorageBackend() as storage_backend: + old_dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) + + async with SqlStorageBackend() as storage_backend: + dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) + await dataset.push_data({'key': 'value'}) + + assert dataset is not old_dataset + + +async def test_new_backend_purges_default_dataset(tmp_path: Path) -> None: + """A new backend with `purge_on_start` enabled purges the default dataset an earlier backend wrote.""" + configuration = Configuration(storage_dir=str(tmp_path), purge_on_start=True) + + async with SqlStorageBackend() as storage_backend: + dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) + await dataset.push_data({'key': 'value'}) + + async with SqlStorageBackend() as storage_backend: + dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) + data = await dataset.get_data() + + assert data.items == [] + def test_import_error_handled() -> None: blocked = {