Repository navigation
fix(sql)!: Keep a passed-in engine open and open storages per backend #2464
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: v2
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,14 +2,15 @@ | |
|
|
||
| import sys | ||
| from typing import TYPE_CHECKING | ||
| from unittest.mock import patch | ||
| from unittest.mock import AsyncMock, patch | ||
|
|
||
| import pytest | ||
| from sqlalchemy import text | ||
| from sqlalchemy.ext.asyncio import create_async_engine | ||
| from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine | ||
|
|
||
| from crawlee.configuration import Configuration | ||
| from crawlee.storage_backends import SqlStorageBackend | ||
| from crawlee.storages import Dataset | ||
|
|
||
| if TYPE_CHECKING: | ||
| from pathlib import Path | ||
|
|
@@ -53,6 +54,89 @@ async def test_sqlite_wal_mode_not_applied_with_custom_engine(tmp_path: Path) -> | |
| result = await conn.execute(text('PRAGMA journal_mode')) | ||
| assert result.scalar() != 'wal' | ||
|
|
||
| await engine.dispose() | ||
|
|
||
|
|
||
| async def test_closed_backend_raises(tmp_path: Path) -> None: | ||
| """A closed backend refuses to open storages and to serve the ones opened before.""" | ||
| configuration = Configuration(storage_dir=str(tmp_path)) | ||
|
|
||
| async with SqlStorageBackend() as storage_backend: | ||
| kvs_backend = await storage_backend.create_kvs_backend(configuration=configuration) | ||
|
|
||
| with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): | ||
| await kvs_backend.set_value(key='key', value='value') | ||
|
|
||
| with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): | ||
| await storage_backend.create_kvs_backend(configuration=configuration) | ||
|
|
||
| with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): | ||
| _ = storage_backend.engine | ||
|
|
||
|
|
||
| async def test_backend_closed_before_first_use_refuses_initialization(tmp_path: Path) -> None: | ||
| """A backend closed before its first use doesn't create an engine afterwards.""" | ||
| async with SqlStorageBackend() as storage_backend: | ||
| pass | ||
|
|
||
| with pytest.raises(RuntimeError, match=r'SqlStorageBackend is closed'): | ||
| await storage_backend.initialize(Configuration(storage_dir=str(tmp_path))) | ||
|
|
||
|
|
||
| async def test_close_keeps_passed_engine(tmp_path: Path) -> None: | ||
| """An engine passed in belongs to the caller, so the backend doesn't dispose it.""" | ||
| engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "test.db"}') | ||
|
|
||
| with patch.object(AsyncEngine, 'dispose', AsyncMock()) as dispose: | ||
| async with SqlStorageBackend(engine=engine): | ||
| pass | ||
|
|
||
| dispose.assert_not_awaited() | ||
| await engine.dispose() | ||
|
|
||
|
|
||
| async def test_close_disposes_own_engine(tmp_path: Path) -> None: | ||
| """An engine created from the connection string is disposed with the backend, and only once.""" | ||
| storage_backend = SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{tmp_path / "test.db"}') | ||
| await storage_backend.initialize(Configuration(storage_dir=str(tmp_path))) | ||
| engine = storage_backend.engine | ||
|
|
||
| with patch.object(AsyncEngine, 'dispose', AsyncMock()) as dispose: | ||
| await storage_backend.close() | ||
| await storage_backend.close() | ||
|
|
||
| dispose.assert_awaited_once() | ||
| await engine.dispose() | ||
|
|
||
|
|
||
| async def test_new_backend_skips_storages_of_closed_one(tmp_path: Path) -> None: | ||
| """A new backend opens its own storage instead of the cached one bound to a closed backend.""" | ||
| configuration = Configuration(storage_dir=str(tmp_path), purge_on_start=False) | ||
|
|
||
| async with SqlStorageBackend() as storage_backend: | ||
| old_dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) | ||
|
|
||
| async with SqlStorageBackend() as storage_backend: | ||
| dataset = await Dataset.open(name='results', storage_backend=storage_backend, configuration=configuration) | ||
| await dataset.push_data({'key': 'value'}) | ||
|
|
||
| assert dataset is not old_dataset | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Suggestion: The docs and the upgrading guide warn that a new backend with ✍️ Drafted by Claude Code |
||
|
|
||
|
|
||
| async def test_new_backend_purges_default_dataset(tmp_path: Path) -> None: | ||
| """A new backend with `purge_on_start` enabled purges the default dataset an earlier backend wrote.""" | ||
| configuration = Configuration(storage_dir=str(tmp_path), purge_on_start=True) | ||
|
|
||
| async with SqlStorageBackend() as storage_backend: | ||
| dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) | ||
| await dataset.push_data({'key': 'value'}) | ||
|
|
||
| async with SqlStorageBackend() as storage_backend: | ||
| dataset = await Dataset.open(storage_backend=storage_backend, configuration=configuration) | ||
| data = await dataset.get_data() | ||
|
|
||
| assert data.items == [] | ||
|
|
||
|
|
||
| def test_import_error_handled() -> None: | ||
| blocked = { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Question: With
id(self)in the key, every backend instance adds its own entries to the global storage cache, andclose()never removes them. A closed backend and its storages stay in memory for the life of the process. Also,Dataset.open(storage_backend=closed_backend)returns the cached instance without an error, and the error only comes on the first operation. Shouldclose()remove its cache entries?✍️ Drafted by Claude Code
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'd keep the current behavior. Storage backends don't know about the storage cache, only the storages do, and an entry leaves the cache on
drop(). Removing entries inclose()would make the backend reach into the global cache and turn that dependency around. What stays behind is a few objects per backend instance with no open connections, and a process normally creates one backend.