From 2b10d960eee2645d25126ba9df1bf6834ca63a49 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 20 Nov 2026 16:35:58 +0100 Subject: [PATCH] perf: take component sets from the listing Calendars piled up on a Synology server, as the caldav library's tests added calendars but could not delete them. With half a thousand of them, listing went really slow: collections() sent one PROPFIND per calendar, 268 s against 6 s for the listing alone. The calendars were deleted some other way, but the number of PROPFINDs was excessive anyway. caldav (from https://github.com/python-caldav/caldav/pull/872, not yet released) keeps the listing's properties on each calendar, and the component sets are now read from there. Bedework 5 also lists "ghost collections" that do not exist. A compatibility hint for this has been added to the caldav library, and where it is set, such ghost collections are weeded out. Prompts: > [pick from the task list: "calendaring: list collections with one > PROPFIND, not one per calendar"] > so let's go for yet another caldav compatibility hint on the ghost > collections > (on the commit message) Please do a QA on the edited text. Didn't > we already from earlier cache propfind results in the caldav library? Assisted-By: Claude Opus 5.5 Reviewed-by: Tobias Brox --- news/+one-propfind-listing.bugfix.rst | 1 + src/calendaring/_async/caldav.py | 8 +++- src/calendaring/_async/dav.py | 11 ++++- src/calendaring/_caldav_features.py | 5 +++ src/calendaring/_caldav_io.py | 10 +++++ src/calendaring/_sync/caldav.py | 8 +++- src/calendaring/_sync/dav.py | 11 ++++- tests/_async/test_caldav_backend.py | 59 ++++++++++++++++++++++++++- tests/_sync/test_caldav_backend.py | 59 ++++++++++++++++++++++++++- 9 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 news/+one-propfind-listing.bugfix.rst diff --git a/news/+one-propfind-listing.bugfix.rst b/news/+one-propfind-listing.bugfix.rst new file mode 100644 index 0000000..790ecb6 --- /dev/null +++ b/news/+one-propfind-listing.bugfix.rst @@ -0,0 +1 @@ +On CalDAV, ``collections()`` no longer sends one PROPFIND per calendar for its component set, where caldav keeps the listing's properties (after 3.7): 469 calendars on a Synology took 268 s, the listing alone 6 s. With caldav 3.7, or a server whose listing leaves the component set out, each calendar is still asked; so is each on a server whose listing names calendars that are not there (Bedework 5, by caldav's ``propfind.listed-members-exist`` hint), to leave those out. That takes caldav's Bedework 5 profile (``features: bedework_5_0_0``): without hints, Bedework 5's ghost ``tasks/`` is listed. diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 8f1b32f..99e1cfe 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -24,6 +24,7 @@ from .._caldav_features import ( backend_capabilities, collection_capabilities, + lists_missing_collections, multiget_serves_deleted, stored_components, ) @@ -272,11 +273,14 @@ async def _open(cls, url: str, **options: Any) -> AsyncCalDAVBackend: return cls(AsyncDAV(url, **options), id=backend_id, loss=loss, verify=verify) async def _wrap(self, calendar: Calendar) -> AsyncCalDAVCollection: + features = self._dav.client.features + # Where the listing names calendars that are not there, a fresh read of each is how they are found. + name = await self._dav.display_name(calendar, cached=not lists_missing_collections(features)) listed = frozenset( Component(c) for c in await self._dav.supported_components(calendar) if c in {k.value for k in Component} ) - components = stored_components(self._dav.client.features, listed) - return self.collection_class(self, calendar, name=await self._dav.display_name(calendar), components=components) + components = stored_components(features, listed) + return self.collection_class(self, calendar, name=name, components=components) async def collections(self) -> list[AsyncCollection]: found: list[AsyncCollection] = [] diff --git a/src/calendaring/_async/dav.py b/src/calendaring/_async/dav.py index 3bf85c3..c2299ac 100644 --- a/src/calendaring/_async/dav.py +++ b/src/calendaring/_async/dav.py @@ -15,6 +15,7 @@ SyncResult, _awaited, _etags, + _listed_components, _raw, _resource, _strong, @@ -57,12 +58,18 @@ async def delete_calendar(self, calendar: Calendar) -> None: await _awaited(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated async def supported_components(self, calendar: Calendar) -> list[str]: + # From the listing if it said, or one PROPFIND per calendar: 268 s for 469 on a Synology. + if (listed := _listed_components(calendar)) is not None: + return listed with translated(): return [str(c) for c in await _awaited(calendar.get_supported_components())] - async def display_name(self, calendar: Calendar) -> str | None: + async def display_name(self, calendar: Calendar, *, cached: bool = True) -> str | None: + """The listed name, or with ``cached=False`` a fresh one: ``NotFoundError`` if the calendar is gone.""" with translated(): - return await _awaited(calendar.get_display_name()) + if cached: + return await _awaited(calendar.get_display_name()) + return await _awaited(calendar.get_property(DisplayName(), use_cached=False)) async def set_display_name(self, calendar: Calendar, name: str) -> None: with translated(): diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index ff334d0..cabf49f 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -288,6 +288,11 @@ def multiget_serves_deleted(features: Any) -> bool: return _level(features, "multiget.deleted") is U +def lists_missing_collections(features: Any) -> bool: + """Bedework 5 lists a calendar, display name and component set and all, that answers 404.""" + return _level(features, "propfind.listed-members-exist") is U + + def backend_capabilities(features: Any) -> Capabilities: return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index 27e9c15..94d5561 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -27,6 +27,7 @@ from caldav.calendarobjectresource import CalendarObjectResource, Event, Journal, Todo from caldav.collection import Calendar from caldav.davclient import DAVClient +from caldav.elements.cdav import SupportedCalendarComponentSet from caldav.elements.dav import DisplayName, GetEtag from caldav.lib import error as dav_error from caldav.lib.url import URL @@ -69,6 +70,15 @@ VALID_SYNC_TOKEN = "{DAV:}valid-sync-token" # RFC 6578 §3.2 + +def _listed_components(calendar: Calendar) -> list[str] | None: + """The component set the calendar listing answered, if caldav kept it (after 3.7): None to ask. + + Read from ``props`` rather than ``get_supported_components(use_cached=True)``, which 3.7 does not take.""" + listed = calendar.props.get(SupportedCalendarComponentSet.tag) + return [str(c) for c in listed] if listed else None + + # A sync-collection report: hrefs changed, hrefs removed, the new token (None if the server gave # none), and whether it is incomplete (truncated beyond what caldav itself follows, RFC 6578 §3.6). SyncResult = tuple[list[str], list[str], str | None, bool] diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index d1bbb31..98f91ab 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -24,6 +24,7 @@ from .._caldav_features import ( backend_capabilities, collection_capabilities, + lists_missing_collections, multiget_serves_deleted, stored_components, ) @@ -272,11 +273,14 @@ def _open(cls, url: str, **options: Any) -> CalDAVBackend: return cls(DAV(url, **options), id=backend_id, loss=loss, verify=verify) def _wrap(self, calendar: Calendar) -> CalDAVCollection: + features = self._dav.client.features + # Where the listing names calendars that are not there, a fresh read of each is how they are found. + name = self._dav.display_name(calendar, cached=not lists_missing_collections(features)) listed = frozenset( Component(c) for c in self._dav.supported_components(calendar) if c in {k.value for k in Component} ) - components = stored_components(self._dav.client.features, listed) - return self.collection_class(self, calendar, name=self._dav.display_name(calendar), components=components) + components = stored_components(features, listed) + return self.collection_class(self, calendar, name=name, components=components) def collections(self) -> list[Collection]: found: list[Collection] = [] diff --git a/src/calendaring/_sync/dav.py b/src/calendaring/_sync/dav.py index e01f137..6473da2 100644 --- a/src/calendaring/_sync/dav.py +++ b/src/calendaring/_sync/dav.py @@ -15,6 +15,7 @@ SyncResult, _returned, _etags, + _listed_components, _raw, _resource, _strong, @@ -57,12 +58,18 @@ def delete_calendar(self, calendar: Calendar) -> None: _returned(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated def supported_components(self, calendar: Calendar) -> list[str]: + # From the listing if it said, or one PROPFIND per calendar: 268 s for 469 on a Synology. + if (listed := _listed_components(calendar)) is not None: + return listed with translated(): return [str(c) for c in _returned(calendar.get_supported_components())] - def display_name(self, calendar: Calendar) -> str | None: + def display_name(self, calendar: Calendar, *, cached: bool = True) -> str | None: + """The listed name, or with ``cached=False`` a fresh one: ``NotFoundError`` if the calendar is gone.""" with translated(): - return _returned(calendar.get_display_name()) + if cached: + return _returned(calendar.get_display_name()) + return _returned(calendar.get_property(DisplayName(), use_cached=False)) def set_display_name(self, calendar: Calendar, name: str) -> None: with translated(): diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index 00deda8..3074999 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -31,10 +31,13 @@ from caldav.collection import Calendar # noqa: E402 from caldav.compatibility_hints import FeatureSet # noqa: E402 +from caldav.elements.cdav import SupportedCalendarComponentSet # noqa: E402 +from caldav.elements.dav import DisplayName # noqa: E402 from caldav.lib.url import URL # noqa: E402 from calendaring._async.caldav import _name_of # noqa: E402 -from calendaring._caldav_io import _resource # noqa: E402 +from calendaring._caldav_features import lists_missing_collections # noqa: E402 +from calendaring._caldav_io import _listed_components, _resource # noqa: E402 @pytest.fixture @@ -90,10 +93,64 @@ async def test_a_listed_calendar_that_is_not_there_is_left_out( assert len(await backend.collections()) == len(real) +async def test_a_listed_calendar_that_is_not_there_is_left_out_where_the_listing_says_otherwise( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + """Bedework 5 lists its ghost ``tasks/`` with a display name and a component set, all a listing could say.""" + caldav_collection: Any = collection + backend = caldav_collection._backend + features = backend._dav.client.features + if "propfind.listed-members-exist" not in features.FEATURES: + pytest.skip("caldav up to 3.7 has no propfind.listed-members-exist hint") + features.set_feature("propfind.listed-members-exist", {"support": "broken"}) + real = await backend._dav.calendars() + ghost = Calendar( + client=backend._dav.client, + url=URL.objectify(caldav_collection.url).join("../ghost/"), + props={DisplayName.tag: "tasks", SupportedCalendarComponentSet.tag: ["VTODO"]}, + ) + monkeypatch.setattr(backend._dav, "calendars", lambda: _returning([*real, ghost])) + assert [c.id for c in await backend.collections()] == [_name_of(str(c.url)) for c in real] + + async def _returning(value: Any) -> Any: return value +async def test_listing_collections_costs_the_same_however_many_there_are( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + """One PROPFIND per calendar took 268 s for 469 calendars on a Synology; the listing alone takes 6 s.""" + backend: Any = collection._backend + if not all(_listed_components(c) for c in await backend._dav.calendars()): + pytest.skip("the listing kept no component set: caldav up to 3.7 keeps only the display name") + if lists_missing_collections(backend._dav.client.features): + pytest.skip("where the listing names calendars that are not there, each is asked") + client = backend._dav.client + request = client.request + methods: list[str] = [] + + async def counted(url: Any, method: str = "GET", *args: Any, **kwargs: Any) -> Any: + methods.append(method) + return await request(url, method, *args, **kwargs) + + monkeypatch.setattr(client, "request", counted) + one = await backend.collections() + one_propfinds, methods[:] = methods.count("PROPFIND"), [] + assert one_propfinds >= 1 # the counting sees the requests at all + monkeypatch.setattr(client, "request", request) + other = await backend.create_collection("backend-tests-2", components=(Component.TASK,)) + try: + monkeypatch.setattr(client, "request", counted) + two = await backend.collections() + assert len(two) == len(one) + 1 + assert methods.count("PROPFIND") == one_propfinds + assert {c.id: c.components for c in two}[collection.id] == {Component.TASK} + finally: + monkeypatch.setattr(client, "request", request) + await other.delete_collection() + + async def test_a_token_is_bearer_auth() -> None: backend = await AsyncBackend.open("https://dav.example.com/", kind="caldav", token="s3cret") async with backend: diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 2527e27..053b526 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -31,10 +31,13 @@ from caldav.collection import Calendar # noqa: E402 from caldav.compatibility_hints import FeatureSet # noqa: E402 +from caldav.elements.cdav import SupportedCalendarComponentSet # noqa: E402 +from caldav.elements.dav import DisplayName # noqa: E402 from caldav.lib.url import URL # noqa: E402 from calendaring._sync.caldav import _name_of # noqa: E402 -from calendaring._caldav_io import _resource # noqa: E402 +from calendaring._caldav_features import lists_missing_collections # noqa: E402 +from calendaring._caldav_io import _listed_components, _resource # noqa: E402 @pytest.fixture @@ -90,10 +93,64 @@ def test_a_listed_calendar_that_is_not_there_is_left_out( assert len(backend.collections()) == len(real) +def test_a_listed_calendar_that_is_not_there_is_left_out_where_the_listing_says_otherwise( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + """Bedework 5 lists its ghost ``tasks/`` with a display name and a component set, all a listing could say.""" + caldav_collection: Any = collection + backend = caldav_collection._backend + features = backend._dav.client.features + if "propfind.listed-members-exist" not in features.FEATURES: + pytest.skip("caldav up to 3.7 has no propfind.listed-members-exist hint") + features.set_feature("propfind.listed-members-exist", {"support": "broken"}) + real = backend._dav.calendars() + ghost = Calendar( + client=backend._dav.client, + url=URL.objectify(caldav_collection.url).join("../ghost/"), + props={DisplayName.tag: "tasks", SupportedCalendarComponentSet.tag: ["VTODO"]}, + ) + monkeypatch.setattr(backend._dav, "calendars", lambda: _returning([*real, ghost])) + assert [c.id for c in backend.collections()] == [_name_of(str(c.url)) for c in real] + + def _returning(value: Any) -> Any: return value +def test_listing_collections_costs_the_same_however_many_there_are( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + """One PROPFIND per calendar took 268 s for 469 calendars on a Synology; the listing alone takes 6 s.""" + backend: Any = collection._backend + if not all(_listed_components(c) for c in backend._dav.calendars()): + pytest.skip("the listing kept no component set: caldav up to 3.7 keeps only the display name") + if lists_missing_collections(backend._dav.client.features): + pytest.skip("where the listing names calendars that are not there, each is asked") + client = backend._dav.client + request = client.request + methods: list[str] = [] + + def counted(url: Any, method: str = "GET", *args: Any, **kwargs: Any) -> Any: + methods.append(method) + return request(url, method, *args, **kwargs) + + monkeypatch.setattr(client, "request", counted) + one = backend.collections() + one_propfinds, methods[:] = methods.count("PROPFIND"), [] + assert one_propfinds >= 1 # the counting sees the requests at all + monkeypatch.setattr(client, "request", request) + other = backend.create_collection("backend-tests-2", components=(Component.TASK,)) + try: + monkeypatch.setattr(client, "request", counted) + two = backend.collections() + assert len(two) == len(one) + 1 + assert methods.count("PROPFIND") == one_propfinds + assert {c.id: c.components for c in two}[collection.id] == {Component.TASK} + finally: + monkeypatch.setattr(client, "request", request) + other.delete_collection() + + def test_a_token_is_bearer_auth() -> None: backend = Backend.open("https://dav.example.com/", kind="caldav", token="s3cret") with backend: