Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions caldav/collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -1640,15 +1640,16 @@ def _assign_multiget_data(
obj._keep_raw_data(data)
obj.props.update(tags)

def _batch_load_objects(self, objects: list) -> None:
def _batch_load_objects(self, objects: list, reload: bool = False) -> None:
"""Load unloaded objects from the list in a single calendar-multiget REPORT.

Already-loaded objects are skipped. If the REPORT fails, falls back to
individual obj.load(only_if_unloaded=True) calls per object; a per-object
failure is logged at debug level and otherwise swallowed, so callers can
filter on is_loaded() afterward.
Already-loaded objects are skipped, unless ``reload``: then all are
loaded again. If the REPORT fails, falls back to individual
obj.load() calls per object; a per-object failure is logged at debug
level and otherwise swallowed, so callers can filter on is_loaded()
afterward.
"""
unloaded = [o for o in objects if not o.is_loaded()]
unloaded = list(objects) if reload else [o for o in objects if not o.is_loaded()]
if not unloaded:
return
try:
Expand All @@ -1657,7 +1658,7 @@ def _batch_load_objects(self, objects: list) -> None:
logging.error("Batch multiget failed, falling back to individual loads", exc_info=True)
for obj in unloaded:
try:
obj.load(only_if_unloaded=True)
obj.load(only_if_unloaded=not reload)
except Exception:
## Deliberate: this method's contract is that the caller
## filters on is_loaded() afterwards, so one object that
Expand All @@ -1667,14 +1668,14 @@ def _batch_load_objects(self, objects: list) -> None:
## otherwise produce a wall of warnings.
log.debug("Individual load failed for %s", obj.url, exc_info=True)

async def _async_batch_load_objects(self, objects: list) -> None:
async def _async_batch_load_objects(self, objects: list, reload: bool = False) -> None:
"""Async version of _batch_load_objects.

The post-processing is shared via _assign_multiget_data(); the only
sync/async difference is the await on the multiget REPORT and on the
per-object fallback load().
"""
unloaded = [o for o in objects if not o.is_loaded()]
unloaded = list(objects) if reload else [o for o in objects if not o.is_loaded()]
if not unloaded:
return
try:
Expand All @@ -1687,7 +1688,7 @@ async def _async_batch_load_objects(self, objects: list) -> None:
)
for obj in unloaded:
try:
load_result = obj.load(only_if_unloaded=True)
load_result = obj.load(only_if_unloaded=not reload)
if inspect.isawaitable(load_result):
await load_result
except Exception:
Expand Down
17 changes: 17 additions & 0 deletions caldav/compatibility_hints.py
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,15 @@ class FeatureSet:
"search.recurrences.expanded.exception": {
"description": "Server expand should work correctly also if a recurrence set with exceptions is given"
},
"search.recurrences.exception": {
"description": "A search handles the exceptions of a recurrence set (the components with a RECURRENCE-ID) as it handles any other component. The children say in what way it does not",
"default": {"support": "full"},
},
"search.recurrences.exception.in-calendar-data": {
"description": "A search returns a recurring object with all its components: the exceptions (the components with a RECURRENCE-ID) along with the master, as a GET has them. RFC 4791 section 9.6 makes a calendar-data element with no children ask for the whole object. 'unsupported' when the calendar-data of a search holds the master only (Bedework 3.10 and 5): a client expanding the recurrence set itself then gets each moved or changed occurrence as a plain one. search() then loads the recurring objects it found again with a multiget",
"default": {"support": "full"},
"links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.6"],
},
"sync-token": {
"description": "RFC6578 sync-collection reports are supported. Server provides sync tokens that can be used to efficiently retrieve only changed objects since last sync. Support can be 'full', 'fragile' (occasionally returns more content than expected), or 'unsupported'. Behaviour 'time-based' indicates second-precision tokens requiring sleep(1) between operations",
## Independent feature (directly probed): the default marks it so the
Expand Down Expand Up @@ -2041,6 +2050,10 @@ def __getattr__(name: str) -> Any:
## 5.x, hence the version-stamped name - a plain `bedework` would be claiming
## far more than we have measured.
bedework_3_10_3 = {
## Measured by hand 2026-11-14 (the conformance suite of calendaring):
## a calendar-query returns the master without the moved occurrence,
## which a GET of the same object has.
"search.recurrences.exception.in-calendar-data": {"support": "unsupported", "behaviour": "the calendar-data of a search holds the master only"},
## Measured with caldav-server-tester, 2026-10-27: the UID filter works although search.text does not.
"search.text.uid": {"support": "full"},
## Probed 2026-10-26 by caldav-server-tester: RFC 4791 section 5.2.3 makes
Expand Down Expand Up @@ -2133,6 +2146,10 @@ def __getattr__(name: str) -> Any:
## deletes calendars, its sync-token and text search behave differently - so
## nothing is inherited from that profile.
bedework_5_0_0 = {
## Measured by hand 2026-11-14 (the conformance suite of calendaring):
## a calendar-query returns the master without the moved occurrence,
## which a GET of the same object has.
"search.recurrences.exception.in-calendar-data": {"support": "unsupported", "behaviour": "the calendar-data of a search holds the master only"},
## Probed 2026-11-12 by caldav-server-tester, and by hand on a fresh
## container: a PUT with one ATTENDEE gives two identical ATTENDEE lines
## in GET and calendar-multiget alike.
Expand Down
43 changes: 43 additions & 0 deletions caldav/search.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import inspect
import logging
import re
from copy import deepcopy
from dataclasses import dataclass, field, replace
from datetime import datetime, timedelta, timezone
Expand All @@ -15,6 +16,7 @@
from .collection import Calendar
from .elements import cdav, dav
from .lib import error
from .lib.python_utilities import to_normal_str

if TYPE_CHECKING:
from collections.abc import Generator
Expand Down Expand Up @@ -383,6 +385,20 @@ def _is_not_defined_supported(features: Any, prop: str) -> bool:
)


_VTIMEZONE = re.compile(r"^BEGIN:VTIMEZONE\b.*?^END:VTIMEZONE\b", re.I | re.M | re.S)
_RECURRENCE = re.compile(r"^(RRULE|RDATE|RECURRENCE-ID)[;:]", re.I | re.M)


def _is_recurring(obj: CalendarObjectResource) -> bool:
"""Whether a loaded object is (part of) a recurrence set, by a look at its text.

The rules of a VTIMEZONE don't count.
"""
if not obj.is_loaded():
return False
return bool(_RECURRENCE.search(_VTIMEZONE.sub("", to_normal_str(obj.data))))


class SearchAction(Enum):
"""Actions yielded by the search generator to be executed by sync/async wrappers."""

Expand All @@ -393,6 +409,9 @@ class SearchAction(Enum):
LOAD_OBJECTS_BATCH = (
auto()
) # (calendar, objects) -> batch-load via calendar._batch_load_objects
RELOAD_OBJECTS_BATCH = (
auto()
) # (calendar, objects) -> load again via calendar._batch_load_objects(reload=True)
RETURN = auto() # (result) -> return this value


Expand Down Expand Up @@ -640,6 +659,8 @@ def _search_impl(
## and the query is sent verbatim (used by the compatibility checker to
## observe raw server behaviour).
cw = self._compatibility_workarounds
## What the caller asked the server for, before the query is built
caller_chose_data = props is not None or xml is not None

## Workaround for servers where REPORT without a time range only returns
## objects within a sliding window (search.unlimited-time-range: broken).
Expand Down Expand Up @@ -1106,8 +1127,22 @@ def _search_impl(
)
return

# Bedework leaves the exceptions out of the calendar-data of a search.
# Only what the search itself returned; the batch load below fetches whole objects
partial = []
if (
cw
and not caller_chose_data
and not calendar.client.features.is_supported(
"search.recurrences.exception.in-calendar-data"
)
):
partial = [o for o in objects if _is_recurring(o)]

# Post-process: batch-load unloaded objects in one REPORT instead of N GETs
yield (SearchAction.LOAD_OBJECTS_BATCH, (calendar, objects))
if partial:
yield (SearchAction.RELOAD_OBJECTS_BATCH, (calendar, partial))
objects = [o for o in objects if o.is_loaded() or o.has_component()]

# Google sometimes returns empty objects
Expand Down Expand Up @@ -1221,6 +1256,10 @@ def _dispatch_search_action(self, action: SearchAction, data: Any) -> Any:
cal, objs = data
cal._batch_load_objects(objs)
return None
if action == SearchAction.RELOAD_OBJECTS_BATCH:
cal, objs = data
cal._batch_load_objects(objs, reload=True)
return None
raise AssertionError(f"unhandled search action {action!r}")

def _search_with_comptypes(
Expand Down Expand Up @@ -1335,6 +1374,10 @@ async def _async_dispatch_search_action(self, action: SearchAction, data: Any) -
cal, objs = data
await cal._async_batch_load_objects(objs)
return None
if action == SearchAction.RELOAD_OBJECTS_BATCH:
cal, objs = data
await cal._async_batch_load_objects(objs, reload=True)
return None
raise AssertionError(f"unhandled search action {action!r}")

async def _async_search_with_comptypes(
Expand Down
1 change: 1 addition & 0 deletions changes/+search-exceptions.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
New server hint `search.recurrences.exception.in-calendar-data`: whether a search returns a recurring object with its exceptions. Bedework 3.10 and 5 return the master only, so `search()` loads such objects again there, and a moved occurrence is no longer expanded as a plain one.
186 changes: 186 additions & 0 deletions tests/test_caldav_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -9955,3 +9955,189 @@ def test_an_empty_etag_is_no_etag(self, text) -> None:
response = MockedDAVResponse(self.XML.format(text))
[found] = response.expand_simple_props(multi_value_props=[dav.GetEtag()]).values()
assert found[dav.GetEtag.tag] == []


class TestSearchDropsExceptions:
"""Where ``search.recurrences.exception.in-calendar-data`` is unsupported, a calendar-query
returns a recurring object without its exceptions, the VEVENTs with a
RECURRENCE-ID (Bedework 3.10 and 5), while a GET or a multiget has them.
search() then loads the recurring objects it found again, so the moved
occurrence is not expanded as a plain one."""

URL = "http://cal.example.com/dav/user/mycal/"
MASTER = (
"BEGIN:VEVENT\r\nUID:series-1\r\nDTSTAMP:20260101T000000Z\r\n"
"DTSTART:20261101T090000Z\r\nDTEND:20261101T100000Z\r\n"
"RRULE:FREQ=DAILY;COUNT=5\r\nSUMMARY:R: daily\r\nEND:VEVENT\r\n"
)
MOVED = (
"BEGIN:VEVENT\r\nUID:series-1\r\nDTSTAMP:20260101T000000Z\r\n"
"RECURRENCE-ID:20261103T090000Z\r\n"
"DTSTART:20261103T140000Z\r\nDTEND:20261103T150000Z\r\n"
"SUMMARY:R: moved\r\nEND:VEVENT\r\n"
)
## A one-off event in a zone with summer time: its VTIMEZONE has RRULEs
ONE_OFF = (
"BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nBEGIN:STANDARD\r\n"
"DTSTART:19701025T030000\r\nRRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU\r\n"
"TZOFFSETFROM:+0200\r\nTZOFFSETTO:+0100\r\nEND:STANDARD\r\nBEGIN:DAYLIGHT\r\n"
"DTSTART:19700329T020000\r\nRRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU\r\n"
"TZOFFSETFROM:+0100\r\nTZOFFSETTO:+0200\r\nEND:DAYLIGHT\r\nEND:VTIMEZONE\r\n"
"BEGIN:VEVENT\r\nUID:one-off\r\nDTSTAMP:20260101T000000Z\r\n"
"DTSTART;TZID=Europe/Berlin:20261102T100000\r\n"
"DTEND;TZID=Europe/Berlin:20261102T110000\r\nSUMMARY:once\r\nEND:VEVENT\r\n"
)
HEAD = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//Example//EN\r\n"
TAIL = "END:VCALENDAR\r\n"
FEATURES = {
"search.recurrences.expanded": {"support": "unsupported"},
"search.recurrences.exception.in-calendar-data": {"support": "unsupported"},
}

@pytest.fixture(params=["sync", "async"])
def base(self, request):
if request.param == "sync":
return DAVClient
from caldav.async_davclient import AsyncDAVClient

return AsyncDAVClient

@staticmethod
def _run(result):
import asyncio

if asyncio.iscoroutine(result):
return asyncio.run(result)
return result

@staticmethod
def _multistatus(data: str | None) -> bytes:
calendar_data = (
""
if data is None
else f"<cal:calendar-data>{data.replace(chr(13), '&#13;')}</cal:calendar-data>"
)
return (
'<d:multistatus xmlns:d="DAV:" xmlns:cal="urn:ietf:params:xml:ns:caldav">'
"<d:response><d:href>/dav/user/mycal/series-1.ics</d:href><d:propstat><d:prop>"
f'<d:getetag>"etag-1"</d:getetag>{calendar_data}'
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>"
"</d:multistatus>"
).encode()

def _calendar(
self, base, features, query=None, full=None, multiget_fails=False, query_has_data=True
):
"""A fake Bedework: the calendar-query answers ``query``, a multiget or GET ``full``."""
query = self.HEAD + self.MASTER + self.TAIL if query is None else query
full = self.HEAD + self.MASTER + self.MOVED + self.TAIL if full is None else full
requests = []
client = base(url="http://cal.example.com/dav/", features=features)

def request(url, method="GET", body="", headers=None):
text = body.decode() if isinstance(body, bytes) else (body or "")
resp = mock.MagicMock()
if method == "GET":
requests.append("get")
resp.status_code, resp.reason = 200, "OK"
resp.headers = {"Content-Type": "text/calendar", "Etag": '"etag-g"'}
resp.content = full.encode()
return DAVResponse(resp, client)
assert method == "REPORT", method
multiget = "calendar-multiget" in text
requests.append("multiget" if multiget else "query")
if multiget and multiget_fails:
resp.status_code, resp.reason = 500, "Internal Server Error"
resp.headers = {}
resp.content = b""
return DAVResponse(resp, client)
resp.status_code, resp.reason = 207, "Multi-Status"
resp.headers = {"Content-Type": "text/xml"}
resp.content = self._multistatus(
full if multiget else (query if query_has_data else None)
)
return DAVResponse(resp, client)

if base is DAVClient:
client.request = request
else:

async def async_request(*args, **kwargs):
return request(*args, **kwargs)

client.request = async_request
return Calendar(client, url=self.URL), requests

def _search(self, base, features=None, expand=True, **kwargs):
search_kwargs = kwargs.pop("search_kwargs", {})
calendar, requests = self._calendar(
base, self.FEATURES if features is None else features, **kwargs
)
found = self._run(
calendar.search(
event=True,
start=datetime(2026, 11, 1, tzinfo=timezone.utc),
end=datetime(2026, 11, 8, tzinfo=timezone.utc),
expand=expand,
**search_kwargs,
)
)
summaries = sorted(
(str(c.get("RECURRENCE-ID") and c["RECURRENCE-ID"].dt), str(c["SUMMARY"]))
for obj in found
for c in obj.icalendar_instance.walk("VEVENT")
)
return summaries, requests

def test_the_moved_occurrence_is_found(self, base) -> None:
summaries, requests = self._search(base)
assert requests == ["query", "multiget"]
assert ("2026-11-03 09:00:00+00:00", "R: moved") in summaries
assert ("2026-11-03 09:00:00+00:00", "R: daily") not in summaries
assert len(summaries) == 5

def test_no_second_request_elsewhere(self, base) -> None:
_, requests = self._search(
base, {"search.recurrences.expanded": {"support": "unsupported"}}
)
assert requests == ["query"]

def test_a_one_off_event_in_a_zone_is_not_loaded_again(self, base) -> None:
one_off = self.HEAD + self.ONE_OFF + self.TAIL
summaries, requests = self._search(base, query=one_off, full=one_off)
assert requests == ["query"]
assert summaries == [("None", "once")]

def test_lower_case_names_count(self, base) -> None:
query = self.HEAD + self.MASTER.replace("RRULE:", "rrule:") + self.TAIL
_, requests = self._search(base, query=query)
assert requests == ["query", "multiget"]

def test_an_object_of_overrides_only_is_loaded_again(self, base) -> None:
moved_only = self.HEAD + self.MOVED + self.TAIL
summaries, requests = self._search(base, expand=False, query=moved_only, full=moved_only)
assert requests == ["query", "multiget"]

def test_an_object_without_data_is_loaded_once(self, base) -> None:
summaries, requests = self._search(base, query_has_data=False)
assert requests == ["query", "multiget"]
assert ("2026-11-03 09:00:00+00:00", "R: moved") in summaries

def test_a_failed_multiget_falls_back_to_get(self, base) -> None:
summaries, requests = self._search(base, multiget_fails=True)
assert requests == ["query", "multiget", "get"]
assert ("2026-11-03 09:00:00+00:00", "R: moved") in summaries

def test_not_without_compatibility_workarounds(self, base) -> None:
_, requests = self._search(base, search_kwargs={"compatibility_workarounds": False})
assert requests == ["query"]

def test_not_when_the_caller_wrote_the_query(self, base) -> None:
query = (
'<C:calendar-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:caldav">'
"<D:prop><C:calendar-data/></D:prop>"
'<C:filter><C:comp-filter name="VCALENDAR"><C:comp-filter name="VEVENT"/></C:comp-filter></C:filter>'
"</C:calendar-query>"
)
_, requests = self._search(base, expand=False, search_kwargs={"xml": query})
assert requests == ["query"]
Loading