Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions caldav/compatibility_hints.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,8 @@ class FeatureSet:
"interval": "Rate limiting window, in seconds",
"count": "Max number of requests to send within the interval",
"max_sleep": "Max sleep when hitting a 429 or 503 with retry-after, in seconds",
"default_sleep": "Sleep for this long when hitting a 429, in seconds"
"default_sleep": "Sleep for this long when hitting a 429, in seconds",
"create_calendar_per_hour": "How many calendars the account may make within an hour (Nextcloud's default period); MKCALENDAR beyond that gets a 429 (Nextcloud's rateLimitCalendarCreation, observed as about 10 on ecloud). A client that makes many calendars, like a test suite, should reuse them rather than make new ones. Read on its own, whatever 'enable' says: a server with only this limit needs no pacing or sleep-and-retry",
}},
"search-cache": {
"type": "server-peculiarity",
Expand Down Expand Up @@ -1864,12 +1865,21 @@ def compare(self, observed):
#'search.text.case-sensitive': {'support': 'unsupported'}, ## observed to work at 4bc0de765a2b53e6f223e0b9ac51c653bac11fb7 (caldav) / 3cae24cf99da1702b851b5a74a9b88c8e5317dad (server checker)
'delete-calendar.free-namespace': False,
## TODO: this applies only to test runs, not to ordinary usage
## 2026-11-12: two test runs at one request per 2 s made 22 calendars in
## 3 hours (up to 9 in one hour) and got no 429 at all, so the pace is
## doubled to one request per second. Only the sync suite paces by
## interval/count. Revisit: lower it further while no 429 shows up,
## raise it again if one does.
'rate-limit': {
'enable': True,
'interval': 2,
'interval': 1,
'count': 1,
'default_sleep': 4,
'max_sleep': 120,
## Nextcloud's own default (rateLimitCalendarCreation); 2026-11-11 a
## dozen creations within the hour got a 429, 2026-11-12 nine in one
## hour did not.
'create_calendar_per_hour': 10,
'description': "It's needed to manually empty trashbin frequently when running tests. Since this operation takes some time and/or there are some caches, it's needed to run tests slowly, even when hammering the 'empty thrashbin' frequently",
},
'auto-connect.url': {
Expand Down Expand Up @@ -2530,7 +2540,14 @@ def __getattr__(name: str) -> Any:
## through its own UI/API and refuses a CalDAV DELETE on the collection,
## where upstream DAViCal allows it. Consequence for callers: Calendar
## .delete() degrades to wiping the objects, so a cal_id is never freed for
## reuse and a later MKCALENDAR at the same id gets 405.
## reuse and a later MKCALENDAR at the same id gets 405. DSM's own web
## API does delete them, with the account's CalDAV credentials: log in
## at /webapi/entry.cgi (api=SYNO.API.Auth, version=6, method=login,
## session=Calendar, format=sid), then api=SYNO.Cal.Cal, version=1,
## method=delete, cal_id='"/<user>/<segment>/"' (a JSON string) with the
## sid as _sid. It works on calendars made over CalDAV too, although
## SYNO.Cal.Cal's list method does not show those (2026-11-18, 384
## leftover test calendars deleted).
'delete-calendar': False,
## sync-token (fragile, inherited from davical) and sync-token.delete
## both came out full in a server-tester run on 2026-10-23.
Expand Down
1 change: 1 addition & 0 deletions changes/+create-calendar-rate-limit.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
New extra key `create_calendar_per_hour` on the `rate-limit` client-feature: how many calendars the account may make within an hour before MKCALENDAR gets a 429. Set to 10 for ecloud, Nextcloud's default. A client making many calendars, like a test suite, can reuse them instead.
4 changes: 2 additions & 2 deletions docs/design/TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ On Nextcloud a deleted calendar moves to the trashbin, keeps its cal_id occupied
## A recognisable cal_id for every test calendar

The tests and the caldav-server-tester leave calendars behind, and on servers that cap the number of calendars per principal (ecloud, see above) that eventually blocks all creation. On 2026-10-23 ecloud held 15 empty calendars with bare-uuid cal_ids and the uuid as display name; which code made them was not found. Every calendar the tests or the tester create should carry a cal_id prefix, like the caldav-server-tester's `csc_` UIDs, so a cleanup step can delete leftovers safely - with the purge from the section above where the server has a trashbin.
* caldav tests: there is no common cal_id prefix on every calendar, and no sweep. Several tests create calendars by name only (`test_caldav.py` "CalConfigA"/"CalConfigB", "XandikosMetaCal"/"RadicaleMetaCal", `test_get_calendars_sections.py`, `test_examples.py`), so `make_calendar()` gives them a bare uuid. Give all of them a `pythoncaldav-test` prefixed cal_id and add a session-level sweep.
* caldav-server-tester: has `PROBE_CALENDAR_PREFIXES` and a sweep matching cal_id or display name, but `CheckDuplicateUID` creates its second calendar by name only, so its cal_id is a bare uuid that the sweep cannot recognise when the server drops the display name. Pass a prefixed cal_id. Its sweep also only moves calendars to the Nextcloud trashbin.
* caldav tests: a session-level sweep now deletes every live `pythoncaldav-` calendar on external servers (`tests/fixture_helpers.py`), but not every calendar carries the prefix. Several tests create calendars by name only (`test_caldav.py` "CalConfigA"/"CalConfigB", "XandikosMetaCal"/"RadicaleMetaCal", `test_get_calendars_sections.py`, `test_examples.py`), so `make_calendar()` gives them a bare uuid. Give all of them a `pythoncaldav-test` prefixed cal_id, so the sweep covers them.
* caldav-server-tester: has `PROBE_CALENDAR_PREFIXES` and a sweep matching cal_id or display name, but `CheckDuplicateUID` and the MOVE check both create their second calendar by name only (`make_calendar(name=...)`), so its cal_id is a bare uuid that the sweep cannot recognise when the server drops the display name. On 2026-11-12 every compatibility run against ecloud left two such calendars behind, their display name reading back as the uuid. Pass a prefixed cal_id and delete by it. Its sweep also only moves calendars to the Nextcloud trashbin.

## An installed caldav.testing looks for the checkout in site-packages

Expand Down
12 changes: 12 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,18 @@ test-servers:
- no-sync-token
```

Most test calendars have ids starting with `pythoncaldav-`. On an external
server every such calendar is deleted when a test session first uses the
server and again when the session ends, bypassing the Nextcloud trashbin. Do
not keep calendars of your own under that prefix. Nothing is deleted when the
server cannot create or cannot delete calendars, when `test-calendar` in the
features names a calendar for the tests, or when its `cleanup-regime` is
`none`. Calendars already in the Nextcloud trashbin are not listed, so the
sweep does not reach them; the tests themselves delete past the trashbin
where a plain DELETE would leave the id taken. The scheduling tests' calendars (`schedulingnosetestcalendar*`,
`asyncschedtest*`) are kept apart from this on purpose: they are reused from
run to run, with their contents wiped.

## Test Structure

### Unit Tests (no server required)
Expand Down
7 changes: 7 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ def pytest_configure(config):
os.environ["PYTHON_CALDAV_DEBUGMODE"] = "DEBUG_PDB"


def pytest_sessionfinish(session, exitstatus):
"""Remove the test calendars from the external servers this session used."""
from .fixture_helpers import sweep_used_servers

sweep_used_servers()


@pytest.fixture
def process_tz(request, monkeypatch):
"""Run the test with the Python process in the time zone given by
Expand Down
177 changes: 174 additions & 3 deletions tests/fixture_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@
import asyncio
import inspect
import logging
import time
from typing import Any
from urllib.parse import unquote, urlparse

from caldav.compatibility_hints import write_delay
from caldav.lib.error import DAVError, NotFoundError


async def _maybe_await(result: Any) -> Any:
"""Await if result is awaitable, otherwise return as-is."""
Expand Down Expand Up @@ -430,14 +434,17 @@ async def cleanup_calendar_objects(calendar: Any) -> None:
pass


async def adelete_calendar_if_present(principal: Any, cal_id: str) -> None:
async def adelete_calendar_if_present(principal: Any, cal_id: str, client: Any = None) -> None:
"""Best-effort removal of a leftover test calendar from a previous run.

A test that recreates a calendar with a fixed ``cal_id`` must first clear
any leftover, or the recreate MKCALENDAR 405s ("resource already exists").
Given the ``client``, the leftover goes as adelete_test_calendar() deletes
it: past the trashbin where a plain DELETE would keep the id taken.

Only ``NotFoundError`` (the calendar isn't there) is swallowed - everything
else propagates. A previous incarnation wrapped this in a bare
else propagates, except on that past-the-trashbin path, where a refused
DELETE is logged rather than raised. A previous incarnation wrapped this in a bare
``except Exception: pass``, which silently hid a real bug (async
``principal.calendar()`` raising ``TypeError``), so the cleanup never ran
and calendars leaked. Keep the catch narrow so that can't recur.
Expand All @@ -447,8 +454,172 @@ async def adelete_calendar_if_present(principal: Any, cal_id: str) -> None:
calendar = await _maybe_await(principal.calendar(cal_id=cal_id))
await cleanup_calendar_objects(calendar)
try:
await _maybe_await(calendar.delete())
if client is None:
await _maybe_await(calendar.delete())
else:
await adelete_test_calendar(client, calendar)
except error.NotFoundError:
# Already gone, which is exactly what this function wants. Nothing
# wider is caught on purpose - see the docstring.
pass


## Most calendars the functional tests create have an id starting with this.
TEST_CALENDAR_PREFIX = "pythoncaldav-"

## Nextcloud moves a deleted calendar to a trashbin, where it still counts
## towards the account's calendar limit and still holds the id; this header
## deletes it for good. Other servers ignore it.
NO_TRASHBIN_HEADERS = {"X-NC-CalDAV-No-Trashbin": "1"}

## Servers already swept in this pytest session, by registry name.
_swept_servers: set[str] = set()


def sweep_test_calendars(client: Any, principal: Any) -> list[str]:
"""Delete every test calendar (id starting with ``pythoncaldav-``) for good.

The fixtures leave calendars behind on purpose where deleting them would
not free the id (wipe-calendar regime), and an interrupted run leaves them
behind anywhere. On accounts allowing only a handful of calendars that
adds up, so the session sweeps them away at its start and end.

Nothing is touched where the calendars could not be recreated: without
calendar creation or deletion, with a calendar configured for the tests, or
with cleanup switched off. Returns the ids deleted.
"""
if not (_supports(client, "create-calendar") and _supports(client, "delete-calendar")):
return []
test_cal_info = client.features.is_supported("test-calendar", return_type=dict)
if test_cal_info.get("cleanup-regime") == "none" or any(
k in test_cal_info for k in ("name", "cal_id", "cal_url")
):
return []
deleted = [
cal_id
for cal in principal.get_calendars()
if (cal_id := _cal_id(cal)).startswith(TEST_CALENDAR_PREFIX)
and delete_calendar_for_good(client, cal)
]
## A server deleting calendars asynchronously (Infomaniak) answers before
## the calendar is gone; a fixture re-creating the id right after would
## meet the old one, as Calendar.delete() knows.
delay = write_delay(client.features, "synchronous-write.delete-calendar")
deadline = time.monotonic() + delay
while deleted and delay:
if not any(_cal_id(cal) in deleted for cal in principal.get_calendars()):
break
if time.monotonic() >= deadline:
logging.warning("sweep: %s still listed after %ss", deleted, delay)
break
time.sleep(0.5)
return deleted


def _cal_id(cal: Any) -> str:
return unquote(str(cal.url).rstrip("/").rsplit("/", 1)[-1])


def delete_calendar_for_good(client: Any, cal: Any) -> bool:
"""DELETE a calendar past the Nextcloud trashbin; False if refused.

A refusal is logged, not raised: the callers clean up, and one calendar
the server will not let go of must not stop them.
"""
try:
response = client.delete(str(cal.url), headers=NO_TRASHBIN_HEADERS)
except DAVError as e:
return _deleted_for_good(cal, e)
return _deleted_for_good(cal, response)


async def adelete_calendar_for_good(client: Any, cal: Any) -> bool:
"""Async twin of delete_calendar_for_good()."""
try:
response = await _maybe_await(client.delete(str(cal.url), headers=NO_TRASHBIN_HEADERS))
except DAVError as e:
return _deleted_for_good(cal, e)
return _deleted_for_good(cal, response)


def delete_test_calendar(client: Any, cal: Any) -> None:
"""Delete a calendar the tests made, freeing its id where that takes more.

Where a plain DELETE leaves the id taken (``delete-calendar.free-namespace``
unsupported: Nextcloud's trashbin), the calendar goes for good, so that it
neither counts towards the account's calendar limit nor blocks the next
MKCALENDAR of the same id. Elsewhere Calendar.delete() does it, with its
retries and its wait for an asynchronous deletion, or by wiping the
objects where the server refuses to delete calendars (Synology).
"""
if _plain_delete_will_do(client):
cal.delete()
else:
delete_calendar_for_good(client, cal)


async def adelete_test_calendar(client: Any, cal: Any) -> None:
"""Async twin of delete_test_calendar()."""
if _plain_delete_will_do(client):
await _maybe_await(cal.delete())
else:
await adelete_calendar_for_good(client, cal)


def _plain_delete_will_do(client: Any) -> bool:
return not _supports(client, "delete-calendar") or _supports(
client, "delete-calendar.free-namespace"
)


def _deleted_for_good(cal: Any, outcome: Any) -> bool:
"""Read a DELETE's response, or the error it raised; log a refusal."""
if isinstance(outcome, NotFoundError):
return True
if isinstance(outcome, DAVError):
logging.warning("DELETE %s refused: %s", cal.url, outcome)
return False
if outcome.status in (200, 204, 404):
return True
logging.warning("DELETE %s answered %s", cal.url, outcome.status)
return False


def sweep_server(server_name: str) -> None:
"""Sweep the test calendars off an external test server.

Docker and embedded servers are thrown away after the run anyway. Best
effort: a sweep that fails must not fail a test.
"""
from caldav.testing.base import ExternalTestServer
from caldav.testing.registry import get_registry

server = get_registry().get(server_name)
if not isinstance(server, ExternalTestServer):
return
try:
with server.get_sync_client() as client:
deleted = sweep_test_calendars(client, client.principal())
except Exception as e:
logging.warning("sweep of %s failed: %s", server_name, e)
return
if deleted:
logging.info("sweep of %s deleted %s", server_name, deleted)


def sweep_server_once(server_name: str | None) -> None:
"""Sweep a server the first time a test in this session uses it.

Clears leftovers from interrupted runs; conftest sweeps the same servers
again when the session ends.
"""
if not server_name or server_name in _swept_servers:
return
_swept_servers.add(server_name)
sweep_server(server_name)


def sweep_used_servers() -> None:
"""Sweep every server swept at its first use, at the end of the session."""
for server_name in sorted(_swept_servers):
sweep_server(server_name)
Loading
Loading