diff --git a/CHANGELOG.md b/CHANGELOG.md index fe6cfdfc..404a06cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,9 +30,11 @@ This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0 * A `features="icloud"` profile, from a caldav-server-tester run by a user - the documentation already referred to it, but it did not exist. https://github.com/python-caldav/caldav/issues/730 * Compatibility feature `sync-token.invalid-token`: whether a sync token the server never issued is refused with `DAV:valid-sync-token` as RFC 6578 section 3.2 says. Xandikos and CCS answer some such tokens with a 500, and Stalwart and Ox with a report and no error. https://github.com/jelmer/xandikos/issues/743 * Compatibility feature `propfind.principal-depth-1`: whether a Depth: 1 PROPFIND on the principal URL works. GMX answers it with a 400. https://github.com/python-caldav/caldav/pull/755 +* Negated searches: `add_property_filter()` passes the `!=` (alias `<>`), `!contains` and `def` operators of icalendar-searcher 1.1.0 on to the search. They are decided client-side (unless `post_filter=False`): the server query leaves them out, as no server has been probed for `negate-condition` support yet. `build_search_xml_query()` renders `!contains` as a negated `text-match`, and `!=` and `def` as a bare `prop-filter` - a negated `text-match` is a negated *substring* match, so it cannot express `!=`. https://github.com/python-caldav/caldav/issues/568 ### Changed +* icalendar-searcher 1.1.0 or later is required (for the negated searches). * `search()` and `multiget()` (also used to load search results in bulk) ask the server for the ETag and the Schedule-Tag of each object, so a `save()` after a search or multiget now sends `If-Match` (or `If-Schedule-Tag-Match`, RFC 6638 section 3.2), and raises `ETagMismatchError` (or `ScheduleTagMismatchError`) if someone else changed the object in the meantime. Previously such a save overwrote the other change silently. * Compatibility features `sync-token.limit` and `sync-token.limit.initial-sync`, for DAV:limit and the checkpoint tokens of a truncated sync-collection report (RFC 6578 sections 3.6 and 3.7). Hints for Stalwart, Xandikos, Radicale and Baïkal. diff --git a/caldav/search.py b/caldav/search.py index 8750dc6a..1c465460 100644 --- a/caldav/search.py +++ b/caldav/search.py @@ -113,6 +113,21 @@ def _flag_component_type(searcher: "Searcher", flag: str) -> None: setattr(searcher, flag, True) +## Operators the server is never trusted with: negations and "defined". +## Whether servers honour negate-condition, or a bare prop-filter, has not +## been probed yet (caldav-server-tester), so search() decides these +## client-side. https://github.com/python-caldav/caldav/issues/568 +_NEGATED_OR_PRESENCE_OPERATORS = ("!=", "!contains", "def") + + +def _collation_str(searcher: "Searcher", property: str) -> str: + """The CalDAV collation name for a property filter.""" + if hasattr(searcher, "_property_collation") and property in searcher._property_collation: + case_sensitive = searcher._property_case_sensitive.get(property, True) + return _collation_to_caldav(searcher._property_collation[property], case_sensitive) + return "i;octet" + + def _build_search_xml_query( searcher: "Searcher", server_expand: bool = False, @@ -208,10 +223,32 @@ def _build_search_xml_query( comp_filter = None for property in searcher._property_operator: - if searcher._property_operator[property] == "undef": - match = cdav.NotDefined() - prop_name = "CATEGORIES" if property.lower() == "category" else property.upper() - filters.append(cdav.PropFilter(prop_name) + match) + operator = searcher._property_operator[property] + prop_name = ( + "CATEGORIES" if property.lower() in ("category", "categories") else property.upper() + ) + if operator == "undef": + filters.append(cdav.PropFilter(prop_name) + cdav.NotDefined()) + elif operator == "!contains" and prop_name != "CATEGORIES": + ## RFC 4791 9.7.5: a negated text-match is a negated substring match + value = searcher._property_filters[property] + if hasattr(value, "to_ical"): + value = value.to_ical() + filters.append( + cdav.PropFilter(prop_name) + + cdav.TextMatch(value, collation=_collation_str(searcher, property), negate=True) + ) + elif operator in _NEGATED_OR_PRESENCE_OPERATORS: + ## A bare prop-filter matches any defined property (RFC 4791 + ## 9.7.2). That is all CalDAV can say about != (a negated + ## text-match means "does not contain", not "not equal") and + ## about def. Nor does a negated text-match fit !contains on + ## categories: the plural form means "not all of these", which + ## is not the AND of one negation per category, and the + ## singular form matches within one category, which a match + ## against the whole comma-joined property value is not. The + ## client-side filter has to decide the rest. + filters.append(cdav.PropFilter(prop_name)) else: value = searcher._property_filters[property] property_ = property.upper() @@ -226,17 +263,7 @@ def _build_search_xml_query( if hasattr(value, "to_ical"): value = value.to_ical() - collation_str = "i;octet" - if ( - hasattr(searcher, "_property_collation") - and property in searcher._property_collation - ): - case_sensitive = searcher._property_case_sensitive.get(property, True) - collation_str = _collation_to_caldav( - searcher._property_collation[property], case_sensitive - ) - - match = cdav.TextMatch(value, collation=collation_str) + match = cdav.TextMatch(value, collation=_collation_str(searcher, property)) filters.append(cdav.PropFilter(property_) + match) if comp_filter and filters: @@ -435,7 +462,8 @@ def add_property_filter( Special virtual property "category" (singular) is also supported for substring matching within category names :param value: Filter value, should adhere to the type defined in the RFC - :param operator: Comparison operator ("contains", "==", "undef"). If not + :param operator: Comparison operator ("contains", "==", "!contains", + "!=", "def", "undef"; see below). If not specified, the server decides the matching behavior (usually substring search per RFC). If explicitly set to "contains" and the server doesn't support substring search, client-side @@ -454,18 +482,31 @@ def add_property_filter( * **contains** - substring match (e.g., "rain" matches "Training session" and "Singing in the rain") * **==** - exact match required, enforced client-side + * **!contains** - no substring match; the property must be defined + * **!=** (or **<>**) - not an exact match; the property must be defined + * **def** - matches if property is defined (value parameter ignored) * **undef** - matches if property is not defined (value parameter ignored) + The negations and ``def`` need icalendar-searcher 1.1.0 or later. + search() decides them client-side and leaves them out of the + server query, unless ``post_filter=False``; then, as with + build_search_xml_query() on its own, they are sent as the + closest CalDAV filter that never excludes a match, and != and + def match any object where the property is defined. + **Special handling for categories:** - **"categories"** (plural): Exact category name matching - "contains": subset check (all filter categories must be in component) - "==": exact set equality (same categories, order doesn't matter) + - "!contains": not all of the filter categories are in the component + - "!=": set inequality - Commas in filter values split into multiple categories - **"category"** (singular): Substring matching within category names - "contains": substring match (e.g., "out" matches "outdoor") - "==": exact match to at least one category name + - "!contains" / "!=": no category name contains / equals the value - Commas in filter values treated as literal characters Examples: @@ -629,7 +670,10 @@ def _search_impl( or self.expand or "categories" in self._property_filters or "category" in self._property_filters - or any(op == "==" for op in self._property_operator.values()) + or any( + op == "==" or op in _NEGATED_OR_PRESENCE_OPERATORS + for op in self._property_operator.values() + ) or not calendar.client.features.is_supported("search.text.case-sensitive") or not calendar.client.features.is_supported("search.time-range.accurate") ) @@ -699,6 +743,31 @@ def _search_impl( ) return + ## Negated and "defined" filters are left out of the server query and + ## decided client-side; see _NEGATED_OR_PRESENCE_OPERATORS. + if cw and post_filter is not False: + untrusted = [ + prop + for prop, op in self._property_operator.items() + if op in _NEGATED_OR_PRESENCE_OPERATORS + ] + if untrusted: + clone = self._clone_without_filters(untrusted) + objects = yield ( + SearchAction.RECURSIVE_SEARCH, + (clone, calendar, server_expand, split_expanded, props, xml, None, None), + ) + yield ( + SearchAction.RETURN, + self.filter( + objects, + post_filter=True, + split_expanded=split_expanded, + server_expand=server_expand, + ), + ) + return + ## special compatibility-case for servers that do not support is-not-defined ## for specific properties (e.g. search.is-not-defined.category or .dtend) if cw and post_filter is not False: diff --git a/docs/design/TODO.md b/docs/design/TODO.md index eff53404..2062349c 100644 --- a/docs/design/TODO.md +++ b/docs/design/TODO.md @@ -1,5 +1,9 @@ # Known Issues and TODO Items +## Server-side negated searches + +`search()` decides the `!=` / `<>`, `!contains` and `def` operators client-side and leaves them out of the server query (https://github.com/python-caldav/caldav/issues/568). A caldav-server-tester probe for `negate-condition` support, and the compatibility_hints flag it would feed, would let `search()` send `!contains` as a negated `text-match` to the servers that honour it. + ## Calendar cleanup in integration tests For servers where calendars cannot be created, the test will take the first available calendar it finds and wipe it. That is not acceptable without the user explicitly configuring that this is OK. diff --git a/pyproject.toml b/pyproject.toml index 25eb3163..9c228817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ dependencies = [ "recurring-ical-events>=2.0.0", "typing_extensions;python_version<'3.11'", "icalendar>6.0.0", - "icalendar-searcher>=1.0.5,<2", + "icalendar-searcher>=1.1.0,<2", "dnspython", "python-dateutil", "PyYAML", diff --git a/tests/test_search.py b/tests/test_search.py index a5ace8f2..9456d054 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1839,3 +1839,208 @@ def test_search_results_populated_after_batch_load(self) -> None: results = searcher.search(cal) assert len(results) == 1 + + +class TestNegatedOperators: + """The negated and presence operators: ``!=``, ``!contains`` and ``def``. + + https://github.com/python-caldav/caldav/issues/568 + + A CalDAV ``text-match`` is a substring match, so ``negate-condition`` + means "does not contain". ``!contains`` maps onto it; ``!=`` does not - + sending a negated substring match for ``SUMMARY != "Training"`` would + drop "Training session". Server support for either is unknown until + caldav-server-tester probes it, so search() decides all three + client-side, and the server query is never narrower than the answer. + """ + + _events = { + "training": "Training session", + "rain": "rain", + "none": None, + } + + _category_events = { + "work": "work", + "both": "work,home", + "none": None, + } + + def _event_data(self, uid: str, value: str | None, prop: str = "SUMMARY") -> str: + summary_line = f"{prop}:{value}\n" if value is not None else "" + return ( + "BEGIN:VCALENDAR\nVERSION:2.0\nPRODID:-//Test//Test//EN\nBEGIN:VEVENT\n" + f"UID:{uid}@example.com\nDTSTAMP:20240101T120000Z\n" + f"DTSTART:20240615T140000Z\n{summary_line}END:VEVENT\nEND:VCALENDAR" + ) + + def _calendar(self, supported: bool | str, prop: str = "SUMMARY") -> mock.Mock: + """supported: True/False for every feature, or the one feature unsupported.""" + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + client.features = mock.Mock() + if isinstance(supported, str): + unsupported = supported + + def is_supported(feature: str, return_type: type = bool) -> object: + if return_type is str: + return "unsupported" if feature == unsupported else "full" + return feature != unsupported + + client.features.is_supported = mock.Mock(side_effect=is_supported) + else: + client.features.is_supported = mock.Mock(return_value=supported) + client.features.backward_compatibility_mode = False + cal = mock.Mock() + cal.client = client + cal.url = URL("https://cal.example.com/cal/") + objects = [ + Event( + client=client, + url=f"https://cal.example.com/cal/{uid}.ics", + data=self._event_data(uid, value, prop), + parent=cal, + ) + for uid, value in ( + self._category_events if prop == "CATEGORIES" else self._events + ).items() + ] + ## The server ignores the filter and returns everything; the + ## client-side filter has to produce the answer. + cal._request_report_build_resultlist = mock.Mock(return_value=(mock.MagicMock(), objects)) + return cal + + @staticmethod + def _xml(searcher: CalDAVSearcher) -> bytes: + from lxml import etree + + xml, _ = searcher.build_search_xml_query() + return etree.tostring(xml.xmlelement()) + + @pytest.mark.parametrize( + ("operator", "value", "expected"), + [ + ("!=", "rain", {"training"}), + ("<>", "rain", {"training"}), + ("!=", "Training", {"training", "rain"}), + ("!contains", "rain", set()), + ("!contains", "session", {"rain"}), + ("def", None, {"training", "rain"}), + ], + ) + @pytest.mark.parametrize( + "supported", [True, False, "search.text"], ids=["all", "none", "no-text-search"] + ) + def test_search_filters_client_side( + self, operator: str, value: str | None, expected: set[str], supported: bool | str + ) -> None: + cal = self._calendar(supported) + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", value, operator=operator) + results = searcher.search(cal) + uids = {str(r.icalendar_component["UID"]).split("@")[0] for r in results} + assert uids == expected + + @pytest.mark.parametrize("operator", ["!=", "!contains", "def"]) + def test_search_sends_no_positive_text_match(self, operator: str) -> None: + cal = self._calendar(True) + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator=operator) + searcher.search(cal) + from lxml import etree + + for call in cal._request_report_build_resultlist.call_args_list: + xml = call.args[0] + body = etree.tostring(xml.xmlelement()) if hasattr(xml, "xmlelement") else xml + assert b"text-match" not in body + + def test_search_keeps_positive_filter_beside_negation(self) -> None: + """Stripping a negation leaves the other filters in the server query.""" + from lxml import etree + + cal = self._calendar(True) + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="!=") + searcher.add_property_filter("LOCATION", "Oslo", operator="contains") + searcher.search(cal) + bodies = [ + etree.tostring(call.args[0].xmlelement()) + for call in cal._request_report_build_resultlist.call_args_list + ] + assert bodies + for body in bodies: + assert b'prop-filter name="LOCATION"' in body + assert b">Oslo<" in body + assert b"SUMMARY" not in body + + def test_xml_not_equal_is_a_bare_prop_filter(self) -> None: + """!= cannot be a negated substring match; only "defined" is sent.""" + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="!=") + xml = self._xml(searcher) + assert b'prop-filter name="SUMMARY"' in xml + assert b"text-match" not in xml + + def test_xml_def_is_a_bare_prop_filter(self) -> None: + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", None, operator="def") + xml = self._xml(searcher) + assert b'prop-filter name="SUMMARY"' in xml + assert b"text-match" not in xml + assert b"is-not-defined" not in xml + + def test_xml_not_contains_is_a_negated_text_match(self) -> None: + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="!contains") + xml = self._xml(searcher) + assert b'negate-condition="yes"' in xml + assert b">rain<" in xml + + @pytest.mark.parametrize("key", ["categories", "category"]) + def test_xml_not_contains_on_categories_is_a_bare_prop_filter(self, key: str) -> None: + """Negating each category separately would AND the negations, which is + not the negation of "has all of them"; send only "defined".""" + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter(key, "work,home", operator="!contains") + xml = self._xml(searcher) + assert b'prop-filter name="CATEGORIES"' in xml + assert b"text-match" not in xml + + @pytest.mark.parametrize( + ("key", "operator", "value", "expected"), + [ + ("categories", "!contains", "work,home", {"work"}), + ("categories", "!contains", "work", set()), + ("categories", "!=", "work", {"both"}), + ("categories", "def", None, {"work", "both"}), + ("category", "!contains", "or", set()), + ("category", "!contains", "hom", {"work"}), + ("category", "!=", "home", {"work"}), + ], + ) + @pytest.mark.parametrize( + "supported", + [True, False, "search.text.category"], + ids=["all", "none", "no-category-search"], + ) + def test_search_filters_categories_client_side( + self, + key: str, + operator: str, + value: str | None, + expected: set[str], + supported: bool | str, + ) -> None: + cal = self._calendar(supported, prop="CATEGORIES") + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter(key, value, operator=operator) + results = searcher.search(cal) + uids = {str(r.icalendar_component["UID"]).split("@")[0] for r in results} + assert uids == expected + + def test_xml_not_contains_case_insensitive_collation(self) -> None: + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="!contains", case_sensitive=False) + xml = self._xml(searcher) + assert b'negate-condition="yes"' in xml + assert b'collation="i;ascii-casemap"' in xml