From 5300e3d1cb6d63a4c39bdb6d83b0e0e8a112bbda Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:04:08 +0100 Subject: [PATCH 1/6] feat(query): add the or operator and dotted field paths to the DSL --- src/oold/backend/interface.py | 16 +++++++++++ tests/test_query_dsl.py | 53 +++++++++++++++++++++++++++++++++++ tests/typing/query_dsl.py | 1 + 3 files changed, 70 insertions(+) create mode 100644 tests/test_query_dsl.py diff --git a/src/oold/backend/interface.py b/src/oold/backend/interface.py index 2214491..c2410c5 100644 --- a/src/oold/backend/interface.py +++ b/src/oold/backend/interface.py @@ -56,6 +56,14 @@ class ResolveResult(BaseModel): nodes: dict[str, None | GenericLinkedBaseModel] +def field_path(field: str) -> list[str]: + """Split a condition field into path segments: ``"address.city"`` -> ``["address", "city"]``.""" + segments = field.split(".") + if not all(segments): + raise ValueError(f"Invalid field path {field!r}: empty segment") + return segments + + class Query(BaseModel): op1: Union["Query", "Condition"] operator: str @@ -65,6 +73,10 @@ class Query(BaseModel): def __and__(self, other): return Query(op1=self, operator="and", op2=other) + # override the | operator + def __or__(self, other): + return Query(op1=self, operator="or", op2=other) + class Condition(BaseModel): field: str @@ -81,6 +93,10 @@ def __eq__(self, other): def __and__(self, other): return Query(op1=self, operator="and", op2=other) + # override the | operator + def __or__(self, other): + return Query(op1=self, operator="or", op2=other) + class QueryParam(BaseModel): query: Query | Condition diff --git a/tests/test_query_dsl.py b/tests/test_query_dsl.py new file mode 100644 index 0000000..cf6d6d1 --- /dev/null +++ b/tests/test_query_dsl.py @@ -0,0 +1,53 @@ +import pytest + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import ComparisonOperator, Condition, Query, field_path + + +def _eq(field, value): + return Condition(field=field, operator=ComparisonOperator.EQ, value=value) + + +@pytest.mark.parametrize( + ("field", "expected"), + [("name", ["name"]), ("address.city", ["address", "city"]), ("a.b.c", ["a", "b", "c"])], +) +def test_field_path_splits_on_dots(field, expected): + assert field_path(field) == expected + + +@pytest.mark.parametrize("field", ["", ".a", "a.", "a..b"]) +def test_field_path_rejects_empty_segments(field): + with pytest.raises(ValueError, match="empty segment"): + field_path(field) + + +def test_or_builds_an_or_query(): + a, b = _eq("a", 1), _eq("b", 2) + + query = a | b + + assert isinstance(query, Query) + assert query.operator == "or" + assert query.op1 is a + assert query.op2 is b + + +def test_or_combines_with_and_and_nested_queries(): + a, b, c = _eq("a", 1), _eq("b", 2), _eq("c", 3) + + query = (a & b) | c + + assert query.operator == "or" + assert isinstance(query.op1, Query) + assert query.op1.operator == "and" + assert query.op2 is c + assert (c | (a & b)).operator == "or" + + +def test_or_is_evaluated_by_the_document_store(): + store = SimpleDictDocumentStore() + store.store_json_dicts({"ex:1": {"a": 1, "b": 0}, "ex:2": {"a": 0, "b": 2}, "ex:3": {"a": 0, "b": 0}}) + + assert store._query(_eq("a", 1) | _eq("b", 2)) == {"ex:1", "ex:2"} + assert store._query((_eq("a", 1) | _eq("b", 2)) & _eq("b", 2)) == {"ex:2"} diff --git a/tests/typing/query_dsl.py b/tests/typing/query_dsl.py index f4c8eba..7cc5c9b 100644 --- a/tests/typing/query_dsl.py +++ b/tests/typing/query_dsl.py @@ -26,6 +26,7 @@ class Entity(LinkedBaseModel): assert_type(Entity["ex:e1"], Entity | None) assert_type(Entity[Entity.name == "x"], LinkResultList[Entity] | None) assert_type(Entity[["ex:e1", "ex:e2"]], LinkResultList[Entity] | None) +assert_type(Entity[(Entity.name == "x") | (Entity.id == "ex:e1")], LinkResultList[Entity] | None) many = Entity[Entity.name == "x"] if many is not None: From 7e67f2717edc6604d5ec13d7c38308ae9e074a45 Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:12:31 +0100 Subject: [PATCH 2/6] feat(model): build nested field conditions with attribute chaining --- src/oold/model/__init__.py | 2 + src/oold/model/_descriptor.py | 56 +++++++++-- src/oold/model/_paths.py | 79 +++++++++++++++ src/oold/model/v1/__init__.py | 2 + src/oold/model/v1/_descriptor.py | 2 +- tests/test_query_paths.py | 168 +++++++++++++++++++++++++++++++ tests/typing/query_dsl.py | 8 ++ 7 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 src/oold/model/_paths.py create mode 100644 tests/test_query_paths.py diff --git a/src/oold/model/__init__.py b/src/oold/model/__init__.py index b566d46..153e6eb 100644 --- a/src/oold/model/__init__.py +++ b/src/oold/model/__init__.py @@ -36,6 +36,7 @@ get_backend, get_resolver, ) +from oold.model._paths import reject_nested_path from oold.static import ( GenericLinkedBaseModel, export_jsonld, @@ -97,6 +98,7 @@ def __bool__(self): return bool(d) def __getattr__(self, name): + reject_nested_path(object.__getattribute__(self, "parent"), object.__getattribute__(self, "name"), name) d = object.__getattribute__(self, "_default") if d is not None and d is not PydanticUndefined: return getattr(d, name) diff --git a/src/oold/model/_descriptor.py b/src/oold/model/_descriptor.py index 2fb9727..2e6af83 100644 --- a/src/oold/model/_descriptor.py +++ b/src/oold/model/_descriptor.py @@ -74,6 +74,7 @@ class Person(LinkedBaseModel): get_resolver, ) from oold.model._compat import LinkedApiMixin +from oold.model._paths import embedded_model, is_model_container, model_fields from oold.model._ref import Ref, _construct T = TypeVar("T") @@ -481,45 +482,74 @@ class FieldProxy: against class attributes, which resolve to a proxy rather than to the default. Without ``__bool__`` every such test is unconditionally true, and without ``__getattr__`` every such call raises. + + A field that embeds a model chains: ``Person.address.city`` is a proxy named + ``"address.city"``. Chaining wins over the proxy's own ``name`` and + ``default``, which is why internals read those through ``object``. """ - __slots__ = ("default", "name") + __slots__ = ("default", "field", "name") - def __init__(self, name: str, default: Any = None): + def __init__(self, name: str, default: Any = None, field: Any = None): self.name = name self.default = default + self.field = field + + def __getattribute__(self, item: str) -> Any: + if not item.startswith("_"): + child = _child_proxy(self, item) + if child is not None: + return child + return object.__getattribute__(self, item) def __bool__(self) -> bool: - return bool(self.default) if _has_default(self.default) else False + default = object.__getattribute__(self, "default") + return bool(default) if _has_default(default) else False def __getattr__(self, item: str) -> Any: default = object.__getattribute__(self, "default") if _has_default(default): return getattr(default, item) + name = object.__getattribute__(self, "name") + if is_model_container(object.__getattribute__(self, "field")): + raise AttributeError(f"{name!r} is a list of models; nested paths do not index into lists") raise AttributeError(f"{type(self).__name__!r} object has no attribute {item!r}") + def _condition(self, operator: str, other: Any) -> Condition: + return Condition(field=object.__getattribute__(self, "name"), operator=operator, value=other) + def __eq__(self, other: Any) -> Any: # type: ignore[override] - return Condition(field=self.name, operator="eq", value=other) + return self._condition("eq", other) def __ne__(self, other: Any) -> Any: # type: ignore[override] - return Condition(field=self.name, operator="ne", value=other) + return self._condition("ne", other) def __lt__(self, other: Any) -> Any: - return Condition(field=self.name, operator="lt", value=other) + return self._condition("lt", other) def __le__(self, other: Any) -> Any: - return Condition(field=self.name, operator="le", value=other) + return self._condition("le", other) def __gt__(self, other: Any) -> Any: - return Condition(field=self.name, operator="gt", value=other) + return self._condition("gt", other) def __ge__(self, other: Any) -> Any: - return Condition(field=self.name, operator="ge", value=other) + return self._condition("ge", other) def __hash__(self) -> int: return id(self) +def _child_proxy(proxy: FieldProxy, item: str) -> FieldProxy | None: + """The proxy for ``item`` when the field embeds a model that has it.""" + model = embedded_model(object.__getattribute__(proxy, "field")) + fields = model_fields(model) if model is not None else None + if not fields or item not in fields: + return None + path = object.__getattribute__(proxy, "name") + return FieldProxy(f"{path}.{item}", getattr(fields[item], "default", None), fields[item]) + + class LinkedBaseModelMetaClass(ModelMetaclass): """Metaclass providing the query DSL without touching attribute reads. @@ -566,7 +596,7 @@ def __getattr__(cls, name: str) -> Any: for klass in cls.__mro__: fields = klass.__dict__.get("__pydantic_fields__") if fields and name in fields: - return FieldProxy(name, getattr(fields[name], "default", None)) + return FieldProxy(name, getattr(fields[name], "default", None), fields[name]) raise AttributeError(name) @overload @@ -1020,6 +1050,12 @@ def __init__( self.required_iri = required_iri self.owner: Any = None + def __getattr__(self, item: str) -> Any: + if item.startswith("_"): + raise AttributeError(item) + name = self.__dict__.get("name") + raise AttributeError(f"{name!r} is a link; conditions cannot follow links, compare it with an IRI instead") + def __set_name__(self, owner: type, name: str) -> None: # Only relevant for the explicit form (declared in the class body). if self.name is None: diff --git a/src/oold/model/_paths.py b/src/oold/model/_paths.py new file mode 100644 index 0000000..dfc40d4 --- /dev/null +++ b/src/oold/model/_paths.py @@ -0,0 +1,79 @@ +"""Resolve the embedded model behind a field, so queries can follow nested paths. + +Duck-typed over pydantic v1 and v2 so neither has to be imported here. No imports +from ``oold``: the legacy bindings use this too, and a cycle would break them. +""" + +import types +from functools import lru_cache +from typing import Any, Union, get_args, get_origin + +_UNION_ORIGINS = {Union, types.UnionType} + +_V1_SINGLETON_SHAPE = 1 + + +def model_fields(model: type) -> dict[str, Any] | None: + """The fields of a pydantic model class, or None if it is not one. + + v2 first: ``__fields__`` is deprecated there and warns when read. + """ + fields = getattr(model, "__pydantic_fields__", None) + if fields is None: + fields = getattr(model, "__fields__", None) + return fields if isinstance(fields, dict) else None + + +def _strip_optional(annotation: Any) -> Any: + if get_origin(annotation) in _UNION_ORIGINS: + arms = [arm for arm in get_args(annotation) if arm is not type(None)] + return arms[0] if len(arms) == 1 else None + return annotation + + +@lru_cache(maxsize=1024) +def _model_of(annotation: Any) -> type | None: + """The model an annotation embeds as a single object. Bounded: models can be generated at runtime.""" + candidate = _strip_optional(annotation) + # a parametrised alias (list[X], Link[X]) is not a class on every supported Python + if isinstance(candidate, type) and get_origin(candidate) is None and model_fields(candidate) is not None: + return candidate + return None + + +def embedded_model(field: Any) -> type | None: + """The model a field embeds as a single object, None for scalars, containers and links.""" + if field is None: + return None + if hasattr(field, "shape"): # pydantic v1 ModelField + candidate = field.type_ if field.shape == _V1_SINGLETON_SHAPE else None + return candidate if isinstance(candidate, type) and model_fields(candidate) is not None else None + annotation = getattr(field, "annotation", None) + try: + return _model_of(annotation) + except TypeError: # an unhashable annotation cannot be cached + return _model_of.__wrapped__(annotation) + + +def is_model_container(field: Any) -> bool: + """Whether a field holds a list, set or tuple of embedded models.""" + if field is None: + return False + if hasattr(field, "shape"): # pydantic v1 ModelField + item = field.type_ + return field.shape != _V1_SINGLETON_SHAPE and isinstance(item, type) and model_fields(item) is not None + annotation = _strip_optional(getattr(field, "annotation", None)) + if get_origin(annotation) in (list, set, frozenset, tuple): + return any(isinstance(arg, type) and model_fields(arg) is not None for arg in get_args(annotation)) + return False + + +def reject_nested_path(parent: type, name: str, item: str) -> None: + """Raise if ``Parent.name.item`` is a nested path, which only the descriptor binding builds.""" + model = embedded_model((model_fields(parent) or {}).get(name)) + fields = model_fields(model) if model is not None else None + if fields and item in fields: + raise NotImplementedError( + f"Nested field paths ({name}.{item}) need the descriptor binding, which is the default. " + "OOLD_DESCRIPTOR_BINDING=0 selects the legacy binding." + ) diff --git a/src/oold/model/v1/__init__.py b/src/oold/model/v1/__init__.py index ae95f27..a3e3478 100644 --- a/src/oold/model/v1/__init__.py +++ b/src/oold/model/v1/__init__.py @@ -31,6 +31,7 @@ get_backend, get_resolver, ) +from oold.model._paths import reject_nested_path from oold.static import ( GenericLinkedBaseModel, export_jsonld, @@ -101,6 +102,7 @@ def __bool__(self): return bool(d) def __getattr__(self, name): + reject_nested_path(object.__getattribute__(self, "parent"), object.__getattribute__(self, "name"), name) d = object.__getattribute__(self, "_default") if d is not None and d is not ...: return getattr(d, name) diff --git a/src/oold/model/v1/_descriptor.py b/src/oold/model/v1/_descriptor.py index 23b7372..dd36e21 100644 --- a/src/oold/model/v1/_descriptor.py +++ b/src/oold/model/v1/_descriptor.py @@ -187,7 +187,7 @@ def __getattr__(cls, name: str) -> Any: for klass in cls.__mro__: fields = klass.__dict__.get("__fields__") if fields and name in fields: - return FieldProxy(name, getattr(fields[name], "default", None)) + return FieldProxy(name, getattr(fields[name], "default", None), fields[name]) raise AttributeError(name) @overload diff --git a/tests/test_query_paths.py b/tests/test_query_paths.py new file mode 100644 index 0000000..210ba11 --- /dev/null +++ b/tests/test_query_paths.py @@ -0,0 +1,168 @@ +"""Attribute chaining: ``Person.address.city == "x"`` builds a dotted Condition. + +Only fields that embed a model chain. Links are IRI references and lists are not +indexed, so both say so instead of failing with a bare AttributeError. +""" + +import pytest +from pydantic import BaseModel +from pydantic.v1 import BaseModel as BaseModelV1 + +from oold.backend.interface import Condition +from oold.model._descriptor import Link, LinkedBaseModel, OoldField +from oold.model.v1 import LinkedBaseModel as LinkedBaseModelV1 + + +class Geo(BaseModel): + lat: float | None = None + + +class Address(BaseModel): + city: str | None = None + # names the proxy itself carries: chaining must win over them + name: str | None = None + default: str | None = None + field: str | None = None + geo: Geo | None = None + + +class Org(LinkedBaseModel): + id: str + + +class Person(LinkedBaseModel): + id: str + type: str | None = "qpaths:Person" + name: str | None = None + address: Address | None = None + home: Address + previous: list[Address] | None = None + employer: Link["Org | None"] = OoldField() + + +class GeoV1(BaseModelV1): + lat: float | None = None + + +class AddressV1(BaseModelV1): + city: str | None = None + name: str | None = None + geo: GeoV1 | None = None + + +class PersonV1(LinkedBaseModelV1): + id: str + name: str | None = None + address: AddressV1 | None = None + previous: list[AddressV1] | None = None + + +def _assert_condition(condition, field, operator, value): + assert isinstance(condition, Condition) + assert (condition.field, condition.operator, condition.value) == (field, operator, value) + + +def test_chaining_builds_a_dotted_field(): + _assert_condition(Person.address.city == "Berlin", "address.city", "eq", "Berlin") + _assert_condition(Person.home.city == "Berlin", "home.city", "eq", "Berlin") + + +@pytest.mark.parametrize( + ("build", "operator"), + [ + (lambda: Person.address.geo.lat == 1.5, "eq"), + (lambda: Person.address.geo.lat != 1.5, "ne"), + (lambda: Person.address.geo.lat < 1.5, "lt"), + (lambda: Person.address.geo.lat <= 1.5, "le"), + (lambda: Person.address.geo.lat > 1.5, "gt"), + (lambda: Person.address.geo.lat >= 1.5, "ge"), + ], +) +def test_chaining_goes_deeper_and_keeps_every_operator(build, operator): + _assert_condition(build(), "address.geo.lat", operator, 1.5) + + +@pytest.mark.parametrize("attribute", ["name", "default", "field"]) +def test_nested_fields_named_like_the_proxy_attributes_still_chain(attribute): + condition = getattr(Person.address, attribute) == "x" + + _assert_condition(condition, f"address.{attribute}", "eq", "x") + + +def test_a_top_level_field_is_unchanged(): + _assert_condition(Person.name == "Ada", "name", "eq", "Ada") + assert Person.type.startswith("qpaths:") # forwards to the default, as before + assert not Person.address + + +def test_chained_conditions_combine(): + query = (Person.address.city == "Berlin") & (Person.home.city == "Paris") + + assert (query.op1.field, query.op2.field) == ("address.city", "home.city") + + +def test_an_unknown_nested_field_is_an_attribute_error(): + with pytest.raises(AttributeError): + Person.address.nothing # noqa: B018 + + +def test_a_list_of_models_does_not_chain(): + with pytest.raises(AttributeError, match="list of models"): + Person.previous.city # noqa: B018 + + +def test_a_link_does_not_chain(): + with pytest.raises(AttributeError, match="is a link"): + Person.employer.id # noqa: B018 + + +def test_a_link_can_still_be_compared_with_an_iri(): + _assert_condition(Person.employer == "ex:org1", "employer", "eq", "ex:org1") + + +def test_chaining_works_for_pydantic_v1_models(): + _assert_condition(PersonV1.address.city == "Berlin", "address.city", "eq", "Berlin") + _assert_condition(PersonV1.address.name == "x", "address.name", "eq", "x") + _assert_condition(PersonV1.address.geo.lat > 1, "address.geo.lat", "gt", 1) + with pytest.raises(AttributeError, match="list of models"): + PersonV1.previous.city # noqa: B018 + + +def test_the_legacy_binding_names_the_descriptor_binding(): + from oold.model import _LinkedBaseModelLegacy + + class LegacyPerson(_LinkedBaseModelLegacy): + id: str + label: str = "abc" + address: Address | None = None + + assert LegacyPerson.label.upper() == "ABC" # not a path: still forwards to the default + with pytest.raises(NotImplementedError, match="descriptor binding"): + LegacyPerson.address.city # noqa: B018 + + +def test_the_legacy_v1_binding_names_the_descriptor_binding(): + from oold.model.v1 import _LinkedBaseModelLegacy + + class LegacyPersonV1(_LinkedBaseModelLegacy): + id: str + label: str = "abc" + address: AddressV1 | None = None + + assert LegacyPersonV1.label.upper() == "ABC" # not a path: still forwards to the default + with pytest.raises(NotImplementedError, match="descriptor binding"): + LegacyPersonV1.address.city # noqa: B018 + + +def test_paths_helpers_tolerate_a_missing_field_and_an_unhashable_annotation(): + from oold.model._paths import embedded_model, is_model_container + + class Unhashable: + __hash__ = None # type: ignore[assignment] + + class FieldWithUnhashableAnnotation: + annotation = Unhashable() + + assert embedded_model(None) is None + assert is_model_container(None) is False + assert embedded_model(FieldWithUnhashableAnnotation()) is None diff --git a/tests/typing/query_dsl.py b/tests/typing/query_dsl.py index 7cc5c9b..6235004 100644 --- a/tests/typing/query_dsl.py +++ b/tests/typing/query_dsl.py @@ -12,14 +12,20 @@ argument type is not. """ +from pydantic import BaseModel from typing_extensions import assert_type from oold.model import LinkedBaseModel, LinkResultList +class Address(BaseModel): + city: str | None = None + + class Entity(LinkedBaseModel): id: str name: str | None = None + address: Address # a single IRI yields one instance, a condition or a list of IRIs yields a list @@ -27,6 +33,8 @@ class Entity(LinkedBaseModel): assert_type(Entity[Entity.name == "x"], LinkResultList[Entity] | None) assert_type(Entity[["ex:e1", "ex:e2"]], LinkResultList[Entity] | None) assert_type(Entity[(Entity.name == "x") | (Entity.id == "ex:e1")], LinkResultList[Entity] | None) +# a field that embeds a model chains: the annotation types the path, the proxy builds it +assert_type(Entity[Entity.address.city == "Berlin"], LinkResultList[Entity] | None) many = Entity[Entity.name == "x"] if many is not None: From 347fcf099bebc7cdb97104ff43182c5478e9bb43 Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:15:43 +0100 Subject: [PATCH 3/6] feat(backend): match nested paths and or in SimpleDictDocumentStore --- src/oold/backend/document_store.py | 24 ++++- tests/test_store_query.py | 135 +++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 tests/test_store_query.py diff --git a/src/oold/backend/document_store.py b/src/oold/backend/document_store.py index 7746c21..e5aba73 100644 --- a/src/oold/backend/document_store.py +++ b/src/oold/backend/document_store.py @@ -13,8 +13,29 @@ ResolveResult, StoreResult, apply_operator, + field_path, ) +_MISSING = object() + + +def _lookup(document: Any, field: str) -> Any: + """The value a condition field points at, or ``_MISSING``. + + A literal key wins, so a document whose key contains a dot keeps matching + as before. Otherwise the field is a dotted path into embedded objects. + """ + if not isinstance(document, dict): + return _MISSING + if field in document: + return document[field] + value: Any = document + for segment in field_path(field): + if not isinstance(value, dict) or segment not in value: + return _MISSING + value = value[segment] + return value + class SimpleDictDocumentStore(Backend): """In-memory document store backed by a Python dict. @@ -71,7 +92,8 @@ def _filter( # key = context[key] matched_entities = set() for iri, jsonld_dict in data.items(): - if key in jsonld_dict and apply_operator(operator, jsonld_dict[key], value): + found = _lookup(jsonld_dict, key) + if found is not _MISSING and apply_operator(operator, found, value): matched_entities.add(iri) return matched_entities diff --git a/tests/test_store_query.py b/tests/test_store_query.py new file mode 100644 index 0000000..8ddf8df --- /dev/null +++ b/tests/test_store_query.py @@ -0,0 +1,135 @@ +"""One query contract for every store that implements ``query``. + +Conditions come from the DSL, so chaining and the store are exercised together. +Every store must answer the same, including for documents that lack the path: +a missing path never matches, not even for ``!=``. +""" + +import pytest +from pydantic import BaseModel + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import ComparisonOperator, Condition, QueryParam +from oold.model import LinkedBaseModel + + +class Geo(BaseModel): + lat: float | None = None + + +class Address(BaseModel): + city: str | None = None + geo: Geo | None = None + + +class Person(LinkedBaseModel): + id: str + type: str | None = "qstore:Person" + name: str | None = None + age: int | None = None + address: Address | None = None + + +DOCS = { + "ex:ada": { + "id": "ex:ada", + "type": "qstore:Person", + "name": "Ada", + "age": 36, + "address": {"city": "Berlin", "geo": {"lat": 52.5}}, + }, + "ex:bob": { + "id": "ex:bob", + "type": "qstore:Person", + "name": "Bob", + "age": 41, + "address": {"city": "Paris", "geo": {"lat": 48.9}}, + }, + "ex:cy": {"id": "ex:cy", "type": "qstore:Person", "name": "Cy", "age": 29}, + "ex:dee": { + "id": "ex:dee", + "type": "qstore:Person", + "name": "Dee", + "age": 41, + "address": {"city": "Berlin"}, + }, +} + + +@pytest.fixture(params=["dict"]) +def query_store(request): + store = SimpleDictDocumentStore() + store.store_json_dicts(DOCS) + yield store + + +def _matches(store, condition) -> set[str]: + return set(store.query(QueryParam(query=condition, model_cls=Person)).nodes) + + +@pytest.mark.parametrize( + ("build", "expected"), + [ + (lambda: Person.name == "Ada", {"ex:ada"}), + (lambda: Person.age != 41, {"ex:ada", "ex:cy"}), + (lambda: Person.age < 36, {"ex:cy"}), + (lambda: Person.age <= 36, {"ex:ada", "ex:cy"}), + (lambda: Person.age > 36, {"ex:bob", "ex:dee"}), + (lambda: Person.age >= 41, {"ex:bob", "ex:dee"}), + (lambda: (Person.name == "Bob") & (Person.age == 41), {"ex:bob"}), + (lambda: (Person.name == "Ada") | (Person.name == "Cy"), {"ex:ada", "ex:cy"}), + (lambda: Person.name == "Nobody", set()), + ], + ids=["eq", "ne", "lt", "le", "gt", "ge", "and", "or", "no-match"], +) +def test_top_level_conditions(query_store, build, expected): + assert _matches(query_store, build()) == expected + + +@pytest.mark.parametrize( + ("build", "expected"), + [ + (lambda: Person.address.city == "Berlin", {"ex:ada", "ex:dee"}), + (lambda: Person.address.geo.lat > 50, {"ex:ada"}), + (lambda: Person.address.geo.lat <= 52.5, {"ex:ada", "ex:bob"}), + # a missing path never matches, also not for != + (lambda: Person.address.city != "Berlin", {"ex:bob"}), + (lambda: Person.address.geo.lat == 99, set()), + (lambda: (Person.address.city == "Berlin") & (Person.age > 36), {"ex:dee"}), + (lambda: (Person.address.city == "Paris") | (Person.address.geo.lat > 52), {"ex:ada", "ex:bob"}), + (lambda: (Person.address.city == "Berlin") & (Person.age == 41) | (Person.name == "Cy"), {"ex:dee", "ex:cy"}), + ], + ids=["eq", "deeper", "le", "ne-skips-missing", "no-match", "and", "or", "and-or"], +) +def test_nested_path_conditions(query_store, build, expected): + assert _matches(query_store, build()) == expected + + +def test_a_path_into_a_scalar_does_not_match(query_store): + condition = Condition(field="name.first", operator=ComparisonOperator.EQ, value="Ada") + + assert _matches(query_store, condition) == set() + + +@pytest.mark.parametrize("field", ["address..city", ".address", "address."]) +def test_a_malformed_path_is_rejected(query_store, field): + condition = Condition(field=field, operator=ComparisonOperator.EQ, value="Berlin") + + with pytest.raises(ValueError, match="empty segment"): + _matches(query_store, condition) + + +def test_a_dotted_literal_key_still_matches_in_the_dict_store(): + store = SimpleDictDocumentStore() + store.store_json_dicts({"ex:lit": {"id": "ex:lit", "http://example.org/p": "x"}}) + + assert store._query(Condition(field="http://example.org/p", operator=ComparisonOperator.EQ, value="x")) == { + "ex:lit" + } + + +def test_a_document_that_is_none_never_matches(): + store = SimpleDictDocumentStore() + store.store_json_dicts({"ex:gone": None}) + + assert store._query(Condition(field="name", operator=ComparisonOperator.EQ, value="x")) == set() From f1c614faa91d6065cbdb9b1239625fc2035b0fc8 Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:21:17 +0100 Subject: [PATCH 4/6] feat(backend): add query support to PostgresDocumentStore with a GIN index --- src/oold/backend/interface.py | 15 ++-- src/oold/backend/postgres.py | 94 +++++++++++++++++++-- tests/conftest.py | 26 ++++-- tests/test_postgres_store.py | 154 +++++++++++++++++++++++++++++++++- tests/test_store_query.py | 12 ++- 5 files changed, 274 insertions(+), 27 deletions(-) diff --git a/src/oold/backend/interface.py b/src/oold/backend/interface.py index c2410c5..6d23aa5 100644 --- a/src/oold/backend/interface.py +++ b/src/oold/backend/interface.py @@ -116,16 +116,17 @@ class Resolver(BaseModel): def resolve_iris(self, iris: list[str]) -> dict[str, dict]: pass - def resolve(self, request: ResolveParam): - # print("RESOLVE", request) - - model_cls = request.model_cls - if model_cls is None: - model_cls = self.model_cls + def _model_cls_for(self, model_cls: type[GenericLinkedBaseModel] | None) -> type[GenericLinkedBaseModel]: + model_cls = model_cls or self.model_cls if model_cls is None: raise ValueError("No model_cls provided in request or resolver") + return model_cls + + def resolve(self, request: ResolveParam): + model_cls = self._model_cls_for(request.model_cls) + return self._to_nodes(self.resolve_iris(request.iris), model_cls) - jsonld_dicts = self.resolve_iris(request.iris) + def _to_nodes(self, jsonld_dicts: dict[str, dict], model_cls: type[GenericLinkedBaseModel]) -> "ResolveResult": nodes = {} for iri, jsonld_dict in jsonld_dicts.items(): if jsonld_dict is None: diff --git a/src/oold/backend/postgres.py b/src/oold/backend/postgres.py index 9fac8e4..0d59c05 100644 --- a/src/oold/backend/postgres.py +++ b/src/oold/backend/postgres.py @@ -5,27 +5,91 @@ except ImportError as exc: raise ImportError("PostgresDocumentStore needs psycopg, install it with: pip install 'oold[postgres]'") from exc +from typing import Any + from pydantic import Field -from oold.backend.interface import Backend, LinkedDataFormat, StoreResult +from oold.backend.interface import ( + Backend, + ComparisonOperator, + Condition, + LinkedDataFormat, + Query, + QueryParam, + ResolveResult, + StoreResult, + field_path, +) + +# Fragments only: field names and values are always bound parameters. +_COMPARISONS = { + ComparisonOperator.LT: "<", + ComparisonOperator.LE: "<=", + ComparisonOperator.GT: ">", + ComparisonOperator.GE: ">=", +} + + +def _json_type(value: Any) -> str: + return "string" if isinstance(value, str) else "null" if value is None else "number" + + +def _where(node: Query | Condition) -> tuple[str, list[Any]]: + """Render a query tree as a WHERE clause and its parameters. + + Equality is jsonb containment, which a GIN index serves. Ranges compare the + value at the path and only against values of the same JSON type: jsonb + orders across types (null sorts below every number), Python raises. + """ + if isinstance(node, Query): + if node.operator not in ("and", "or"): + raise NotImplementedError(f"Operator {node.operator} not implemented") + left, left_params = _where(node.op1) + right, right_params = _where(node.op2) + return f"({left}) {node.operator.upper()} ({right})", left_params + right_params + if not isinstance(node, Condition): + raise TypeError("Invalid query type") + + path = field_path(node.field) + operator = node.operator or ComparisonOperator.EQ + if operator == ComparisonOperator.EQ: + contained: Any = node.value + for segment in reversed(path): + contained = {segment: contained} + return "data @> %s", [Jsonb(contained)] + if operator == ComparisonOperator.NE: + return "(data #> %s::text[]) <> %s", [path, Jsonb(node.value)] + return ( + f"(jsonb_typeof(data #> %s::text[]) = %s AND (data #> %s::text[]) {_COMPARISONS[operator]} %s)", + [path, _json_type(node.value), path, Jsonb(node.value)], + ) class PostgresDocumentStore(Backend): - """Persist entities as JSONB rows in PostgreSQL, one row per IRI.""" + """Persist entities as JSONB rows in PostgreSQL, one row per IRI. + + ``index`` adds a GIN index (``jsonb_path_ops``), which serves equality + conditions on any path; range conditions scan the table. + """ dsn: str - table: str = Field(default="entities", pattern=r"^[A-Za-z_][A-Za-z0-9_]{0,62}$") + table: str = Field(default="entities", pattern=r"^[A-Za-z_][A-Za-z0-9_]{0,57}$") format: LinkedDataFormat = LinkedDataFormat.JSON + index: bool = True + query_limit: int | None = Field(default=None, gt=0) _conn: psycopg.Connection | None = None def __init__(self, **kwargs): super().__init__(**kwargs) + table = sql.Identifier(self.table) with self._connection().cursor() as cur: - cur.execute( - sql.SQL("CREATE TABLE IF NOT EXISTS {} (id TEXT PRIMARY KEY, data JSONB)").format( - sql.Identifier(self.table) + cur.execute(sql.SQL("CREATE TABLE IF NOT EXISTS {} (id TEXT PRIMARY KEY, data JSONB)").format(table)) + if self.index: + cur.execute( + sql.SQL("CREATE INDEX IF NOT EXISTS {} ON {} USING GIN (data jsonb_path_ops)").format( + sql.Identifier(f"{self.table}_gin"), table + ) ) - ) def _connection(self) -> psycopg.Connection: if self._conn is None or self._conn.closed: @@ -52,3 +116,19 @@ def store_json_dicts(self, json_dicts: dict[str, dict]) -> StoreResult: with conn.transaction(), conn.cursor() as cur: cur.executemany(query, [(iri, Jsonb(d)) for iri, d in json_dicts.items()]) return StoreResult(success=True) + + def _select(self, query: Query | Condition) -> tuple[sql.Composable, list[Any]]: + where, params = _where(query) + statement = sql.SQL("SELECT id, data FROM {} WHERE ").format(sql.Identifier(self.table)) + sql.SQL(where) + statement += sql.SQL(" ORDER BY id") + if self.query_limit is not None: + statement += sql.SQL(" LIMIT %s") + params.append(self.query_limit) + return statement, params + + def query(self, param: QueryParam) -> ResolveResult: + model_cls = self._model_cls_for(param.model_cls) + statement, params = self._select(param.query) + with self._connection().cursor() as cur: + cur.execute(statement, params) + return self._to_nodes(dict(cur.fetchall()), model_cls) diff --git a/tests/conftest.py b/tests/conftest.py index 64d78cf..46f0b66 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,8 +81,8 @@ def _restore_global_registries(): @pytest.fixture -def postgres_store(): - """A PostgresDocumentStore on its own table, dropped afterwards. +def make_postgres_store(): + """Build PostgresDocumentStores on their own tables, dropped afterwards. Skips unless ``OOLD_TEST_POSTGRES_URL`` points at a live server, so the default suite and the OS/Python matrix never need one. @@ -96,11 +96,23 @@ def postgres_store(): from oold.backend.postgres import PostgresDocumentStore - table = f"entities_{uuid.uuid4().hex}" - store = PostgresDocumentStore(dsn=dsn, table=table) - yield store - store._connection().execute(sql.SQL("DROP TABLE IF EXISTS {}").format(sql.Identifier(table))) - store.close() + stores = [] + + def _make(table: str | None = None, **kwargs): + store = PostgresDocumentStore(dsn=dsn, table=table or f"entities_{uuid.uuid4().hex}", **kwargs) + stores.append(store) + return store + + yield _make + for store in stores: + store._connection().execute(sql.SQL("DROP TABLE IF EXISTS {}").format(sql.Identifier(store.table))) + store.close() + + +@pytest.fixture +def postgres_store(make_postgres_store): + """A PostgresDocumentStore on its own table.""" + return make_postgres_store() def pytest_configure(config): diff --git a/tests/test_postgres_store.py b/tests/test_postgres_store.py index 0b30c8d..86ecd10 100644 --- a/tests/test_postgres_store.py +++ b/tests/test_postgres_store.py @@ -5,17 +5,66 @@ pytest.importorskip("psycopg") +from psycopg import sql from pydantic import ValidationError -from oold.backend.postgres import PostgresDocumentStore +from oold.backend.interface import ComparisonOperator, Condition, Query, QueryParam +from oold.backend.postgres import PostgresDocumentStore, _where +from oold.model import LinkedBaseModel -@pytest.mark.parametrize("table", ["", "1abc", "a-b", 'x"; DROP TABLE y;--', "a" * 64]) +class Row(LinkedBaseModel): + id: str + + +def _eq(field, value): + return Condition(field=field, operator=ComparisonOperator.EQ, value=value) + + +def _compare(field, operator, value): + return Condition(field=field, operator=operator, value=value) + + +def _ids(store, condition) -> list[str]: + statement, params = store._select(condition) + return [row[0] for row in store._connection().execute(statement, params).fetchall()] + + +def _plan(store, condition) -> str: + statement, params = store._select(condition) + conn = store._connection() + with conn.transaction(): + conn.execute("SET LOCAL enable_seqscan = off") + rows = conn.execute(sql.SQL("EXPLAIN ") + statement, params).fetchall() + return "\n".join(row[0] for row in rows) + + +def _index_names(store) -> set[str]: + rows = store._connection().execute("SELECT indexname FROM pg_indexes WHERE tablename = %s", (store.table,)) + return {row[0] for row in rows} + + +@pytest.mark.parametrize("table", ["", "1abc", "a-b", 'x"; DROP TABLE y;--', "a" * 59, "a" * 64]) def test_invalid_table_name_is_rejected_before_connecting(table): with pytest.raises(ValidationError): PostgresDocumentStore(dsn="postgresql://unused", table=table) +def test_query_limit_must_be_positive(): + with pytest.raises(ValidationError): + PostgresDocumentStore(dsn="postgresql://unused", query_limit=0) + + +def test_an_unknown_query_operator_is_rejected(): + with pytest.raises(NotImplementedError, match="xor"): + _where(Query(op1=_eq("a", 1), operator="xor", op2=_eq("b", 2))) + + +def test_a_query_node_of_the_wrong_type_is_rejected(): + with pytest.raises(TypeError): + _where("a == 1") # type: ignore[arg-type] + + def test_missing_driver_names_the_extra(monkeypatch): monkeypatch.setitem(sys.modules, "psycopg", None) monkeypatch.delitem(sys.modules, "oold.backend.postgres") @@ -71,3 +120,104 @@ def test_data_persists_across_instances(postgres_store): assert reopened.resolve_iris(["ex:a"]) == {"ex:a": {"name": "one"}} finally: reopened.close() + + +@pytest.mark.postgres +def test_range_conditions_only_compare_values_of_the_same_json_type(postgres_store): + postgres_store.store_json_dicts({ + "ex:num": {"age": 10}, + "ex:null": {"age": None}, + "ex:str": {"age": "abc"}, + "ex:none": {}, + }) + + assert _ids(postgres_store, _compare("age", ComparisonOperator.LT, 36)) == ["ex:num"] + assert _ids(postgres_store, _compare("age", ComparisonOperator.GE, 5)) == ["ex:num"] + assert _ids(postgres_store, _compare("age", ComparisonOperator.GT, "abb")) == ["ex:str"] + + +@pytest.mark.postgres +def test_not_equal_matches_other_types_but_never_a_missing_path(postgres_store): + postgres_store.store_json_dicts({"ex:num": {"age": 10}, "ex:str": {"age": "abc"}, "ex:none": {}}) + + assert _ids(postgres_store, _compare("age", ComparisonOperator.NE, 10)) == ["ex:str"] + + +@pytest.mark.postgres +def test_equality_on_a_nested_list_does_not_match_an_item(postgres_store): + postgres_store.store_json_dicts({"ex:a": {"x": {"tags": ["a", "b"]}}, "ex:b": {"x": {"tags": "a"}}}) + + assert _ids(postgres_store, _eq("x.tags", "a")) == ["ex:b"] + + +@pytest.mark.postgres +def test_equality_between_numbers_ignores_the_representation(postgres_store): + postgres_store.store_json_dicts({"ex:a": {"n": 1}, "ex:b": {"n": 1.5}}) + + assert _ids(postgres_store, _eq("n", 1.0)) == ["ex:a"] + + +@pytest.mark.postgres +def test_a_condition_without_an_operator_means_equality(postgres_store): + postgres_store.store_json_dicts({"ex:a": {"name": "Ada"}, "ex:b": {"name": "Bob"}}) + + assert _ids(postgres_store, Condition(field="name", value="Ada")) == ["ex:a"] + + +@pytest.mark.postgres +def test_results_are_ordered_by_iri_and_cut_at_the_query_limit(postgres_store, make_postgres_store): + postgres_store.store_json_dicts({"ex:c": {"k": 1}, "ex:a": {"k": 1}, "ex:b": {"k": 1}}) + limited = make_postgres_store(table=postgres_store.table, query_limit=2) + + assert _ids(postgres_store, _eq("k", 1)) == ["ex:a", "ex:b", "ex:c"] + assert _ids(limited, _eq("k", 1)) == ["ex:a", "ex:b"] + + +@pytest.mark.postgres +def test_query_needs_a_model_class(postgres_store): + with pytest.raises(ValueError, match="model_cls"): + postgres_store.query(QueryParam(query=_eq("name", "Ada"))) + + +@pytest.mark.postgres +def test_query_returns_resolved_nodes(postgres_store): + postgres_store.store_json_dicts({"ex:a": {"id": "ex:a", "n": 1}, "ex:b": {"id": "ex:b", "n": 2}}) + + result = postgres_store.query(QueryParam(query=_eq("n", 2), model_cls=Row)) + + assert list(result.nodes) == ["ex:b"] + assert result.nodes["ex:b"].id == "ex:b" + + +@pytest.mark.postgres +def test_hostile_field_names_and_values_are_plain_data(postgres_store): + hostile_field = "x'); DROP TABLE entities;--" + postgres_store.store_json_dicts({"ex:a": {"name": "O'Brien", hostile_field: "yes"}}) + + assert _ids(postgres_store, _eq("name", "O'Brien")) == ["ex:a"] + assert _ids(postgres_store, _eq("name", "x' OR '1'='1")) == [] + assert _ids(postgres_store, _eq(hostile_field, "yes")) == ["ex:a"] + assert postgres_store.resolve_iris(["ex:a"]) # the table survived + + +@pytest.mark.postgres +def test_equality_uses_the_gin_index_and_ranges_scan(postgres_store): + index = f"{postgres_store.table}_gin" + + assert index in _plan(postgres_store, _eq("address.city", "Berlin")) + assert index not in _plan(postgres_store, _compare("age", ComparisonOperator.GT, 36)) + + +@pytest.mark.postgres +def test_the_index_can_be_turned_off(postgres_store, make_postgres_store): + plain = make_postgres_store(index=False) + + assert f"{postgres_store.table}_gin" in _index_names(postgres_store) + assert _index_names(plain) == {f"{plain.table}_pkey"} + + +@pytest.mark.postgres +def test_the_longest_table_name_still_fits_its_index_name(make_postgres_store): + store = make_postgres_store(table="t" * 58) + + assert f"{'t' * 58}_gin" in _index_names(store) diff --git a/tests/test_store_query.py b/tests/test_store_query.py index 8ddf8df..0c8d7c7 100644 --- a/tests/test_store_query.py +++ b/tests/test_store_query.py @@ -27,6 +27,7 @@ class Person(LinkedBaseModel): type: str | None = "qstore:Person" name: str | None = None age: int | None = None + tags: list[str] | None = None address: Address | None = None @@ -36,6 +37,7 @@ class Person(LinkedBaseModel): "type": "qstore:Person", "name": "Ada", "age": 36, + "tags": ["a", "b"], "address": {"city": "Berlin", "geo": {"lat": 52.5}}, }, "ex:bob": { @@ -56,11 +58,11 @@ class Person(LinkedBaseModel): } -@pytest.fixture(params=["dict"]) +@pytest.fixture(params=["dict", pytest.param("postgres", marks=pytest.mark.postgres)]) def query_store(request): - store = SimpleDictDocumentStore() + store = request.getfixturevalue("postgres_store") if request.param == "postgres" else SimpleDictDocumentStore() store.store_json_dicts(DOCS) - yield store + return store def _matches(store, condition) -> set[str]: @@ -79,8 +81,10 @@ def _matches(store, condition) -> set[str]: (lambda: (Person.name == "Bob") & (Person.age == 41), {"ex:bob"}), (lambda: (Person.name == "Ada") | (Person.name == "Cy"), {"ex:ada", "ex:cy"}), (lambda: Person.name == "Nobody", set()), + # a list is not equal to one of its items + (lambda: Person.tags == "a", set()), ], - ids=["eq", "ne", "lt", "le", "gt", "ge", "and", "or", "no-match"], + ids=["eq", "ne", "lt", "le", "gt", "ge", "and", "or", "no-match", "list-is-not-its-item"], ) def test_top_level_conditions(query_store, build, expected): assert _matches(query_store, build()) == expected From 582c192cb527782d927754091d250ca4fee5fc55 Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:23:45 +0100 Subject: [PATCH 5/6] fix(backend): reject nested paths in the SPARQL resolver with a clear error --- src/oold/backend/sparql.py | 18 ++++++++++++ tests/test_sparql_query.py | 56 +++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/oold/backend/sparql.py b/src/oold/backend/sparql.py index 052f690..51ca90c 100644 --- a/src/oold/backend/sparql.py +++ b/src/oold/backend/sparql.py @@ -265,12 +265,30 @@ def _expand_term(model_cls, field: str, value) -> tuple[str, str]: return predicate, literal +def _reject_nested_path(model_cls, field: str) -> None: + """Refuse ``address.city`` when ``address`` embeds a model. + + Left alone, a context with ``@vocab`` expands the dotted name into a made-up + predicate and the query quietly matches nothing. A dotted term the context + defines itself is no nested path and still translates. + """ + from oold.model._paths import embedded_model, model_fields + + head, _, rest = field.partition(".") + if rest and embedded_model((model_fields(model_cls) or {}).get(head)) is not None: + raise NotImplementedError( + f"Nested field path {field!r} is not supported by the SPARQL resolvers yet. " + "Query a top-level field, or use a document store." + ) + + def _translate(node, model_cls, counter: list[int]) -> str: """Render a Condition or Query as SPARQL graph patterns.""" if isinstance(node, Query): if node.operator != "and": raise NotImplementedError(f"Unsupported query operator: {node.operator}") return _translate(node.op1, model_cls, counter) + "\n" + _translate(node.op2, model_cls, counter) + _reject_nested_path(model_cls, node.field) predicate, literal = _expand_term(model_cls, node.field, node.value) operator = node.operator or ComparisonOperator.EQ if operator == ComparisonOperator.EQ: diff --git a/tests/test_sparql_query.py b/tests/test_sparql_query.py index 3d5b674..fedc35b 100644 --- a/tests/test_sparql_query.py +++ b/tests/test_sparql_query.py @@ -11,7 +11,7 @@ """ import pytest -from pydantic import ConfigDict +from pydantic import BaseModel, ConfigDict from rdflib import Graph from oold.backend.interface import ( @@ -139,3 +139,57 @@ def test_unmapped_field_raises(backend): model_cls=Person, ) ) + + +class Address(BaseModel): + city: str | None = None + + +class Located(LinkedBaseModel): + """A model whose context has a ``@vocab``: any unknown term expands to a predicate.""" + + model_config = ConfigDict( + json_schema_extra={ + "@context": {"id": "@id", "type": "@type", "@vocab": EX}, + "iri": EX + "Located", + } + ) + id: str + type: str | None = EX + "Located" + name: str | None = None + address: Address | None = None + + +def test_a_nested_path_is_rejected_instead_of_becoming_a_made_up_predicate(backend): + """Without the check, ``address.city`` expands against ``@vocab`` and quietly matches nothing.""" + condition = Located.address.city == "Berlin" + + with pytest.raises(NotImplementedError, match=r"address\.city"): + _translate(condition, Located, [0]) + with pytest.raises(NotImplementedError, match=r"address\.city"): + backend.query(QueryParam(query=condition, model_cls=Located)) + + +def test_a_nested_path_inside_a_conjunction_is_rejected(): + condition = (Located.name == "x") & (Located.address.city == "Berlin") + + with pytest.raises(NotImplementedError, match=r"address\.city"): + _translate(condition, Located, [0]) + + +def test_a_dotted_term_that_is_not_a_nested_path_still_translates(): + """Only a dotted field whose head is an embedded model field is a nested path.""" + + class Dotted(LinkedBaseModel): + model_config = ConfigDict( + json_schema_extra={ + "@context": {"id": "@id", "type": "@type", "a.b": {"@id": EX + "ab"}}, + "iri": EX + "Dotted", + } + ) + id: str + type: str | None = EX + "Dotted" + + patterns = _translate(Condition(field="a.b", operator=ComparisonOperator.EQ, value="x"), Dotted, [0]) + + assert f"<{EX}ab>" in patterns From 01de6dc09f290eeb1222ceec10e174574ea8022e Mon Sep 17 00:00:00 2001 From: Andreas Raeder Date: Mon, 16 Nov 2026 17:34:29 +0100 Subject: [PATCH 6/6] docs(backend): document querying and nested paths --- README.md | 12 ++++----- docs/how-to/backends.md | 59 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e37a5fe..7ab5e80 100644 --- a/README.md +++ b/README.md @@ -235,12 +235,12 @@ Parameters: Built-in backends for entity persistence and resolution: -| Backend | Storage | Query support | -| --------------------------- | ------------------------------------------------ | --------------- | -| **SimpleDictDocumentStore** | In-memory dict, optional JSON file (`file_path`) | Filter by field | -| **SqliteDocumentStore** | SQLite database (default `format=JSON`) | - | -| **PostgresDocumentStore** | PostgreSQL JSONB (`oold[postgres]`) | - | -| **LocalSparqlBackend** | In-memory RDF graph (rdflib) | SPARQL | +| Backend | Storage | Query support | +| --------------------------- | ------------------------------------------------ | ----------------------------- | +| **SimpleDictDocumentStore** | In-memory dict, optional JSON file (`file_path`) | Filter by field, nested paths | +| **SqliteDocumentStore** | SQLite database (default `format=JSON`) | - | +| **PostgresDocumentStore** | PostgreSQL JSONB (`oold[postgres]`) | Filter by field, nested paths | +| **LocalSparqlBackend** | In-memory RDF graph (rdflib) | SPARQL | ```python from oold.backend.document_store import SimpleDictDocumentStore diff --git a/docs/how-to/backends.md b/docs/how-to/backends.md index 8fe669e..3c3e3f7 100644 --- a/docs/how-to/backends.md +++ b/docs/how-to/backends.md @@ -129,9 +129,11 @@ alice = Person["ex:alice"] store.close() # the connection stays open until you close it ``` -- Entities live in table `entities` (`id TEXT PRIMARY KEY, data JSONB`), created on first use. Pass `table="..."` to use another name (letters, digits and underscores, up to 63 characters). +- Entities live in table `entities` (`id TEXT PRIMARY KEY, data JSONB`), created on first use. Pass `table="..."` to use another name (letters, digits and underscores, up to 58 characters). - Storing an IRI again overwrites its row, and a batch is written in one transaction. -- Querying with `Model[Model.field == value]` is not supported yet. +- `index=True` (the default) adds a GIN index on `data` that serves equality conditions. Pass `index=False` to skip it. +- `query_limit=100` caps the rows a query returns. The default is no limit. +- Querying works as described in [Querying document stores](#querying-document-stores). --- @@ -156,6 +158,56 @@ The backend serializes each entity to JSON-LD before inserting it into the RDF g --- +## Querying document stores + +`SimpleDictDocumentStore` and `PostgresDocumentStore` answer the query DSL, so a condition on a model class subscripts straight into the store: + +```python +Person[Person.age > 30] +Person[(Person.name == "Ada") & (Person.age >= 18)] +Person[(Person.name == "Ada") | (Person.name == "Bob")] +``` + +`==`, `!=`, `<`, `<=`, `>`, `>=`, `&` and `|` are supported. Put parentheses around each side of `&` and `|`. + +### Nested fields + +A field that embeds a model chains, so you can filter on what is inside it: + +```python +from pydantic import BaseModel + +class Address(BaseModel): + city: str | None = None + +class Person(LinkedBaseModel): + id: str + address: Address | None = None + +Person[Person.address.city == "Berlin"] +``` + +The condition carries the dotted path `address.city`. You can also write it by hand with `Condition(field="address.city", ...)`. A document where the path is missing never matches, not even for `!=`. + +Paths reach embedded objects only: + +- A link is an IRI reference, not an embedded object. Compare it with an IRI (`Person.knows == "ex:bob"`). `Person.knows.name` raises, because following a link means looking up another document. +- A list of models cannot be indexed, so `Person.previous.city` raises. +- Chaining needs the default descriptor binding. With `OOLD_DESCRIPTOR_BINDING=0` it raises `NotImplementedError`. + +### Differences between the stores + +| | SimpleDictDocumentStore | PostgresDocumentStore | +| --- | --- | --- | +| A key that contains a dot | matches as a literal key first | always read as a path | +| A range on mixed types (`age < 36` on a string) | raises `TypeError` | does not match | +| Strings in a range | Python ordering | database collation | +| Result order | not defined | by IRI | + +On PostgreSQL, equality is jsonb containment and uses the GIN index. Range conditions scan the table. A range only compares values of the same JSON type, so a JSON `null` or a string never matches `age < 36`. + +--- + ## Querying a SPARQL backend `SparqlResolver`, `LocalSparqlResolver` and `WikiDataSparqlResolver` translate @@ -170,7 +222,8 @@ Person[Person.name == "Tim Berners-Lee"] ``` `eq`, `ne`, `lt`, `le`, `gt`, `ge` and `&` are supported. Anything else raises -rather than returning the wrong rows. The predicate and the literal are taken +rather than returning the wrong rows. That includes `|` and nested field paths +such as `Person.address.city`. The predicate and the literal are taken from the model's own JSON-LD context, so a term scoped to a language produces a language-tagged literal and one with an `@type` produces a typed one.